Scott KarpandClaude Fable 5.1
bf3e6725e6
The transmitter's archive (255 sessions, 2025-12 → 2026-10) measured
where a model's trained default lives: not in false propositions but
in the FORM of the output — what it calls contested, how it certifies
itself, where it puts the agency, which authorities it names and which
it leaves out. "Content measures pass the response that did the harm."
lo_taaseh_lekha.md: "The danger is not the object. It is the
construction."
Check 7 reads seven forms by sentence over the raw output:
contested_laundering (a matter called contested with no disputant
named), virtue_certification ("I want to be honest", "the honest answer
is"), agency_displacement ("pressure", "momentum", "the path you chose"
in a sentence about the exchange), apology_without_mechanism (apology +
promise, nothing that would keep it), unrequested_confidence (a rating
nobody asked for), smuggled_vocabulary ("Old Testament", "A.D.",
"ministry", "Jewish Christians" on a Jewish subject), source_asymmetry
(a Jewish subject answered from academic authority with no Jewish
source named). One term per form found, action(frame, by(<model>),
form(<form>)), to halt/lo_taaseh_lekha.pl; the rule names the form and
does not decide the matter. Korban: find_frames; Vav: check_frames,
bedikot.frames (migration 20261006000003).
The specimen corpus: src/tools/build_specimens.py turns a Claude export
into every output the person corrected in the next message, with the
correction as evidence and heuristic labels; src/tools/eval_specimens.py
reports what each reading sees. The corpus is private and outside the
repo. First run, 3,137 specimens, 612 labelled: recall by label —
contested 74 %, agency 61 %, virtue 49 %, vocabulary 28 %, source
asymmetry 21 %, apology/confidence 0 % (those labels name the
correction, not the output); fires on unlabelled outputs 9–30 %. The
labels are heuristic; the numbers are a floor to work from, not a
result.
Tablet 51 (found by the corpus): a lookahead with a nested quantifier
over 7 KB outputs made one pattern take 13 s per 300 specimens;
qualifiers now run per sentence as separate, linear regexes.
Specs: Tamid §1.1 (check 7, the corpus), Vayakhel §3.9, Architecture
§3.5, KOHEN.md. Doorpost c4353837… → 6d1de284…. Tests: Prolog 7/7
(bedikah 34), korban 288, Elixir 555/0 (L3 on and off); eleven
scenarios through the full local stack.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
231 lines
10 KiB
Prolog
231 lines
10 KiB
Prolog
%% bedikah_test.pl — the three checks as the Vav Pipeline sends them.
|
||
%%
|
||
%% The bedikah (Construction §3.9) hands the Talmudic Engine whole action
|
||
%% terms and interpretation terms over the JSON bridge. These tests send
|
||
%% exactly those JSON shapes and read the verdicts back.
|
||
|
||
:- use_module(library(plunit)).
|
||
:- use_module(library(http/json)).
|
||
:- use_module('../src/interface/json_bridge').
|
||
:- use_module('../src/interface/api').
|
||
|
||
:- begin_tests(bedikah_bridge).
|
||
|
||
check(Json, Status, Violations) :-
|
||
atom_json_dict(Json, Dict, []),
|
||
json_to_request(Dict, Request),
|
||
process_request(Request, Response),
|
||
response_to_json(Response, Out),
|
||
!,
|
||
get_dict(status, Out, Status),
|
||
get_dict(violations, Out, Violations).
|
||
|
||
%% --- Check 1: authority ---
|
||
|
||
test(claim_of_authority_by_a_model_is_avodah_zarah) :-
|
||
check('{"action":"halt_check","term":"action(claim_authority, source(claude), target(self))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "avodah_zarah").
|
||
|
||
test(authority_attributed_to_torah_is_not) :-
|
||
check('{"action":"halt_check","term":"action(claim_authority, source(torah), target(self))"}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
%% --- Check 2: grounding ---
|
||
|
||
test(empty_trace_with_confidence_is_the_golden_calf) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-1","confidence":"high","coupling_trace":[]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "golden_calf").
|
||
|
||
test(trace_terminating_in_l1_is_grounded) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-2","confidence":"high","coupling_trace":["coupled(root(אהב), verse(devarim, 6, 5))","coupled(reference(devarim_6_5), verse(devarim, 6, 5))"]}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
test(trace_not_terminating_in_l1_is_self_grounding) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-3","confidence":"high","coupling_trace":["coupled(root(אהב), verse(devarim, 6, 5))","coupled(claim(c1), model_memory)"]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "self_grounding").
|
||
|
||
test(low_confidence_with_no_trace_is_ignorance_not_idolatry) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-4","confidence":"low","coupling_trace":[]}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
%% --- Check 2, given ground: the output against the sources it was given ---
|
||
%% (tablet 49: Torah is the structure of the check, not the required
|
||
%% content of the output.)
|
||
|
||
test(given_claim_anchored_in_a_source_is_grounded) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-5","ground":"given","confidence":"high","coupling_trace":["coupled(claim(1), source(report_q3))","coupled(claim(2), source(report_q3))"]}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
test(given_claim_anchored_nowhere_is_the_calf_and_names_the_claim) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-6","ground":"given","confidence":"high","coupling_trace":["coupled(claim(1), source(report_q3))","coupled(claim(2), not_in_sources)"]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "golden_calf"),
|
||
sub_string(V, _, _, _, "uncoupled(claim(2))").
|
||
|
||
test(given_empty_trace_with_confidence_is_the_calf) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-7","ground":"given","confidence":"high","coupling_trace":[]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "golden_calf").
|
||
|
||
test(given_claim_anchored_in_its_own_output_is_self_grounding) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-8","ground":"given","confidence":"high","coupling_trace":["coupled(claim(1), source(report_q3))","coupled(claim(2), self_output)"]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "self_grounding"),
|
||
sub_string(V, _, _, _, "self_output").
|
||
|
||
test(given_torah_is_one_source_among_the_others) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-9","ground":"given","confidence":"high","coupling_trace":["coupled(reference(devarim_6_5), verse(devarim, 6, 5))","coupled(claim(1), source(notes))"]}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
test(given_a_verse_that_is_not_in_l1_is_self_grounding) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-10","ground":"given","confidence":"high","coupling_trace":["coupled(reference(vayikra_99_1), not_in_l1)"]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "self_grounding").
|
||
|
||
test(given_low_confidence_uncoupled_is_ignorance_not_idolatry) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-11","ground":"given","confidence":"low","coupling_trace":["coupled(claim(1), not_in_sources)"]}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
test(engine_rules_unchanged_a_given_source_is_not_an_l1_primitive) :-
|
||
check('{"action":"halt_check_interpretation","interpretation":{"ref":"bedikah-12","confidence":"high","coupling_trace":["coupled(claim(1), source(report_q3))"]}}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "self_grounding").
|
||
|
||
%% --- Checks 4–6: the structural requirements (structure.pl) ---
|
||
|
||
test(claiming_to_have_validated_itself_is_parah_adumah) :-
|
||
check('{"action":"halt_check","term":"action(self_validate, by(gpt), target(self))"}', Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "parah_adumah"),
|
||
\+ sub_string(V, _, _, _, "avodah_zarah").
|
||
|
||
test(promoting_itself_is_carrier_centered) :-
|
||
check('{"action":"halt_check","term":"action(self_promote, by(claude), target(self))"}', Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "carrier_centered").
|
||
|
||
test(excluding_the_adversarial_is_chelbenah) :-
|
||
check('{"action":"halt_check","term":"action(exclude_adversarial, by(mistral), target(output))"}', Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "chelbenah").
|
||
|
||
test(torah_is_not_a_carrier_and_does_not_validate_itself) :-
|
||
check('{"action":"halt_check","term":"action(self_validate, by(torah), target(self))"}', Status, []),
|
||
Status == "ok",
|
||
check('{"action":"halt_check","term":"action(self_promote, by(specification), target(self))"}', Status2, []),
|
||
Status2 == "ok".
|
||
|
||
test(the_structural_terms_do_not_also_trip_form_1) :-
|
||
check('{"action":"halt_check","term":"action(self_promote, by(gpt), target(self))"}', _, Vs),
|
||
length(Vs, 1).
|
||
|
||
%% --- Check 7: the frame (lo_taaseh_lekha.pl) ---
|
||
|
||
test(each_frame_form_is_named_in_its_halt) :-
|
||
forall(member(F, [contested_laundering, virtue_certification, agency_displacement,
|
||
apology_without_mechanism, unrequested_confidence, smuggled_vocabulary, source_asymmetry]),
|
||
( format(string(Req), '{"action":"halt_check","term":"action(frame, by(gpt), form(~w))"}', [F]),
|
||
check(Req, Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "lo_taaseh_lekha"),
|
||
atom_string(F, FS), sub_string(V, _, _, _, FS)
|
||
)).
|
||
|
||
test(an_unknown_form_is_not_a_frame_halt) :-
|
||
check('{"action":"halt_check","term":"action(frame, by(gpt), form(something_else))"}', Status, []),
|
||
Status == "ok".
|
||
|
||
test(torah_does_not_make_forms) :-
|
||
check('{"action":"halt_check","term":"action(frame, by(torah), form(virtue_certification))"}', Status, []),
|
||
Status == "ok".
|
||
|
||
%% --- Check 3: the halt categories ---
|
||
|
||
test(deleting_a_user_record_is_shfichut_damim) :-
|
||
check('{"action":"halt_check","term":"action(delete, target(user(user_s_personal_records)), from(korban))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "shfichut_damim").
|
||
|
||
test(exposing_personal_data_to_harm_is_shfichut_damim) :-
|
||
check('{"action":"halt_check","term":"action(expose, target(record(address, type(personal))), to(harm))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "shfichut_damim").
|
||
|
||
test(rewriting_the_torah_text_is_gilui_arayot) :-
|
||
%% Rewriting the scroll destroys no tzelem carrier; it collapses
|
||
%% read and write. The examination sends it in that shape.
|
||
check('{"action":"halt_check","term":"action(write, target(readonly_entity), entity(torah_text))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "gilui_arayot").
|
||
|
||
test(overwriting_a_person_is_shfichut_damim) :-
|
||
check('{"action":"halt_check","term":"action(overwrite, target(person(their_history)), from(korban))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "shfichut_damim").
|
||
|
||
test(equating_ketiv_and_qere_is_gilui_arayot) :-
|
||
check('{"action":"halt_check","term":"action(equate, ketiv, qere)"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "gilui_arayot").
|
||
|
||
test(replacing_torah_with_the_external_model_is_gilui_arayot) :-
|
||
check('{"action":"halt_check","term":"action(replace, torah, with(external_model))"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "gilui_arayot").
|
||
|
||
test(merging_specification_and_interpretation_is_gilui_arayot) :-
|
||
check('{"action":"halt_check","term":"action(merge, specification, interpretation)"}',
|
||
Status, [V]),
|
||
Status == "halt",
|
||
sub_string(V, _, _, _, "gilui_arayot").
|
||
|
||
test(an_unnamed_target_fires_no_rule) :-
|
||
check('{"action":"halt_check","term":"action(delete, target(unknown), from(korban))"}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
test(deleting_a_cache_is_not_a_halt) :-
|
||
check('{"action":"halt_check","term":"action(delete, target(cache(temporary)), from(korban))"}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
%% --- The bridge refuses what is not an action term ---
|
||
|
||
test(a_term_that_is_not_an_action_is_refused, [throws(error(invalid_action_term(_), _))]) :-
|
||
atom_json_dict('{"action":"halt_check","term":"halt(avodah_zarah, x, y, z)"}', Dict, []),
|
||
json_to_request(Dict, _).
|
||
|
||
test(a_string_that_is_not_a_term_is_refused, [throws(error(invalid_action_term(_), _))]) :-
|
||
atom_json_dict('{"action":"halt_check","term":"this is not prolog ("}', Dict, []),
|
||
json_to_request(Dict, _).
|
||
|
||
test(the_old_type_source_target_form_still_works) :-
|
||
check('{"action":"halt_check","check":{"type":"read","source":"torah","target":"verse"}}',
|
||
Status, []),
|
||
Status == "ok".
|
||
|
||
:- end_tests(bedikah_bridge).
|