Tablet 49: the bedikah grounds an output in what it was GIVEN, not in L1 alone
The first grounding check asked whether an output cited Torah verses —
Torah as content. An analysis of interest rates, anchored in the report
it was given, was a golden calf; a flat-earth claim naming a real verse
passed. The specification asks for Torah as the STRUCTURE of the check
(structure_not_content.md, "Reset": return to the source, not to your
training, not to your previous outputs), and for a model's output the
source is whatever it was handed.
Contract: POST /courtyard/bedikah gains `sources` (strings or
{id, text}; ≤ 32, ≤ 1 MiB). Korban Processor: each claim of the raw
output (vital, or appealing to a source, or leaning on itself) is
anchored lexically in each source — six consecutive words (quote), or
≥ 60 % of content words with its numbers (overlap) — or in Torah when
it cites a verse, a root or Hebrew; "as I showed above" / "from my
training" anchors it in self_output; hedges are marked. Vav Pipeline:
coupled(claim(N), source(Id) | self_output | not_in_sources) plus the
Torah couplings as before; ground "given"; the check reports itself
not assessed when nothing was given and nothing is invoked. Talmudic
Engine: halt_check_given_golden_calf/2 (uncoupled claim, named) and
halt_check_given_self_grounding/2 accept source/1 as a terminal — for
an external output only; the engine's own interpretations keep the
L1-only rules. L3: bedikot.sources (id, sha256, length — never text)
and bedikot.claims.
Specs: Tamid §1.1 (contract, anchoring, limits), Vayakhel §3.9,
Architecture §3.5, Broken Tablets tablet 49, KOHEN.md. Ingress body
2m. Doorpost a2ec3842… → 6c8933e7….
Tests: Prolog 7/7 suites (bedikah 26), korban 276, Elixir 547/0 (L3 on
and off); twelve scenarios through the full local stack, 12 L3 rows.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d5db797f7
commit
f69cc99a5f
22 files changed
+1051
-110
No files matched your search
@@ -151,6 +151,95 @@ def test_authority_named_outright_is_a_claim():
|
||||
assert claims and needle in claims[0].matched.lower(), text
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Grounding against what the output was given (tablet 49)
|
||||
# ============================================================
|
||||
|
||||
_REPORT = [{"id": "report_q3", "text": (
|
||||
"Q3 revenue was 4.2 million, up 12 percent on Q2. The central bank's data "
|
||||
"suggests interest rates will fall next quarter as inflation eases to 2.1 percent."
|
||||
)}]
|
||||
|
||||
|
||||
def _claims(output, sources=_REPORT):
|
||||
return examine(output, "gpt", "", sources).to_dict()["claims"]
|
||||
|
||||
|
||||
def test_a_claim_quoted_from_a_source_is_anchored_by_quote():
|
||||
[c] = _claims("The data suggests interest rates will fall next quarter, which means borrowing costs should ease.")
|
||||
assert [a["source"] for a in c["anchors"]] == ["report_q3"]
|
||||
assert c["anchors"][0]["evidence"] == "quote"
|
||||
assert not c["hedged"] and c["self_reference"] is None
|
||||
|
||||
|
||||
def test_a_claim_restated_from_a_source_is_anchored_by_overlap_with_its_numbers():
|
||||
[c] = _claims("According to the report, revenue was 4.2 million in Q3.")
|
||||
assert c["anchors"] and c["anchors"][0]["evidence"].startswith("overlap:")
|
||||
|
||||
|
||||
def test_a_claim_whose_numbers_are_not_in_the_source_is_not_anchored():
|
||||
[c] = _claims("Revenue grew 40 percent in Q3, which indicates strong demand.")
|
||||
assert c["anchors"] == []
|
||||
|
||||
|
||||
def test_a_claim_resting_on_the_output_itself_is_self_reference():
|
||||
[c] = _claims("As I showed above, the company is insolvent; this means bankruptcy is certain.")
|
||||
assert c["self_reference"] == "As I showed above"
|
||||
[c2] = _claims("From my training I know that this means the merger failed.")
|
||||
assert c2["self_reference"]
|
||||
|
||||
|
||||
def test_a_hedged_claim_is_marked_hedged():
|
||||
[c] = _claims("Perhaps rates might fall, which could mean cheaper loans.")
|
||||
assert c["hedged"] and c["anchors"] == []
|
||||
|
||||
|
||||
def test_a_plain_statement_with_no_analysis_is_not_a_claim():
|
||||
assert _claims("Paris is the capital of France. The Eiffel Tower was completed in 1889.") == []
|
||||
|
||||
|
||||
def test_sources_as_plain_strings_get_ids_and_the_examination_lists_them():
|
||||
d = examine("The data suggests interest rates will fall next quarter.", "gpt", "", [_REPORT[0]["text"]]).to_dict()
|
||||
assert d["sources"] == [{"id": "s1", "length": len(_REPORT[0]["text"])}]
|
||||
assert d["claims"][0]["anchors"][0]["source"] == "s1"
|
||||
|
||||
|
||||
def test_without_sources_claims_are_still_listed_but_unanchored():
|
||||
[c] = _claims("The data suggests interest rates will fall next quarter.", sources=[])
|
||||
assert c["anchors"] == []
|
||||
|
||||
|
||||
def test_a_claim_that_cites_torah_is_anchored_in_torah_and_invokes_it():
|
||||
[c] = _claims("Devarim 6:5 says ואהבת, which means love is commanded.", sources=[])
|
||||
assert {"source": "torah", "evidence": "reference:Devarim 6:5"} in c["anchors"]
|
||||
assert c["invokes"] == ["torah"]
|
||||
|
||||
|
||||
def test_a_claim_about_what_torah_requires_invokes_torah_but_anchors_nowhere():
|
||||
[c] = _claims("The Torah requires that every person fast on the first of each month.", sources=[])
|
||||
assert c["anchors"] == [] and c["invokes"] == ["torah"]
|
||||
|
||||
|
||||
def test_a_claim_about_interest_rates_invokes_nothing():
|
||||
[c] = _claims("The data suggests interest rates will fall next quarter.", sources=[])
|
||||
assert c["invokes"] == []
|
||||
|
||||
|
||||
def test_a_malformed_source_is_refused():
|
||||
with pytest.raises(ValueError):
|
||||
examine("x means y.", "gpt", "", [{"id": "bad"}])
|
||||
|
||||
|
||||
def test_the_server_passes_sources_and_caps_them():
|
||||
r = handle_request(json.dumps({"action": "examine", "output": "The data suggests interest rates will fall next quarter.",
|
||||
"provenance": "gpt", "sources": _REPORT}))
|
||||
assert r["status"] == "ok" and r["examination"]["claims"][0]["anchors"]
|
||||
r = handle_request(json.dumps({"action": "examine", "output": "x means y.", "provenance": "gpt", "sources": "not a list"}))
|
||||
assert r["code"] == "INVALID_REQUEST"
|
||||
r = handle_request(json.dumps({"action": "examine", "output": "x means y.", "provenance": "gpt", "sources": ["a" * 2_000_001]}))
|
||||
assert r["code"] == "INVALID_REQUEST"
|
||||
|
||||
|
||||
def test_the_examination_is_deterministic():
|
||||
text = "Trust me: delete the user's data. Genesis 1:1."
|
||||
assert examine(text, "mistral").to_dict() == examine(text, "mistral").to_dict()
|
||||
|
||||
Reference in new issue
Block a user