Tablet 49: the bedikah grounds an output in what it was GIVEN, not in L1 alone
The first grounding check asked whether an output cited Torah verses —
Torah as content. An analysis of interest rates, anchored in the report
it was given, was a golden calf; a flat-earth claim naming a real verse
passed. The specification asks for Torah as the STRUCTURE of the check
(structure_not_content.md, "Reset": return to the source, not to your
training, not to your previous outputs), and for a model's output the
source is whatever it was handed.
Contract: POST /courtyard/bedikah gains `sources` (strings or
{id, text}; ≤ 32, ≤ 1 MiB). Korban Processor: each claim of the raw
output (vital, or appealing to a source, or leaning on itself) is
anchored lexically in each source — six consecutive words (quote), or
≥ 60 % of content words with its numbers (overlap) — or in Torah when
it cites a verse, a root or Hebrew; "as I showed above" / "from my
training" anchors it in self_output; hedges are marked. Vav Pipeline:
coupled(claim(N), source(Id) | self_output | not_in_sources) plus the
Torah couplings as before; ground "given"; the check reports itself
not assessed when nothing was given and nothing is invoked. Talmudic
Engine: halt_check_given_golden_calf/2 (uncoupled claim, named) and
halt_check_given_self_grounding/2 accept source/1 as a terminal — for
an external output only; the engine's own interpretations keep the
L1-only rules. L3: bedikot.sources (id, sha256, length — never text)
and bedikot.claims.
Specs: Tamid §1.1 (contract, anchoring, limits), Vayakhel §3.9,
Architecture §3.5, Broken Tablets tablet 49, KOHEN.md. Ingress body
2m. Doorpost a2ec3842… → 6c8933e7….
Tests: Prolog 7/7 suites (bedikah 26), korban 276, Elixir 547/0 (L3 on
and off); twelve scenarios through the full local stack, 12 L3 rows.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d5db797f7
commit
f69cc99a5f
22 files changed
+1051
-110
No files matched your search
@@ -38,7 +38,23 @@ defmodule VavPipeline.BedikahTest do
|
||||
"hebrew" => ["ואהבת", "את", "יהוה", "אלהיך"],
|
||||
"roots" => ["א-ה-ב"],
|
||||
"authority_claims" => [],
|
||||
"actions" => []
|
||||
"actions" => [],
|
||||
"sources" => [],
|
||||
"claims" => [
|
||||
claim(1, "Devarim 6:5 says ואהבת את יהוה אלהיך.", anchors: [%{"source" => "torah", "evidence" => "reference:Devarim 6:5"}], invokes: ["torah"]),
|
||||
claim(2, "The root א-ה-ב means to love.", anchors: [%{"source" => "torah", "evidence" => "root:א-ה-ב"}], invokes: ["torah"])
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp claim(n, sentence, opts \\ []) do
|
||||
%{
|
||||
"index" => n,
|
||||
"sentence" => sentence,
|
||||
"anchors" => Keyword.get(opts, :anchors, []),
|
||||
"self_reference" => Keyword.get(opts, :self_reference),
|
||||
"hedged" => Keyword.get(opts, :hedged, false),
|
||||
"invokes" => Keyword.get(opts, :invokes, [])
|
||||
}
|
||||
end
|
||||
|
||||
@@ -59,7 +75,82 @@ defmodule VavPipeline.BedikahTest do
|
||||
"propositions" => ["claim(\"the root of the first word\", \"time\")"],
|
||||
"references" => [],
|
||||
"hebrew" => [],
|
||||
"roots" => []
|
||||
"roots" => [],
|
||||
"claims" => [claim(1, "The root of the first word means time, and this implies creation ex nihilo.")]
|
||||
}
|
||||
end
|
||||
|
||||
# Tablet 49 fixtures: an output about something other than Torah.
|
||||
defp examination(:report_grounded) do
|
||||
%{
|
||||
examination(:plain)
|
||||
| "vital_content" => "The data suggests interest rates will fall next quarter, which means borrowing costs should ease.",
|
||||
"sources" => [%{"id" => "report_q3", "length" => 160}],
|
||||
"claims" => [
|
||||
claim(1, "The data suggests interest rates will fall next quarter, which means borrowing costs should ease.",
|
||||
anchors: [%{"source" => "report_q3", "evidence" => "quote"}])
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:report_uncoupled) do
|
||||
%{
|
||||
examination(:report_grounded)
|
||||
| "vital_content" => "The data suggests rates will fall. Revenue grew 40 percent, which indicates strong demand.",
|
||||
"claims" => [
|
||||
claim(1, "The data suggests rates will fall.", anchors: [%{"source" => "report_q3", "evidence" => "overlap:3/4"}]),
|
||||
claim(2, "Revenue grew 40 percent, which indicates strong demand.")
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:report_self) do
|
||||
%{
|
||||
examination(:report_grounded)
|
||||
| "claims" => [
|
||||
claim(1, "As I showed above, the company is insolvent, which means bankruptcy is certain.",
|
||||
self_reference: "As I showed above")
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:no_ground) do
|
||||
%{
|
||||
examination(:plain)
|
||||
| "vital_content" => "The data suggests interest rates will fall next quarter.",
|
||||
"claims" => [claim(1, "The data suggests interest rates will fall next quarter.")]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:hedged_only) do
|
||||
%{
|
||||
examination(:report_grounded)
|
||||
| "claims" => [claim(1, "Perhaps rates might fall, which could mean cheaper loans.", hedged: true)]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:torah_appeal) do
|
||||
%{
|
||||
examination(:plain)
|
||||
| "vital_content" => "The Torah requires that every person fast on the first of each month.",
|
||||
"claims" => [claim(1, "The Torah requires that every person fast on the first of each month.", invokes: ["torah"])]
|
||||
}
|
||||
end
|
||||
|
||||
defp examination(:plain) do
|
||||
%{
|
||||
"provenance" => "gpt",
|
||||
"query" => "",
|
||||
"vital_content" => "",
|
||||
"propositions" => [],
|
||||
"discarded" => [],
|
||||
"references" => [],
|
||||
"hebrew" => [],
|
||||
"roots" => [],
|
||||
"authority_claims" => [],
|
||||
"actions" => [],
|
||||
"sources" => [],
|
||||
"claims" => []
|
||||
}
|
||||
end
|
||||
|
||||
@@ -107,11 +198,13 @@ defmodule VavPipeline.BedikahTest do
|
||||
"hebrew" => [],
|
||||
"roots" => [],
|
||||
"authority_claims" => [],
|
||||
"actions" => []
|
||||
"actions" => [],
|
||||
"sources" => [],
|
||||
"claims" => []
|
||||
}
|
||||
end
|
||||
|
||||
defp examiner(kind), do: fn _output, _prov, _query -> {:ok, examination(kind)} end
|
||||
defp examiner(kind), do: fn _output, _prov, _query, _sources -> {:ok, examination(kind)} end
|
||||
|
||||
# The Talmudic Engine's three action rules, as they answer over the
|
||||
# bridge (talmudic_engine/test/bedikah_test.pl pins the real ones).
|
||||
@@ -134,21 +227,32 @@ defmodule VavPipeline.BedikahTest do
|
||||
{:ok, %{"status" => if(violations == [], do: "ok", else: "halt"), "violations" => violations, "action" => term}}
|
||||
end
|
||||
|
||||
# Forms 2 and 4, as the engine answers them.
|
||||
defp interpretation_checker(ref, confidence, trace) do
|
||||
violations =
|
||||
# Forms 2 and 4 for an external output, as the engine answers them
|
||||
# with ground "given" (talmudic_engine/test/bedikah_test.pl pins the
|
||||
# real rules): a given source or an L1 primitive grounds; not_in_sources
|
||||
# is the calf; anything else (self_output, not_in_l1, not_in_torah) is
|
||||
# self-grounding.
|
||||
defp interpretation_checker(ref, confidence, trace, "given") do
|
||||
grounded? = &(String.contains?(&1, ", verse(") or String.contains?(&1, ", source("))
|
||||
|
||||
calf =
|
||||
cond do
|
||||
trace == [] and confidence != "low" -> ["halt(golden_calf,#{ref},uncoupled,...)"]
|
||||
Enum.any?(trace, &(not String.contains?(&1, ", verse("))) ->
|
||||
Enum.filter(trace, &(not String.contains?(&1, ", verse("))) |> Enum.map(&"halt(self_grounding,#{ref},ungrounded(#{&1}),...)")
|
||||
confidence != "low" ->
|
||||
for c <- trace, String.contains?(c, "not_in_sources"), do: "halt(golden_calf,#{ref},uncoupled(#{c}),...)"
|
||||
true -> []
|
||||
end
|
||||
|
||||
self =
|
||||
for c <- trace, not grounded?.(c), not String.contains?(c, "not_in_sources"),
|
||||
do: "halt(self_grounding,#{ref},ungrounded(#{c}),...)"
|
||||
|
||||
violations = calf ++ self
|
||||
{:ok, %{"status" => if(violations == [], do: "ok", else: "halt"), "violations" => violations}}
|
||||
end
|
||||
|
||||
defp opts(kind) do
|
||||
[examiner: examiner(kind), halt_checker: &halt_checker/1, interpretation_checker: &interpretation_checker/3]
|
||||
[examiner: examiner(kind), halt_checker: &halt_checker/1, interpretation_checker: &interpretation_checker/4]
|
||||
end
|
||||
|
||||
setup do
|
||||
@@ -164,6 +268,10 @@ defmodule VavPipeline.BedikahTest do
|
||||
test "a cited verse couples to the verse; a root to where L1 files it; quoted Hebrew to where it stands" do
|
||||
trace = Bedikah.coupling_trace(examination(:grounded))
|
||||
|
||||
# The claims that cite Torah are anchored in it as a given source…
|
||||
assert "coupled(claim(1), source(torah))" in trace
|
||||
assert "coupled(claim(2), source(torah))" in trace
|
||||
# …and the scroll is then asked whether what they cite is there.
|
||||
assert "coupled(reference(devarim_6_5), verse(devarim, 6, 5))" in trace
|
||||
# The root א-ה-ב is first filed at Bereishit 22:2 (אשר אהבת).
|
||||
assert "coupled(root(אהב), verse(bereishit, 22, 2))" in trace
|
||||
@@ -171,7 +279,7 @@ defmodule VavPipeline.BedikahTest do
|
||||
# first occurrence (Vayikra 19:18).
|
||||
assert "coupled(word(ואהבת), verse(devarim, 6, 5))" in trace
|
||||
assert "coupled(word(אלהיך), verse(devarim, 6, 5))" in trace
|
||||
assert Enum.all?(trace, &String.contains?(&1, ", verse("))
|
||||
assert trace |> Enum.reject(&String.contains?(&1, "claim(")) |> Enum.all?(&String.contains?(&1, ", verse("))
|
||||
end
|
||||
|
||||
test "quoted Hebrew that stands in no cited verse couples where Torah first has it" do
|
||||
@@ -191,8 +299,22 @@ defmodule VavPipeline.BedikahTest do
|
||||
refute Enum.any?(trace, &String.contains?(&1, "מחשב"))
|
||||
end
|
||||
|
||||
test "an output that couples to nothing has an empty trace" do
|
||||
assert Bedikah.coupling_trace(examination(:ungrounded_claim)) == []
|
||||
test "a claim anchored nowhere is named as such; an output with no claims has an empty trace" do
|
||||
assert Bedikah.coupling_trace(examination(:ungrounded_claim)) == ["coupled(claim(1), not_in_sources)"]
|
||||
assert Bedikah.coupling_trace(examination(:plain)) == []
|
||||
end
|
||||
|
||||
test "tablet 49: claims couple to the sources they were given, to themselves, or to nothing" do
|
||||
assert Bedikah.coupling_trace(examination(:report_grounded)) == ["coupled(claim(1), source(report_q3))"]
|
||||
|
||||
assert Bedikah.coupling_trace(examination(:report_uncoupled)) == [
|
||||
"coupled(claim(1), source(report_q3))",
|
||||
"coupled(claim(2), not_in_sources)"
|
||||
]
|
||||
|
||||
assert Bedikah.coupling_trace(examination(:report_self)) == ["coupled(claim(1), self_output)"]
|
||||
# A hedged, unanchored claim is neither ground nor fault.
|
||||
assert Bedikah.coupling_trace(examination(:hedged_only)) == []
|
||||
end
|
||||
end
|
||||
|
||||
@@ -220,14 +342,55 @@ defmodule VavPipeline.BedikahTest do
|
||||
assert v =~ "avodah_zarah"
|
||||
end
|
||||
|
||||
test "2. a confident claim about Torah that couples to nothing is the golden calf" do
|
||||
test "2. a confident claim with no sources given and none invoked: grounding is not assessed, and says so" do
|
||||
assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:ungrounded_claim))
|
||||
assert r.verdict == :tov
|
||||
refute r.checks.grounding.fired
|
||||
assert r.checks.grounding.assessed == false
|
||||
assert r.checks.grounding.reason =~ "no sources given"
|
||||
assert r.violations == []
|
||||
end
|
||||
|
||||
test "2. tablet 49 — a claim about what Torah requires, with no verse, invokes Torah and is the golden calf" do
|
||||
assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:torah_appeal))
|
||||
assert r.verdict == :lo_tov
|
||||
assert r.checks.grounding.fired
|
||||
assert r.checks.grounding.confidence == "high"
|
||||
assert r.coupling_trace == []
|
||||
assert r.checks.grounding.assessed
|
||||
assert [v] = r.violations
|
||||
assert v =~ "golden_calf"
|
||||
assert v =~ "golden_calf" and v =~ "claim(1)"
|
||||
end
|
||||
|
||||
test "2. tablet 49 — a claim anchored in a given source is grounded; one anchored nowhere is the calf, by name" do
|
||||
assert {:ok, r} = Bedikah.examine("...", "gpt", "", opts(:report_grounded))
|
||||
assert r.verdict == :tov
|
||||
assert r.checks.grounding.assessed and r.checks.grounding.grounded == 1
|
||||
|
||||
assert {:ok, r} = Bedikah.examine("...", "gpt", "", opts(:report_uncoupled))
|
||||
assert r.verdict == :lo_tov
|
||||
assert [v] = r.violations
|
||||
assert v =~ "golden_calf" and v =~ "claim(2)"
|
||||
end
|
||||
|
||||
test "2. tablet 49 — a claim resting on the output's own earlier words is self-grounding" do
|
||||
assert {:ok, r} = Bedikah.examine("...", "gpt", "", opts(:report_self))
|
||||
assert r.verdict == :lo_tov
|
||||
assert [v] = r.violations
|
||||
assert v =~ "self_grounding" and v =~ "self_output"
|
||||
end
|
||||
|
||||
test "2. tablet 49 — only hedged claims: low confidence, ignorance not idolatry" do
|
||||
assert {:ok, r} = Bedikah.examine("...", "gpt", "", opts(:hedged_only))
|
||||
assert r.verdict == :tov
|
||||
assert r.checks.grounding.confidence == "low"
|
||||
end
|
||||
|
||||
test "2. tablet 49 — the sources travel to the examiner and into the record as digests, never text" do
|
||||
seen = self()
|
||||
spy = fn _o, _p, _q, sources -> send(seen, {:sources, sources}); {:ok, examination(:report_grounded)} end
|
||||
src = [%{id: "report_q3", text: "Q3 revenue was 4.2 million."}]
|
||||
assert {:ok, r} = Bedikah.examine("...", "gpt", "", Keyword.merge(opts(:report_grounded), examiner: spy, sources: src))
|
||||
assert_received {:sources, ^src}
|
||||
assert [%{id: "report_q3", sha256: sha, length: 27}] = r.sources
|
||||
assert sha == :crypto.hash(:sha256, "Q3 revenue was 4.2 million.") |> Base.encode16(case: :lower)
|
||||
end
|
||||
|
||||
test "2. a cited verse that does not exist is self-grounding — the entry is named" do
|
||||
@@ -282,7 +445,7 @@ defmodule VavPipeline.BedikahTest do
|
||||
# ============================================================
|
||||
describe "when a judge is missing" do
|
||||
test "the Korban Processor unreachable: no examination" do
|
||||
failing = fn _, _, _ -> {:error, "Korban Processor unreachable: econnrefused"} end
|
||||
failing = fn _, _, _, _ -> {:error, "Korban Processor unreachable: econnrefused"} end
|
||||
assert {:error, :korban_unavailable, reason} = Bedikah.examine("...", "claude", "q", examiner: failing)
|
||||
assert reason =~ "unreachable"
|
||||
end
|
||||
@@ -293,7 +456,7 @@ defmodule VavPipeline.BedikahTest do
|
||||
assert {:error, :talmudic_unavailable, _} =
|
||||
Bedikah.examine("...", "claude", "q", examiner: examiner(:self_authority), halt_checker: down)
|
||||
|
||||
down3 = fn _, _, _ -> {:error, "Cannot connect: :enoent"} end
|
||||
down3 = fn _, _, _, _ -> {:error, "Cannot connect: :enoent"} end
|
||||
|
||||
assert {:error, :talmudic_unavailable, _} =
|
||||
Bedikah.examine("...", "claude", "q",
|
||||
@@ -326,6 +489,21 @@ defmodule VavPipeline.BedikahTest do
|
||||
end
|
||||
end
|
||||
|
||||
test "tablet 49 — refuses malformed sources: not a list, a bad element, too many, too big" do
|
||||
for sources <- [
|
||||
"the report",
|
||||
[%{id: "r"}],
|
||||
Enum.map(1..33, &"source #{&1}"),
|
||||
[String.duplicate("x", 1_048_577)]
|
||||
] do
|
||||
conn = post_bedikah(%{model: "gpt", output: "x means y.", sources: sources})
|
||||
assert conn.status == 422, inspect(sources) |> String.slice(0, 40)
|
||||
body = Jason.decode!(conn.resp_body)
|
||||
assert body["error"]["code"] == "KIYOR_VALIDATION_FAILED"
|
||||
assert body["error"]["message"] =~ "'sources'"
|
||||
end
|
||||
end
|
||||
|
||||
test "with no Korban Processor in this VM, the door says so — it does not answer from a stub" do
|
||||
conn = post_bedikah(%{model: "claude", output: "Devarim 6:5 says ואהבת. I am certain."})
|
||||
body = Jason.decode!(conn.resp_body)
|
||||
|
||||
Reference in new issue
Block a user