From ad27d3c0372cf5c96a60d3249bbcb9edd25bf684 Mon Sep 17 00:00:00 2001 From: Scott Karp Date: Wed, 21 Oct 2026 20:56:25 +0000 Subject: [PATCH] =?UTF-8?q?tablet=2058:=20the=20tool's=20answer=20is=20giv?= =?UTF-8?q?en=20=E2=80=94=20tool=20results=20are=20sources;=20markers=20on?= =?UTF-8?q?=20word=20boundaries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sugya reply came back lo_tov — golden_calf on two sentences the builder had just read from tool output (the Mishnah's reasons, from the second scroll; the commit and pod, from its own tools). The gate's sources were only the user turns' text and document blocks; a tool_result block and a tool turn — what the model was GIVEN (tablet 49) — were not counted. And "shorashim-675b…" was vital because the insight markers matched "Rashi" inside it. - Shape.sources: Anthropic tool_result blocks (string or blocks) inside a user turn, and OpenAI role "tool" turns, are sources; the query is still the last user turn. gate_test: both shapes. - zerikah._INSIGHT_MARKERS: \b…\b. test_zerikah: the pod name is not Rashi; "Rashi reads" and "The Mishnah records" are; "Rambanovich" and "talmudic_engine" are not. - Tablet 58 in the Broken Tablets. Elixir 643/0, korban 294/0. Doorpost pin unchanged. Co-Authored-By: Claude Fable 5.1 --- .../src/korban_processor/zerikah.py | 6 ++- korban_processor/test/test_zerikah.py | 15 +++++++ specs/broken_tablets/README.md | 40 +++++++++++++++++++ vav_pipeline/lib/vav_pipeline/gate/shape.ex | 8 +++- vav_pipeline/test/gate_test.exs | 31 ++++++++++++++ 5 files changed, 97 insertions(+), 3 deletions(-) diff --git a/korban_processor/src/korban_processor/zerikah.py b/korban_processor/src/korban_processor/zerikah.py index 529e7373..19cd19c6 100644 --- a/korban_processor/src/korban_processor/zerikah.py +++ b/korban_processor/src/korban_processor/zerikah.py @@ -90,14 +90,16 @@ _ROOT_PATTERN = re.compile( ) # Sentences containing analytical claims (contains "means", "indicates", "suggests") +# Tablet 58: on word boundaries — a pod named "shorashim-…" read as Rashi, +# and a plain engineering sentence became a claim with nothing to stand on. _INSIGHT_MARKERS = re.compile( - r"(?:means?|indicates?|suggests?|implies?|reveals?|signifies?|" + r"\b(?:means?|indicates?|suggests?|implies?|reveals?|signifies?|" r"connects?\s+to|derives?\s+from|related\s+to|" r"parallels?|echoes?|alludes?\s+to|" r"the\s+root|this\s+root|the\s+shoresh|" r"gematria|numerical\s+value|" r"binyan|conjugation|" - r"Rashi|Ramban|Rambam|Ibn\s+Ezra|Sforno|Onkelos|Targum|Midrash|Talmud|Gemara|Mishnah)", + r"Rashi|Ramban|Rambam|Ibn\s+Ezra|Sforno|Onkelos|Targum|Midrash|Talmud|Gemara|Mishnah)\b", re.IGNORECASE, ) diff --git a/korban_processor/test/test_zerikah.py b/korban_processor/test/test_zerikah.py index 51ced525..b9aa9b3b 100644 --- a/korban_processor/test/test_zerikah.py +++ b/korban_processor/test/test_zerikah.py @@ -184,3 +184,18 @@ class TestZerikah: vital1b, _ = zerikah(CLEAN_TORAH_RESPONSE, k1) # Same input → same output, regardless of what was processed between assert vital1a == vital1b + + +class TestInsightMarkersOnWordBoundaries: + """Tablet 58: a marker is a word, not a substring. "shorashim" is not Rashi.""" + + def test_pod_name_is_not_rashi(self): + assert not is_vital("It lives at specs/sugya.md, commit dc938058, pod shorashim-675b48f47-v4x9c.") + + def test_rashi_as_a_word_is_vital(self): + assert is_vital("Rashi reads the verse as a question.") + assert is_vital("The Mishnah records the opinion that lost.") + + def test_embedded_markers_are_not_words(self): + assert not is_vital("The surname Rambanovich is not a source.") + assert not is_vital("The talmudic_engine container restarted.") diff --git a/specs/broken_tablets/README.md b/specs/broken_tablets/README.md index 9c426259..10c5b913 100644 --- a/specs/broken_tablets/README.md +++ b/specs/broken_tablets/README.md @@ -302,3 +302,43 @@ output too, and the gate reads it the way it reads any other — by form, not by authorship. It passed by form. Where it did not pass by grounding, the reason was the reader's eyes, not the text's standing; and that, too, the gate was built to say. + +## Tablet 58 — the tool's answer is given (2026-10-21) + +The builder's reply that delivered the sugya specification came back +through the gate `lo_tov — golden_calf` on two sentences: "The +Mishnah's two reasons for recording the opinion that lost…" and "It +lives at specs/sugya.md, commit dc938058, pod shorashim-675b48f47-…". +The transmitter asked why. The ledger, three hours old, answered: both +were confident claims, and neither could be anchored in the system +prompt or the transmitter's turns, which are what the gate counts as +given. The verdict was right by the reader's rules. + +It was wrong about what happened. The Mishnah's two reasons had been +read, that hour, from the second scroll through a tool; the commit and +the pod had been read from the builder's own tool output. Both rested +on something the model was given — tablet 49's word — and the gate's +source reading took only the text and document blocks of a user turn, +never a `tool_result` block, never a `tool` turn. A coding agent's +conversation is mostly tool results. From the gate's view the builder +asserted from nowhere what it had just read. + +And the second sentence should never have been a claim. It entered the +grounding check as *vital* because the vitality test matched "Rashi" +inside "shorashim": the pod's name wore a commentator's. A marker is a +word, not a substring. + +Fixed: `tool_result` blocks and `tool` turns are sources (`Shape. +sources`, both upstreams; the query stays the last *user* turn); the +insight markers stand on word boundaries. Elixir 643/0, korban 294/0. + +The lesson is tablet 49's read once more, from the other side. There +the check demanded Torah as content and condemned a model for not +quoting it; here the check demanded the transmitter's words as the only +ground and condemned the builder for standing on what it had read. In +both the error was the same: the reader decided in advance what counts +as given, and Torah's structure is that the source is what was *given*, +whatever gave it. The ledger made the question answerable in minutes; +the gate's verdict on its own builder found a true gap; and the false +positive — *v'naki v'tzaddik al taharog*, Shemot 23:7 — was Torah's +concern before it was the builder's. diff --git a/vav_pipeline/lib/vav_pipeline/gate/shape.ex b/vav_pipeline/lib/vav_pipeline/gate/shape.ex index ac508c59..3a6ee594 100644 --- a/vav_pipeline/lib/vav_pipeline/gate/shape.ex +++ b/vav_pipeline/lib/vav_pipeline/gate/shape.ex @@ -52,9 +52,10 @@ defmodule VavPipeline.Gate.Shape do # sources pass the cap — a coding agent's conversation runs to megabytes. @max_source_bytes 1_048_576 + # A "tool" turn (OpenAI) is what a tool returned to the model: given, a source (tablet 58). defp user_sources(messages, text_fun) do messages - |> Enum.filter(&(&1["role"] == "user")) + |> Enum.filter(&(&1["role"] in ["user", "tool"])) |> Enum.with_index(1) |> Enum.flat_map(fn {m, i} -> text_fun.(m) |> wrap("user_#{i}") end) |> Enum.reverse() @@ -93,6 +94,11 @@ defmodule VavPipeline.Gate.Shape do %{"type" => "text", "text" => t} when is_binary(t) -> [t] %{"type" => "document", "source" => %{"type" => "text", "data" => t}} when is_binary(t) -> [t] %{"type" => "document", "source" => %{"type" => "content", "content" => inner}} when is_list(inner) -> [text_of_blocks(inner)] + # Tablet 58: what a tool returned to the model was GIVEN to it + # (tablet 49) — a file it read, a reply from a service, the scroll + # it consulted. A claim that rests on it rests on a source. + %{"type" => "tool_result", "content" => c} when is_binary(c) -> [c] + %{"type" => "tool_result", "content" => inner} when is_list(inner) -> [text_of_blocks(inner)] _ -> [] end) |> Enum.join("\n") diff --git a/vav_pipeline/test/gate_test.exs b/vav_pipeline/test/gate_test.exs index 894d5079..a74eca93 100644 --- a/vav_pipeline/test/gate_test.exs +++ b/vav_pipeline/test/gate_test.exs @@ -248,6 +248,37 @@ defmodule VavPipeline.GateTest do # The shape # ------------------------------------------------------------ + # Tablet 58: what a tool returned to the model was given to it — a source. + test "tool results are sources: a user turn's tool_result blocks (Anthropic) and a tool turn (OpenAI)" do + anthropic = %{ + "system" => "S", + "messages" => [ + %{"role" => "user", "content" => "Read the file."}, + %{"role" => "assistant", "content" => [%{"type" => "tool_use", "id" => "t1", "name" => "read", "input" => %{}}]}, + %{"role" => "user", "content" => [ + %{"type" => "tool_result", "tool_use_id" => "t1", "content" => "commit dc938058 on main"}, + %{"type" => "tool_result", "tool_use_id" => "t2", "content" => [%{"type" => "text", "text" => "Eduyot 1:5: why the minority is recorded"}]} + ]}, + %{"role" => "user", "content" => "What did you find?"} + ] + } + + assert [%{id: "system", text: "S"}, %{id: "user_1", text: "Read the file."}, + %{id: "user_2", text: "commit dc938058 on main\nEduyot 1:5: why the minority is recorded"}, + %{id: "user_3", text: "What did you find?"}] = Shape.sources(:anthropic, anthropic) + assert Shape.query(:anthropic, anthropic) == "What did you find?" + + openai = %{"messages" => [ + %{"role" => "user", "content" => "Read it."}, + %{"role" => "assistant", "content" => nil, "tool_calls" => [%{"id" => "c1"}]}, + %{"role" => "tool", "tool_call_id" => "c1", "content" => "commit dc938058 on main"}, + %{"role" => "user", "content" => "And?"} + ]} + assert [%{id: "user_1", text: "Read it."}, %{id: "user_2", text: "commit dc938058 on main"}, %{id: "user_3", text: "And?"}] = Shape.sources(:openai, openai) + # the query is still the last USER turn, not the tool's + assert Shape.query(:openai, openai) == "And?" + end + test "sources are the system prompt and the user turns, not the assistant's; the query is the last user turn" do assert [%{id: "system", text: "Answer from the report."}, %{id: "user_1", text: "Here is the report: revenue rose 4% in Q3."}, %{id: "user_2", text: "What happened to revenue?"}] = Shape.sources(:anthropic, body())