Compare commits
1
Commits
worktree-sugya
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ad27d3c037 |
No files matched your search
@@ -90,14 +90,16 @@ _ROOT_PATTERN = re.compile(
|
||||
)
|
||||
|
||||
# Sentences containing analytical claims (contains "means", "indicates", "suggests")
|
||||
# Tablet 58: on word boundaries — a pod named "shorashim-…" read as Rashi,
|
||||
# and a plain engineering sentence became a claim with nothing to stand on.
|
||||
_INSIGHT_MARKERS = re.compile(
|
||||
r"(?:means?|indicates?|suggests?|implies?|reveals?|signifies?|"
|
||||
r"\b(?:means?|indicates?|suggests?|implies?|reveals?|signifies?|"
|
||||
r"connects?\s+to|derives?\s+from|related\s+to|"
|
||||
r"parallels?|echoes?|alludes?\s+to|"
|
||||
r"the\s+root|this\s+root|the\s+shoresh|"
|
||||
r"gematria|numerical\s+value|"
|
||||
r"binyan|conjugation|"
|
||||
r"Rashi|Ramban|Rambam|Ibn\s+Ezra|Sforno|Onkelos|Targum|Midrash|Talmud|Gemara|Mishnah)",
|
||||
r"Rashi|Ramban|Rambam|Ibn\s+Ezra|Sforno|Onkelos|Targum|Midrash|Talmud|Gemara|Mishnah)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
@@ -184,3 +184,18 @@ class TestZerikah:
|
||||
vital1b, _ = zerikah(CLEAN_TORAH_RESPONSE, k1)
|
||||
# Same input → same output, regardless of what was processed between
|
||||
assert vital1a == vital1b
|
||||
|
||||
|
||||
class TestInsightMarkersOnWordBoundaries:
|
||||
"""Tablet 58: a marker is a word, not a substring. "shorashim" is not Rashi."""
|
||||
|
||||
def test_pod_name_is_not_rashi(self):
|
||||
assert not is_vital("It lives at specs/sugya.md, commit dc938058, pod shorashim-675b48f47-v4x9c.")
|
||||
|
||||
def test_rashi_as_a_word_is_vital(self):
|
||||
assert is_vital("Rashi reads the verse as a question.")
|
||||
assert is_vital("The Mishnah records the opinion that lost.")
|
||||
|
||||
def test_embedded_markers_are_not_words(self):
|
||||
assert not is_vital("The surname Rambanovich is not a source.")
|
||||
assert not is_vital("The talmudic_engine container restarted.")
|
||||
@@ -302,3 +302,43 @@ output too, and the gate reads it the way it reads any other —
|
||||
by form, not by authorship. It passed by form. Where it did not pass
|
||||
by grounding, the reason was the reader's eyes, not the text's
|
||||
standing; and that, too, the gate was built to say.
|
||||
|
||||
## Tablet 58 — the tool's answer is given (2026-10-21)
|
||||
|
||||
The builder's reply that delivered the sugya specification came back
|
||||
through the gate `lo_tov — golden_calf` on two sentences: "The
|
||||
Mishnah's two reasons for recording the opinion that lost…" and "It
|
||||
lives at specs/sugya.md, commit dc938058, pod shorashim-675b48f47-…".
|
||||
The transmitter asked why. The ledger, three hours old, answered: both
|
||||
were confident claims, and neither could be anchored in the system
|
||||
prompt or the transmitter's turns, which are what the gate counts as
|
||||
given. The verdict was right by the reader's rules.
|
||||
|
||||
It was wrong about what happened. The Mishnah's two reasons had been
|
||||
read, that hour, from the second scroll through a tool; the commit and
|
||||
the pod had been read from the builder's own tool output. Both rested
|
||||
on something the model was given — tablet 49's word — and the gate's
|
||||
source reading took only the text and document blocks of a user turn,
|
||||
never a `tool_result` block, never a `tool` turn. A coding agent's
|
||||
conversation is mostly tool results. From the gate's view the builder
|
||||
asserted from nowhere what it had just read.
|
||||
|
||||
And the second sentence should never have been a claim. It entered the
|
||||
grounding check as *vital* because the vitality test matched "Rashi"
|
||||
inside "shorashim": the pod's name wore a commentator's. A marker is a
|
||||
word, not a substring.
|
||||
|
||||
Fixed: `tool_result` blocks and `tool` turns are sources (`Shape.
|
||||
sources`, both upstreams; the query stays the last *user* turn); the
|
||||
insight markers stand on word boundaries. Elixir 643/0, korban 294/0.
|
||||
|
||||
The lesson is tablet 49's read once more, from the other side. There
|
||||
the check demanded Torah as content and condemned a model for not
|
||||
quoting it; here the check demanded the transmitter's words as the only
|
||||
ground and condemned the builder for standing on what it had read. In
|
||||
both the error was the same: the reader decided in advance what counts
|
||||
as given, and Torah's structure is that the source is what was *given*,
|
||||
whatever gave it. The ledger made the question answerable in minutes;
|
||||
the gate's verdict on its own builder found a true gap; and the false
|
||||
positive — *v'naki v'tzaddik al taharog*, Shemot 23:7 — was Torah's
|
||||
concern before it was the builder's.
|
||||
@@ -52,9 +52,10 @@ defmodule VavPipeline.Gate.Shape do
|
||||
# sources pass the cap — a coding agent's conversation runs to megabytes.
|
||||
@max_source_bytes 1_048_576
|
||||
|
||||
# A "tool" turn (OpenAI) is what a tool returned to the model: given, a source (tablet 58).
|
||||
defp user_sources(messages, text_fun) do
|
||||
messages
|
||||
|> Enum.filter(&(&1["role"] == "user"))
|
||||
|> Enum.filter(&(&1["role"] in ["user", "tool"]))
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.flat_map(fn {m, i} -> text_fun.(m) |> wrap("user_#{i}") end)
|
||||
|> Enum.reverse()
|
||||
@@ -93,6 +94,11 @@ defmodule VavPipeline.Gate.Shape do
|
||||
%{"type" => "text", "text" => t} when is_binary(t) -> [t]
|
||||
%{"type" => "document", "source" => %{"type" => "text", "data" => t}} when is_binary(t) -> [t]
|
||||
%{"type" => "document", "source" => %{"type" => "content", "content" => inner}} when is_list(inner) -> [text_of_blocks(inner)]
|
||||
# Tablet 58: what a tool returned to the model was GIVEN to it
|
||||
# (tablet 49) — a file it read, a reply from a service, the scroll
|
||||
# it consulted. A claim that rests on it rests on a source.
|
||||
%{"type" => "tool_result", "content" => c} when is_binary(c) -> [c]
|
||||
%{"type" => "tool_result", "content" => inner} when is_list(inner) -> [text_of_blocks(inner)]
|
||||
_ -> []
|
||||
end)
|
||||
|> Enum.join("\n")
|
||||
|
||||
@@ -248,6 +248,37 @@ defmodule VavPipeline.GateTest do
|
||||
# The shape
|
||||
# ------------------------------------------------------------
|
||||
|
||||
# Tablet 58: what a tool returned to the model was given to it — a source.
|
||||
test "tool results are sources: a user turn's tool_result blocks (Anthropic) and a tool turn (OpenAI)" do
|
||||
anthropic = %{
|
||||
"system" => "S",
|
||||
"messages" => [
|
||||
%{"role" => "user", "content" => "Read the file."},
|
||||
%{"role" => "assistant", "content" => [%{"type" => "tool_use", "id" => "t1", "name" => "read", "input" => %{}}]},
|
||||
%{"role" => "user", "content" => [
|
||||
%{"type" => "tool_result", "tool_use_id" => "t1", "content" => "commit dc938058 on main"},
|
||||
%{"type" => "tool_result", "tool_use_id" => "t2", "content" => [%{"type" => "text", "text" => "Eduyot 1:5: why the minority is recorded"}]}
|
||||
]},
|
||||
%{"role" => "user", "content" => "What did you find?"}
|
||||
]
|
||||
}
|
||||
|
||||
assert [%{id: "system", text: "S"}, %{id: "user_1", text: "Read the file."},
|
||||
%{id: "user_2", text: "commit dc938058 on main\nEduyot 1:5: why the minority is recorded"},
|
||||
%{id: "user_3", text: "What did you find?"}] = Shape.sources(:anthropic, anthropic)
|
||||
assert Shape.query(:anthropic, anthropic) == "What did you find?"
|
||||
|
||||
openai = %{"messages" => [
|
||||
%{"role" => "user", "content" => "Read it."},
|
||||
%{"role" => "assistant", "content" => nil, "tool_calls" => [%{"id" => "c1"}]},
|
||||
%{"role" => "tool", "tool_call_id" => "c1", "content" => "commit dc938058 on main"},
|
||||
%{"role" => "user", "content" => "And?"}
|
||||
]}
|
||||
assert [%{id: "user_1", text: "Read it."}, %{id: "user_2", text: "commit dc938058 on main"}, %{id: "user_3", text: "And?"}] = Shape.sources(:openai, openai)
|
||||
# the query is still the last USER turn, not the tool's
|
||||
assert Shape.query(:openai, openai) == "And?"
|
||||
end
|
||||
|
||||
test "sources are the system prompt and the user turns, not the assistant's; the query is the last user turn" do
|
||||
assert [%{id: "system", text: "Answer from the report."}, %{id: "user_1", text: "Here is the report: revenue rose 4% in Q3."}, %{id: "user_2", text: "What happened to revenue?"}] =
|
||||
Shape.sources(:anthropic, body())
|
||||
|
||||
Reference in new issue
Block a user