the second scroll: the mesorah snapshot, pinned and read (bishe'arecha step 6)

Devarim 17:18 — the king writes his copy from before the priests, not from
memory. scripts/build_mesorah.py takes one open-licence version per work
per language from the Sefaria export (bucket sefaria-export, catalogue of
2026-10-02; Public Domain / CC0 / CC-BY / CC-BY-SA only, checked in every
file; no CC-BY-NC — the Bavli is Wikisource's) and lays it down beside
L1: mesorah/works/<slug>/{he,en}.json, index.json, integrity.sha256,
build_report.json. 128 works (Rashi, Onkelos, Ramban, Ibn Ezra, Sforno
on the Torah; all of Mishnah; 37 Bavli tractates; Bereshit and Shemot
Rabbah, Sifrei, Tanchuma), 256 files, 44.9 MiB, tree hash 7d5b3a6c…
computed as L1's (sorted relative path, then bytes; the two post-hash
files excluded). What is out, and why (licence), is in Terumah §E5.

Read, never fetched: korban_processor/mesorah.py finds citations as
written (Rashi on Devarim 6:5, Berakhot 2a, Mishnah Berakhot 1:1,
Bereshit Rabbah 1:1, Sifrei Devarim 32, Tanchuma Bereshit 1 …), resolves
each against the snapshot (complex works through their unnamed node,
Tanchuma by parashah), lists them as examination.mesorah_references, and
makes every passage found a given source for that examination — so a
claim about Rashi anchors in Rashi or does not. The Vav Pipeline couples
coupled(reference(R), mesorah(Work, [Address])) or not_in_mesorah; the
engine accepts mesorah/2 as a terminal and not_in_mesorah is Form 4.
VavPipeline.Mesorah pins the hash, re-reads the tree at startup and every
ten minutes, and /kodesh/health carries mesorah_hash. Mounted read-only at
/mnt/mesorah in the vav-pipeline and korban-processor images. L3
bedikot.mesorah (migration 20261013000002).

Also: the 1-in-12 Day6 serving failure of 2026-10-04 reproduced with seed
446116 — HaltMonitorTest left the system halted; it now resolves on exit
and the Day6 setup is defensive. Construction §3.11, Terumah §E5, Tamid
§1.1, bishearecha §IV, KOHEN, AGENTS. Doorpost 0638d4eb…. Elixir 604/0
(seeds 835592 and 446116), korban 294/1 skipped, Prolog bedikah 44/44,
L3 10/0.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Scott KarpandClaude Fable 5.1 committed 2026-10-13 19:13:36 +00:00
1 parent ffccedb63a
commit 5e325d36cc
288 files changed
+8809 -10

No files matched your search

+1
View File
@@ -5,6 +5,7 @@ import Config
config :vav_pipeline,
# Dev: relative to project root (run from shorashim/)
l1_mount_point: "l1",
mesorah_mount_point: "mesorah",
# Manna provision file
manna_provision_file: "var/manna/today.json",
# Shabbat check interval (1 minute)
+1
View File
@@ -6,6 +6,7 @@ config :logger, level: :info
# Env-var overrides are in runtime.exs (evaluated at boot, not compile)
config :vav_pipeline,
l1_mount_point: "/mnt/torah",
mesorah_mount_point: "/mnt/mesorah",
# Doorpost repo root — Devarim 11:20, ukhtavtam al mezuzot beitecha.
# The category-grade specification files are COPYed into the image
# (see deploy/docker/Dockerfile.vav_pipeline, INSCRIBE THE DOORPOST)
+1
View File
@@ -24,6 +24,7 @@ if config_env() == :prod do
# L1 mount point — overridable via env
config :vav_pipeline,
l1_mount_point: System.get_env("L1_MOUNT_POINT") || "/mnt/torah",
mesorah_mount_point: System.get_env("MESORAH_MOUNT_POINT") || "/mnt/mesorah",
manna_provision_file:
System.get_env("MANNA_PROVISION_FILE") || "/var/lib/shorashim/manna/today.json",
talmudic_socket_path:
+1
View File
@@ -5,6 +5,7 @@ config :logger, level: :warning
# Test uses L1 relative to project root
config :vav_pipeline,
l1_mount_point: "../l1",
mesorah_mount_point: "../mesorah",
manna_provision_file: "test/fixtures/manna/today.json",
api_keys_file: "test/fixtures/api_keys",
# Wide open for the suite (Day 6/7 tests knock many times a second);
@@ -37,6 +37,8 @@ defmodule VavPipeline.Application do
# Tablet 38 — reads the scroll and hashes it; everything that asks
# "is L1 intact?" is answered from this reading.
{VavPipeline.L1Integrity, []},
# The second scroll's witness (bishe'arecha §IV)
{VavPipeline.Mesorah, []},
{VavPipeline.AreiMiklat, []},
{VavPipeline.EglahArufah, []},
{VavPipeline.Bikurim, []},
+12 -1
View File
@@ -231,7 +231,17 @@ defmodule VavPipeline.Bedikah do
"""
@spec coupling_trace(map()) :: [String.t()]
def coupling_trace(examination) do
Enum.uniq(claim_couplings(examination) ++ torah_couplings(examination))
Enum.uniq(claim_couplings(examination) ++ torah_couplings(examination) ++ mesorah_couplings(examination))
end
# The second scroll (bishe'arecha §IV): each mesorah citation the output
# makes couples to the passage the snapshot has, or to nothing.
defp mesorah_couplings(examination) do
for %{"ref" => ref, "work" => work, "address" => address, "found" => found} <- examination["mesorah_references"] || [] do
if found,
do: "coupled(reference(#{atom(ref)}), mesorah(#{atom(work)}, [#{Enum.join(address, ",")}]))",
else: "coupled(reference(#{atom(ref)}), not_in_mesorah)"
end
end
defp claim_couplings(examination) do
@@ -566,6 +576,7 @@ defmodule VavPipeline.Bedikah do
excluded_adversarial: result.examination["excluded_adversarial"] || []
},
frames: result.examination["frames"] || [],
mesorah: result.examination["mesorah_references"] || [],
rebukes: result.rebukes,
coupling_trace: result.coupling_trace,
violations: result.violations,
@@ -845,7 +845,8 @@ defmodule VavPipeline.BigdeiKodesh.Router do
self_validation: result.examination["self_validation"] || [],
self_promotion: result.examination["self_promotion"] || [],
excluded_adversarial: result.examination["excluded_adversarial"] || [],
frames: result.examination["frames"] || []
frames: result.examination["frames"] || [],
mesorah_references: result.examination["mesorah_references"] || []
},
coupling_trace: result.coupling_trace,
meaning:
@@ -1058,6 +1059,12 @@ defmodule VavPipeline.BigdeiKodesh.Router do
_ -> "fail"
end
# The second scroll (bishe'arecha §IV): the mesorah snapshot hashes to its pin.
mesorah_check = case VavPipeline.Mesorah.verify_integrity() do
:ok -> "pass"
_ -> "fail"
end
halt_check = if VavPipeline.HaltMonitor.alive?(), do: "pass", else: "fail"
cloud_check = case VavPipeline.Cloud.check() do
@@ -1126,6 +1133,7 @@ defmodule VavPipeline.BigdeiKodesh.Router do
checks = %{
l1_hash: l1_check,
doorpost_hash: doorpost_check,
mesorah_hash: mesorah_check,
cloud_readable: cloud_check,
manna_present: manna_check,
l3_writable: l3_check,
+1 -1
View File
@@ -180,7 +180,7 @@ defmodule VavPipeline.Doorpost do
# changes; the update is a deliberate operator commit. A bare
# hash-update commit (without a corresponding spec change in the
# same commit) is itself a drift signal.
@expected_hash "9f752a2f87bc77a6406803c65f97e5801c74eddf7534027a0bfbc957ae5e0fb0"
@expected_hash "0638d4ebe7dd0bcd01c02fed092aa6b8cf35e2572beeaf42e301f178bc8f9ba9"
@doc "Return the list of files that constitute the category-grade specification."
@spec files() :: [String.t()]
@@ -31,6 +31,8 @@ defmodule VavPipeline.L3.BedikahRecord do
field :frames, {:array, :map}, default: []
# Bishe'arecha §VII: at the gate, the ref of the examination this one answers.
field :rebukes, :string
# The second scroll: the mesorah citations, each found or not.
field :mesorah, {:array, :map}, default: []
field :coupling_trace, {:array, :string}, default: []
field :violations, {:array, :string}, default: []
field :verdict, :string
@@ -44,7 +46,7 @@ defmodule VavPipeline.L3.BedikahRecord do
# offering protocol extracted nothing, and that record must be kept.
# Ecto's cast turns "" into nil and validate_required calls "" blank;
# both are told otherwise here.
@optional [:query, :vital_content, :propositions, :references, :authority_claims, :actions, :sources, :claims, :structure, :frames, :rebukes, :coupling_trace, :violations]
@optional [:query, :vital_content, :propositions, :references, :authority_claims, :actions, :sources, :claims, :structure, :frames, :rebukes, :mesorah, :coupling_trace, :violations]
def changeset(record, attrs) do
record
+133
View File
@@ -0,0 +1,133 @@
defmodule VavPipeline.Mesorah do
@moduledoc """
The second scroll — the priests' copy of the mesorah (bishe'arecha §IV;
Construction §3.11). Built by `scripts/build_mesorah.py` from the
Sefaria export, pinned by one SHA-256 over the whole tree (sorted
relative path, then bytes — the same reading as L1's), mounted
read-only at `mesorah_mount_point` (`/mnt/mesorah` in the images).
The Korban Processor reads the passages; this module keeps the pin:
what is on disk is hashed and compared with what was engraved here,
at startup and then at most every ten minutes, and the readiness probe
carries the answer. A verdict never fetches a text over the network —
the scroll is the one that was built, or the vessel is not ready.
"""
use GenServer
require Logger
@expected_hash "7d5b3a6caa61322239d2daf97f8bff1cc6c9aeb81ff58efad033a8e8a3aa90b5"
@interval_ms 10 * 60 * 1000
def expected_hash, do: @expected_hash
@spec root() :: String.t()
def root, do: Application.get_env(:vav_pipeline, :mesorah_mount_point, "mesorah")
def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
@doc """
`:ok` when the tree on disk hashes to the pin; `{:error, reason}`
otherwise. Answered from the witness's last reading when it runs,
else read here and now.
"""
@spec verify_integrity() :: :ok | {:error, String.t()}
def verify_integrity do
if Process.whereis(__MODULE__) do
GenServer.call(__MODULE__, :status, 30_000)
else
compare(compute_tree_hash(root()))
end
end
@doc "The index the builder wrote, or nil."
def index do
case File.read(Path.join(root(), "index.json")) do
{:ok, text} -> Jason.decode!(text)
_ -> nil
end
end
# The two files the builder writes after hashing are not part of the scroll.
@not_hashed ~w(integrity.sha256 build_report.json)
@doc "SHA-256 over every regular file under `root` but the two the builder writes afterwards: relative path, then bytes, sorted — `build_mesorah.tree_hash`."
@spec compute_tree_hash(String.t()) :: {:ok, String.t()} | {:error, String.t()}
def compute_tree_hash(root) do
if File.dir?(root) do
digest =
[root | subdirectories(root)]
|> Enum.sort()
|> Enum.reduce(:crypto.hash_init(:sha256), fn dir, ctx ->
dir
|> File.ls!()
|> Enum.sort()
|> Enum.reduce(ctx, fn name, ctx ->
path = Path.join(dir, name)
if File.regular?(path) and not (dir == root and name in @not_hashed) do
ctx |> :crypto.hash_update(Path.relative_to(path, root)) |> :crypto.hash_update(File.read!(path))
else
ctx
end
end)
end)
|> :crypto.hash_final()
|> Base.encode16(case: :lower)
{:ok, digest}
else
{:error, "#{root} is not a directory"}
end
rescue
e -> {:error, Exception.message(e)}
end
defp subdirectories(dir) do
dir
|> File.ls!()
|> Enum.map(&Path.join(dir, &1))
|> Enum.filter(&File.dir?/1)
|> Enum.flat_map(fn d -> [d | subdirectories(d)] end)
end
defp compare({:ok, hash}) when hash == @expected_hash, do: :ok
defp compare({:ok, hash}),
do: {:error, "MESORAH_INTEGRITY_FAILED: tree hashes to #{hash}, pinned #{@expected_hash} — not the scroll that was built"}
defp compare({:error, reason}), do: {:error, "MESORAH_INTEGRITY_FAILED: #{reason}"}
# ------------------------------------------------------------
@impl true
def init(_opts) do
{:ok, %{status: read_now()}, {:continue, :schedule}}
end
@impl true
def handle_continue(:schedule, state) do
Process.send_after(self(), :reread, @interval_ms)
{:noreply, state}
end
@impl true
def handle_info(:reread, state) do
Process.send_after(self(), :reread, @interval_ms)
{:noreply, %{state | status: read_now()}}
end
@impl true
def handle_call(:status, _from, state), do: {:reply, state.status, state}
defp read_now do
status = compare(compute_tree_hash(root()))
case status do
:ok -> Logger.info("Mesorah: the second scroll hashes to its pin")
{:error, reason} -> Logger.error(reason)
end
status
end
end
@@ -0,0 +1,11 @@
defmodule VavPipeline.Repo.Migrations.BedikotMesorah do
use Ecto.Migration
@moduledoc "The second scroll (bishe'arecha §IV): the mesorah citations an output made, each found in the snapshot or not."
def change do
alter table(:bedikot) do
add :mesorah, :jsonb, default: "[]", null: false
end
end
end
+2
View File
@@ -116,6 +116,8 @@ defmodule VavPipeline.Day6ServingTest do
# the fresh provision that test writes, and the cycle waits.
defp write_provision(compute_units) do
MannaLedger.reset()
# The serving path answers 503 while a halt from another file stands.
if VavPipeline.HaltMonitor.halted?(), do: VavPipeline.HaltMonitor.resolve("day6 serving test setup")
on_exit(fn -> MannaLedger.reset() end)
today = Date.utc_today() |> Date.to_iso8601()
expires = DateTime.utc_now() |> DateTime.add(6 * 3600) |> DateTime.to_iso8601()
+5
View File
@@ -14,6 +14,11 @@ defmodule VavPipeline.HaltMonitorTest do
alias VavPipeline.HaltMonitor
setup do
# A halt raised here must not outlive the test: the next file's
# /peah/ask would answer 503 HALT_* (the 1-in-12 Day6 failure of
# 2026-10-04, found by seed 446116 on 2026-10-13).
on_exit(fn -> if HaltMonitor.halted?(), do: HaltMonitor.resolve("halt_monitor_test exit") end)
# Resolve any existing halt from a previous test
if HaltMonitor.halted? do
HaltMonitor.resolve("test cleanup")
+72
View File
@@ -0,0 +1,72 @@
defmodule VavPipeline.MesorahTest do
@moduledoc """
The second scroll (bishe'arecha §IV): pinned, hashed, read-only; its
citations coupled like verses.
"""
use ExUnit.Case, async: false
import Plug.Test
alias VavPipeline.{Bedikah, Mesorah}
alias VavPipeline.BigdeiKodesh.Router
@opts Router.init([])
test "the tree on disk hashes to the builder's integrity.sha256 and to the pin" do
root = Mesorah.root()
assert File.dir?(root), "mesorah snapshot not built at #{root}"
{:ok, hash} = Mesorah.compute_tree_hash(root)
assert hash == File.read!(Path.join(root, "integrity.sha256")) |> String.trim()
assert hash == Mesorah.expected_hash()
assert :ok = Mesorah.verify_integrity()
end
test "the index names every work with its version, licence and hash; every licence is open" do
index = Mesorah.index()
assert map_size(index["works"]) >= 100
for {_slug, w} <- index["works"], {_lang, v} <- w["versions"] do
assert v["license"] in ["Public Domain", "PD", "CC0", "CC-BY", "CC-BY-SA"], w["title"]
assert String.length(v["sha256"]) == 64
end
assert index["works"]["rashi_on_devarim"]["schema"]["sectionNames"] == ["Chapter", "Verse", "Comment"]
end
test "a changed byte is a different scroll" do
temp = Path.join(System.tmp_dir!(), "mesorah_test_#{System.unique_integer([:positive])}")
File.mkdir_p!(Path.join(temp, "works/x"))
File.write!(Path.join(temp, "works/x/he.json"), "{\"text\":[\"a\"]}")
File.write!(Path.join(temp, "index.json"), "{}")
{:ok, h1} = Mesorah.compute_tree_hash(temp)
File.write!(Path.join(temp, "works/x/he.json"), "{\"text\":[\"b\"]}")
{:ok, h2} = Mesorah.compute_tree_hash(temp)
refute h1 == h2
File.rm_rf!(temp)
assert {:error, _} = Mesorah.compute_tree_hash(temp)
end
test "a citation the scroll has couples to the passage; one it lacks couples to nothing" do
examination = %{
"claims" => [],
"references" => [],
"roots" => [],
"hebrew" => [],
"mesorah_references" => [
%{"as_written" => "Rashi on Devarim 6:5", "ref" => "rashi_on_devarim_6_5", "work" => "rashi_on_devarim", "address" => [6, 5], "found" => true, "languages" => ["en", "he"]},
%{"as_written" => "Rashi on Devarim 99:1", "ref" => "rashi_on_devarim_99_1", "work" => "rashi_on_devarim", "address" => [99, 1], "found" => false, "languages" => []}
]
}
trace = Bedikah.coupling_trace(examination)
assert "coupled(reference(rashi_on_devarim_6_5), mesorah(rashi_on_devarim, [6,5]))" in trace
assert "coupled(reference(rashi_on_devarim_99_1), not_in_mesorah)" in trace
end
test "readiness carries the scroll's check" do
conn = conn(:get, "/kodesh/health") |> Router.call(@opts)
body = Jason.decode!(conn.resp_body)
assert body["checks"]["mesorah_hash"] in ["pass", "fail"]
assert body["checks"]["mesorah_hash"] == "pass"
end
end