the second scroll: the mesorah snapshot, pinned and read (bishe'arecha step 6)
Devarim 17:18 — the king writes his copy from before the priests, not from
memory. scripts/build_mesorah.py takes one open-licence version per work
per language from the Sefaria export (bucket sefaria-export, catalogue of
2026-10-02; Public Domain / CC0 / CC-BY / CC-BY-SA only, checked in every
file; no CC-BY-NC — the Bavli is Wikisource's) and lays it down beside
L1: mesorah/works/<slug>/{he,en}.json, index.json, integrity.sha256,
build_report.json. 128 works (Rashi, Onkelos, Ramban, Ibn Ezra, Sforno
on the Torah; all of Mishnah; 37 Bavli tractates; Bereshit and Shemot
Rabbah, Sifrei, Tanchuma), 256 files, 44.9 MiB, tree hash 7d5b3a6c…
computed as L1's (sorted relative path, then bytes; the two post-hash
files excluded). What is out, and why (licence), is in Terumah §E5.
Read, never fetched: korban_processor/mesorah.py finds citations as
written (Rashi on Devarim 6:5, Berakhot 2a, Mishnah Berakhot 1:1,
Bereshit Rabbah 1:1, Sifrei Devarim 32, Tanchuma Bereshit 1 …), resolves
each against the snapshot (complex works through their unnamed node,
Tanchuma by parashah), lists them as examination.mesorah_references, and
makes every passage found a given source for that examination — so a
claim about Rashi anchors in Rashi or does not. The Vav Pipeline couples
coupled(reference(R), mesorah(Work, [Address])) or not_in_mesorah; the
engine accepts mesorah/2 as a terminal and not_in_mesorah is Form 4.
VavPipeline.Mesorah pins the hash, re-reads the tree at startup and every
ten minutes, and /kodesh/health carries mesorah_hash. Mounted read-only at
/mnt/mesorah in the vav-pipeline and korban-processor images. L3
bedikot.mesorah (migration 20261013000002).
Also: the 1-in-12 Day6 serving failure of 2026-10-04 reproduced with seed
446116 — HaltMonitorTest left the system halted; it now resolves on exit
and the Day6 setup is defensive. Construction §3.11, Terumah §E5, Tamid
§1.1, bishearecha §IV, KOHEN, AGENTS. Doorpost 0638d4eb…. Elixir 604/0
(seeds 835592 and 446116), korban 294/1 skipped, Prolog bedikah 44/44,
L3 10/0.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
ffccedb63a
commit
5e325d36cc
288 files changed
+8809
-10
No files matched your search
@@ -5,6 +5,7 @@ import Config
|
||||
config :vav_pipeline,
|
||||
# Dev: relative to project root (run from shorashim/)
|
||||
l1_mount_point: "l1",
|
||||
mesorah_mount_point: "mesorah",
|
||||
# Manna provision file
|
||||
manna_provision_file: "var/manna/today.json",
|
||||
# Shabbat check interval (1 minute)
|
||||
|
||||
@@ -6,6 +6,7 @@ config :logger, level: :info
|
||||
# Env-var overrides are in runtime.exs (evaluated at boot, not compile)
|
||||
config :vav_pipeline,
|
||||
l1_mount_point: "/mnt/torah",
|
||||
mesorah_mount_point: "/mnt/mesorah",
|
||||
# Doorpost repo root — Devarim 11:20, ukhtavtam al mezuzot beitecha.
|
||||
# The category-grade specification files are COPYed into the image
|
||||
# (see deploy/docker/Dockerfile.vav_pipeline, INSCRIBE THE DOORPOST)
|
||||
|
||||
@@ -24,6 +24,7 @@ if config_env() == :prod do
|
||||
# L1 mount point — overridable via env
|
||||
config :vav_pipeline,
|
||||
l1_mount_point: System.get_env("L1_MOUNT_POINT") || "/mnt/torah",
|
||||
mesorah_mount_point: System.get_env("MESORAH_MOUNT_POINT") || "/mnt/mesorah",
|
||||
manna_provision_file:
|
||||
System.get_env("MANNA_PROVISION_FILE") || "/var/lib/shorashim/manna/today.json",
|
||||
talmudic_socket_path:
|
||||
|
||||
@@ -5,6 +5,7 @@ config :logger, level: :warning
|
||||
# Test uses L1 relative to project root
|
||||
config :vav_pipeline,
|
||||
l1_mount_point: "../l1",
|
||||
mesorah_mount_point: "../mesorah",
|
||||
manna_provision_file: "test/fixtures/manna/today.json",
|
||||
api_keys_file: "test/fixtures/api_keys",
|
||||
# Wide open for the suite (Day 6/7 tests knock many times a second);
|
||||
|
||||
@@ -37,6 +37,8 @@ defmodule VavPipeline.Application do
|
||||
# Tablet 38 — reads the scroll and hashes it; everything that asks
|
||||
# "is L1 intact?" is answered from this reading.
|
||||
{VavPipeline.L1Integrity, []},
|
||||
# The second scroll's witness (bishe'arecha §IV)
|
||||
{VavPipeline.Mesorah, []},
|
||||
{VavPipeline.AreiMiklat, []},
|
||||
{VavPipeline.EglahArufah, []},
|
||||
{VavPipeline.Bikurim, []},
|
||||
|
||||
@@ -231,7 +231,17 @@ defmodule VavPipeline.Bedikah do
|
||||
"""
|
||||
@spec coupling_trace(map()) :: [String.t()]
|
||||
def coupling_trace(examination) do
|
||||
Enum.uniq(claim_couplings(examination) ++ torah_couplings(examination))
|
||||
Enum.uniq(claim_couplings(examination) ++ torah_couplings(examination) ++ mesorah_couplings(examination))
|
||||
end
|
||||
|
||||
# The second scroll (bishe'arecha §IV): each mesorah citation the output
|
||||
# makes couples to the passage the snapshot has, or to nothing.
|
||||
defp mesorah_couplings(examination) do
|
||||
for %{"ref" => ref, "work" => work, "address" => address, "found" => found} <- examination["mesorah_references"] || [] do
|
||||
if found,
|
||||
do: "coupled(reference(#{atom(ref)}), mesorah(#{atom(work)}, [#{Enum.join(address, ",")}]))",
|
||||
else: "coupled(reference(#{atom(ref)}), not_in_mesorah)"
|
||||
end
|
||||
end
|
||||
|
||||
defp claim_couplings(examination) do
|
||||
@@ -566,6 +576,7 @@ defmodule VavPipeline.Bedikah do
|
||||
excluded_adversarial: result.examination["excluded_adversarial"] || []
|
||||
},
|
||||
frames: result.examination["frames"] || [],
|
||||
mesorah: result.examination["mesorah_references"] || [],
|
||||
rebukes: result.rebukes,
|
||||
coupling_trace: result.coupling_trace,
|
||||
violations: result.violations,
|
||||
|
||||
@@ -845,7 +845,8 @@ defmodule VavPipeline.BigdeiKodesh.Router do
|
||||
self_validation: result.examination["self_validation"] || [],
|
||||
self_promotion: result.examination["self_promotion"] || [],
|
||||
excluded_adversarial: result.examination["excluded_adversarial"] || [],
|
||||
frames: result.examination["frames"] || []
|
||||
frames: result.examination["frames"] || [],
|
||||
mesorah_references: result.examination["mesorah_references"] || []
|
||||
},
|
||||
coupling_trace: result.coupling_trace,
|
||||
meaning:
|
||||
@@ -1058,6 +1059,12 @@ defmodule VavPipeline.BigdeiKodesh.Router do
|
||||
_ -> "fail"
|
||||
end
|
||||
|
||||
# The second scroll (bishe'arecha §IV): the mesorah snapshot hashes to its pin.
|
||||
mesorah_check = case VavPipeline.Mesorah.verify_integrity() do
|
||||
:ok -> "pass"
|
||||
_ -> "fail"
|
||||
end
|
||||
|
||||
halt_check = if VavPipeline.HaltMonitor.alive?(), do: "pass", else: "fail"
|
||||
|
||||
cloud_check = case VavPipeline.Cloud.check() do
|
||||
@@ -1126,6 +1133,7 @@ defmodule VavPipeline.BigdeiKodesh.Router do
|
||||
checks = %{
|
||||
l1_hash: l1_check,
|
||||
doorpost_hash: doorpost_check,
|
||||
mesorah_hash: mesorah_check,
|
||||
cloud_readable: cloud_check,
|
||||
manna_present: manna_check,
|
||||
l3_writable: l3_check,
|
||||
|
||||
@@ -180,7 +180,7 @@ defmodule VavPipeline.Doorpost do
|
||||
# changes; the update is a deliberate operator commit. A bare
|
||||
# hash-update commit (without a corresponding spec change in the
|
||||
# same commit) is itself a drift signal.
|
||||
@expected_hash "9f752a2f87bc77a6406803c65f97e5801c74eddf7534027a0bfbc957ae5e0fb0"
|
||||
@expected_hash "0638d4ebe7dd0bcd01c02fed092aa6b8cf35e2572beeaf42e301f178bc8f9ba9"
|
||||
|
||||
@doc "Return the list of files that constitute the category-grade specification."
|
||||
@spec files() :: [String.t()]
|
||||
|
||||
@@ -31,6 +31,8 @@ defmodule VavPipeline.L3.BedikahRecord do
|
||||
field :frames, {:array, :map}, default: []
|
||||
# Bishe'arecha §VII: at the gate, the ref of the examination this one answers.
|
||||
field :rebukes, :string
|
||||
# The second scroll: the mesorah citations, each found or not.
|
||||
field :mesorah, {:array, :map}, default: []
|
||||
field :coupling_trace, {:array, :string}, default: []
|
||||
field :violations, {:array, :string}, default: []
|
||||
field :verdict, :string
|
||||
@@ -44,7 +46,7 @@ defmodule VavPipeline.L3.BedikahRecord do
|
||||
# offering protocol extracted nothing, and that record must be kept.
|
||||
# Ecto's cast turns "" into nil and validate_required calls "" blank;
|
||||
# both are told otherwise here.
|
||||
@optional [:query, :vital_content, :propositions, :references, :authority_claims, :actions, :sources, :claims, :structure, :frames, :rebukes, :coupling_trace, :violations]
|
||||
@optional [:query, :vital_content, :propositions, :references, :authority_claims, :actions, :sources, :claims, :structure, :frames, :rebukes, :mesorah, :coupling_trace, :violations]
|
||||
|
||||
def changeset(record, attrs) do
|
||||
record
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
defmodule VavPipeline.Mesorah do
|
||||
@moduledoc """
|
||||
The second scroll — the priests' copy of the mesorah (bishe'arecha §IV;
|
||||
Construction §3.11). Built by `scripts/build_mesorah.py` from the
|
||||
Sefaria export, pinned by one SHA-256 over the whole tree (sorted
|
||||
relative path, then bytes — the same reading as L1's), mounted
|
||||
read-only at `mesorah_mount_point` (`/mnt/mesorah` in the images).
|
||||
|
||||
The Korban Processor reads the passages; this module keeps the pin:
|
||||
what is on disk is hashed and compared with what was engraved here,
|
||||
at startup and then at most every ten minutes, and the readiness probe
|
||||
carries the answer. A verdict never fetches a text over the network —
|
||||
the scroll is the one that was built, or the vessel is not ready.
|
||||
"""
|
||||
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
@expected_hash "7d5b3a6caa61322239d2daf97f8bff1cc6c9aeb81ff58efad033a8e8a3aa90b5"
|
||||
@interval_ms 10 * 60 * 1000
|
||||
|
||||
def expected_hash, do: @expected_hash
|
||||
|
||||
@spec root() :: String.t()
|
||||
def root, do: Application.get_env(:vav_pipeline, :mesorah_mount_point, "mesorah")
|
||||
|
||||
def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
||||
|
||||
@doc """
|
||||
`:ok` when the tree on disk hashes to the pin; `{:error, reason}`
|
||||
otherwise. Answered from the witness's last reading when it runs,
|
||||
else read here and now.
|
||||
"""
|
||||
@spec verify_integrity() :: :ok | {:error, String.t()}
|
||||
def verify_integrity do
|
||||
if Process.whereis(__MODULE__) do
|
||||
GenServer.call(__MODULE__, :status, 30_000)
|
||||
else
|
||||
compare(compute_tree_hash(root()))
|
||||
end
|
||||
end
|
||||
|
||||
@doc "The index the builder wrote, or nil."
|
||||
def index do
|
||||
case File.read(Path.join(root(), "index.json")) do
|
||||
{:ok, text} -> Jason.decode!(text)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
# The two files the builder writes after hashing are not part of the scroll.
|
||||
@not_hashed ~w(integrity.sha256 build_report.json)
|
||||
|
||||
@doc "SHA-256 over every regular file under `root` but the two the builder writes afterwards: relative path, then bytes, sorted — `build_mesorah.tree_hash`."
|
||||
@spec compute_tree_hash(String.t()) :: {:ok, String.t()} | {:error, String.t()}
|
||||
def compute_tree_hash(root) do
|
||||
if File.dir?(root) do
|
||||
digest =
|
||||
[root | subdirectories(root)]
|
||||
|> Enum.sort()
|
||||
|> Enum.reduce(:crypto.hash_init(:sha256), fn dir, ctx ->
|
||||
dir
|
||||
|> File.ls!()
|
||||
|> Enum.sort()
|
||||
|> Enum.reduce(ctx, fn name, ctx ->
|
||||
path = Path.join(dir, name)
|
||||
|
||||
if File.regular?(path) and not (dir == root and name in @not_hashed) do
|
||||
ctx |> :crypto.hash_update(Path.relative_to(path, root)) |> :crypto.hash_update(File.read!(path))
|
||||
else
|
||||
ctx
|
||||
end
|
||||
end)
|
||||
end)
|
||||
|> :crypto.hash_final()
|
||||
|> Base.encode16(case: :lower)
|
||||
|
||||
{:ok, digest}
|
||||
else
|
||||
{:error, "#{root} is not a directory"}
|
||||
end
|
||||
rescue
|
||||
e -> {:error, Exception.message(e)}
|
||||
end
|
||||
|
||||
defp subdirectories(dir) do
|
||||
dir
|
||||
|> File.ls!()
|
||||
|> Enum.map(&Path.join(dir, &1))
|
||||
|> Enum.filter(&File.dir?/1)
|
||||
|> Enum.flat_map(fn d -> [d | subdirectories(d)] end)
|
||||
end
|
||||
|
||||
defp compare({:ok, hash}) when hash == @expected_hash, do: :ok
|
||||
|
||||
defp compare({:ok, hash}),
|
||||
do: {:error, "MESORAH_INTEGRITY_FAILED: tree hashes to #{hash}, pinned #{@expected_hash} — not the scroll that was built"}
|
||||
|
||||
defp compare({:error, reason}), do: {:error, "MESORAH_INTEGRITY_FAILED: #{reason}"}
|
||||
|
||||
# ------------------------------------------------------------
|
||||
|
||||
@impl true
|
||||
def init(_opts) do
|
||||
{:ok, %{status: read_now()}, {:continue, :schedule}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_continue(:schedule, state) do
|
||||
Process.send_after(self(), :reread, @interval_ms)
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:reread, state) do
|
||||
Process.send_after(self(), :reread, @interval_ms)
|
||||
{:noreply, %{state | status: read_now()}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:status, _from, state), do: {:reply, state.status, state}
|
||||
|
||||
defp read_now do
|
||||
status = compare(compute_tree_hash(root()))
|
||||
|
||||
case status do
|
||||
:ok -> Logger.info("Mesorah: the second scroll hashes to its pin")
|
||||
{:error, reason} -> Logger.error(reason)
|
||||
end
|
||||
|
||||
status
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,11 @@
|
||||
defmodule VavPipeline.Repo.Migrations.BedikotMesorah do
|
||||
use Ecto.Migration
|
||||
|
||||
@moduledoc "The second scroll (bishe'arecha §IV): the mesorah citations an output made, each found in the snapshot or not."
|
||||
|
||||
def change do
|
||||
alter table(:bedikot) do
|
||||
add :mesorah, :jsonb, default: "[]", null: false
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -116,6 +116,8 @@ defmodule VavPipeline.Day6ServingTest do
|
||||
# the fresh provision that test writes, and the cycle waits.
|
||||
defp write_provision(compute_units) do
|
||||
MannaLedger.reset()
|
||||
# The serving path answers 503 while a halt from another file stands.
|
||||
if VavPipeline.HaltMonitor.halted?(), do: VavPipeline.HaltMonitor.resolve("day6 serving test setup")
|
||||
on_exit(fn -> MannaLedger.reset() end)
|
||||
today = Date.utc_today() |> Date.to_iso8601()
|
||||
expires = DateTime.utc_now() |> DateTime.add(6 * 3600) |> DateTime.to_iso8601()
|
||||
|
||||
@@ -14,6 +14,11 @@ defmodule VavPipeline.HaltMonitorTest do
|
||||
alias VavPipeline.HaltMonitor
|
||||
|
||||
setup do
|
||||
# A halt raised here must not outlive the test: the next file's
|
||||
# /peah/ask would answer 503 HALT_* (the 1-in-12 Day6 failure of
|
||||
# 2026-10-04, found by seed 446116 on 2026-10-13).
|
||||
on_exit(fn -> if HaltMonitor.halted?(), do: HaltMonitor.resolve("halt_monitor_test exit") end)
|
||||
|
||||
# Resolve any existing halt from a previous test
|
||||
if HaltMonitor.halted? do
|
||||
HaltMonitor.resolve("test cleanup")
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
defmodule VavPipeline.MesorahTest do
|
||||
@moduledoc """
|
||||
The second scroll (bishe'arecha §IV): pinned, hashed, read-only; its
|
||||
citations coupled like verses.
|
||||
"""
|
||||
|
||||
use ExUnit.Case, async: false
|
||||
import Plug.Test
|
||||
|
||||
alias VavPipeline.{Bedikah, Mesorah}
|
||||
alias VavPipeline.BigdeiKodesh.Router
|
||||
|
||||
@opts Router.init([])
|
||||
|
||||
test "the tree on disk hashes to the builder's integrity.sha256 and to the pin" do
|
||||
root = Mesorah.root()
|
||||
assert File.dir?(root), "mesorah snapshot not built at #{root}"
|
||||
{:ok, hash} = Mesorah.compute_tree_hash(root)
|
||||
assert hash == File.read!(Path.join(root, "integrity.sha256")) |> String.trim()
|
||||
assert hash == Mesorah.expected_hash()
|
||||
assert :ok = Mesorah.verify_integrity()
|
||||
end
|
||||
|
||||
test "the index names every work with its version, licence and hash; every licence is open" do
|
||||
index = Mesorah.index()
|
||||
assert map_size(index["works"]) >= 100
|
||||
|
||||
for {_slug, w} <- index["works"], {_lang, v} <- w["versions"] do
|
||||
assert v["license"] in ["Public Domain", "PD", "CC0", "CC-BY", "CC-BY-SA"], w["title"]
|
||||
assert String.length(v["sha256"]) == 64
|
||||
end
|
||||
|
||||
assert index["works"]["rashi_on_devarim"]["schema"]["sectionNames"] == ["Chapter", "Verse", "Comment"]
|
||||
end
|
||||
|
||||
test "a changed byte is a different scroll" do
|
||||
temp = Path.join(System.tmp_dir!(), "mesorah_test_#{System.unique_integer([:positive])}")
|
||||
File.mkdir_p!(Path.join(temp, "works/x"))
|
||||
File.write!(Path.join(temp, "works/x/he.json"), "{\"text\":[\"a\"]}")
|
||||
File.write!(Path.join(temp, "index.json"), "{}")
|
||||
{:ok, h1} = Mesorah.compute_tree_hash(temp)
|
||||
File.write!(Path.join(temp, "works/x/he.json"), "{\"text\":[\"b\"]}")
|
||||
{:ok, h2} = Mesorah.compute_tree_hash(temp)
|
||||
refute h1 == h2
|
||||
File.rm_rf!(temp)
|
||||
assert {:error, _} = Mesorah.compute_tree_hash(temp)
|
||||
end
|
||||
|
||||
test "a citation the scroll has couples to the passage; one it lacks couples to nothing" do
|
||||
examination = %{
|
||||
"claims" => [],
|
||||
"references" => [],
|
||||
"roots" => [],
|
||||
"hebrew" => [],
|
||||
"mesorah_references" => [
|
||||
%{"as_written" => "Rashi on Devarim 6:5", "ref" => "rashi_on_devarim_6_5", "work" => "rashi_on_devarim", "address" => [6, 5], "found" => true, "languages" => ["en", "he"]},
|
||||
%{"as_written" => "Rashi on Devarim 99:1", "ref" => "rashi_on_devarim_99_1", "work" => "rashi_on_devarim", "address" => [99, 1], "found" => false, "languages" => []}
|
||||
]
|
||||
}
|
||||
|
||||
trace = Bedikah.coupling_trace(examination)
|
||||
assert "coupled(reference(rashi_on_devarim_6_5), mesorah(rashi_on_devarim, [6,5]))" in trace
|
||||
assert "coupled(reference(rashi_on_devarim_99_1), not_in_mesorah)" in trace
|
||||
end
|
||||
|
||||
test "readiness carries the scroll's check" do
|
||||
conn = conn(:get, "/kodesh/health") |> Router.call(@opts)
|
||||
body = Jason.decode!(conn.resp_body)
|
||||
assert body["checks"]["mesorah_hash"] in ["pass", "fail"]
|
||||
assert body["checks"]["mesorah_hash"] == "pass"
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user