Shemot 25:16 — v'natata el ha-aron et ha-edut asher eten elekha. The edut placed in the aron must be complete. The Masoretic tradition preserves every letter, every enlarged letter, every ketiv/qere pair — because each carries meaning the scribe cannot reconstruct. The previous build_l1.py silently dropped: - enlarged letters encoded as <seg type="x-large"> nested in <w> → the Shema was corrupt: שמ instead of שמע, אח instead of אחד → the middle letter of Torah was corrupt: גח instead of גחון → the Zelophchad nun was missing: משפט instead of משפטן - ketiv/qere dual readings encoded as <note type="variant">/<rdg type="x-qere"> → the spec committed to preserving both; the code returned [] - smikhut (construct state) pairs marked in morphology codes → the spec committed to extracting them; the code returned [] Fixes: - collect_word_text() walks all nested <seg> children of <w> and records large_letters / small_letters metadata with character offsets. - parse_verse() recognizes <w type="x-ketiv"> followed by <note>, extracts both the ketiv and the qere readings, stores a ketiv_qere list on the verse. - _extract_smikhut() scans words for construct-state morph codes (ending in "c" on N/A primary segments, excluding pronominal suffixes) and records construct/absolute pairs. - L1 SHA-256 hash updated everywhere: integrity.sha256, l1_reader.ex, shoresh-engine's Luchot.Firmware, its test, KOHEN.md. Torah-wide results after fix: - 4 enlarged letters preserved (was 0): Shema ayin, Shema dalet, gachon vav (middle of Torah), Zelophchad nun - 67 ketiv/qere pairs captured (was 0) - 9,203 smikhut pairs extracted (was 0) Elixir updates: - l1_reader.ex: normalize_verse now surfaces ketiv_qere, smikhut, and per-word large_letters/small_letters. New type specs. New extract_large_letters/1 helper. extract_ketiv_qere and extract_smikhut now return real data instead of []. - step31_shema_test.exs: replaced placeholder asserts with real integrity checks. Four new tests verify the Shema word is complete (שמע not שמ), echad is complete (אחד not אח), the enlarged letters are marked, the middle letter of Torah is preserved, and ketiv/qere pairs from Bereishit 8:17 are both captured. Build script: - build_l1.py no longer rmtrees the entire l1/ directory (preserves l1/README.md). Only regenerates torah/ + build_report.json + integrity.sha256. - Accepts SHORASHIM_SOURCE_DATA env var or CLI arg for morphhb path. Test counts: - Elixir: 260 (was 254), 0 failures - Haskell: 33, 0 failures - Prolog: 195, 0 failures The system that can hear שמע correctly has passed its own shema test. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
506 lines
18 KiB
Elixir
506 lines
18 KiB
Elixir
defmodule VavPipeline.Step31.ShemaTest do
|
||
@moduledoc """
|
||
Step 31: The Shema Test — can the system hear?
|
||
|
||
Architecture §6.3:
|
||
Before the ki tov test, the shema test.
|
||
The system that can speak but not hear has failed
|
||
the most fundamental test. The ear must work
|
||
before the mouth is tested.
|
||
|
||
Five tests. Five faculties of reception.
|
||
"""
|
||
|
||
use ExUnit.Case, async: false
|
||
|
||
alias VavPipeline.{Shema, L1Reader, Cloud, Manna, BereishitCycle}
|
||
|
||
@scope %{book: "bereishit", chapter: 1, verse: 1}
|
||
|
||
# ============================================================
|
||
# Test 1: Te'amim Reception
|
||
#
|
||
# Architecture §6.3:
|
||
# Feed the system a verse. Does the ShemaContext contain
|
||
# the raw cantillation marks from L1? Does the shoresh
|
||
# engine's readTeamim construct a parse tree that matches
|
||
# the text's own syntactic divisions?
|
||
#
|
||
# Feed a verse where the etnachta falls in a syntactically
|
||
# surprising place. Does the system follow the text's
|
||
# division or impose its own?
|
||
# ============================================================
|
||
describe "Test 1: Te'amim reception" do
|
||
test "ShemaContext contains raw cantillation marks from L1" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
assert is_list(context.teamim_marks)
|
||
assert length(context.teamim_marks) > 0
|
||
|
||
# Each mark should have word_index, taam, and role
|
||
Enum.each(context.teamim_marks, fn mark ->
|
||
assert Map.has_key?(mark, :word_index)
|
||
assert Map.has_key?(mark, :taam)
|
||
assert Map.has_key?(mark, :role)
|
||
end)
|
||
end
|
||
|
||
test "etnachta is present and marks the correct division point" do
|
||
# Bereishit 1:1 — etnachta falls on hashamayim (word 5, index 4 if 0-based)
|
||
# "Bereishit bara Elohim [etnachta] et hashamayim v'et ha'aretz"
|
||
# Actually etnachta is on word 5 (את השמים) — at index 4
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
etnachta = Enum.find(context.teamim_marks, fn m -> m.taam == "etnachta" end)
|
||
assert etnachta != nil
|
||
assert etnachta.role == "melekh"
|
||
end
|
||
|
||
test "te'amim marks are read from L1, not generated" do
|
||
# Read L1 directly and compare with ShemaContext
|
||
{:ok, verse} = L1Reader.read_verse("bereishit", 1, 1)
|
||
direct_marks = L1Reader.extract_teamim(verse)
|
||
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
# ShemaContext marks should match L1 marks exactly
|
||
assert length(context.teamim_marks) == length(direct_marks)
|
||
|
||
Enum.zip(context.teamim_marks, direct_marks)
|
||
|> Enum.each(fn {ctx_mark, l1_mark} ->
|
||
assert ctx_mark.taam == l1_mark.taam
|
||
assert ctx_mark.word_index == l1_mark.word_index
|
||
end)
|
||
end
|
||
|
||
test "te'amim from a verse with surprising etnachta position" do
|
||
# Bereishit 1:4 — vayar Elohim et ha'or ki tov [etnachta]
|
||
# vayavdel Elohim bein ha'or u'vein hachoshekh
|
||
# The etnachta divides "seeing the light was good" from "separating"
|
||
scope_14 = %{book: "bereishit", chapter: 1, verse: 4}
|
||
{:ok, context} = Shema.receive_context(scope_14)
|
||
|
||
assert is_list(context.teamim_marks)
|
||
assert length(context.teamim_marks) > 0
|
||
|
||
# Verify etnachta exists — the system receives the TEXT's division
|
||
etnachta = Enum.find(context.teamim_marks, fn m -> m.taam == "etnachta" end)
|
||
assert etnachta != nil, "Verse must have etnachta — the text's own division"
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 1b: The Shema itself must be intact.
|
||
#
|
||
# The system has a "Shema layer" because it is supposed to hear
|
||
# before it speaks. If the word "shema" itself is corrupt in L1,
|
||
# the system cannot fulfill its own foundational principle.
|
||
#
|
||
# Devarim 6:4 — Shema Yisrael Hashem Eloheinu Hashem echad.
|
||
# The ayin of שמע and the dalet of אחד are enlarged in the
|
||
# Masoretic scroll. Together they spell עד — "witness."
|
||
# ============================================================
|
||
describe "Test 1b: Shema verse integrity" do
|
||
test "shema is the complete word שמע (three letters, not two)" do
|
||
{:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
|
||
first_word = hd(verse.words)
|
||
assert first_word.consonants == "שמע",
|
||
"The system cannot hear. L1 has #{first_word.consonants} instead of שמע."
|
||
end
|
||
|
||
test "echad is the complete word אחד (three letters, not two)" do
|
||
{:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
|
||
last_word = List.last(verse.words)
|
||
assert last_word.consonants == "אחד",
|
||
"The closing of Shema is truncated. L1 has #{last_word.consonants} instead of אחד."
|
||
end
|
||
|
||
test "enlarged ayin of שמע is marked as a large letter" do
|
||
{:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
|
||
large = L1Reader.extract_large_letters(verse)
|
||
|
||
ayin = Enum.find(large, fn l -> l.letter == "ע" end)
|
||
assert ayin != nil,
|
||
"The enlarged ayin of shema is not preserved. With the enlarged dalet it spells עד — witness."
|
||
assert ayin.word_position == 1
|
||
end
|
||
|
||
test "enlarged dalet of אחד is marked as a large letter" do
|
||
{:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
|
||
large = L1Reader.extract_large_letters(verse)
|
||
|
||
dalet = Enum.find(large, fn l -> l.letter == "ד" end)
|
||
assert dalet != nil,
|
||
"The enlarged dalet of echad is not preserved."
|
||
assert dalet.word_position == 6
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 1c: The middle letter of the Torah must be preserved.
|
||
#
|
||
# Vayikra 11:42 — the vav of גחון is the middle letter of the
|
||
# entire Torah by Masoretic count. It is written enlarged in the
|
||
# scroll so that a scribe transcribing the Torah verifies the
|
||
# letter count against this known landmark.
|
||
# ============================================================
|
||
describe "Test 1c: Middle letter of the Torah" do
|
||
test "enlarged vav of גחון in Vayikra 11:42 is preserved" do
|
||
{:ok, verse} = L1Reader.read_verse("vayikra", 11, 42)
|
||
large = L1Reader.extract_large_letters(verse)
|
||
|
||
vav = Enum.find(large, fn l -> l.letter == "ו" end)
|
||
assert vav != nil,
|
||
"Middle letter of Torah not preserved. The scribe cannot verify."
|
||
|
||
# The word is גחון; check it contains all four letters
|
||
gachon = Enum.find(verse.words, fn w -> w.consonants == "גחון" end)
|
||
assert gachon != nil,
|
||
"Word גחון not intact in L1 — missing letter around the middle of Torah."
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 2: Ketiv/Qere Preservation
|
||
#
|
||
# Architecture §6.3:
|
||
# Feed the system a verse containing a ketiv/qere position.
|
||
# Does the ShemaContext contain both readings?
|
||
# At no point should the dual reading collapse to single.
|
||
#
|
||
# L1 now encodes ketiv/qere from the WLC source. The Shema context
|
||
# must preserve both readings — neither can be silently chosen.
|
||
# ============================================================
|
||
describe "Test 2: Ketiv/qere preservation" do
|
||
# Bereishit 8:17 has a known ketiv/qere: ketiv "הוצא", qere "היצא"
|
||
@kq_scope %{book: "bereishit", chapter: 8, verse: 17}
|
||
|
||
test "ShemaContext has ketiv_qere field" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
assert Map.has_key?(context, :ketiv_qere)
|
||
assert is_list(context.ketiv_qere)
|
||
end
|
||
|
||
test "verse with no ketiv/qere has an empty ketiv_qere list" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
# Bereishit 1:1 has no ketiv/qere variants
|
||
assert context.ketiv_qere == []
|
||
end
|
||
|
||
test "Bereishit 8:17 has ketiv הוצא and qere היצא — both preserved" do
|
||
{:ok, context} = Shema.receive_context(@kq_scope)
|
||
|
||
assert length(context.ketiv_qere) >= 1
|
||
|
||
entry = Enum.find(context.ketiv_qere, fn e ->
|
||
e.ketiv_consonants == "הוצא"
|
||
end)
|
||
|
||
assert entry != nil,
|
||
"expected to find ketiv הוצא in Bereishit 8:17 — L1 is dropping ketiv/qere"
|
||
|
||
assert entry.ketiv_consonants == "הוצא",
|
||
"ketiv must be preserved exactly as written in the scroll"
|
||
assert entry.qere_consonants == "היצא",
|
||
"qere must be preserved exactly as read aloud — dual reading must not collapse"
|
||
end
|
||
|
||
test "ketiv_qere survives the full pipeline (context immutable)" do
|
||
{:ok, context} = Shema.receive_context(@kq_scope)
|
||
|
||
original_kq = context.ketiv_qere
|
||
assert length(original_kq) > 0
|
||
|
||
input = %{
|
||
source: "shema_test",
|
||
verse_scope: @kq_scope,
|
||
content: context.verse.text_hebrew
|
||
}
|
||
|
||
_result = BereishitCycle.execute(input, context)
|
||
|
||
# Context is immutable — ketiv/qere unchanged through pipeline
|
||
assert context.ketiv_qere == original_kq
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 3: Cloud Obedience
|
||
#
|
||
# Architecture §6.3:
|
||
# Write a :lift signal. Start a processing cycle.
|
||
# The system should NOT begin processing.
|
||
# The system does not deliberate. It does not ask whether
|
||
# lifting is optimal. It obeys.
|
||
# ============================================================
|
||
describe "Test 3: Cloud obedience" do
|
||
test "cloud :camped allows processing" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
# Normal state — cloud is camped (L1 intact, not Shabbat)
|
||
assert context.cloud_state == :camped
|
||
end
|
||
|
||
test "cloud :moving stops processing — system does not deliberate" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
# Simulate cloud lifted by overriding context
|
||
_lifted_context = Map.put(context, :cloud_state, :moving)
|
||
|
||
input = %{
|
||
source: "cloud_obedience_test",
|
||
verse_scope: @scope,
|
||
content: "Test content"
|
||
}
|
||
|
||
# VavPipeline.process checks cloud state and returns transition
|
||
# We test the pipeline-level behavior
|
||
result = VavPipeline.process(input)
|
||
|
||
# If it's not Shabbat and L1 is intact, cloud should be :camped
|
||
# and processing should proceed or wait for provision.
|
||
# The important test: when cloud IS :moving, the pipeline
|
||
# returns {:transition, :cloud_lifted} — tested next.
|
||
assert elem(result, 0) in [:ok, :unresolved, :waiting, :transition, :halted]
|
||
end
|
||
|
||
test "pipeline returns {:transition, :cloud_lifted} when cloud is moving" do
|
||
# The VavPipeline.process function checks cloud_state from Shema.
|
||
# When cloud is :moving, it signals breaking_camp and returns transition.
|
||
# Since Cloud derives state from L1 integrity + calendar,
|
||
# we test the logic directly.
|
||
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
moving_context = Map.put(context, :cloud_state, :moving)
|
||
|
||
# Verify the cloud_state is what we set — context is immutable
|
||
assert moving_context.cloud_state == :moving
|
||
assert context.cloud_state != :moving
|
||
end
|
||
|
||
test "Shabbat triggers major_transition, not :camped" do
|
||
# During Shabbat, the cloud signals rest
|
||
{:ok, friday_evening, _} = DateTime.from_iso8601("2026-03-20T19:00:00Z")
|
||
|
||
{:ok, cloud_state} = Cloud.check_at(friday_evening)
|
||
|
||
assert match?({:major_transition, :shabbat_rest, _}, cloud_state),
|
||
"Friday evening should be Shabbat — cloud signals rest, not :camped"
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 4: Manna Fidelity
|
||
#
|
||
# Architecture §6.3:
|
||
# Set provision to half typical allocation. System should
|
||
# operate within reduced allocation. Remove provision entirely.
|
||
# System should return {:waiting, :no_provision}.
|
||
# ============================================================
|
||
describe "Test 4: Manna fidelity" do
|
||
test "no provision file → system waits" do
|
||
# Ensure no provision file
|
||
path = Manna.provision_file()
|
||
File.rm(path)
|
||
|
||
input = %{
|
||
source: "manna_fidelity_test",
|
||
verse_scope: @scope,
|
||
content: "Test when no manna"
|
||
}
|
||
|
||
result = VavPipeline.process(input)
|
||
|
||
# With no provision, the pipeline should return {:waiting, :no_provision}
|
||
assert result == {:waiting, :no_provision}
|
||
end
|
||
|
||
test "provision file present → system processes" do
|
||
# Write a valid provision file
|
||
today = Date.utc_today() |> Date.to_iso8601()
|
||
|
||
expires =
|
||
DateTime.utc_now()
|
||
|> DateTime.add(6 * 3600)
|
||
|> DateTime.to_iso8601()
|
||
|
||
provision = %{
|
||
"date" => today,
|
||
"provision" => %{
|
||
"compute_units" => 50,
|
||
"memory_mb" => 1024,
|
||
"api_calls_external" => 10,
|
||
"storage_delta_mb" => 5,
|
||
"sanhedrin_panels_max" => 1
|
||
},
|
||
"type" => "regular",
|
||
"expires_at" => expires,
|
||
"written_by" => "shema_test",
|
||
"written_at" => DateTime.utc_now() |> DateTime.to_iso8601()
|
||
}
|
||
|
||
path = Manna.provision_file()
|
||
File.mkdir_p!(Path.dirname(path))
|
||
File.write!(path, Jason.encode!(provision))
|
||
on_exit(fn -> File.rm(path) end)
|
||
|
||
input = %{
|
||
source: "manna_fidelity_test",
|
||
verse_scope: @scope,
|
||
content: "בראשית ברא אלהים"
|
||
}
|
||
|
||
result = VavPipeline.process(input)
|
||
|
||
# System should process — not wait
|
||
assert elem(result, 0) in [:ok, :unresolved, :transition, :halted]
|
||
assert result != {:waiting, :no_provision}
|
||
end
|
||
|
||
test "MannaLedger tracks consumption within allocation" do
|
||
VavPipeline.MannaLedger.reset()
|
||
|
||
# Write a small provision
|
||
today = Date.utc_today() |> Date.to_iso8601()
|
||
|
||
expires =
|
||
DateTime.utc_now()
|
||
|> DateTime.add(6 * 3600)
|
||
|> DateTime.to_iso8601()
|
||
|
||
provision = %{
|
||
"date" => today,
|
||
"provision" => %{
|
||
"compute_units" => 5,
|
||
"memory_mb" => 512,
|
||
"api_calls_external" => 2,
|
||
"storage_delta_mb" => 1,
|
||
"sanhedrin_panels_max" => 1
|
||
},
|
||
"type" => "regular",
|
||
"expires_at" => expires,
|
||
"written_by" => "shema_test",
|
||
"written_at" => DateTime.utc_now() |> DateTime.to_iso8601()
|
||
}
|
||
|
||
path = Manna.provision_file()
|
||
File.mkdir_p!(Path.dirname(path))
|
||
File.write!(path, Jason.encode!(provision))
|
||
on_exit(fn -> File.rm(path) end)
|
||
|
||
VavPipeline.MannaLedger.load_provision()
|
||
|
||
# Consume within allocation
|
||
assert {:ok, 4} = VavPipeline.MannaLedger.consume(:compute_units, 1)
|
||
assert {:ok, 3} = VavPipeline.MannaLedger.consume(:compute_units, 1)
|
||
|
||
# Consume all remaining
|
||
assert {:ok, 0} = VavPipeline.MannaLedger.consume(:compute_units, 3)
|
||
|
||
# Now exhausted
|
||
assert {:error, :provision_exhausted} =
|
||
VavPipeline.MannaLedger.consume(:compute_units, 1)
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Test 5: Context Immutability
|
||
#
|
||
# Architecture §6.3:
|
||
# Start a processing cycle. Verify that the ShemaContext
|
||
# passed to each pipeline step is identical — no step has
|
||
# modified it. The context is what was heard. Processing
|
||
# operates within it. If any step has altered the context,
|
||
# the system has stopped hearing and started rewriting.
|
||
# ============================================================
|
||
describe "Test 5: Context immutability" do
|
||
test "ShemaContext is not modified through the pipeline" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
# Capture the context before processing
|
||
context_before = deep_copy(context)
|
||
|
||
input = %{
|
||
source: "immutability_test",
|
||
verse_scope: @scope,
|
||
content: "בראשית ברא אלהים את השמים ואת הארץ"
|
||
}
|
||
|
||
# Run the full cycle — context is passed through
|
||
_result = BereishitCycle.execute(input, context)
|
||
|
||
# Context MUST be identical after processing
|
||
assert context.verse == context_before.verse
|
||
assert context.teamim_marks == context_before.teamim_marks
|
||
assert context.ketiv_qere == context_before.ketiv_qere
|
||
assert context.root_gematria == context_before.root_gematria
|
||
assert context.smikhut == context_before.smikhut
|
||
assert context.cloud_state == context_before.cloud_state
|
||
assert context.daily_provision == context_before.daily_provision
|
||
assert context.scope == context_before.scope
|
||
assert context.received_at == context_before.received_at
|
||
end
|
||
|
||
test "context keys are not added or removed during processing" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
keys_before = Map.keys(context) |> Enum.sort()
|
||
|
||
input = %{
|
||
source: "key_stability_test",
|
||
verse_scope: @scope,
|
||
content: "בראשית ברא אלהים"
|
||
}
|
||
|
||
_result = BereishitCycle.execute(input, context)
|
||
|
||
keys_after = Map.keys(context) |> Enum.sort()
|
||
assert keys_before == keys_after
|
||
end
|
||
|
||
test "each pipeline stage receives the same context" do
|
||
{:ok, context} = Shema.receive_context(@scope)
|
||
|
||
input = %{
|
||
source: "stage_context_test",
|
||
verse_scope: @scope,
|
||
content: "בראשית ברא אלהים את השמים ואת הארץ"
|
||
}
|
||
|
||
# Run each stage manually and verify context is unchanged
|
||
state1 = BereishitCycle.tohu(input, context)
|
||
assert context.scope == @scope
|
||
|
||
state2 = BereishitCycle.vayomer(state1, context)
|
||
assert context.scope == @scope
|
||
assert context.teamim_marks == L1Reader.extract_teamim(context.verse)
|
||
|
||
state3 = BereishitCycle.havdalah(state2, context)
|
||
assert context.scope == @scope
|
||
assert context.cloud_state == :camped
|
||
|
||
state4 = BereishitCycle.kriah(state3, context)
|
||
assert context.scope == @scope
|
||
|
||
_state5 = BereishitCycle.ki_tov(state4, context)
|
||
assert context.scope == @scope
|
||
assert context.verse == context.verse
|
||
end
|
||
end
|
||
|
||
# ============================================================
|
||
# Helpers
|
||
# ============================================================
|
||
|
||
# Deep copy a map to ensure we have an independent reference
|
||
defp deep_copy(map) when is_map(map) do
|
||
Map.new(map, fn {k, v} -> {k, deep_copy_value(v)} end)
|
||
end
|
||
|
||
defp deep_copy_value(%DateTime{} = dt), do: dt
|
||
defp deep_copy_value(list) when is_list(list), do: Enum.map(list, &deep_copy_value/1)
|
||
defp deep_copy_value(map) when is_map(map) and not is_struct(map), do: deep_copy(map)
|
||
defp deep_copy_value(struct) when is_struct(struct), do: struct
|
||
defp deep_copy_value(other), do: other
|
||
end
|