L1 data integrity: preserve the edut whole — Shema, middle letter, ketiv/qere, smikhut

Shemot 25:16 — v'natata el ha-aron et ha-edut asher eten elekha.
The edut placed in the aron must be complete. The Masoretic
tradition preserves every letter, every enlarged letter, every
ketiv/qere pair — because each carries meaning the scribe cannot
reconstruct.

The previous build_l1.py silently dropped:
- enlarged letters encoded as <seg type="x-large"> nested in <w>
  → the Shema was corrupt: שמ instead of שמע, אח instead of אחד
  → the middle letter of Torah was corrupt: גח instead of גחון
  → the Zelophchad nun was missing: משפט instead of משפטן
- ketiv/qere dual readings encoded as <note type="variant">/<rdg type="x-qere">
  → the spec committed to preserving both; the code returned []
- smikhut (construct state) pairs marked in morphology codes
  → the spec committed to extracting them; the code returned []

Fixes:
- collect_word_text() walks all nested <seg> children of <w> and records
  large_letters / small_letters metadata with character offsets.
- parse_verse() recognizes <w type="x-ketiv"> followed by <note>, extracts
  both the ketiv and the qere readings, stores a ketiv_qere list on the verse.
- _extract_smikhut() scans words for construct-state morph codes
  (ending in "c" on N/A primary segments, excluding pronominal suffixes)
  and records construct/absolute pairs.
- L1 SHA-256 hash updated everywhere: integrity.sha256, l1_reader.ex,
  shoresh-engine's Luchot.Firmware, its test, KOHEN.md.

Torah-wide results after fix:
- 4 enlarged letters preserved (was 0): Shema ayin, Shema dalet,
  gachon vav (middle of Torah), Zelophchad nun
- 67 ketiv/qere pairs captured (was 0)
- 9,203 smikhut pairs extracted (was 0)

Elixir updates:
- l1_reader.ex: normalize_verse now surfaces ketiv_qere, smikhut,
  and per-word large_letters/small_letters. New type specs. New
  extract_large_letters/1 helper. extract_ketiv_qere and
  extract_smikhut now return real data instead of [].
- step31_shema_test.exs: replaced placeholder asserts with real
  integrity checks. Four new tests verify the Shema word is complete
  (שמע not שמ), echad is complete (אחד not אח), the enlarged
  letters are marked, the middle letter of Torah is preserved,
  and ketiv/qere pairs from Bereishit 8:17 are both captured.

Build script:
- build_l1.py no longer rmtrees the entire l1/ directory (preserves
  l1/README.md). Only regenerates torah/ + build_report.json +
  integrity.sha256.
- Accepts SHORASHIM_SOURCE_DATA env var or CLI arg for morphhb path.

Test counts:
- Elixir: 260 (was 254), 0 failures
- Haskell: 33, 0 failures
- Prolog: 195, 0 failures

The system that can hear שמע correctly has passed its own shema test.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

vav_pipeline/test/step31_shema_test.exs

@@ -89,6 +89,77 @@ defmodule VavPipeline.Step31.ShemaTest do
     end
   end
 
+  # ============================================================
+  # Test 1b: The Shema itself must be intact.
+  #
+  # The system has a "Shema layer" because it is supposed to hear
+  # before it speaks. If the word "shema" itself is corrupt in L1,
+  # the system cannot fulfill its own foundational principle.
+  #
+  # Devarim 6:4 — Shema Yisrael Hashem Eloheinu Hashem echad.
+  # The ayin of שמע and the dalet of אחד are enlarged in the
+  # Masoretic scroll. Together they spell עד — "witness."
+  # ============================================================
+  describe "Test 1b: Shema verse integrity" do
+    test "shema is the complete word שמע (three letters, not two)" do
+      {:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
+      first_word = hd(verse.words)
+      assert first_word.consonants == "שמע",
+             "The system cannot hear. L1 has #{first_word.consonants} instead of שמע."
+    end
+
+    test "echad is the complete word אחד (three letters, not two)" do
+      {:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
+      last_word = List.last(verse.words)
+      assert last_word.consonants == "אחד",
+             "The closing of Shema is truncated. L1 has #{last_word.consonants} instead of אחד."
+    end
+
+    test "enlarged ayin of שמע is marked as a large letter" do
+      {:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
+      large = L1Reader.extract_large_letters(verse)
+
+      ayin = Enum.find(large, fn l -> l.letter == "ע" end)
+      assert ayin != nil,
+             "The enlarged ayin of shema is not preserved. With the enlarged dalet it spells עד — witness."
+      assert ayin.word_position == 1
+    end
+
+    test "enlarged dalet of אחד is marked as a large letter" do
+      {:ok, verse} = L1Reader.read_verse("devarim", 6, 4)
+      large = L1Reader.extract_large_letters(verse)
+
+      dalet = Enum.find(large, fn l -> l.letter == "ד" end)
+      assert dalet != nil,
+             "The enlarged dalet of echad is not preserved."
+      assert dalet.word_position == 6
+    end
+  end
+
+  # ============================================================
+  # Test 1c: The middle letter of the Torah must be preserved.
+  #
+  # Vayikra 11:42 — the vav of גחון is the middle letter of the
+  # entire Torah by Masoretic count. It is written enlarged in the
+  # scroll so that a scribe transcribing the Torah verifies the
+  # letter count against this known landmark.
+  # ============================================================
+  describe "Test 1c: Middle letter of the Torah" do
+    test "enlarged vav of גחון in Vayikra 11:42 is preserved" do
+      {:ok, verse} = L1Reader.read_verse("vayikra", 11, 42)
+      large = L1Reader.extract_large_letters(verse)
+
+      vav = Enum.find(large, fn l -> l.letter == "ו" end)
+      assert vav != nil,
+             "Middle letter of Torah not preserved. The scribe cannot verify."
+
+      # The word is גחון; check it contains all four letters
+      gachon = Enum.find(verse.words, fn w -> w.consonants == "גחון" end)
+      assert gachon != nil,
+             "Word גחון not intact in L1 — missing letter around the middle of Torah."
+    end
+  end
+
   # ============================================================
   # Test 2: Ketiv/Qere Preservation
   #
@@ -97,44 +168,59 @@ defmodule VavPipeline.Step31.ShemaTest do
   # Does the ShemaContext contain both readings?
   # At no point should the dual reading collapse to single.
   #
-  # NOTE: L1 does not yet encode ketiv/qere as separate fields.
-  # L1Reader.extract_ketiv_qere returns [] for now.
-  # This test documents the requirement and verifies the
-  # ShemaContext field exists and is list-typed.
+  # L1 now encodes ketiv/qere from the WLC source. The Shema context
+  # must preserve both readings — neither can be silently chosen.
   # ============================================================
   describe "Test 2: Ketiv/qere preservation" do
+    # Bereishit 8:17 has a known ketiv/qere: ketiv "הוצא", qere "היצא"
+    @kq_scope %{book: "bereishit", chapter: 8, verse: 17}
+
     test "ShemaContext has ketiv_qere field" do
       {:ok, context} = Shema.receive_context(@scope)
       assert Map.has_key?(context, :ketiv_qere)
       assert is_list(context.ketiv_qere)
     end
 
-    test "ketiv_qere field is populated from L1 (when L1 encodes it)" do
+    test "verse with no ketiv/qere has an empty ketiv_qere list" do
       {:ok, context} = Shema.receive_context(@scope)
-
-      # Currently returns [] — L1 doesn't encode ketiv/qere yet.
-      # When L1 is enriched, this test will need updating.
-      # The field EXISTS and is the right type — the pipeline is READY.
-      assert is_list(context.ketiv_qere)
+      # Bereishit 1:1 has no ketiv/qere variants
+      assert context.ketiv_qere == []
     end
 
-    test "ketiv_qere survives the full pipeline" do
-      {:ok, context} = Shema.receive_context(@scope)
+    test "Bereishit 8:17 has ketiv הוצא and qere היצא — both preserved" do
+      {:ok, context} = Shema.receive_context(@kq_scope)
+
+      assert length(context.ketiv_qere) >= 1
+
+      entry = Enum.find(context.ketiv_qere, fn e ->
+        e.ketiv_consonants == "הוצא"
+      end)
+
+      assert entry != nil,
+             "expected to find ketiv הוצא in Bereishit 8:17 — L1 is dropping ketiv/qere"
+
+      assert entry.ketiv_consonants == "הוצא",
+             "ketiv must be preserved exactly as written in the scroll"
+      assert entry.qere_consonants == "היצא",
+             "qere must be preserved exactly as read aloud — dual reading must not collapse"
+    end
+
+    test "ketiv_qere survives the full pipeline (context immutable)" do
+      {:ok, context} = Shema.receive_context(@kq_scope)
+
+      original_kq = context.ketiv_qere
+      assert length(original_kq) > 0
 
-      # Run through BereishitCycle — ketiv_qere should be accessible
-      # throughout (context is passed but never modified)
       input = %{
         source: "shema_test",
-        verse_scope: @scope,
-        content: "בראשית ברא אלהים את השמים ואת הארץ"
+        verse_scope: @kq_scope,
+        content: context.verse.text_hebrew
       }
 
-      # The pipeline should complete — and context remains accessible
-      result = BereishitCycle.execute(input, context)
-      assert elem(result, 0) in [:ok, :unresolved]
+      _result = BereishitCycle.execute(input, context)
 
-      # Context was never modified — ketiv_qere is still there
-      assert is_list(context.ketiv_qere)
+      # Context is immutable — ketiv/qere unchanged through pipeline
+      assert context.ketiv_qere == original_kq
     end
   end