bikoret/vav_pipeline/lib/vav_pipeline/l1_reader.ex

defmodule VavPipeline.L1Reader do
  @moduledoc """
  Reads from the L1 Torah filesystem.

  L1 is read-only. This module reads, it never writes.
  The aron is sealed. The scroll is read, not rewritten.

  File structure:
    l1/torah/{book}/{chapter}/{verse}.json
    l1/torah/index.json
    l1/integrity.sha256
  """

  @l1_root Application.compile_env(:vav_pipeline, :l1_mount_point, "l1")

  # L0 firmware constant — from Luchot.Firmware (Haskell)
  # Updated after build_l1.py fix: enlarged letters (Shema's ayin/dalet, gachon's
  # vav, mishpatan's nun), ketiv/qere pairs, and smikhut construct pairs are now
  # preserved in L1 rather than silently dropped.
  @l1_expected_hash "9e1abe4edabac825d8a7037606ff247c3e8f8167650f04553f46c34e4f3c3086"

  @type letter_mark :: %{letter: String.t(), offset: non_neg_integer()}

  @type word :: %{
          position: pos_integer(),
          text: String.t(),
          consonants: String.t(),
          lemma: String.t(),
          morph: String.t(),
          teamim: [String.t()],
          prefix: String.t() | nil,
          shoresh: String.t() | nil,
          large_letters: [letter_mark()],
          small_letters: [letter_mark()]
        }

  @type ketiv_qere_entry :: %{
          word_position: pos_integer(),
          ketiv: String.t(),
          ketiv_consonants: String.t(),
          qere: String.t(),
          qere_consonants: String.t()
        }

  @type smikhut_pair :: %{
          position: pos_integer(),
          construct: String.t(),
          construct_consonants: String.t(),
          absolute: String.t(),
          absolute_consonants: String.t()
        }

  # A petuchah (open) or setumah (closed) paragraph break, recorded by the
  # word it follows. `after_word` equal to the verse's word count is a
  # break at the end of the verse; anything less is a break within it.
  @type parashah_break :: %{after_word: pos_integer(), type: :petuchah | :setumah}

  @type verse :: %{
          ref: String.t(),
          book: String.t(),
          chapter: pos_integer(),
          verse: pos_integer(),
          text_hebrew: String.t(),
          words: [word()],
          ketiv_qere: [ketiv_qere_entry()],
          smikhut: [smikhut_pair()],
          parashah_breaks: [parashah_break()],
          inverted_nuns: [%{after_word: pos_integer()}]
        }

  @doc """
  Read a verse from L1.

  Returns the parsed verse data or an error.
  """
  @spec read_verse(String.t(), pos_integer(), pos_integer()) ::
          {:ok, verse()} | {:error, String.t()}
  def read_verse(book, chapter, verse) do
    path = verse_path(book, chapter, verse)

    case File.read(path) do
      {:ok, contents} ->
        case Jason.decode(contents) do
          {:ok, data} -> {:ok, normalize_verse(data)}
          {:error, _} -> {:error, "L1_READ_FAILED: corrupt JSON at #{path}"}
        end

      {:error, :enoent} ->
        {:error, "INVALID_SCOPE: #{book} #{chapter}:#{verse} not found in L1"}

      {:error, reason} ->
        {:error, "L1_READ_FAILED: #{inspect(reason)} at #{path}"}
    end
  end

  @doc """
  Extract teamim marks from a verse, with word indices.

  Returns a list of %{word_index, taam, role} maps matching
  the ShemaContext schema from Construction spec §3.1.
  """
  @spec extract_teamim(verse()) :: [map()]
  def extract_teamim(verse) do
    verse.words
    |> Enum.flat_map(fn word ->
      Enum.map(word.teamim, fn taam ->
        %{
          word_index: word.position - 1,
          taam: taam,
          role: taam_role(taam)
        }
      end)
    end)
  end

  @doc """
  Extract ketiv/qere dual readings from a verse.

  The Masoretic tradition preserves both readings — ketiv (what is written)
  and qere (what is read aloud). Neither may be silently collapsed.
  Each entry has: word_position, ketiv, ketiv_consonants, qere, qere_consonants.
  """
  @spec extract_ketiv_qere(verse()) :: [map()]
  def extract_ketiv_qere(verse) do
    Map.get(verse, :ketiv_qere, [])
  end

  @doc """
  Extract roots from a verse with gematria values.

  PROVENANCE NOTE — single-opinion derivation:

  The shoresh value on each word comes from the OSHB (Open Scriptures
  Hebrew Bible) lemma database via build_l1.py. OSHB pins one root
  per word at build time; the L1 JSON does not record which
  derivational opinion produced it. For triliteral roots the choice
  is rarely contested, but for weak roots and biliterals it matters
  constantly — e.g. for וְנָתַתִּי (Devarim 11:14) OSHB records the
  biliteral נת, while classical analysis often gives נתן with
  assimilated nun. Both are defensible; OSHB picked one.

  Downstream consumers MUST NOT treat the returned shoresh as
  unique-truth. The Talmudic Engine should record the derivational
  source as part of its coupling_trace when a shoresh-based ruling
  enters L3 — otherwise a future re-derivation cannot be reconciled
  with prior opinions stored on top of the OSHB choice.

  This function returns a flat root → gematria map. To preserve
  derivational alternatives, that work belongs in the build_l1.py
  enrichment pass, not here.
  """
  @spec extract_roots(verse()) :: map()
  def extract_roots(verse) do
    verse.words
    |> Enum.filter(fn w -> w.shoresh != nil end)
    |> Enum.map(fn w -> {w.shoresh, gematria(w.shoresh)} end)
    |> Enum.uniq_by(fn {root, _} -> root end)
    |> Map.new()
  end

  @doc """
  Extract smikhut (construct-state) pairs from a verse.

  A word whose morph code ends in "c" (construct state) binds to the following
  word as an absolute. Example: "mishkan ha-edut" (tabernacle of the testimony).
  Each entry has: position, construct, construct_consonants, absolute,
  absolute_consonants.
  """
  @spec extract_smikhut(verse()) :: [map()]
  def extract_smikhut(verse) do
    Map.get(verse, :smikhut, [])
  end

  @doc """
  Extract grammatical voice profile from a verse.

  Walks every word and reads the OSHB morphology codes, returning the
  person/gender/number of every verb (V…), pronominal suffix (Sp…),
  and independent personal pronoun (Pp…) in the verse.

  Why this exists (anti-capture):
    The Shema (Devarim 6:4-9, 11:13-21, Bamidbar 15:37-41) deliberately
    oscillates between singular ("you, sg.") and plural ("you, pl.")
    voice — and Devarim 11:14 famously switches mid-verse from plural
    (אַרְצְכֶם, "your land") to singular (וְאָסַפְתָּ דְגָנֶךָ, "and YOU
    [sg.] gather YOUR [sg.] grain"). Voice that stabilizes at either
    pole is anti-Shema:
      • stuck at second-singular → private capture (gnostic individualism)
      • stuck at second-plural   → collective capture (mob theology)
      • stuck at first-anything  → operator-as-deity (the calf)
    The trajectory IS the anti-capture mechanism. The system records
    voice per-verse so downstream stages can detect stabilization.

  Returns:
    %{
      markers: [%{position, source, person, gender, number, morph}],
      summary: %{voice_key => count},
      dominant: voice_key | :mixed | :none,
      oscillates_within_verse: boolean    # true iff both 2-sg AND 2-pl appear
    }

  voice_key is one of:
    :first_singular, :first_plural,
    :second_singular, :second_plural,
    :third_singular, :third_plural
  """
  @spec extract_voice(verse()) :: map()
  def extract_voice(verse) do
    markers =
      verse.words
      |> Enum.flat_map(&voice_markers_in_word/1)

    summary =
      markers
      |> Enum.frequencies_by(fn m -> voice_key(m.person, m.number) end)

    oscillates =
      Enum.any?(markers, fn m -> m.person == 2 and m.number == :singular end) and
        Enum.any?(markers, fn m -> m.person == 2 and m.number == :plural end)

    %{
      markers: markers,
      summary: summary,
      dominant: dominant_voice(summary),
      oscillates_within_verse: oscillates
    }
  end

  @doc """
  Extract enlarged letters (otiot gedolot) from a verse.

  Returns a list of %{word_position, letter, offset} for each enlarged letter.
  These letters carry halakhic and exegetical weight: the enlarged ayin/dalet
  of Shema spell "ed" (witness); the enlarged vav of gachon marks the middle
  letter of Torah.
  """
  @spec extract_large_letters(verse()) :: [map()]
  def extract_large_letters(verse) do
    verse.words
    |> Enum.flat_map(fn w ->
      case Map.get(w, :large_letters, []) do
        [] ->
          []

        letters ->
          Enum.map(letters, fn l ->
            %{word_position: w.position, letter: l.letter, offset: l.offset}
          end)
      end
    end)
  end

  @doc """
  Verify L1 integrity: the scroll on disk must hash to the pinned hash.

  This is a critical check — if L1 is corrupted, the pipeline
  must halt. The specification store is inviolable.

  Two things are asked, and both must hold:

    1. The hash recorded beside the scroll (`integrity.sha256`) is the
       pinned one — this L1 is the L1 this build was made for.
    2. The scroll itself, read from disk and hashed, gives that hash
       (`VavPipeline.L1Integrity`, which reads it at startup and again
       every ten minutes).

  Until tablet 38 only the first was asked, and the first compares a
  recorded string with a compiled-in string: it could not notice a
  changed verse.
  """
  @spec verify_integrity() :: :ok | {:error, String.t()}
  def verify_integrity do
    hash_file = Path.join(l1_root(), "integrity.sha256")

    case File.read(hash_file) do
      {:ok, contents} ->
        stored_hash = contents |> String.trim()

        if stored_hash == @l1_expected_hash do
          verify_scroll()
        else
          {:error, "L1_INTEGRITY_FAILED: hash mismatch — stored #{stored_hash}, expected #{@l1_expected_hash}"}
        end

      {:error, _} ->
        {:error, "L1_INTEGRITY_FAILED: cannot read integrity.sha256"}
    end
  end

  # What the scroll on disk actually hashes to. The supervised witness
  # holds a recent reading; where it is not running (a bare unit test,
  # a one-off `eval`), the scroll is read here and now.
  defp verify_scroll do
    if Process.whereis(VavPipeline.L1Integrity) do
      VavPipeline.L1Integrity.status()
    else
      case compute_tree_hash() do
        {:ok, @l1_expected_hash} ->
          :ok

        {:ok, other} ->
          {:error,
           "L1_INTEGRITY_FAILED: the scroll on disk hashes to #{other}, expected #{@l1_expected_hash}"}

        {:error, reason} ->
          {:error, "L1_INTEGRITY_FAILED: the scroll could not be read — #{reason}"}
      end
    end
  end

  @doc """
  Hash the scroll: SHA-256 over every file under `<root>/torah`, each
  contributing its path relative to `torah/` and then its bytes.

  The order is the order `scripts/build_l1.py` (`generate_integrity_hash`)
  used when L1 was engraved, and must stay so or the two will disagree:
  directories in ascending order of their path, and within a directory the
  files in ascending order of name — plain string order, so `10.json`
  comes before `2.json`.
  """
  @spec compute_tree_hash(String.t()) :: {:ok, String.t()} | {:error, String.t()}
  def compute_tree_hash(root \\ l1_root()) do
    torah = Path.join(root, "torah")

    if File.dir?(torah) do
      digest =
        [torah | subdirectories(torah)]
        |> Enum.sort()
        |> Enum.reduce(:crypto.hash_init(:sha256), fn dir, ctx ->
          dir
          |> File.ls!()
          |> Enum.sort()
          |> Enum.reduce(ctx, fn name, ctx ->
            path = Path.join(dir, name)

            if File.regular?(path) do
              ctx
              |> :crypto.hash_update(Path.relative_to(path, torah))
              |> :crypto.hash_update(File.read!(path))
            else
              ctx
            end
          end)
        end)
        |> :crypto.hash_final()
        |> Base.encode16(case: :lower)

      {:ok, digest}
    else
      {:error, "#{torah} is not a directory"}
    end
  rescue
    e in [File.Error] -> {:error, Exception.message(e)}
  end

  defp subdirectories(dir) do
    dir
    |> File.ls!()
    |> Enum.map(&Path.join(dir, &1))
    |> Enum.filter(&File.dir?/1)
    |> Enum.flat_map(fn sub -> [sub | subdirectories(sub)] end)
  end

  @doc "The pinned L1 hash this build expects."
  @spec expected_hash() :: String.t()
  def expected_hash, do: @l1_expected_hash

  @doc """
  Get the root path of L1.
  """
  @spec l1_root() :: String.t()
  def l1_root, do: @l1_root

  # --- Private ---

  defp verse_path(book, chapter, verse) do
    Path.join([l1_root(), "torah", book, to_string(chapter), "#{verse}.json"])
  end

  defp normalize_verse(data) do
    %{
      ref: data["ref"],
      book: data["book"],
      chapter: data["chapter"],
      verse: data["verse"],
      text_hebrew: data["text_hebrew"],
      words: Enum.map(data["words"] || [], &normalize_word/1),
      ketiv_qere: Enum.map(data["ketiv_qere"] || [], &normalize_ketiv_qere/1),
      smikhut: Enum.map(data["smikhut"] || [], &normalize_smikhut/1),
      parashah_breaks: Enum.flat_map(data["parashah_breaks"] || [], &normalize_parashah_break/1),
      # Nun hafukha (׆) — recorded, like a paragraph break, by the word it
      # follows. Two in Torah: after Bamidbar 10:34 and after 10:36.
      inverted_nuns:
        for %{"after_word" => n} when is_integer(n) <- data["inverted_nuns"] || [] do
          %{after_word: n}
        end
    }
  end

  # Only the two kinds the scroll has. Anything else in that field is not
  # a paragraph break and is not passed on as one.
  defp normalize_parashah_break(%{"after_word" => n, "type" => "petuchah"}) when is_integer(n),
    do: [%{after_word: n, type: :petuchah}]

  defp normalize_parashah_break(%{"after_word" => n, "type" => "setumah"}) when is_integer(n),
    do: [%{after_word: n, type: :setumah}]

  defp normalize_parashah_break(_), do: []

  defp normalize_word(w) do
    %{
      position: w["position"],
      text: w["text"],
      consonants: w["consonants"],
      lemma: w["lemma"],
      morph: w["morph"],
      teamim: w["teamim"] || [],
      prefix: w["prefix"],
      shoresh: w["shoresh"],
      large_letters: Enum.map(w["large_letters"] || [], &normalize_letter_mark/1),
      small_letters: Enum.map(w["small_letters"] || [], &normalize_letter_mark/1)
    }
  end

  defp normalize_letter_mark(l) do
    %{letter: l["letter"], offset: l["offset"]}
  end

  defp normalize_ketiv_qere(kq) do
    %{
      word_position: kq["word_position"],
      ketiv: kq["ketiv"],
      ketiv_consonants: kq["ketiv_consonants"],
      qere: kq["qere"],
      qere_consonants: kq["qere_consonants"]
    }
  end

  defp normalize_smikhut(s) do
    %{
      position: s["position"],
      construct: s["construct"],
      construct_consonants: s["construct_consonants"],
      absolute: s["absolute"],
      absolute_consonants: s["absolute_consonants"]
    }
  end

  # Te'amim classification — mirrors Shoresh Engine Teamim.hs
  # Melekh = disjunctor (emperor/king), Mesharet = conjunctor (servant)
  # Spellings must match what build_l1.py writes to L1 JSON.
  # Verified against actual L1 data (bereishit-devarim ch1 sample).
  #
  # Classification principle (the WHY behind the two lists):
  #
  # Each ta'am is either MELEKH or MESHARET. The classification is
  # not a stylistic preference — it is structural. A melekh stands
  # alone and ends a phrase; a mesharet only joins to the following
  # word and cannot stand alone. The diagnostic test:
  #
  #   "Can this ta'am occupy the position of sof-pasuk-equivalent
  #    in some valid context?" → if yes, melekh.
  #   "Does this ta'am only occur in service of a following melekh,
  #    binding two words into one syntactic unit?" → if yes, mesharet.
  #
  # The hierarchy of melakhim (sof_pasuk → etnachta → zaqef → tipcha
  # → revia → …) corresponds to syntactic depth: each level marks
  # the cut for one level of clausal subdivision. The mesharetim
  # (munach, mercha, mahapakh, darga, qadma, telisha_qetana,
  # yerah_ben_yomo) precede their melakhim — the servant goes
  # before the king.
  #
  # This classification is binding for the pipeline's havdalah
  # (separation) operation: only melakhim are valid phrase-cut
  # points. Treating a mesharet as a phrase-end would falsify the
  # text's own divisions — a small gilui_arayot (collapsing the
  # disjunctor/conjunctor distinction).
  #
  # Source authority: this is the consensus Masoretic position
  # documented in Yeivin's "Introduction to the Tiberian
  # Masorah" (1980) §174-§268. The lists below match what
  # build_l1.py emits from the OSHB JSON; if OSHB ever encodes a
  # ta'am with a different name, BOTH lists must be updated.
  @melekh_marks ~w(sof_pasuk etnachta segol_accent shalshelet zaqef_qatan
    zaqef_gadol tipcha revia zarqa pashta yetiv tevir geresh
    gershayim pazer telisha_gedola meteg_siluq zinor geresh_muqdam)
  @mesharet_marks ~w(munach mercha mahapakh darga qadma
    telisha_qetana yerah_ben_yomo)

  defp taam_role(taam) when taam in @melekh_marks, do: "melekh"
  defp taam_role(taam) when taam in @mesharet_marks, do: "mesharet"
  defp taam_role(_), do: "unknown"

  # Simple gematria — consonant values
  @gematria_values %{
    "א" => 1,
    "ב" => 2,
    "ג" => 3,
    "ד" => 4,
    "ה" => 5,
    "ו" => 6,
    "ז" => 7,
    "ח" => 8,
    "ט" => 9,
    "י" => 10,
    "כ" => 20,
    "ך" => 20,
    "ל" => 30,
    "מ" => 40,
    "ם" => 40,
    "נ" => 50,
    "ן" => 50,
    "ס" => 60,
    "ע" => 70,
    "פ" => 80,
    "ף" => 80,
    "צ" => 90,
    "ץ" => 90,
    "ק" => 100,
    "ר" => 200,
    "ש" => 300,
    "ת" => 400
  }

  defp gematria(text) do
    text
    |> String.graphemes()
    |> Enum.reduce(0, fn char, acc ->
      acc + Map.get(@gematria_values, char, 0)
    end)
  end

  # --- Voice profile (OSHB morphology code parser) ---
  #
  # OSHB morph format uses '/' to separate language-segment groups, e.g.
  #   "HC/Vqq2ms"      — H = Hebrew, C = conjunction prefix, then Vqq2ms verb
  #   "HNcmsc/Sp2ms"   — Hebrew noun construct, then 2ms pronominal suffix
  #   "HC/Vqq2mp"      — verb 2nd masc plural
  # We extract person/gender/number from segments starting with V (verb),
  # Sp (pronominal suffix), or Pp (independent personal pronoun).

  defp voice_markers_in_word(word) do
    case word.morph do
      nil -> []
      "" -> []
      morph ->
        morph
        |> String.split("/")
        |> Enum.flat_map(fn seg ->
          case parse_morph_segment(seg) do
            nil -> []
            parsed -> [Map.merge(parsed, %{position: word.position, morph: morph})]
          end
        end)
    end
  end

  defp parse_morph_segment(seg) do
    # OSHB: the first segment of a morph string is prefixed with H (Hebrew)
    # or A (Aramaic) language indicator. Strip it before POS detection;
    # the same `/`-segment can be e.g. "HVqi2mp" (תִּשְׁמְעוּ) — without
    # stripping, it would miss every verse-initial verb in the Torah.
    stripped =
      case seg do
        "H" <> rest -> rest
        "A" <> rest -> rest
        other -> other
      end

    cond do
      String.starts_with?(stripped, "Sp") -> attempt_pgn(stripped, :suffix)
      String.starts_with?(stripped, "Pp") -> attempt_pgn(stripped, :pronoun)
      String.starts_with?(stripped, "V")  -> attempt_pgn(stripped, :verb)
      true -> nil
    end
  end

  defp attempt_pgn(seg, source) when byte_size(seg) >= 3 do
    pgn = String.slice(seg, -3, 3)
    case parse_pgn(pgn) do
      {:ok, p, g, n} -> %{source: source, person: p, gender: g, number: n}
      :error -> nil
    end
  end
  defp attempt_pgn(_, _), do: nil

  defp parse_pgn(<<p, g, n>>) when p in ?1..?3 and g in [?m, ?f, ?c] and n in [?s, ?p] do
    person = p - ?0
    gender =
      case g do
        ?m -> :masculine
        ?f -> :feminine
        ?c -> :common
      end
    number =
      case n do
        ?s -> :singular
        ?p -> :plural
      end
    {:ok, person, gender, number}
  end
  defp parse_pgn(_), do: :error

  defp voice_key(1, :singular), do: :first_singular
  defp voice_key(1, :plural),   do: :first_plural
  defp voice_key(2, :singular), do: :second_singular
  defp voice_key(2, :plural),   do: :second_plural
  defp voice_key(3, :singular), do: :third_singular
  defp voice_key(3, :plural),   do: :third_plural

  defp dominant_voice(summary) when map_size(summary) == 0, do: :none
  defp dominant_voice(summary) do
    sorted = Enum.sort_by(summary, fn {_k, c} -> -c end)
    case sorted do
      [{key, _}] -> key
      [{key1, c1}, {_key2, c2} | _] when c1 > c2 -> key1
      _ -> :mixed
    end
  end
end