bikoret/scripts/build_l1.py

#!/usr/bin/env python3
"""
L1 Torah Filesystem Builder — Bezalel's Chisel

Parses the Westminster Leningrad Codex (morphhb OSIS XML) and produces
the read-only Torah filesystem for Shorashim.

This is a construction tool. It runs once. It produces JSON files.
The consumers (Haskell L0, Erlang Shoresh Engine) will read these files.

Two quarries, both outside the repo (see __main__):
  morphhb        — the text: WLC with te'amim, lemma and morphology.
  HebrewLexicon  — the roots: the OSHB lexical index, which arranges
                   every Strong's lemma under its Brown-Driver-Briggs
                   root (specs/shorashim_terumah.md §E2).

Ka'asher tzivah Hashem et Moshe.
"""

import json
import hashlib
import os
import re
import sys
import unicodedata
import xml.etree.ElementTree as ET
from pathlib import Path
from collections import defaultdict

# === CONSTANTS ===

OSIS_NS = "http://www.bibletechnologies.net/2003/OSIS/namespace"

TORAH_BOOKS = [
    ("Gen", "bereishit"),
    ("Exod", "shemot"),
    ("Lev", "vayikra"),
    ("Num", "bamidbar"),
    ("Deut", "devarim"),
]

# Hebrew prefix letters and their morphhb codes
PREFIX_MAP = {
    "b": "ב",   # bet - in/with
    "c": "ו",   # vav-consecutive/conjunctive
    "d": "ה",   # he ha-yedia (definite article)
    "e": "ה",   # he interrogative
    "f": "ו",   # vav conjunctive (variant)
    "k": "כ",   # kaf - like/as
    "l": "ל",   # lamed - to/for
    "m": "מ",   # mem - from
    "s": "ש",   # shin - that/which (relative)
}

# Unicode ranges for Hebrew cantillation marks (te'amim)
# Range: U+0591 to U+05AF
TEAMIM_MAP = {
    "\u0591": "etnachta",
    "\u0592": "segol_accent",       # segol (accent, not vowel)
    "\u0593": "shalshelet",
    "\u0594": "zaqef_qatan",
    "\u0595": "zaqef_gadol",
    "\u0596": "tipcha",
    "\u0597": "revia",
    "\u0598": "zarqa",
    "\u0599": "pashta",
    "\u059A": "yetiv",
    "\u059B": "tevir",
    "\u059C": "geresh",
    "\u059D": "geresh_muqdam",
    "\u059E": "gershayim",
    "\u059F": "qarney_para",
    "\u05A0": "telisha_gedola",
    "\u05A1": "pazer",
    "\u05A2": "atnah_hafukh",
    "\u05A3": "munach",
    "\u05A4": "mahapakh",
    "\u05A5": "mercha",
    "\u05A6": "mercha_kefula",
    "\u05A7": "darga",
    "\u05A8": "qadma",
    "\u05A9": "telisha_qetana",
    "\u05AA": "yerah_ben_yomo",
    "\u05AB": "ole",
    "\u05AC": "iluy",
    "\u05AD": "dehi",
    "\u05AE": "zinor",
    "\u05AF": "masora_circle",
}

# U+05BD (meteg) doubles as siluq (cantillation mark on final word before sof-pasuq).
# We treat it as a cantillation mark so it shows up in te'amim extraction.
TEAMIM_MAP["\u05BD"] = "meteg_siluq"

# Unicode ranges for Hebrew vowel points (nikkud)
# Range: U+05B0 to U+05BC, U+05BF, U+05C1, U+05C2, U+05C7
# NOTE: U+05BD (meteg/siluq) is in TEAMIM_MAP, not here
VOWEL_POINTS = set(
    "\u05B0\u05B1\u05B2\u05B3\u05B4\u05B5\u05B6\u05B7"
    "\u05B8\u05B9\u05BA\u05BB\u05BC\u05BF\u05C1\u05C2\u05C7"
)

# Sof pasuq and maqaf
# OSIS <seg type="…"> → the paragraph break it marks.
PARASHAH_BREAKS = {
    "x-pe": "petuchah",      # פ — open: the rest of the line is left blank
    "x-samekh": "setumah",   # ס — closed: a space within the line
}

SOF_PASUQ = "\u05C3"  # ׃
MAQAF = "\u05BE"       # ־
PASEQ = "\u05C0"       # ׀


def strip_cantillation(text: str) -> str:
    """Remove cantillation marks, keep vowels and consonants."""
    return "".join(c for c in text if c not in TEAMIM_MAP)


def strip_vowels_and_cantillation(text: str) -> str:
    """Extract consonantal text only — strip vowels, cantillation, and dagesh."""
    result = []
    for c in text:
        if c in TEAMIM_MAP:
            continue
        if c in VOWEL_POINTS:
            continue
        if c == "/":
            continue
        if c == MAQAF or c == SOF_PASUQ or c == PASEQ:
            continue
        result.append(c)
    return "".join(result)


def extract_teamim(text: str) -> list[str]:
    """Extract cantillation mark names from Hebrew text."""
    marks = []
    for c in text:
        if c in TEAMIM_MAP:
            marks.append(TEAMIM_MAP[c])
    return marks


def parse_prefixes(lemma_str: str, morph_str: str) -> tuple[list[str], str, str]:
    """
    Parse prefix codes from lemma and morph strings.

    Returns: (prefix_hebrew_letters, core_lemma, core_morph)

    lemma format: "b/7225" or "c/d/776" (prefix codes / Strong's number)
    morph format: "HR/Ncfsa" or "HC/Td/Ncbsa" (H + prefix types / core morph)
    """
    prefixes = []

    # Parse lemma prefixes
    lemma_parts = lemma_str.split("/")
    core_lemma = lemma_parts[-1]  # Last part is always the Strong's number

    for part in lemma_parts[:-1]:
        if part in PREFIX_MAP:
            prefixes.append(PREFIX_MAP[part])

    # Parse morph — strip the language letter and split on '/'.
    #
    # The segments run [prefix…, WORD, pronominal-suffix?] and the lemma
    # has one part per prefix plus the word. So the word's own morphology
    # is the segment at the same index as the lemma's last part — NOT the
    # last segment. Until tablet 34 this took the last segment, which for
    # לְבָבְךָ (HNcmsc/Sp2ms) is the suffix "Sp2ms": the word was classed
    # as a suffix, and every suffixed noun and verb in Torah — 28,146
    # words — was written without a shoresh.
    core_morph = morph_str
    if morph_str[:1] in ("H", "A"):
        morph_parts = morph_str[1:].split("/")
        word_index = len(lemma_parts) - 1
        core_morph = morph_parts[word_index] if word_index < len(morph_parts) else morph_parts[-1]

    return prefixes, core_lemma, core_morph


def parse_morph_code(morph: str) -> dict:
    """
    Parse a morphhb morphological code into a structured dict.

    Code format (after stripping H prefix and prefixes):
    First char = part of speech:
      V=verb, N=noun, A=adjective, P=pronoun, R=preposition,
      T=particle, C=conjunction, D=adverb, S=suffix

    For verbs: V + stem(q/N/p/P/h/H/t/etc) + conjugation(p/i/w/j/v/r/c/a)
               + person(1/2/3) + gender(m/f/c) + number(s/p/d)
    For nouns: N + type(c/p) + gender(m/f/b) + number(s/p/d) + state(a/c/d)
    """
    if not morph:
        return {"raw": morph}

    result = {"raw": morph}

    if len(morph) < 1:
        return result

    pos = morph[0]
    pos_map = {
        "V": "verb", "N": "noun", "A": "adjective",
        "P": "pronoun", "R": "preposition", "T": "particle",
        "C": "conjunction", "D": "adverb", "S": "suffix",
    }
    result["pos"] = pos_map.get(pos, pos)

    if pos == "V" and len(morph) >= 3:
        stem_map = {
            "q": "qal", "N": "nifal", "p": "piel", "P": "pual",
            "h": "hifil", "H": "hofal", "t": "hitpael",
            "o": "polel", "O": "polal", "r": "hitpolel",
            "m": "poel", "M": "poal", "k": "palel",
            "K": "pulal", "Q": "qal_passive", "l": "pilpel",
            "L": "polpal", "f": "hithpalpel", "D": "nithpael",
            "j": "pealal", "i": "pilel", "u": "hothpaal",
            "c": "tiphil", "v": "hishtaphel", "w": "nithpalel",
            "y": "nithpoel", "z": "hithpoel",
        }
        result["stem"] = stem_map.get(morph[1], morph[1])

        conj_map = {
            "p": "perfect", "i": "imperfect", "w": "vav_consecutive",
            "j": "jussive", "v": "imperative", "r": "participle_active",
            "s": "participle_passive", "a": "infinitive_absolute",
            "c": "infinitive_construct",
        }
        if len(morph) >= 3:
            result["conjugation"] = conj_map.get(morph[2], morph[2])

    elif pos == "N" and len(morph) >= 2:
        type_map = {"c": "common", "p": "proper"}
        result["noun_type"] = type_map.get(morph[1], morph[1])

    return result


def strong_to_lemma_text(strong_num: str) -> str:
    """
    Clean a Strong's number string for use as lemma identifier.
    Handles formats like "1254 a", "430", "7225"
    """
    return strong_num.strip()


# === THE ROOT LEXICON ===
#
# Broken tablet 34 (2026-10-04). Until now `shoresh` was guessed from the
# consonants: strip a prefix letter, strip a suffix letter, keep the first
# three. וַיַּרְא "and he saw" was engraved as ירא ("fear"); וַיְהִי as
# יהי; שָׁמַיִם as שמ; מִזְבֵּחַ as מזב; הֵבֵאתִי as בא. 2,608 "unique
# roots", most of them fragments.
#
# Every word in the WLC already carries its lemma — a Strong's number —
# and the OSHB lexical index files each lemma under its Brown-Driver-
# Briggs root. The root is not derived here. It is read.
#
#   <entry id="gqv"> מִזְבֵּחַ  strong="4196"  <etym type="sub">dfp</etym>
#   <entry id="dfp"> זָבַח      strong="2076"  <etym type="main" root="זבח">
#
#   main    the entry IS a root. Its `root` attribute is the root; where
#           BDB gives none (primitive nouns — יד, בן, דם) the word stands
#           as its own root.
#   sub     the entry is filed under another; follow it up to its main.
#   single  BDB files the word alone, under no root (משה, שם, עיר). It is
#           its own heading, and is engraved as such.
#   (none)  the index records no etymology at all — mostly proper names
#           (יוסף, מצרים). No shoresh. The builder does not supply one.

LEXICON_NS = "{http://openscriptures.github.com/morphhb/namespace}"

# Set by build_l1() before any verse is parsed.
_LEXICON: dict[str, tuple[str | None, str]] = {}
# How each word's shoresh was decided, and the content words left without.
_SHORESH_STATS: dict[str, int] = defaultdict(int)
_ROOTLESS: dict[tuple[str, str, str], int] = defaultdict(int)


def _hebrew_consonants(text: str | None) -> str:
    """Hebrew letters only — drops vowels, shin/sin dots, accents."""
    return re.sub(r"[^\u05D0-\u05EA]", "", text or "")


def load_lexicon(lexicon_dir: str) -> dict[str, tuple[str | None, str]]:
    """
    Read LexicalIndex.xml + AugIndex.xml into
        augmented Strong's key ("1254a", "430") -> (root | None, kind)
    where kind is "main" | "sub" | "single" | "none".
    """
    entries: dict[str, dict] = {}
    index = ET.parse(os.path.join(lexicon_dir, "LexicalIndex.xml")).getroot()
    for e in index.iter(f"{LEXICON_NS}entry"):
        w = e.find(f"{LEXICON_NS}w")
        etym = e.find(f"{LEXICON_NS}etym")
        entries[e.get("id")] = {
            "word": _hebrew_consonants(w.text if w is not None else ""),
            "type": etym.get("type") if etym is not None else None,
            "root": _hebrew_consonants(etym.get("root")) if etym is not None else "",
            "parent": (etym.text or "").strip() if etym is not None else "",
        }

    def root_of(entry_id: str) -> tuple[str | None, str]:
        entry = entries[entry_id]
        kind = entry["type"] or "none"
        seen = {entry_id}
        while entry["type"] == "sub" and entry["parent"] in entries and entry["parent"] not in seen:
            seen.add(entry["parent"])
            entry = entries[entry["parent"]]
        if entry["type"] == "main":
            return (entry["root"] or entry["word"]) or None, kind
        if entry["type"] == "single":
            return entry["word"] or None, kind
        return None, kind

    lexicon = {}
    aug = ET.parse(os.path.join(lexicon_dir, "AugIndex.xml")).getroot()
    for w in aug.iter(f"{LEXICON_NS}w"):
        if w.text in entries:
            lexicon[w.get("aug")] = root_of(w.text)
    return lexicon


def lexicon_key_candidates(core_lemma: str) -> list[str]:
    """
    morphhb writes "1254 a", "1187+", "7451"; the index is keyed "1254a",
    "1187", and — where OSHB split a Strong's number — only "7451a",
    "7451b"…. Try the exact key, then the lettered ones.
    """
    key = re.sub(r"[\s+]", "", core_lemma)
    if not key or not key[0].isdigit():
        return []
    if key[-1].isalpha():
        return [key]
    return [key] + [key + letter for letter in "abcdefgh"]


def shoresh_from_lexicon(core_lemma: str, morph_info: dict) -> tuple[str | None, str]:
    """
    The word's root, read from the lexicon by its lemma.

    Returns (shoresh | None, why) where why is one of
      "main" "sub" "single"   — a root was read
      "none"                  — the lexicon records no etymology
      "particle"              — prepositions, conjunctions, particles and
                                pronouns are engraved without a shoresh,
                                as they always were (את, אשר, כי, הוא)
      "unlisted"              — the lemma is not in the lexical index
    """
    pos = morph_info.get("pos", "")
    if pos in ("preposition", "particle", "conjunction", "pronoun", "suffix"):
        return None, "particle"

    found = [_LEXICON[k] for k in lexicon_key_candidates(core_lemma) if k in _LEXICON]
    if not found:
        return None, "unlisted"

    # An unlettered lemma that OSHB split (7451 → 7451a, 7451b): accept
    # the root only if every sense agrees on it.
    roots = {root for root, _ in found}
    if len(roots) > 1:
        return None, "unlisted"
    return found[0]


def collect_word_text(w_elem) -> tuple[str, list[dict], list[dict]]:
    """
    Gather the full text of a <w> element including nested <seg> children.

    Returns (full_text, large_letters, small_letters) where each letter entry is
    {"letter": str, "offset": int} — the offset is the character position within
    the full_text string (pre prefix-split).

    The Masoretic tradition marks certain letters enlarged (otiot gedolot) or
    reduced (otiot zeirot). WLC encodes these as <seg type="x-large"> or
    <seg type="x-small"> nested inside <w>. The previous parser read only the
    direct text of <w> and silently dropped these letters — corrupting the
    Shema (שמ instead of שמע), the middle letter of Torah (גח instead of גחון),
    and other halakhically significant forms.
    """
    parts = []
    large = []
    small = []

    if w_elem.text:
        parts.append(w_elem.text)

    for child in w_elem:
        tag = child.tag.replace(f"{{{OSIS_NS}}}", "")
        if tag == "seg":
            seg_type = child.get("type", "")
            seg_text = child.text or ""
            offset = sum(len(p) for p in parts)
            parts.append(seg_text)
            # Record letters stripped to consonants only — otiot gedolot/zeirot
            # are a property of specific letters, not of vowels/cantillation.
            letters = strip_vowels_and_cantillation(seg_text)
            if seg_type == "x-large" and letters:
                for i, ch in enumerate(letters):
                    large.append({"letter": ch, "offset": offset + i})
            elif seg_type == "x-small" and letters:
                for i, ch in enumerate(letters):
                    small.append({"letter": ch, "offset": offset + i})
        if child.tail:
            parts.append(child.tail)

    return "".join(parts), large, small


def parse_verse(verse_elem, book_osis: str, book_hebrew: str) -> dict:
    """Parse a single verse element into the L1 JSON format."""
    osis_id = verse_elem.get("osisID")  # e.g., "Gen.1.1"
    parts = osis_id.split(".")
    chapter = int(parts[1])
    verse_num = int(parts[2])

    words = []
    full_text_parts = []
    word_position = 0
    ketiv_qere_entries = []
    parashah_breaks = []

    children = list(verse_elem)
    i = 0
    while i < len(children):
        elem = children[i]
        tag = elem.tag.replace(f"{{{OSIS_NS}}}", "")

        if tag == "w":
            word_position += 1
            word_data = parse_word(elem, word_position)
            words.append(word_data)
            full_text_parts.append(word_data["text"])

            # Ketiv/qere: when a <w type="x-ketiv"> is followed by
            # <note type="variant"><rdg type="x-qere"><w>...</w></rdg></note>,
            # the ketiv is what's written (in the scroll) and the qere is what's
            # read aloud. The Masoretic tradition preserves BOTH. Neither can be
            # silently collapsed — that would be choosing one reading over the other.
            if elem.get("type") == "x-ketiv" and i + 1 < len(children):
                nxt = children[i + 1]
                nxt_tag = nxt.tag.replace(f"{{{OSIS_NS}}}", "")
                if nxt_tag == "note" and nxt.get("type") == "variant":
                    qere_text = _extract_qere(nxt)
                    if qere_text:
                        ketiv_qere_entries.append({
                            "word_position": word_position,
                            "ketiv": word_data["text"],
                            "ketiv_consonants": word_data["consonants"],
                            "qere": qere_text,
                            "qere_consonants": strip_vowels_and_cantillation(
                                qere_text.replace("/", "")
                            ),
                        })
                    i += 1  # Skip the note element

        elif tag == "seg":
            seg_type = elem.get("type", "")
            if seg_type == "x-sof-pasuq":
                full_text_parts.append("׃")
            elif seg_type == "x-maqqef":
                full_text_parts.append("־")
            elif seg_type == "x-paseq":
                full_text_parts.append(" ׀ ")
            elif seg_type in PARASHAH_BREAKS:
                # Petuchah (פ) and setumah (ס): the open and closed
                # paragraph breaks of the scroll. The WLC marks each one
                # where it falls — almost always after the verse's last
                # word, four times in the middle of a verse (pisqa
                # b'emtza pasuq: Bereishit 35:22, Devarim 2:8, 5:21,
                # 23:8). Until tablet 39 the builder read past them and
                # L1 had no paragraphs at all. The break is recorded by
                # the word it follows; `text_hebrew` is left as it was.
                parashah_breaks.append({
                    "after_word": word_position,
                    "type": PARASHAH_BREAKS[seg_type],
                })

        i += 1

    # Reconstruct full verse text
    text_hebrew = " ".join(full_text_parts).strip()
    # Clean up spacing around maqaf
    text_hebrew = text_hebrew.replace(" ־ ", "־").replace(" ־", "־").replace("־ ", "־")
    text_hebrew = re.sub(r"\s+", " ", text_hebrew).strip()

    # Extract smikhut pairs — construct state + next noun.
    # The WLC morph code ending in "c" marks construct state (smikhut).
    # Example: "av hamon" (father of multitude) — av is construct, hamon absolute.
    smikhut = _extract_smikhut(words)

    ref = f"{book_hebrew.capitalize()} {chapter}:{verse_num}"

    verse_out = {
        "ref": ref,
        "book": book_hebrew,
        "chapter": chapter,
        "verse": verse_num,
        "text_hebrew": text_hebrew,
        "words": words,
    }
    if ketiv_qere_entries:
        verse_out["ketiv_qere"] = ketiv_qere_entries
    if smikhut:
        verse_out["smikhut"] = smikhut
    if parashah_breaks:
        verse_out["parashah_breaks"] = parashah_breaks
    return verse_out


def _extract_qere(note_elem) -> str:
    """Extract the qere reading from a <note type="variant"> element."""
    for rdg in note_elem:
        rdg_tag = rdg.tag.replace(f"{{{OSIS_NS}}}", "")
        if rdg_tag == "rdg" and rdg.get("type") == "x-qere":
            # <rdg> contains one or more <w> children
            qere_parts = []
            for w in rdg:
                w_tag = w.tag.replace(f"{{{OSIS_NS}}}", "")
                if w_tag == "w":
                    qere_text, _, _ = collect_word_text(w)
                    qere_parts.append(qere_text.replace("/", ""))
            if qere_parts:
                return " ".join(qere_parts)
    return ""


def _extract_smikhut(words: list[dict]) -> list[dict]:
    """
    Extract smikhut (construct-state) pairs. When a noun or adjective is in
    construct state (morph code ending in "c"), it binds to the following
    word as an absolute. Example: "av hamon" (אב המון) — father of multitude.

    We skip word-internal smikhut (pronominal suffixes like "av-i" = my father),
    which are marked by /Sp... segments. Those are real smikhut but single-word,
    not two-word pairs.
    """
    pairs = []
    for i, w in enumerate(words[:-1]):
        morph = w.get("morph", "")
        if not morph:
            continue
        # Morph format: H + primary-POS + optional /prefix-or-suffix segments.
        # Strip leading H (language marker) and find the primary word segment
        # (the first segment describing N=noun or A=adjective).
        segments = [s.lstrip("H") for s in morph.split("/")]
        primary = None
        for s in segments:
            if s and s[0] in ("N", "A"):
                primary = s
                break
        if not primary:
            continue
        # Skip if the same segment list has a pronominal suffix (Sp...) —
        # that's word-internal smikhut, not a two-word pair.
        if any(s.startswith("Sp") for s in segments):
            continue
        # Construct state is the last character of the primary morph.
        if not primary.endswith("c"):
            continue
        next_w = words[i + 1]
        pairs.append({
            "position": w["position"],
            "construct": w["text"],
            "construct_consonants": w["consonants"],
            "absolute": next_w["text"],
            "absolute_consonants": next_w["consonants"],
        })
    return pairs


def parse_word(w_elem, position: int) -> dict:
    """Parse a single <w> element into word data.

    Collects the full text including nested <seg type="x-large"> and
    <seg type="x-small"> children so enlarged/small Masoretic letters are
    preserved rather than silently dropped (as they were before).
    """
    raw_text, large_letters, small_letters = collect_word_text(w_elem)
    lemma_str = w_elem.get("lemma", "")
    morph_str = w_elem.get("morph", "")
    word_id = w_elem.get("id", "")

    # Parse prefixes
    prefixes, core_lemma, core_morph = parse_prefixes(lemma_str, morph_str)

    # Get text without prefix separators for the full word
    joined_text = raw_text.replace("/", "")

    # Get the word part (after last /) for the core word
    text_parts = raw_text.split("/")
    core_text = text_parts[-1] if text_parts else raw_text

    # Extract consonantal text (strip everything but Hebrew letters)
    consonants_full = strip_vowels_and_cantillation(joined_text)

    # Extract cantillation marks
    teamim = extract_teamim(joined_text)

    # Parse morphology
    morph_info = parse_morph_code(core_morph)

    # Read the shoresh from the lexicon (tablet 34)
    shoresh, why = shoresh_from_lexicon(core_lemma, morph_info)
    _SHORESH_STATS[why] += 1
    if shoresh is None and why in ("none", "unlisted"):
        _ROOTLESS[(core_lemma, strip_vowels_and_cantillation(core_text), why)] += 1

    # Build prefix string
    prefix_str = "".join(prefixes) if prefixes else ""

    word_data = {
        "position": position,
        "text": joined_text,
        "consonants": consonants_full,
        "lemma": core_lemma,
        "morph": morph_str,
        "teamim": teamim,
    }

    if prefix_str:
        word_data["prefix"] = prefix_str

    if shoresh:
        word_data["shoresh"] = shoresh

    # Otiot gedolot / zeirot — preserved as metadata on the word.
    # These are halakhically significant (the enlarged ayin/dalet of Shema
    # spell "ed" — witness; the enlarged vav of gachon marks the middle
    # letter of Torah; the small aleph of vayikra marks Moshe's humility).
    if large_letters:
        word_data["large_letters"] = large_letters
    if small_letters:
        word_data["small_letters"] = small_letters

    return word_data


def parse_book(xml_path: str, book_osis: str, book_hebrew: str) -> list[dict]:
    """Parse an entire OSIS XML book file into verses."""
    tree = ET.parse(xml_path)
    root = tree.getroot()

    verses = []

    # Navigate: osis > osisText > div (book) > chapter > verse
    for elem in root.iter(f"{{{OSIS_NS}}}verse"):
        osis_id = elem.get("osisID", "")
        if not osis_id.startswith(f"{book_osis}."):
            continue

        verse_data = parse_verse(elem, book_osis, book_hebrew)
        verses.append(verse_data)

    return verses


def _source_commit(path: str) -> str:
    """The git commit a quarry stands at — so a rebuild can be repeated."""
    import subprocess
    try:
        out = subprocess.run(
            ["git", "-C", path, "rev-parse", "HEAD"],
            capture_output=True, text=True, check=True,
        )
        return out.stdout.strip()
    except Exception:
        return "unknown"


def build_l1(morphhb_dir: str, output_dir: str, lexicon_dir: str):
    """Build the complete L1 Torah filesystem."""
    wlc_dir = os.path.join(morphhb_dir, "wlc")
    torah_dir = os.path.join(output_dir, "torah")

    _LEXICON.clear()
    _LEXICON.update(load_lexicon(lexicon_dir))
    _SHORESH_STATS.clear()
    _ROOTLESS.clear()

    total_verses = 0
    total_words = 0
    total_roots = set()
    parashah_counts = defaultdict(int)
    missing_teamim = []
    book_stats = {}

    print("=" * 60)
    print("L1 TORAH FILESYSTEM BUILDER")
    print("Bereishit bara Elohim et hashamayim v'et ha'aretz.")
    print("=" * 60)
    print()

    for book_osis, book_hebrew in TORAH_BOOKS:
        xml_path = os.path.join(wlc_dir, f"{book_osis}.xml")

        if not os.path.exists(xml_path):
            print(f"ERROR: {xml_path} not found!")
            sys.exit(1)

        print(f"Parsing {book_osis} ({book_hebrew})...", end=" ", flush=True)

        verses = parse_book(xml_path, book_osis, book_hebrew)

        # Organize by chapter
        chapters = defaultdict(list)
        for v in verses:
            chapters[v["chapter"]].append(v)

        book_dir = os.path.join(torah_dir, book_hebrew)
        book_verse_count = 0
        book_word_count = 0
        chapter_info = {}

        for chapter_num in sorted(chapters.keys()):
            chapter_verses = chapters[chapter_num]
            chapter_dir = os.path.join(book_dir, str(chapter_num))
            os.makedirs(chapter_dir, exist_ok=True)

            for verse_data in chapter_verses:
                verse_num = verse_data["verse"]
                verse_path = os.path.join(chapter_dir, f"{verse_num}.json")

                with open(verse_path, "w", encoding="utf-8") as f:
                    json.dump(verse_data, f, ensure_ascii=False, indent=2)

                book_verse_count += 1
                total_verses += 1
                for brk in verse_data.get("parashah_breaks", []):
                    parashah_counts[brk["type"]] += 1

                # Count words and collect stats
                for word in verse_data["words"]:
                    book_word_count += 1
                    total_words += 1

                    if word.get("shoresh"):
                        total_roots.add(word["shoresh"])

                # Check for te'amim
                verse_has_teamim = any(
                    len(w.get("teamim", [])) > 0 for w in verse_data["words"]
                )
                if not verse_has_teamim:
                    missing_teamim.append(verse_data["ref"])

            chapter_info[chapter_num] = len(chapter_verses)

        # Write book index
        book_index = {
            "book": book_hebrew,
            "osis_id": book_osis,
            "chapters": len(chapters),
            "verses": book_verse_count,
            "words": book_word_count,
            "chapter_verse_counts": chapter_info,
        }

        index_path = os.path.join(book_dir, "index.json")
        with open(index_path, "w", encoding="utf-8") as f:
            json.dump(book_index, f, ensure_ascii=False, indent=2)

        book_stats[book_hebrew] = book_index

        print(f"{len(chapters)} chapters, {book_verse_count} verses, {book_word_count} words")

    # Write master index
    master_index = {
        "name": "Torah",
        "source": "Westminster Leningrad Codex (morphhb)",
        "books": list(book_stats.values()),
        "total_chapters": sum(b["chapters"] for b in book_stats.values()),
        "total_verses": total_verses,
        "total_words": total_words,
        "unique_roots": len(total_roots),
    }

    master_path = os.path.join(torah_dir, "index.json")
    with open(master_path, "w", encoding="utf-8") as f:
        json.dump(master_index, f, ensure_ascii=False, indent=2)

    # Generate SHA-256 integrity hash
    print("\nGenerating integrity hash...")
    sha256 = generate_integrity_hash(torah_dir)

    hash_path = os.path.join(output_dir, "integrity.sha256")
    with open(hash_path, "w") as f:
        f.write(sha256)

    # Print report
    print()
    print("=" * 60)
    print("L1 BUILD REPORT")
    print("=" * 60)
    print(f"Total books:          {len(TORAH_BOOKS)}")
    print(f"Total chapters:       {master_index['total_chapters']}")
    print(f"Total verses:         {total_verses}")
    print(f"Total words:          {total_words}")
    print(f"Unique shoreshim:     {len(total_roots)}")
    print(f"Verses missing te'amim: {len(missing_teamim)}")
    print(f"Parashah breaks:      {parashah_counts['petuchah']} petuchot, "
          f"{parashah_counts['setumah']} setumot")
    words_with_shoresh = sum(_SHORESH_STATS[k] for k in ("main", "sub", "single"))
    rootless = sorted(_ROOTLESS.items(), key=lambda kv: (-kv[1], kv[0]))
    print(f"Words with a shoresh: {words_with_shoresh}")
    print(f"  read from a root entry (main):        {_SHORESH_STATS['main']}")
    print(f"  filed under a root (sub):             {_SHORESH_STATS['sub']}")
    print(f"  standing alone in the lexicon (single): {_SHORESH_STATS['single']}")
    print(f"Particles, pronouns (no shoresh by rule): {_SHORESH_STATS['particle']}")
    print(f"Lexicon records no etymology:           {_SHORESH_STATS['none']}")
    print(f"Lemma not in the lexical index:         {_SHORESH_STATS['unlisted']}")
    print(f"SHA-256:              {sha256}")
    print()

    if missing_teamim:
        print(f"Verses without te'amim ({len(missing_teamim)}):")
        for ref in missing_teamim[:20]:
            print(f"  - {ref}")
        if len(missing_teamim) > 20:
            print(f"  ... and {len(missing_teamim) - 20} more")
        print()

    if rootless:
        print(f"Most frequent words the lexicon gives no root ({len(rootless)} distinct):")
        for (lemma, word, why), n in rootless[:20]:
            print(f"  - {word} (lemma {lemma}, {why}) x{n}")
        print()

    # Write the report as JSON too
    report = {
        "total_books": len(TORAH_BOOKS),
        "total_chapters": master_index["total_chapters"],
        "total_verses": total_verses,
        "total_words": total_words,
        "unique_shoreshim": len(total_roots),
        "verses_missing_teamim": len(missing_teamim),
        "parashah_breaks": {
            "petuchah": parashah_counts["petuchah"],
            "setumah": parashah_counts["setumah"],
        },
        "words_with_shoresh": words_with_shoresh,
        "shoresh_provenance": {k: _SHORESH_STATS[k] for k in
                               ("main", "sub", "single", "particle", "none", "unlisted")},
        "sha256": sha256,
        "missing_teamim_refs": missing_teamim,
        "words_without_lexicon_root": [
            {"lemma": lemma, "word": word, "why": why, "count": n}
            for (lemma, word, why), n in rootless[:100]
        ],
        "sources": {
            "text": {
                "name": "openscriptures/morphhb (Westminster Leningrad Codex)",
                "commit": _source_commit(morphhb_dir),
            },
            "roots": {
                "name": "openscriptures/HebrewLexicon (LexicalIndex.xml, AugIndex.xml — BDB roots)",
                "commit": _source_commit(lexicon_dir),
            },
        },
    }

    report_path = os.path.join(output_dir, "build_report.json")
    with open(report_path, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

    print(f"Report written to: {report_path}")
    print(f"Master index: {master_path}")
    print(f"Integrity hash: {hash_path}")

    return report


def generate_integrity_hash(torah_dir: str) -> str:
    """Generate SHA-256 hash of the entire L1 Torah directory tree."""
    sha256 = hashlib.sha256()

    for root, dirs, files in sorted(os.walk(torah_dir)):
        dirs.sort()
        for filename in sorted(files):
            filepath = os.path.join(root, filename)
            rel_path = os.path.relpath(filepath, torah_dir)
            # Hash the relative path
            sha256.update(rel_path.encode("utf-8"))
            # Hash the file contents
            with open(filepath, "rb") as f:
                while True:
                    chunk = f.read(8192)
                    if not chunk:
                        break
                    sha256.update(chunk)

    return sha256.hexdigest()


if __name__ == "__main__":
    # The quarry (morphhb source) lives outside the repo.
    # Default: ~/shorashim-source-data/morphhb
    # Override: SHORASHIM_SOURCE_DATA env var, or first CLI arg.
    repo_root = Path(__file__).resolve().parent.parent

    default_source = Path.home() / "shorashim-source-data" / "morphhb"
    if len(sys.argv) > 1:
        morphhb_dir = Path(sys.argv[1]).expanduser().resolve()
    elif os.environ.get("SHORASHIM_SOURCE_DATA"):
        morphhb_dir = Path(os.environ["SHORASHIM_SOURCE_DATA"]).expanduser().resolve()
    else:
        morphhb_dir = default_source

    output_dir = repo_root / "l1"

    # The root lexicon sits beside the text quarry.
    # Override: SHORASHIM_LEXICON env var, or second CLI arg.
    if len(sys.argv) > 2:
        lexicon_dir = Path(sys.argv[2]).expanduser().resolve()
    elif os.environ.get("SHORASHIM_LEXICON"):
        lexicon_dir = Path(os.environ["SHORASHIM_LEXICON"]).expanduser().resolve()
    else:
        lexicon_dir = morphhb_dir.parent / "HebrewLexicon"

    if not (lexicon_dir / "LexicalIndex.xml").exists():
        print(f"ERROR: HebrewLexicon not found at {lexicon_dir}")
        print()
        print("The root lexicon (BDB roots by Strong's lemma) lives outside this repo.")
        print("To fetch it:")
        print(f"  git clone https://github.com/openscriptures/HebrewLexicon.git {lexicon_dir}")
        print()
        print("Or pass a path as the second argument, or set SHORASHIM_LEXICON.")
        sys.exit(1)

    if not morphhb_dir.exists():
        print(f"ERROR: morphhb not found at {morphhb_dir}")
        print()
        print("The morphhb source (the quarry) lives outside this repo.")
        print("To fetch it:")
        print(f"  mkdir -p {default_source.parent}")
        print(f"  git clone https://github.com/openscriptures/morphhb.git {default_source}")
        print()
        print("Or pass a path as the first argument, or set SHORASHIM_SOURCE_DATA.")
        sys.exit(1)

    # Clean only the generated artifacts — preserve hand-written files like README.md.
    # The torah/ tree is regenerated. build_report.json and integrity.sha256 too.
    import shutil
    torah_dir = output_dir / "torah"
    if torah_dir.exists():
        shutil.rmtree(torah_dir)
    for artifact in ("build_report.json", "integrity.sha256"):
        p = output_dir / artifact
        if p.exists():
            p.unlink()

    os.makedirs(output_dir, exist_ok=True)

    report = build_l1(str(morphhb_dir), str(output_dir), str(lexicon_dir))

    # Verify counts
    print()
    print("=" * 60)
    print("VERIFICATION")
    print("=" * 60)

    expected_books = 5
    expected_chapters = 187
    # WLC/BHS versification gives 5853 (8 more than the traditional Masoretic
    # count of 5845 due to verse numbering differences in Shemot, Bamidbar, Devarim)
    expected_verses = 5853

    checks = [
        ("Books", report["total_books"], expected_books),
        ("Chapters", report["total_chapters"], expected_chapters),
        ("Verses", report["total_verses"], expected_verses),
    ]

    all_pass = True
    for name, actual, expected in checks:
        status = "PASS" if actual == expected else "FAIL"
        if actual != expected:
            all_pass = False
        print(f"  {name}: {actual} (expected {expected}) [{status}]")

    te_amim_status = "PASS" if report["verses_missing_teamim"] == 0 else "WARN"
    print(f"  Te'amim coverage: {report['verses_missing_teamim']} missing [{te_amim_status}]")

    print()
    if all_pass:
        print("The firmament is in place. The throne sits above it.")
    else:
        print("Verification incomplete. Review the report.")