bikoret/scripts/build_l1.py

#!/usr/bin/env python3
"""
L1 Torah Filesystem Builder — Bezalel's Chisel

Parses the Westminster Leningrad Codex (morphhb OSIS XML) and produces
the read-only Torah filesystem for Shorashim.

This is a construction tool. It runs once. It produces JSON files.
The consumers (Haskell L0, Erlang Shoresh Engine) will read these files.

Ka'asher tzivah Hashem et Moshe.
"""

import json
import hashlib
import os
import re
import sys
import unicodedata
import xml.etree.ElementTree as ET
from pathlib import Path
from collections import defaultdict

# === CONSTANTS ===

OSIS_NS = "http://www.bibletechnologies.net/2003/OSIS/namespace"

TORAH_BOOKS = [
    ("Gen", "bereishit"),
    ("Exod", "shemot"),
    ("Lev", "vayikra"),
    ("Num", "bamidbar"),
    ("Deut", "devarim"),
]

# Hebrew prefix letters and their morphhb codes
PREFIX_MAP = {
    "b": "ב",   # bet - in/with
    "c": "ו",   # vav-consecutive/conjunctive
    "d": "ה",   # he ha-yedia (definite article)
    "e": "ה",   # he interrogative
    "f": "ו",   # vav conjunctive (variant)
    "k": "כ",   # kaf - like/as
    "l": "ל",   # lamed - to/for
    "m": "מ",   # mem - from
    "s": "ש",   # shin - that/which (relative)
}

# Unicode ranges for Hebrew cantillation marks (te'amim)
# Range: U+0591 to U+05AF
TEAMIM_MAP = {
    "\u0591": "etnachta",
    "\u0592": "segol_accent",       # segol (accent, not vowel)
    "\u0593": "shalshelet",
    "\u0594": "zaqef_qatan",
    "\u0595": "zaqef_gadol",
    "\u0596": "tipcha",
    "\u0597": "revia",
    "\u0598": "zarqa",
    "\u0599": "pashta",
    "\u059A": "yetiv",
    "\u059B": "tevir",
    "\u059C": "geresh",
    "\u059D": "geresh_muqdam",
    "\u059E": "gershayim",
    "\u059F": "qarney_para",
    "\u05A0": "telisha_gedola",
    "\u05A1": "pazer",
    "\u05A2": "atnah_hafukh",
    "\u05A3": "munach",
    "\u05A4": "mahapakh",
    "\u05A5": "mercha",
    "\u05A6": "mercha_kefula",
    "\u05A7": "darga",
    "\u05A8": "qadma",
    "\u05A9": "telisha_qetana",
    "\u05AA": "yerah_ben_yomo",
    "\u05AB": "ole",
    "\u05AC": "iluy",
    "\u05AD": "dehi",
    "\u05AE": "zinor",
    "\u05AF": "masora_circle",
}

# U+05BD (meteg) doubles as siluq (cantillation mark on final word before sof-pasuq).
# We treat it as a cantillation mark so it shows up in te'amim extraction.
TEAMIM_MAP["\u05BD"] = "meteg_siluq"

# Unicode ranges for Hebrew vowel points (nikkud)
# Range: U+05B0 to U+05BC, U+05BF, U+05C1, U+05C2, U+05C7
# NOTE: U+05BD (meteg/siluq) is in TEAMIM_MAP, not here
VOWEL_POINTS = set(
    "\u05B0\u05B1\u05B2\u05B3\u05B4\u05B5\u05B6\u05B7"
    "\u05B8\u05B9\u05BA\u05BB\u05BC\u05BF\u05C1\u05C2\u05C7"
)

# Sof pasuq and maqaf
SOF_PASUQ = "\u05C3"  # ׃
MAQAF = "\u05BE"       # ־
PASEQ = "\u05C0"       # ׀


def strip_cantillation(text: str) -> str:
    """Remove cantillation marks, keep vowels and consonants."""
    return "".join(c for c in text if c not in TEAMIM_MAP)


def strip_vowels_and_cantillation(text: str) -> str:
    """Extract consonantal text only — strip vowels, cantillation, and dagesh."""
    result = []
    for c in text:
        if c in TEAMIM_MAP:
            continue
        if c in VOWEL_POINTS:
            continue
        if c == "/":
            continue
        if c == MAQAF or c == SOF_PASUQ or c == PASEQ:
            continue
        result.append(c)
    return "".join(result)


def extract_teamim(text: str) -> list[str]:
    """Extract cantillation mark names from Hebrew text."""
    marks = []
    for c in text:
        if c in TEAMIM_MAP:
            marks.append(TEAMIM_MAP[c])
    return marks


def parse_prefixes(lemma_str: str, morph_str: str) -> tuple[list[str], str, str]:
    """
    Parse prefix codes from lemma and morph strings.

    Returns: (prefix_hebrew_letters, core_lemma, core_morph)

    lemma format: "b/7225" or "c/d/776" (prefix codes / Strong's number)
    morph format: "HR/Ncfsa" or "HC/Td/Ncbsa" (H + prefix types / core morph)
    """
    prefixes = []

    # Parse lemma prefixes
    lemma_parts = lemma_str.split("/")
    core_lemma = lemma_parts[-1]  # Last part is always the Strong's number

    for part in lemma_parts[:-1]:
        if part in PREFIX_MAP:
            prefixes.append(PREFIX_MAP[part])

    # Parse morph — strip leading 'H' and split on '/'
    core_morph = morph_str
    if morph_str.startswith("H"):
        morph_inner = morph_str[1:]
        morph_parts = morph_inner.split("/")
        core_morph = morph_parts[-1]  # Last part is the core morphology

    return prefixes, core_lemma, core_morph


def parse_morph_code(morph: str) -> dict:
    """
    Parse a morphhb morphological code into a structured dict.

    Code format (after stripping H prefix and prefixes):
    First char = part of speech:
      V=verb, N=noun, A=adjective, P=pronoun, R=preposition,
      T=particle, C=conjunction, D=adverb, S=suffix

    For verbs: V + stem(q/N/p/P/h/H/t/etc) + conjugation(p/i/w/j/v/r/c/a)
               + person(1/2/3) + gender(m/f/c) + number(s/p/d)
    For nouns: N + type(c/p) + gender(m/f/b) + number(s/p/d) + state(a/c/d)
    """
    if not morph:
        return {"raw": morph}

    result = {"raw": morph}

    if len(morph) < 1:
        return result

    pos = morph[0]
    pos_map = {
        "V": "verb", "N": "noun", "A": "adjective",
        "P": "pronoun", "R": "preposition", "T": "particle",
        "C": "conjunction", "D": "adverb", "S": "suffix",
    }
    result["pos"] = pos_map.get(pos, pos)

    if pos == "V" and len(morph) >= 3:
        stem_map = {
            "q": "qal", "N": "nifal", "p": "piel", "P": "pual",
            "h": "hifil", "H": "hofal", "t": "hitpael",
            "o": "polel", "O": "polal", "r": "hitpolel",
            "m": "poel", "M": "poal", "k": "palel",
            "K": "pulal", "Q": "qal_passive", "l": "pilpel",
            "L": "polpal", "f": "hithpalpel", "D": "nithpael",
            "j": "pealal", "i": "pilel", "u": "hothpaal",
            "c": "tiphil", "v": "hishtaphel", "w": "nithpalel",
            "y": "nithpoel", "z": "hithpoel",
        }
        result["stem"] = stem_map.get(morph[1], morph[1])

        conj_map = {
            "p": "perfect", "i": "imperfect", "w": "vav_consecutive",
            "j": "jussive", "v": "imperative", "r": "participle_active",
            "s": "participle_passive", "a": "infinitive_absolute",
            "c": "infinitive_construct",
        }
        if len(morph) >= 3:
            result["conjugation"] = conj_map.get(morph[2], morph[2])

    elif pos == "N" and len(morph) >= 2:
        type_map = {"c": "common", "p": "proper"}
        result["noun_type"] = type_map.get(morph[1], morph[1])

    return result


def strong_to_lemma_text(strong_num: str) -> str:
    """
    Clean a Strong's number string for use as lemma identifier.
    Handles formats like "1254 a", "430", "7225"
    """
    return strong_num.strip()


def derive_shoresh_from_word(consonants: str, morph_info: dict) -> str | None:
    """
    Attempt to derive a triliteral root from consonantal text and morphology.

    This is a heuristic — not all words have triliteral roots (particles,
    pronouns, etc). Returns None when derivation fails.
    """
    pos = morph_info.get("pos", "")

    # Particles, prepositions, conjunctions, pronouns don't have shoreshim
    if pos in ("preposition", "particle", "conjunction", "pronoun", "suffix"):
        return None

    # For the consonantal text, we want only Hebrew letters
    hebrew_only = re.sub(r"[^\u05D0-\u05EA]", "", consonants)

    if len(hebrew_only) < 2:
        return None

    stem = morph_info.get("stem", "")
    conjugation = morph_info.get("conjugation", "")

    if pos == "verb":
        # Strip imperfect/jussive/vav-consecutive prefixes: י מ נ ת א
        # These are conjugation prefixes, not part of the root
        if conjugation in ("imperfect", "vav_consecutive", "jussive"):
            if len(hebrew_only) >= 4 and hebrew_only[0] in "ימנתא":
                hebrew_only = hebrew_only[1:]
        # Strip hifil/hofal he prefix
        if stem in ("hifil", "hofal") and len(hebrew_only) >= 4:
            if hebrew_only[0] == "ה":
                hebrew_only = hebrew_only[1:]
            # Imperfect hifil: prefix already stripped, check for yod remnant
        # Strip hitpael tav prefix (and sometimes he)
        elif stem == "hitpael" and len(hebrew_only) >= 4:
            if hebrew_only[0] == "ה" and len(hebrew_only) >= 5 and hebrew_only[1] == "ת":
                hebrew_only = hebrew_only[2:]
            elif hebrew_only[0] == "ת":
                hebrew_only = hebrew_only[1:]
        # Strip nifal nun prefix
        elif stem == "nifal" and len(hebrew_only) >= 4:
            if hebrew_only[0] == "נ":
                hebrew_only = hebrew_only[1:]
        # Strip participle mem prefix
        if conjugation in ("participle_active", "participle_passive"):
            if len(hebrew_only) >= 4 and hebrew_only[0] == "מ":
                hebrew_only = hebrew_only[1:]

    elif pos == "noun":
        # Strip common noun prefixes: מ (place/instrument nouns), ת (abstract nouns)
        if len(hebrew_only) >= 4 and hebrew_only[0] in "מת":
            pass  # Don't strip by default for nouns — too aggressive

    # Strip common suffixes
    for suffix in ["ים", "ות", "ין", "תי", "תם", "תן", "נו"]:
        if hebrew_only.endswith(suffix) and len(hebrew_only) - len(suffix) >= 2:
            hebrew_only = hebrew_only[:-len(suffix)]
            break
    for suffix in ["ה", "ת", "ו", "י", "ך", "ם", "ן"]:
        if hebrew_only.endswith(suffix) and len(hebrew_only) - 1 >= 3:
            hebrew_only = hebrew_only[:-1]
            break

    if len(hebrew_only) == 3:
        return hebrew_only
    elif len(hebrew_only) == 2:
        return hebrew_only
    elif len(hebrew_only) > 3:
        # Take first three as best guess
        return hebrew_only[:3]

    return hebrew_only if hebrew_only else None


def collect_word_text(w_elem) -> tuple[str, list[dict], list[dict]]:
    """
    Gather the full text of a <w> element including nested <seg> children.

    Returns (full_text, large_letters, small_letters) where each letter entry is
    {"letter": str, "offset": int} — the offset is the character position within
    the full_text string (pre prefix-split).

    The Masoretic tradition marks certain letters enlarged (otiot gedolot) or
    reduced (otiot zeirot). WLC encodes these as <seg type="x-large"> or
    <seg type="x-small"> nested inside <w>. The previous parser read only the
    direct text of <w> and silently dropped these letters — corrupting the
    Shema (שמ instead of שמע), the middle letter of Torah (גח instead of גחון),
    and other halakhically significant forms.
    """
    parts = []
    large = []
    small = []

    if w_elem.text:
        parts.append(w_elem.text)

    for child in w_elem:
        tag = child.tag.replace(f"{{{OSIS_NS}}}", "")
        if tag == "seg":
            seg_type = child.get("type", "")
            seg_text = child.text or ""
            offset = sum(len(p) for p in parts)
            parts.append(seg_text)
            # Record letters stripped to consonants only — otiot gedolot/zeirot
            # are a property of specific letters, not of vowels/cantillation.
            letters = strip_vowels_and_cantillation(seg_text)
            if seg_type == "x-large" and letters:
                for i, ch in enumerate(letters):
                    large.append({"letter": ch, "offset": offset + i})
            elif seg_type == "x-small" and letters:
                for i, ch in enumerate(letters):
                    small.append({"letter": ch, "offset": offset + i})
        if child.tail:
            parts.append(child.tail)

    return "".join(parts), large, small


def parse_verse(verse_elem, book_osis: str, book_hebrew: str) -> dict:
    """Parse a single verse element into the L1 JSON format."""
    osis_id = verse_elem.get("osisID")  # e.g., "Gen.1.1"
    parts = osis_id.split(".")
    chapter = int(parts[1])
    verse_num = int(parts[2])

    words = []
    full_text_parts = []
    word_position = 0
    ketiv_qere_entries = []

    children = list(verse_elem)
    i = 0
    while i < len(children):
        elem = children[i]
        tag = elem.tag.replace(f"{{{OSIS_NS}}}", "")

        if tag == "w":
            word_position += 1
            word_data = parse_word(elem, word_position)
            words.append(word_data)
            full_text_parts.append(word_data["text"])

            # Ketiv/qere: when a <w type="x-ketiv"> is followed by
            # <note type="variant"><rdg type="x-qere"><w>...</w></rdg></note>,
            # the ketiv is what's written (in the scroll) and the qere is what's
            # read aloud. The Masoretic tradition preserves BOTH. Neither can be
            # silently collapsed — that would be choosing one reading over the other.
            if elem.get("type") == "x-ketiv" and i + 1 < len(children):
                nxt = children[i + 1]
                nxt_tag = nxt.tag.replace(f"{{{OSIS_NS}}}", "")
                if nxt_tag == "note" and nxt.get("type") == "variant":
                    qere_text = _extract_qere(nxt)
                    if qere_text:
                        ketiv_qere_entries.append({
                            "word_position": word_position,
                            "ketiv": word_data["text"],
                            "ketiv_consonants": word_data["consonants"],
                            "qere": qere_text,
                            "qere_consonants": strip_vowels_and_cantillation(
                                qere_text.replace("/", "")
                            ),
                        })
                    i += 1  # Skip the note element

        elif tag == "seg":
            seg_type = elem.get("type", "")
            if seg_type == "x-sof-pasuq":
                full_text_parts.append("׃")
            elif seg_type == "x-maqqef":
                full_text_parts.append("־")
            elif seg_type == "x-paseq":
                full_text_parts.append(" ׀ ")

        i += 1

    # Reconstruct full verse text
    text_hebrew = " ".join(full_text_parts).strip()
    # Clean up spacing around maqaf
    text_hebrew = text_hebrew.replace(" ־ ", "־").replace(" ־", "־").replace("־ ", "־")
    text_hebrew = re.sub(r"\s+", " ", text_hebrew).strip()

    # Extract smikhut pairs — construct state + next noun.
    # The WLC morph code ending in "c" marks construct state (smikhut).
    # Example: "av hamon" (father of multitude) — av is construct, hamon absolute.
    smikhut = _extract_smikhut(words)

    ref = f"{book_hebrew.capitalize()} {chapter}:{verse_num}"

    verse_out = {
        "ref": ref,
        "book": book_hebrew,
        "chapter": chapter,
        "verse": verse_num,
        "text_hebrew": text_hebrew,
        "words": words,
    }
    if ketiv_qere_entries:
        verse_out["ketiv_qere"] = ketiv_qere_entries
    if smikhut:
        verse_out["smikhut"] = smikhut
    return verse_out


def _extract_qere(note_elem) -> str:
    """Extract the qere reading from a <note type="variant"> element."""
    for rdg in note_elem:
        rdg_tag = rdg.tag.replace(f"{{{OSIS_NS}}}", "")
        if rdg_tag == "rdg" and rdg.get("type") == "x-qere":
            # <rdg> contains one or more <w> children
            qere_parts = []
            for w in rdg:
                w_tag = w.tag.replace(f"{{{OSIS_NS}}}", "")
                if w_tag == "w":
                    qere_text, _, _ = collect_word_text(w)
                    qere_parts.append(qere_text.replace("/", ""))
            if qere_parts:
                return " ".join(qere_parts)
    return ""


def _extract_smikhut(words: list[dict]) -> list[dict]:
    """
    Extract smikhut (construct-state) pairs. When a noun or adjective is in
    construct state (morph code ending in "c"), it binds to the following
    word as an absolute. Example: "av hamon" (אב המון) — father of multitude.

    We skip word-internal smikhut (pronominal suffixes like "av-i" = my father),
    which are marked by /Sp... segments. Those are real smikhut but single-word,
    not two-word pairs.
    """
    pairs = []
    for i, w in enumerate(words[:-1]):
        morph = w.get("morph", "")
        if not morph:
            continue
        # Morph format: H + primary-POS + optional /prefix-or-suffix segments.
        # Strip leading H (language marker) and find the primary word segment
        # (the first segment describing N=noun or A=adjective).
        segments = [s.lstrip("H") for s in morph.split("/")]
        primary = None
        for s in segments:
            if s and s[0] in ("N", "A"):
                primary = s
                break
        if not primary:
            continue
        # Skip if the same segment list has a pronominal suffix (Sp...) —
        # that's word-internal smikhut, not a two-word pair.
        if any(s.startswith("Sp") for s in segments):
            continue
        # Construct state is the last character of the primary morph.
        if not primary.endswith("c"):
            continue
        next_w = words[i + 1]
        pairs.append({
            "position": w["position"],
            "construct": w["text"],
            "construct_consonants": w["consonants"],
            "absolute": next_w["text"],
            "absolute_consonants": next_w["consonants"],
        })
    return pairs


def parse_word(w_elem, position: int) -> dict:
    """Parse a single <w> element into word data.

    Collects the full text including nested <seg type="x-large"> and
    <seg type="x-small"> children so enlarged/small Masoretic letters are
    preserved rather than silently dropped (as they were before).
    """
    raw_text, large_letters, small_letters = collect_word_text(w_elem)
    lemma_str = w_elem.get("lemma", "")
    morph_str = w_elem.get("morph", "")
    word_id = w_elem.get("id", "")

    # Parse prefixes
    prefixes, core_lemma, core_morph = parse_prefixes(lemma_str, morph_str)

    # Get text without prefix separators for the full word
    joined_text = raw_text.replace("/", "")

    # Get the word part (after last /) for the core word
    text_parts = raw_text.split("/")
    core_text = text_parts[-1] if text_parts else raw_text

    # Extract consonantal text (strip everything but Hebrew letters)
    consonants_full = strip_vowels_and_cantillation(joined_text)
    consonants_core = strip_vowels_and_cantillation(core_text)

    # Extract cantillation marks
    teamim = extract_teamim(joined_text)

    # Parse morphology
    morph_info = parse_morph_code(core_morph)

    # Derive shoresh
    shoresh = derive_shoresh_from_word(consonants_core, morph_info)

    # Build prefix string
    prefix_str = "".join(prefixes) if prefixes else ""

    word_data = {
        "position": position,
        "text": joined_text,
        "consonants": consonants_full,
        "lemma": core_lemma,
        "morph": morph_str,
        "teamim": teamim,
    }

    if prefix_str:
        word_data["prefix"] = prefix_str

    if shoresh:
        word_data["shoresh"] = shoresh

    # Otiot gedolot / zeirot — preserved as metadata on the word.
    # These are halakhically significant (the enlarged ayin/dalet of Shema
    # spell "ed" — witness; the enlarged vav of gachon marks the middle
    # letter of Torah; the small aleph of vayikra marks Moshe's humility).
    if large_letters:
        word_data["large_letters"] = large_letters
    if small_letters:
        word_data["small_letters"] = small_letters

    return word_data


def parse_book(xml_path: str, book_osis: str, book_hebrew: str) -> list[dict]:
    """Parse an entire OSIS XML book file into verses."""
    tree = ET.parse(xml_path)
    root = tree.getroot()

    verses = []

    # Navigate: osis > osisText > div (book) > chapter > verse
    for elem in root.iter(f"{{{OSIS_NS}}}verse"):
        osis_id = elem.get("osisID", "")
        if not osis_id.startswith(f"{book_osis}."):
            continue

        verse_data = parse_verse(elem, book_osis, book_hebrew)
        verses.append(verse_data)

    return verses


def build_l1(morphhb_dir: str, output_dir: str):
    """Build the complete L1 Torah filesystem."""
    wlc_dir = os.path.join(morphhb_dir, "wlc")
    torah_dir = os.path.join(output_dir, "torah")

    total_verses = 0
    total_words = 0
    total_roots = set()
    missing_teamim = []
    failed_shoresh = []
    book_stats = {}

    print("=" * 60)
    print("L1 TORAH FILESYSTEM BUILDER")
    print("Bereishit bara Elohim et hashamayim v'et ha'aretz.")
    print("=" * 60)
    print()

    for book_osis, book_hebrew in TORAH_BOOKS:
        xml_path = os.path.join(wlc_dir, f"{book_osis}.xml")

        if not os.path.exists(xml_path):
            print(f"ERROR: {xml_path} not found!")
            sys.exit(1)

        print(f"Parsing {book_osis} ({book_hebrew})...", end=" ", flush=True)

        verses = parse_book(xml_path, book_osis, book_hebrew)

        # Organize by chapter
        chapters = defaultdict(list)
        for v in verses:
            chapters[v["chapter"]].append(v)

        book_dir = os.path.join(torah_dir, book_hebrew)
        book_verse_count = 0
        book_word_count = 0
        chapter_info = {}

        for chapter_num in sorted(chapters.keys()):
            chapter_verses = chapters[chapter_num]
            chapter_dir = os.path.join(book_dir, str(chapter_num))
            os.makedirs(chapter_dir, exist_ok=True)

            for verse_data in chapter_verses:
                verse_num = verse_data["verse"]
                verse_path = os.path.join(chapter_dir, f"{verse_num}.json")

                with open(verse_path, "w", encoding="utf-8") as f:
                    json.dump(verse_data, f, ensure_ascii=False, indent=2)

                book_verse_count += 1
                total_verses += 1

                # Count words and collect stats
                for word in verse_data["words"]:
                    book_word_count += 1
                    total_words += 1

                    if word.get("shoresh"):
                        total_roots.add(word["shoresh"])
                    else:
                        # Only flag content words (not particles) as failed
                        morph_info = parse_morph_code(
                            word["morph"].split("/")[-1].lstrip("H") if word["morph"] else ""
                        )
                        if morph_info.get("pos") in ("verb", "noun", "adjective"):
                            failed_shoresh.append({
                                "ref": verse_data["ref"],
                                "word": word["text"],
                                "morph": word["morph"],
                            })

                # Check for te'amim
                verse_has_teamim = any(
                    len(w.get("teamim", [])) > 0 for w in verse_data["words"]
                )
                if not verse_has_teamim:
                    missing_teamim.append(verse_data["ref"])

            chapter_info[chapter_num] = len(chapter_verses)

        # Write book index
        book_index = {
            "book": book_hebrew,
            "osis_id": book_osis,
            "chapters": len(chapters),
            "verses": book_verse_count,
            "words": book_word_count,
            "chapter_verse_counts": chapter_info,
        }

        index_path = os.path.join(book_dir, "index.json")
        with open(index_path, "w", encoding="utf-8") as f:
            json.dump(book_index, f, ensure_ascii=False, indent=2)

        book_stats[book_hebrew] = book_index

        print(f"{len(chapters)} chapters, {book_verse_count} verses, {book_word_count} words")

    # Write master index
    master_index = {
        "name": "Torah",
        "source": "Westminster Leningrad Codex (morphhb)",
        "books": list(book_stats.values()),
        "total_chapters": sum(b["chapters"] for b in book_stats.values()),
        "total_verses": total_verses,
        "total_words": total_words,
        "unique_roots": len(total_roots),
    }

    master_path = os.path.join(torah_dir, "index.json")
    with open(master_path, "w", encoding="utf-8") as f:
        json.dump(master_index, f, ensure_ascii=False, indent=2)

    # Generate SHA-256 integrity hash
    print("\nGenerating integrity hash...")
    sha256 = generate_integrity_hash(torah_dir)

    hash_path = os.path.join(output_dir, "integrity.sha256")
    with open(hash_path, "w") as f:
        f.write(sha256)

    # Print report
    print()
    print("=" * 60)
    print("L1 BUILD REPORT")
    print("=" * 60)
    print(f"Total books:          {len(TORAH_BOOKS)}")
    print(f"Total chapters:       {master_index['total_chapters']}")
    print(f"Total verses:         {total_verses}")
    print(f"Total words:          {total_words}")
    print(f"Unique shoreshim:     {len(total_roots)}")
    print(f"Verses missing te'amim: {len(missing_teamim)}")
    print(f"Content words missing shoresh: {len(failed_shoresh)}")
    print(f"SHA-256:              {sha256}")
    print()

    if missing_teamim:
        print(f"Verses without te'amim ({len(missing_teamim)}):")
        for ref in missing_teamim[:20]:
            print(f"  - {ref}")
        if len(missing_teamim) > 20:
            print(f"  ... and {len(missing_teamim) - 20} more")
        print()

    if failed_shoresh:
        print(f"Content words without shoresh ({len(failed_shoresh)}):")
        for entry in failed_shoresh[:20]:
            print(f"  - {entry['ref']}: {entry['word']} ({entry['morph']})")
        if len(failed_shoresh) > 20:
            print(f"  ... and {len(failed_shoresh) - 20} more")
        print()

    # Write the report as JSON too
    report = {
        "total_books": len(TORAH_BOOKS),
        "total_chapters": master_index["total_chapters"],
        "total_verses": total_verses,
        "total_words": total_words,
        "unique_shoreshim": len(total_roots),
        "verses_missing_teamim": len(missing_teamim),
        "content_words_missing_shoresh": len(failed_shoresh),
        "sha256": sha256,
        "missing_teamim_refs": missing_teamim,
        "failed_shoresh_samples": failed_shoresh[:100],
    }

    report_path = os.path.join(output_dir, "build_report.json")
    with open(report_path, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

    print(f"Report written to: {report_path}")
    print(f"Master index: {master_path}")
    print(f"Integrity hash: {hash_path}")

    return report


def generate_integrity_hash(torah_dir: str) -> str:
    """Generate SHA-256 hash of the entire L1 Torah directory tree."""
    sha256 = hashlib.sha256()

    for root, dirs, files in sorted(os.walk(torah_dir)):
        dirs.sort()
        for filename in sorted(files):
            filepath = os.path.join(root, filename)
            rel_path = os.path.relpath(filepath, torah_dir)
            # Hash the relative path
            sha256.update(rel_path.encode("utf-8"))
            # Hash the file contents
            with open(filepath, "rb") as f:
                while True:
                    chunk = f.read(8192)
                    if not chunk:
                        break
                    sha256.update(chunk)

    return sha256.hexdigest()


if __name__ == "__main__":
    # The quarry (morphhb source) lives outside the repo.
    # Default: ~/shorashim-source-data/morphhb
    # Override: SHORASHIM_SOURCE_DATA env var, or first CLI arg.
    repo_root = Path(__file__).resolve().parent.parent

    default_source = Path.home() / "shorashim-source-data" / "morphhb"
    if len(sys.argv) > 1:
        morphhb_dir = Path(sys.argv[1]).expanduser().resolve()
    elif os.environ.get("SHORASHIM_SOURCE_DATA"):
        morphhb_dir = Path(os.environ["SHORASHIM_SOURCE_DATA"]).expanduser().resolve()
    else:
        morphhb_dir = default_source

    output_dir = repo_root / "l1"

    if not morphhb_dir.exists():
        print(f"ERROR: morphhb not found at {morphhb_dir}")
        print()
        print("The morphhb source (the quarry) lives outside this repo.")
        print("To fetch it:")
        print(f"  mkdir -p {default_source.parent}")
        print(f"  git clone https://github.com/openscriptures/morphhb.git {default_source}")
        print()
        print("Or pass a path as the first argument, or set SHORASHIM_SOURCE_DATA.")
        sys.exit(1)

    # Clean only the generated artifacts — preserve hand-written files like README.md.
    # The torah/ tree is regenerated. build_report.json and integrity.sha256 too.
    import shutil
    torah_dir = output_dir / "torah"
    if torah_dir.exists():
        shutil.rmtree(torah_dir)
    for artifact in ("build_report.json", "integrity.sha256"):
        p = output_dir / artifact
        if p.exists():
            p.unlink()

    os.makedirs(output_dir, exist_ok=True)

    report = build_l1(str(morphhb_dir), str(output_dir))

    # Verify counts
    print()
    print("=" * 60)
    print("VERIFICATION")
    print("=" * 60)

    expected_books = 5
    expected_chapters = 187
    # WLC/BHS versification gives 5853 (8 more than the traditional Masoretic
    # count of 5845 due to verse numbering differences in Shemot, Bamidbar, Devarim)
    expected_verses = 5853

    checks = [
        ("Books", report["total_books"], expected_books),
        ("Chapters", report["total_chapters"], expected_chapters),
        ("Verses", report["total_verses"], expected_verses),
    ]

    all_pass = True
    for name, actual, expected in checks:
        status = "PASS" if actual == expected else "FAIL"
        if actual != expected:
            all_pass = False
        print(f"  {name}: {actual} (expected {expected}) [{status}]")

    te_amim_status = "PASS" if report["verses_missing_teamim"] == 0 else "WARN"
    print(f"  Te'amim coverage: {report['verses_missing_teamim']} missing [{te_amim_status}]")

    print()
    if all_pass:
        print("The firmament is in place. The throne sits above it.")
    else:
        print("Verification incomplete. Review the report.")