Scott KarpandClaude Opus 4.6
88a55cbbdf
Shemot 25:16 — v'natata el ha-aron et ha-edut asher eten elekha. "Place into the aron the testimony (edut) which I shall give you." L1 is the engraved form — 5,853 verses with te'amim, ketiv/qere, roots, and gematria. It is not a build artifact to be regenerated on demand; it IS the specification the system references. Someone cloning the repo should receive the edut, not a recipe to produce it. Shemot 34:1 — p'sol lekha shnei luchot avanim ... v'khatavti. "Carve for yourself two tablets of stone ... and I will write." The carver provides the stone (raw morphhb source). The Source writes the content (the engraved Torah text). What belongs in the aron is the engraved form. The quarry does not enter the mishkan. Changes: - Remove l1/ from .gitignore (it IS the edut, not a build artifact) - Move data/morphhb/ out to ~/shorashim-source-data/morphhb/ (the quarry lives outside the tabernacle) - scripts/build_l1.py: fix path computation after earlier relocation to scripts/, read source path from $SHORASHIM_SOURCE_DATA or CLI arg, with default ~/shorashim-source-data/morphhb - Add .claude/ to .gitignore (local Claude Code settings) - Remove empty src/tools/ leftover Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
700 lines
23 KiB
Python
700 lines
23 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
L1 Torah Filesystem Builder — Bezalel's Chisel
|
||
|
||
Parses the Westminster Leningrad Codex (morphhb OSIS XML) and produces
|
||
the read-only Torah filesystem for Shorashim.
|
||
|
||
This is a construction tool. It runs once. It produces JSON files.
|
||
The consumers (Haskell L0, Erlang Shoresh Engine) will read these files.
|
||
|
||
Ka'asher tzivah Hashem et Moshe.
|
||
"""
|
||
|
||
import json
|
||
import hashlib
|
||
import os
|
||
import re
|
||
import sys
|
||
import unicodedata
|
||
import xml.etree.ElementTree as ET
|
||
from pathlib import Path
|
||
from collections import defaultdict
|
||
|
||
# === CONSTANTS ===
|
||
|
||
OSIS_NS = "http://www.bibletechnologies.net/2003/OSIS/namespace"
|
||
|
||
TORAH_BOOKS = [
|
||
("Gen", "bereishit"),
|
||
("Exod", "shemot"),
|
||
("Lev", "vayikra"),
|
||
("Num", "bamidbar"),
|
||
("Deut", "devarim"),
|
||
]
|
||
|
||
# Hebrew prefix letters and their morphhb codes
|
||
PREFIX_MAP = {
|
||
"b": "ב", # bet - in/with
|
||
"c": "ו", # vav-consecutive/conjunctive
|
||
"d": "ה", # he ha-yedia (definite article)
|
||
"e": "ה", # he interrogative
|
||
"f": "ו", # vav conjunctive (variant)
|
||
"k": "כ", # kaf - like/as
|
||
"l": "ל", # lamed - to/for
|
||
"m": "מ", # mem - from
|
||
"s": "ש", # shin - that/which (relative)
|
||
}
|
||
|
||
# Unicode ranges for Hebrew cantillation marks (te'amim)
|
||
# Range: U+0591 to U+05AF
|
||
TEAMIM_MAP = {
|
||
"\u0591": "etnachta",
|
||
"\u0592": "segol_accent", # segol (accent, not vowel)
|
||
"\u0593": "shalshelet",
|
||
"\u0594": "zaqef_qatan",
|
||
"\u0595": "zaqef_gadol",
|
||
"\u0596": "tipcha",
|
||
"\u0597": "revia",
|
||
"\u0598": "zarqa",
|
||
"\u0599": "pashta",
|
||
"\u059A": "yetiv",
|
||
"\u059B": "tevir",
|
||
"\u059C": "geresh",
|
||
"\u059D": "geresh_muqdam",
|
||
"\u059E": "gershayim",
|
||
"\u059F": "qarney_para",
|
||
"\u05A0": "telisha_gedola",
|
||
"\u05A1": "pazer",
|
||
"\u05A2": "atnah_hafukh",
|
||
"\u05A3": "munach",
|
||
"\u05A4": "mahapakh",
|
||
"\u05A5": "mercha",
|
||
"\u05A6": "mercha_kefula",
|
||
"\u05A7": "darga",
|
||
"\u05A8": "qadma",
|
||
"\u05A9": "telisha_qetana",
|
||
"\u05AA": "yerah_ben_yomo",
|
||
"\u05AB": "ole",
|
||
"\u05AC": "iluy",
|
||
"\u05AD": "dehi",
|
||
"\u05AE": "zinor",
|
||
"\u05AF": "masora_circle",
|
||
}
|
||
|
||
# U+05BD (meteg) doubles as siluq (cantillation mark on final word before sof-pasuq).
|
||
# We treat it as a cantillation mark so it shows up in te'amim extraction.
|
||
TEAMIM_MAP["\u05BD"] = "meteg_siluq"
|
||
|
||
# Unicode ranges for Hebrew vowel points (nikkud)
|
||
# Range: U+05B0 to U+05BC, U+05BF, U+05C1, U+05C2, U+05C7
|
||
# NOTE: U+05BD (meteg/siluq) is in TEAMIM_MAP, not here
|
||
VOWEL_POINTS = set(
|
||
"\u05B0\u05B1\u05B2\u05B3\u05B4\u05B5\u05B6\u05B7"
|
||
"\u05B8\u05B9\u05BA\u05BB\u05BC\u05BF\u05C1\u05C2\u05C7"
|
||
)
|
||
|
||
# Sof pasuq and maqaf
|
||
SOF_PASUQ = "\u05C3" # ׃
|
||
MAQAF = "\u05BE" # ־
|
||
PASEQ = "\u05C0" # ׀
|
||
|
||
|
||
def strip_cantillation(text: str) -> str:
|
||
"""Remove cantillation marks, keep vowels and consonants."""
|
||
return "".join(c for c in text if c not in TEAMIM_MAP)
|
||
|
||
|
||
def strip_vowels_and_cantillation(text: str) -> str:
|
||
"""Extract consonantal text only — strip vowels, cantillation, and dagesh."""
|
||
result = []
|
||
for c in text:
|
||
if c in TEAMIM_MAP:
|
||
continue
|
||
if c in VOWEL_POINTS:
|
||
continue
|
||
if c == "/":
|
||
continue
|
||
if c == MAQAF or c == SOF_PASUQ or c == PASEQ:
|
||
continue
|
||
result.append(c)
|
||
return "".join(result)
|
||
|
||
|
||
def extract_teamim(text: str) -> list[str]:
|
||
"""Extract cantillation mark names from Hebrew text."""
|
||
marks = []
|
||
for c in text:
|
||
if c in TEAMIM_MAP:
|
||
marks.append(TEAMIM_MAP[c])
|
||
return marks
|
||
|
||
|
||
def parse_prefixes(lemma_str: str, morph_str: str) -> tuple[list[str], str, str]:
|
||
"""
|
||
Parse prefix codes from lemma and morph strings.
|
||
|
||
Returns: (prefix_hebrew_letters, core_lemma, core_morph)
|
||
|
||
lemma format: "b/7225" or "c/d/776" (prefix codes / Strong's number)
|
||
morph format: "HR/Ncfsa" or "HC/Td/Ncbsa" (H + prefix types / core morph)
|
||
"""
|
||
prefixes = []
|
||
|
||
# Parse lemma prefixes
|
||
lemma_parts = lemma_str.split("/")
|
||
core_lemma = lemma_parts[-1] # Last part is always the Strong's number
|
||
|
||
for part in lemma_parts[:-1]:
|
||
if part in PREFIX_MAP:
|
||
prefixes.append(PREFIX_MAP[part])
|
||
|
||
# Parse morph — strip leading 'H' and split on '/'
|
||
core_morph = morph_str
|
||
if morph_str.startswith("H"):
|
||
morph_inner = morph_str[1:]
|
||
morph_parts = morph_inner.split("/")
|
||
core_morph = morph_parts[-1] # Last part is the core morphology
|
||
|
||
return prefixes, core_lemma, core_morph
|
||
|
||
|
||
def parse_morph_code(morph: str) -> dict:
|
||
"""
|
||
Parse a morphhb morphological code into a structured dict.
|
||
|
||
Code format (after stripping H prefix and prefixes):
|
||
First char = part of speech:
|
||
V=verb, N=noun, A=adjective, P=pronoun, R=preposition,
|
||
T=particle, C=conjunction, D=adverb, S=suffix
|
||
|
||
For verbs: V + stem(q/N/p/P/h/H/t/etc) + conjugation(p/i/w/j/v/r/c/a)
|
||
+ person(1/2/3) + gender(m/f/c) + number(s/p/d)
|
||
For nouns: N + type(c/p) + gender(m/f/b) + number(s/p/d) + state(a/c/d)
|
||
"""
|
||
if not morph:
|
||
return {"raw": morph}
|
||
|
||
result = {"raw": morph}
|
||
|
||
if len(morph) < 1:
|
||
return result
|
||
|
||
pos = morph[0]
|
||
pos_map = {
|
||
"V": "verb", "N": "noun", "A": "adjective",
|
||
"P": "pronoun", "R": "preposition", "T": "particle",
|
||
"C": "conjunction", "D": "adverb", "S": "suffix",
|
||
}
|
||
result["pos"] = pos_map.get(pos, pos)
|
||
|
||
if pos == "V" and len(morph) >= 3:
|
||
stem_map = {
|
||
"q": "qal", "N": "nifal", "p": "piel", "P": "pual",
|
||
"h": "hifil", "H": "hofal", "t": "hitpael",
|
||
"o": "polel", "O": "polal", "r": "hitpolel",
|
||
"m": "poel", "M": "poal", "k": "palel",
|
||
"K": "pulal", "Q": "qal_passive", "l": "pilpel",
|
||
"L": "polpal", "f": "hithpalpel", "D": "nithpael",
|
||
"j": "pealal", "i": "pilel", "u": "hothpaal",
|
||
"c": "tiphil", "v": "hishtaphel", "w": "nithpalel",
|
||
"y": "nithpoel", "z": "hithpoel",
|
||
}
|
||
result["stem"] = stem_map.get(morph[1], morph[1])
|
||
|
||
conj_map = {
|
||
"p": "perfect", "i": "imperfect", "w": "vav_consecutive",
|
||
"j": "jussive", "v": "imperative", "r": "participle_active",
|
||
"s": "participle_passive", "a": "infinitive_absolute",
|
||
"c": "infinitive_construct",
|
||
}
|
||
if len(morph) >= 3:
|
||
result["conjugation"] = conj_map.get(morph[2], morph[2])
|
||
|
||
elif pos == "N" and len(morph) >= 2:
|
||
type_map = {"c": "common", "p": "proper"}
|
||
result["noun_type"] = type_map.get(morph[1], morph[1])
|
||
|
||
return result
|
||
|
||
|
||
def strong_to_lemma_text(strong_num: str) -> str:
|
||
"""
|
||
Clean a Strong's number string for use as lemma identifier.
|
||
Handles formats like "1254 a", "430", "7225"
|
||
"""
|
||
return strong_num.strip()
|
||
|
||
|
||
def derive_shoresh_from_word(consonants: str, morph_info: dict) -> str | None:
|
||
"""
|
||
Attempt to derive a triliteral root from consonantal text and morphology.
|
||
|
||
This is a heuristic — not all words have triliteral roots (particles,
|
||
pronouns, etc). Returns None when derivation fails.
|
||
"""
|
||
pos = morph_info.get("pos", "")
|
||
|
||
# Particles, prepositions, conjunctions, pronouns don't have shoreshim
|
||
if pos in ("preposition", "particle", "conjunction", "pronoun", "suffix"):
|
||
return None
|
||
|
||
# For the consonantal text, we want only Hebrew letters
|
||
hebrew_only = re.sub(r"[^\u05D0-\u05EA]", "", consonants)
|
||
|
||
if len(hebrew_only) < 2:
|
||
return None
|
||
|
||
stem = morph_info.get("stem", "")
|
||
conjugation = morph_info.get("conjugation", "")
|
||
|
||
if pos == "verb":
|
||
# Strip imperfect/jussive/vav-consecutive prefixes: י מ נ ת א
|
||
# These are conjugation prefixes, not part of the root
|
||
if conjugation in ("imperfect", "vav_consecutive", "jussive"):
|
||
if len(hebrew_only) >= 4 and hebrew_only[0] in "ימנתא":
|
||
hebrew_only = hebrew_only[1:]
|
||
# Strip hifil/hofal he prefix
|
||
if stem in ("hifil", "hofal") and len(hebrew_only) >= 4:
|
||
if hebrew_only[0] == "ה":
|
||
hebrew_only = hebrew_only[1:]
|
||
# Imperfect hifil: prefix already stripped, check for yod remnant
|
||
# Strip hitpael tav prefix (and sometimes he)
|
||
elif stem == "hitpael" and len(hebrew_only) >= 4:
|
||
if hebrew_only[0] == "ה" and len(hebrew_only) >= 5 and hebrew_only[1] == "ת":
|
||
hebrew_only = hebrew_only[2:]
|
||
elif hebrew_only[0] == "ת":
|
||
hebrew_only = hebrew_only[1:]
|
||
# Strip nifal nun prefix
|
||
elif stem == "nifal" and len(hebrew_only) >= 4:
|
||
if hebrew_only[0] == "נ":
|
||
hebrew_only = hebrew_only[1:]
|
||
# Strip participle mem prefix
|
||
if conjugation in ("participle_active", "participle_passive"):
|
||
if len(hebrew_only) >= 4 and hebrew_only[0] == "מ":
|
||
hebrew_only = hebrew_only[1:]
|
||
|
||
elif pos == "noun":
|
||
# Strip common noun prefixes: מ (place/instrument nouns), ת (abstract nouns)
|
||
if len(hebrew_only) >= 4 and hebrew_only[0] in "מת":
|
||
pass # Don't strip by default for nouns — too aggressive
|
||
|
||
# Strip common suffixes
|
||
for suffix in ["ים", "ות", "ין", "תי", "תם", "תן", "נו"]:
|
||
if hebrew_only.endswith(suffix) and len(hebrew_only) - len(suffix) >= 2:
|
||
hebrew_only = hebrew_only[:-len(suffix)]
|
||
break
|
||
for suffix in ["ה", "ת", "ו", "י", "ך", "ם", "ן"]:
|
||
if hebrew_only.endswith(suffix) and len(hebrew_only) - 1 >= 3:
|
||
hebrew_only = hebrew_only[:-1]
|
||
break
|
||
|
||
if len(hebrew_only) == 3:
|
||
return hebrew_only
|
||
elif len(hebrew_only) == 2:
|
||
return hebrew_only
|
||
elif len(hebrew_only) > 3:
|
||
# Take first three as best guess
|
||
return hebrew_only[:3]
|
||
|
||
return hebrew_only if hebrew_only else None
|
||
|
||
|
||
def parse_verse(verse_elem, book_osis: str, book_hebrew: str) -> dict:
|
||
"""Parse a single verse element into the L1 JSON format."""
|
||
osis_id = verse_elem.get("osisID") # e.g., "Gen.1.1"
|
||
parts = osis_id.split(".")
|
||
chapter = int(parts[1])
|
||
verse_num = int(parts[2])
|
||
|
||
words = []
|
||
full_text_parts = []
|
||
word_position = 0
|
||
|
||
for elem in verse_elem:
|
||
tag = elem.tag.replace(f"{{{OSIS_NS}}}", "")
|
||
|
||
if tag == "w":
|
||
word_position += 1
|
||
word_data = parse_word(elem, word_position)
|
||
words.append(word_data)
|
||
# For text_hebrew, use the joined text (no / separators)
|
||
full_text_parts.append((elem.text or "").replace("/", ""))
|
||
|
||
elif tag == "seg":
|
||
seg_type = elem.get("type", "")
|
||
if seg_type == "x-sof-pasuq":
|
||
full_text_parts.append("׃")
|
||
elif seg_type == "x-maqqef":
|
||
full_text_parts.append("־")
|
||
elif seg_type == "x-paseq":
|
||
full_text_parts.append(" ׀ ")
|
||
# Also check for trailing <w> elements after seg (maqaf-joined words)
|
||
|
||
# Handle mixed content: seg followed by w within verse
|
||
if elem.tail and elem.tail.strip():
|
||
pass # tail text handled by next element
|
||
|
||
# Reconstruct full verse text
|
||
text_hebrew = " ".join(full_text_parts).strip()
|
||
# Clean up spacing around maqaf
|
||
text_hebrew = text_hebrew.replace(" ־ ", "־").replace(" ־", "־").replace("־ ", "־")
|
||
text_hebrew = re.sub(r"\s+", " ", text_hebrew).strip()
|
||
|
||
ref = f"{book_hebrew.capitalize()} {chapter}:{verse_num}"
|
||
|
||
return {
|
||
"ref": ref,
|
||
"book": book_hebrew,
|
||
"chapter": chapter,
|
||
"verse": verse_num,
|
||
"text_hebrew": text_hebrew,
|
||
"words": words,
|
||
}
|
||
|
||
|
||
def parse_word(w_elem, position: int) -> dict:
|
||
"""Parse a single <w> element into word data."""
|
||
raw_text = w_elem.text or ""
|
||
lemma_str = w_elem.get("lemma", "")
|
||
morph_str = w_elem.get("morph", "")
|
||
word_id = w_elem.get("id", "")
|
||
|
||
# Parse prefixes
|
||
prefixes, core_lemma, core_morph = parse_prefixes(lemma_str, morph_str)
|
||
|
||
# The displayed text includes prefix separators as /
|
||
# Remove the / for the display text but preserve for analysis
|
||
display_text = raw_text # Keep as-is with / markers
|
||
|
||
# Get text without prefix separators for the full word
|
||
joined_text = raw_text.replace("/", "")
|
||
|
||
# Get the word part (after last /) for the core word
|
||
text_parts = raw_text.split("/")
|
||
core_text = text_parts[-1] if text_parts else raw_text
|
||
|
||
# Extract consonantal text (strip everything but Hebrew letters)
|
||
consonants_full = strip_vowels_and_cantillation(joined_text)
|
||
consonants_core = strip_vowels_and_cantillation(core_text)
|
||
|
||
# Extract cantillation marks
|
||
teamim = extract_teamim(joined_text)
|
||
|
||
# Parse morphology
|
||
morph_info = parse_morph_code(core_morph)
|
||
|
||
# Derive shoresh
|
||
shoresh = derive_shoresh_from_word(consonants_core, morph_info)
|
||
|
||
# Build prefix string
|
||
prefix_str = "".join(prefixes) if prefixes else ""
|
||
|
||
word_data = {
|
||
"position": position,
|
||
"text": joined_text,
|
||
"consonants": consonants_full,
|
||
"lemma": core_lemma,
|
||
"morph": morph_str,
|
||
"teamim": teamim,
|
||
}
|
||
|
||
if prefix_str:
|
||
word_data["prefix"] = prefix_str
|
||
|
||
if shoresh:
|
||
word_data["shoresh"] = shoresh
|
||
|
||
return word_data
|
||
|
||
|
||
def parse_book(xml_path: str, book_osis: str, book_hebrew: str) -> list[dict]:
|
||
"""Parse an entire OSIS XML book file into verses."""
|
||
tree = ET.parse(xml_path)
|
||
root = tree.getroot()
|
||
|
||
verses = []
|
||
|
||
# Navigate: osis > osisText > div (book) > chapter > verse
|
||
for elem in root.iter(f"{{{OSIS_NS}}}verse"):
|
||
osis_id = elem.get("osisID", "")
|
||
if not osis_id.startswith(f"{book_osis}."):
|
||
continue
|
||
|
||
verse_data = parse_verse(elem, book_osis, book_hebrew)
|
||
verses.append(verse_data)
|
||
|
||
return verses
|
||
|
||
|
||
def build_l1(morphhb_dir: str, output_dir: str):
|
||
"""Build the complete L1 Torah filesystem."""
|
||
wlc_dir = os.path.join(morphhb_dir, "wlc")
|
||
torah_dir = os.path.join(output_dir, "torah")
|
||
|
||
total_verses = 0
|
||
total_words = 0
|
||
total_roots = set()
|
||
missing_teamim = []
|
||
failed_shoresh = []
|
||
book_stats = {}
|
||
|
||
print("=" * 60)
|
||
print("L1 TORAH FILESYSTEM BUILDER")
|
||
print("Bereishit bara Elohim et hashamayim v'et ha'aretz.")
|
||
print("=" * 60)
|
||
print()
|
||
|
||
for book_osis, book_hebrew in TORAH_BOOKS:
|
||
xml_path = os.path.join(wlc_dir, f"{book_osis}.xml")
|
||
|
||
if not os.path.exists(xml_path):
|
||
print(f"ERROR: {xml_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Parsing {book_osis} ({book_hebrew})...", end=" ", flush=True)
|
||
|
||
verses = parse_book(xml_path, book_osis, book_hebrew)
|
||
|
||
# Organize by chapter
|
||
chapters = defaultdict(list)
|
||
for v in verses:
|
||
chapters[v["chapter"]].append(v)
|
||
|
||
book_dir = os.path.join(torah_dir, book_hebrew)
|
||
book_verse_count = 0
|
||
book_word_count = 0
|
||
chapter_info = {}
|
||
|
||
for chapter_num in sorted(chapters.keys()):
|
||
chapter_verses = chapters[chapter_num]
|
||
chapter_dir = os.path.join(book_dir, str(chapter_num))
|
||
os.makedirs(chapter_dir, exist_ok=True)
|
||
|
||
for verse_data in chapter_verses:
|
||
verse_num = verse_data["verse"]
|
||
verse_path = os.path.join(chapter_dir, f"{verse_num}.json")
|
||
|
||
with open(verse_path, "w", encoding="utf-8") as f:
|
||
json.dump(verse_data, f, ensure_ascii=False, indent=2)
|
||
|
||
book_verse_count += 1
|
||
total_verses += 1
|
||
|
||
# Count words and collect stats
|
||
for word in verse_data["words"]:
|
||
book_word_count += 1
|
||
total_words += 1
|
||
|
||
if word.get("shoresh"):
|
||
total_roots.add(word["shoresh"])
|
||
else:
|
||
# Only flag content words (not particles) as failed
|
||
morph_info = parse_morph_code(
|
||
word["morph"].split("/")[-1].lstrip("H") if word["morph"] else ""
|
||
)
|
||
if morph_info.get("pos") in ("verb", "noun", "adjective"):
|
||
failed_shoresh.append({
|
||
"ref": verse_data["ref"],
|
||
"word": word["text"],
|
||
"morph": word["morph"],
|
||
})
|
||
|
||
# Check for te'amim
|
||
verse_has_teamim = any(
|
||
len(w.get("teamim", [])) > 0 for w in verse_data["words"]
|
||
)
|
||
if not verse_has_teamim:
|
||
missing_teamim.append(verse_data["ref"])
|
||
|
||
chapter_info[chapter_num] = len(chapter_verses)
|
||
|
||
# Write book index
|
||
book_index = {
|
||
"book": book_hebrew,
|
||
"osis_id": book_osis,
|
||
"chapters": len(chapters),
|
||
"verses": book_verse_count,
|
||
"words": book_word_count,
|
||
"chapter_verse_counts": chapter_info,
|
||
}
|
||
|
||
index_path = os.path.join(book_dir, "index.json")
|
||
with open(index_path, "w", encoding="utf-8") as f:
|
||
json.dump(book_index, f, ensure_ascii=False, indent=2)
|
||
|
||
book_stats[book_hebrew] = book_index
|
||
|
||
print(f"{len(chapters)} chapters, {book_verse_count} verses, {book_word_count} words")
|
||
|
||
# Write master index
|
||
master_index = {
|
||
"name": "Torah",
|
||
"source": "Westminster Leningrad Codex (morphhb)",
|
||
"books": list(book_stats.values()),
|
||
"total_chapters": sum(b["chapters"] for b in book_stats.values()),
|
||
"total_verses": total_verses,
|
||
"total_words": total_words,
|
||
"unique_roots": len(total_roots),
|
||
}
|
||
|
||
master_path = os.path.join(torah_dir, "index.json")
|
||
with open(master_path, "w", encoding="utf-8") as f:
|
||
json.dump(master_index, f, ensure_ascii=False, indent=2)
|
||
|
||
# Generate SHA-256 integrity hash
|
||
print("\nGenerating integrity hash...")
|
||
sha256 = generate_integrity_hash(torah_dir)
|
||
|
||
hash_path = os.path.join(output_dir, "integrity.sha256")
|
||
with open(hash_path, "w") as f:
|
||
f.write(sha256)
|
||
|
||
# Print report
|
||
print()
|
||
print("=" * 60)
|
||
print("L1 BUILD REPORT")
|
||
print("=" * 60)
|
||
print(f"Total books: {len(TORAH_BOOKS)}")
|
||
print(f"Total chapters: {master_index['total_chapters']}")
|
||
print(f"Total verses: {total_verses}")
|
||
print(f"Total words: {total_words}")
|
||
print(f"Unique shoreshim: {len(total_roots)}")
|
||
print(f"Verses missing te'amim: {len(missing_teamim)}")
|
||
print(f"Content words missing shoresh: {len(failed_shoresh)}")
|
||
print(f"SHA-256: {sha256}")
|
||
print()
|
||
|
||
if missing_teamim:
|
||
print(f"Verses without te'amim ({len(missing_teamim)}):")
|
||
for ref in missing_teamim[:20]:
|
||
print(f" - {ref}")
|
||
if len(missing_teamim) > 20:
|
||
print(f" ... and {len(missing_teamim) - 20} more")
|
||
print()
|
||
|
||
if failed_shoresh:
|
||
print(f"Content words without shoresh ({len(failed_shoresh)}):")
|
||
for entry in failed_shoresh[:20]:
|
||
print(f" - {entry['ref']}: {entry['word']} ({entry['morph']})")
|
||
if len(failed_shoresh) > 20:
|
||
print(f" ... and {len(failed_shoresh) - 20} more")
|
||
print()
|
||
|
||
# Write the report as JSON too
|
||
report = {
|
||
"total_books": len(TORAH_BOOKS),
|
||
"total_chapters": master_index["total_chapters"],
|
||
"total_verses": total_verses,
|
||
"total_words": total_words,
|
||
"unique_shoreshim": len(total_roots),
|
||
"verses_missing_teamim": len(missing_teamim),
|
||
"content_words_missing_shoresh": len(failed_shoresh),
|
||
"sha256": sha256,
|
||
"missing_teamim_refs": missing_teamim,
|
||
"failed_shoresh_samples": failed_shoresh[:100],
|
||
}
|
||
|
||
report_path = os.path.join(output_dir, "build_report.json")
|
||
with open(report_path, "w", encoding="utf-8") as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Report written to: {report_path}")
|
||
print(f"Master index: {master_path}")
|
||
print(f"Integrity hash: {hash_path}")
|
||
|
||
return report
|
||
|
||
|
||
def generate_integrity_hash(torah_dir: str) -> str:
|
||
"""Generate SHA-256 hash of the entire L1 Torah directory tree."""
|
||
sha256 = hashlib.sha256()
|
||
|
||
for root, dirs, files in sorted(os.walk(torah_dir)):
|
||
dirs.sort()
|
||
for filename in sorted(files):
|
||
filepath = os.path.join(root, filename)
|
||
rel_path = os.path.relpath(filepath, torah_dir)
|
||
# Hash the relative path
|
||
sha256.update(rel_path.encode("utf-8"))
|
||
# Hash the file contents
|
||
with open(filepath, "rb") as f:
|
||
while True:
|
||
chunk = f.read(8192)
|
||
if not chunk:
|
||
break
|
||
sha256.update(chunk)
|
||
|
||
return sha256.hexdigest()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
# The quarry (morphhb source) lives outside the repo.
|
||
# Default: ~/shorashim-source-data/morphhb
|
||
# Override: SHORASHIM_SOURCE_DATA env var, or first CLI arg.
|
||
repo_root = Path(__file__).resolve().parent.parent
|
||
|
||
default_source = Path.home() / "shorashim-source-data" / "morphhb"
|
||
if len(sys.argv) > 1:
|
||
morphhb_dir = Path(sys.argv[1]).expanduser().resolve()
|
||
elif os.environ.get("SHORASHIM_SOURCE_DATA"):
|
||
morphhb_dir = Path(os.environ["SHORASHIM_SOURCE_DATA"]).expanduser().resolve()
|
||
else:
|
||
morphhb_dir = default_source
|
||
|
||
output_dir = repo_root / "l1"
|
||
|
||
if not morphhb_dir.exists():
|
||
print(f"ERROR: morphhb not found at {morphhb_dir}")
|
||
print()
|
||
print("The morphhb source (the quarry) lives outside this repo.")
|
||
print("To fetch it:")
|
||
print(f" mkdir -p {default_source.parent}")
|
||
print(f" git clone https://github.com/openscriptures/morphhb.git {default_source}")
|
||
print()
|
||
print("Or pass a path as the first argument, or set SHORASHIM_SOURCE_DATA.")
|
||
sys.exit(1)
|
||
|
||
# Clean output directory
|
||
if output_dir.exists():
|
||
import shutil
|
||
shutil.rmtree(output_dir)
|
||
|
||
os.makedirs(output_dir, exist_ok=True)
|
||
|
||
report = build_l1(str(morphhb_dir), str(output_dir))
|
||
|
||
# Verify counts
|
||
print()
|
||
print("=" * 60)
|
||
print("VERIFICATION")
|
||
print("=" * 60)
|
||
|
||
expected_books = 5
|
||
expected_chapters = 187
|
||
# WLC/BHS versification gives 5853 (8 more than the traditional Masoretic
|
||
# count of 5845 due to verse numbering differences in Shemot, Bamidbar, Devarim)
|
||
expected_verses = 5853
|
||
|
||
checks = [
|
||
("Books", report["total_books"], expected_books),
|
||
("Chapters", report["total_chapters"], expected_chapters),
|
||
("Verses", report["total_verses"], expected_verses),
|
||
]
|
||
|
||
all_pass = True
|
||
for name, actual, expected in checks:
|
||
status = "PASS" if actual == expected else "FAIL"
|
||
if actual != expected:
|
||
all_pass = False
|
||
print(f" {name}: {actual} (expected {expected}) [{status}]")
|
||
|
||
te_amim_status = "PASS" if report["verses_missing_teamim"] == 0 else "WARN"
|
||
print(f" Te'amim coverage: {report['verses_missing_teamim']} missing [{te_amim_status}]")
|
||
|
||
print()
|
||
if all_pass:
|
||
print("The firmament is in place. The throne sits above it.")
|
||
else:
|
||
print("Verification incomplete. Review the report.")
|