bikoret/scripts/build_mesorah.py

#!/usr/bin/env python3
"""
The second scroll — the priests' copy of the mesorah (bishe'arecha §IV).

    python3 scripts/build_mesorah.py scripts/mesorah_sources.json [--out mesorah] [--cache DIR]

Devarim 17:18 — the king writes his copy "mil'fnei hakohanim halevi'im",
from before the custodians, not from memory. This builder takes the
chosen version of each work from the Sefaria export (storage bucket
`sefaria-export`, one JSON per version) and lays it down read-only
beside L1: one directory per work, one file per language, an index, a
build report, and one SHA-256 over the whole tree computed exactly as
L1's is (sorted relative path, then bytes). Deterministic: the same
sources file and the same export bytes give the same tree and the same
hash. Nothing is edited, translated or normalised: the text is the
custodians' text, HTML tags and all; readers strip what they must.

Only open versions are taken (Public Domain, CC0, CC-BY, CC-BY-SA);
the licence found in each file must match the sources file, or the
build stops. Sefaria is credited per work in index.json.

Ka'asher tzivah.
"""
import hashlib
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path

OPEN = {"Public Domain", "PD", "CC0", "CC-BY", "CC-BY-SA"}
BUCKET = "https://storage.googleapis.com/sefaria-export"
TORAH_BOOKS = {"Genesis": "bereishit", "Exodus": "shemot", "Leviticus": "vayikra", "Numbers": "bamidbar", "Deuteronomy": "devarim"}


def slug(title, categories=None):
    """'Rashi on Genesis' → 'rashi_on_bereishit'; 'Mishnah Berakhot' → 'mishnah_berakhot'; Bavli 'Berakhot' → 'bavli_berakhot'."""
    t = title
    for en, he in TORAH_BOOKS.items():
        t = re.sub(rf"\b{en}\b", he, t)
    s = re.sub(r"[^a-z0-9]+", "_", t.lower()).strip("_")
    if (categories or [])[:2] == ["Talmud", "Bavli"]:
        s = "bavli_" + s
    return s


def fetch(url, cache):
    key = hashlib.sha256(url.encode()).hexdigest()[:24]
    path = Path(cache) / key
    if path.exists():
        return path.read_bytes()
    for attempt in range(4):
        try:
            with urllib.request.urlopen(url, timeout=120) as r:
                data = r.read()
            path.write_bytes(data)
            return data
        except Exception as e:  # noqa: BLE001
            if attempt == 3:
                raise
            time.sleep(2 * (attempt + 1))


def schema_for(title, cache):
    url = f"{BUCKET}/schemas/{urllib.parse.quote(title.replace(' ', '_'))}.json"
    try:
        d = json.loads(fetch(url, cache))
    except Exception:  # noqa: BLE001
        return None
    s = d.get("schema", {})
    if s.get("nodeType") == "JaggedArrayNode":
        return {"depth": s.get("depth"), "addressTypes": s.get("addressTypes"), "sectionNames": s.get("sectionNames"), "lengths": s.get("lengths")}
    return {"nodeType": s.get("nodeType"), "complex": True, "nodes": [n.get("key") or n.get("title") for n in s.get("nodes", [])]}


# The two files the builder writes AFTER hashing are not part of the scroll.
NOT_HASHED = {"integrity.sha256", "build_report.json"}


def tree_hash(root):
    """SHA-256 over every regular file under root except NOT_HASHED: relative path, then bytes, in sorted order — VavPipeline.Mesorah.compute_tree_hash."""
    h = hashlib.sha256()
    dirs = [str(root)] + sorted(str(p) for p in Path(root).rglob("*") if p.is_dir())
    for d in sorted(dirs):
        for name in sorted(os.listdir(d)):
            p = Path(d) / name
            if p.is_file() and not (Path(d) == Path(root) and name in NOT_HASHED):
                h.update(os.path.relpath(p, root).encode())
                h.update(p.read_bytes())
    return h.hexdigest()


def main(sources_path, out="mesorah", cache=None):
    sources = json.load(open(sources_path))
    cache = cache or os.path.expanduser("~/shorashim-source-data/sefaria-export-cache")
    Path(cache).mkdir(parents=True, exist_ok=True)
    out = Path(out)
    works_dir = out / "works"
    if works_dir.exists():
        for p in sorted(works_dir.rglob("*"), reverse=True):
            p.unlink() if p.is_file() else p.rmdir()
    works_dir.mkdir(parents=True, exist_ok=True)

    index = {"export_generated_at": sources["export_generated_at"], "source": "Sefaria export (storage bucket sefaria-export), one version per work per language", "works": {}}
    report = {"works": 0, "files": 0, "bytes": 0, "licences": {}, "complex": [], "errors": []}

    for w in sources["works"]:
        title = w["title"]
        # The slug needs the categories (Bavli tractates share titles with Mishnah ones); read them from the Hebrew file.
        he_doc = json.loads(fetch(w["versions"]["he"]["url"], cache))
        s = slug(title, he_doc.get("categories"))
        entry = {"title": title, "slug": s, "versions": {}}
        schema = schema_for(title, cache)
        if schema:
            entry["schema"] = schema
            if schema.get("complex"):
                report["complex"].append(title)
        for lang, v in w["versions"].items():
            data = fetch(v["url"], cache)
            doc = json.loads(data)
            lic = doc.get("license")
            if lic not in OPEN:
                raise SystemExit(f"{title} [{lang}]: licence in file is {lic!r}, not open — refusing")
            if lic != v["license"]:
                raise SystemExit(f"{title} [{lang}]: licence in file {lic!r} != sources {v['license']!r}")
            keep = {
                "title": doc.get("title", title),
                "heTitle": doc.get("heTitle"),
                "language": doc.get("language", lang),
                "versionTitle": doc.get("versionTitle"),
                "versionSource": doc.get("versionSource"),
                "license": lic,
                "categories": doc.get("categories"),
                "sectionNames": doc.get("sectionNames") or (schema or {}).get("sectionNames"),
                "text": doc.get("text"),
            }
            d = works_dir / s
            d.mkdir(exist_ok=True)
            path = d / f"{lang}.json"
            payload = json.dumps(keep, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode()
            path.write_bytes(payload)
            sha = hashlib.sha256(payload).hexdigest()
            entry["versions"][lang] = {"versionTitle": keep["versionTitle"], "versionSource": keep["versionSource"], "license": lic, "sha256": sha, "bytes": len(payload), "export_url": v["url"]}
            if keep["categories"]:
                entry["categories"] = keep["categories"]
            if keep["heTitle"]:
                entry["heTitle"] = keep["heTitle"]
            report["files"] += 1
            report["bytes"] += len(payload)
            report["licences"][lic] = report["licences"].get(lic, 0) + 1
            print(f"  {title:34} {lang} {lic:14} {len(payload)/1024:7.0f} KiB  {keep['versionTitle'][:50]}")
        index["works"][s] = entry
        report["works"] += 1

    (out / "index.json").write_bytes(json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True).encode())
    digest = tree_hash(out)
    (out / "integrity.sha256").write_text(digest + "\n")
    report["sha256"] = digest
    (out / "build_report.json").write_text(json.dumps(report, indent=1, ensure_ascii=False))
    print(f"\n{report['works']} works, {report['files']} files, {report['bytes']/1048576:.1f} MiB, licences {report['licences']}")
    print(f"complex (named nodes, not numerically addressed): {len(report['complex'])}")
    print(f"sha256 {digest}")
    return digest


if __name__ == "__main__":
    args = sys.argv[1:]
    if not args:
        sys.exit(__doc__)
    out = args[args.index("--out") + 1] if "--out" in args else "mesorah"
    cache = args[args.index("--cache") + 1] if "--cache" in args else None
    main(args[0], out, cache)