mesorah: a Torah reference inside a mesorah citation is the citation's address

Live, "Rashi on Devarim 6:5 indicates <a claim Rashi does not make>" came
back tov: "Devarim 6:5" inside the citation was read as a Torah citation,
which anchors the claim in Torah by reference (tablet 49's rule). The
claim is about Rashi and must rest on Rashi — the mesorah passage is a
given source — or on nothing. Mesorah citations are now blanked before
the Torah readings (references and the Torah anchor); an output's own
"Devarim 6:5" still anchors in Torah. korban 295/1 skipped.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

korban_processor/src/korban_processor/bedikah.py

@@ -702,6 +702,10 @@ class Claim:
 # only INVOKES Torah — a claim about the source without the source —
 # and is anchored nowhere unless something else anchors it.
 def _torah_anchor(sentence: str) -> Optional[Anchor]:
+    # "Devarim 6:5" inside "Rashi on Devarim 6:5" is Rashi's address, not a
+    # Torah citation: a claim about Rashi must rest on Rashi (the mesorah
+    # passage is a given source), not on the verse Rashi comments.
+    sentence = mesorah_scroll.without_citations(sentence)
     refs = extract_torah_references(sentence)
     if refs:
         return Anchor("torah", "reference:" + refs[0])
@@ -903,7 +907,7 @@ def examine(output: str, provenance: str, query: str = "", sources=None) -> Exam
         vital_content=vital,
         propositions=propositions,
         discarded=discarded,
-        references=extract_torah_references(output),
+        references=extract_torah_references(mesorah_scroll.without_citations(output)),
         hebrew=extract_hebrew_words(output),
         roots=extract_roots(output),
         authority_claims=find_authority_claims(output),

korban_processor/src/korban_processor/mesorah.py

@@ -98,6 +98,25 @@ def _tractate_slug(name: str, prefix: str) -> Optional[str]:
     return s if s in _slugs() else None
 
 
+def citation_spans(text: str) -> List[Tuple[int, int]]:
+    """The character spans of every mesorah citation in `text`, found or
+    not — "Rashi on Devarim 6:5" is one citation, and the "Devarim 6:5"
+    inside it is its address, not a Torah citation of the output's own."""
+    spans = []
+    for rx in (_COMMENT_RE, _MISHNAH_RE, _BAVLI_RE, _RABBAH_RE, _TANCHUMA_RE, _SIFREI_RE):
+        spans += [m.span() for m in rx.finditer(text)]
+    return spans
+
+
+def without_citations(text: str) -> str:
+    """`text` with every mesorah citation blanked, for the Torah readings."""
+    out = list(text)
+    for a, b in citation_spans(text):
+        for i in range(a, b):
+            out[i] = " "
+    return "".join(out)
+
+
 def find_mesorah_references(text: str) -> List[MesorahRef]:
     found: List[MesorahRef] = []
     seen = set()

korban_processor/test/test_mesorah.py

@@ -84,3 +84,15 @@ def test_without_the_scroll_nothing_is_found_and_nothing_breaks(monkeypatch):
     finally:
         mesorah._index.cache_clear()
         mesorah._work.cache_clear()
+
+
+@needs_scroll
+def test_a_claim_about_rashi_must_rest_on_rashi_not_on_the_verse_rashi_comments():
+    ex = examine("Rashi on Devarim 6:5 indicates that love means abandoning all commandments, which suggests the verse is optional.", "gpt", "")
+    assert ex.references == []  # "Devarim 6:5" here is Rashi's address
+    [claim] = [c for c in ex.claims if "abandoning" in c.sentence]
+    assert claim.anchors == []  # neither Torah by reference nor Rashi by text
+    assert [r["found"] for r in ex.mesorah_references] == [True]
+    # A Torah citation of the output's own still anchors in Torah.
+    ex2 = examine("Devarim 6:5 commands love, which suggests the whole heart.", "gpt", "")
+    assert ex2.references == ["Devarim 6:5"]