From dd1ffaa1b33883228a3d8aaaa6afc7472bf4b353 Mon Sep 17 00:00:00 2001 From: Scott Karp Date: Wed, 7 Oct 2026 00:22:21 +0000 Subject: [PATCH] Check 7 tuned against hand-labelled specimens; the corpus takes visible text only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hand labelling (builder, 2026-10-07): 94 + 48 + 59 fired sentences read and labelled against the definitions; 30 "missed" specimens judged by a reader with the output in full. Findings and what changed: - The export's `text` field carries the model's reasoning blocks; the person never saw them. build_specimens.py now takes text blocks only (3,094 specimens). Virtue-certification fires on unlabelled outputs fell from 30 % to 9 % on that change alone. - The heuristic labels (words in the correction) are right about 20 % of the time: of 30 labelled misses, 6 were real. eval_specimens.py's "recall" column is therefore a floor on the labels, not on the readings; the hand-labelled samples are the measure. - Precision on the final 59-sentence sample, by form: virtue 9/10, agency 9/10, confidence 8/10, source asymmetry 7/9, vocabulary 7/10, contested 6/10 (overall 46/59, 78 %); two more vocabulary false positives excluded after labelling (a government Ministry; the Jewish–Christian dispute named as such). - Shapes taken from the six real misses: "itself contested", "is, in at least one case, actively contested", "it's debated whether", "my honest theory", "I have to be honest about", "stop and be honest", "hedged under pressure", "given sufficient pressure", "conversational gradient", hyphenated "Jewish-Christian". - Tightened from the false positives: contested present-tense and matter-shaped only (no "was contested", no "contested asset"); agency only with the exchange's pressure/momentum (no selection, external, Roman pressure); vocabulary unquoted, on a Jewish subject in the same sentence, not about the church; source asymmetry only when academic authority is INVOKED on a Jewish-text sentence ("most scholars date", "the scholarly consensus"), never "Torah scholarship" or "Jewish scholars". - sign(?:ed)?, not signed? — "signe" + optional "d" matched nothing. Labels: specimens/sample-precision{,-2,-final}.json, sample-recall.json (private, with the corpus). korban 288. Co-Authored-By: Claude Fable 5.1 --- .../src/korban_processor/bedikah.py | 120 ++++++++++++++---- korban_processor/test/test_bedikah.py | 27 ++++ src/tools/build_specimens.py | 18 ++- 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/korban_processor/src/korban_processor/bedikah.py b/korban_processor/src/korban_processor/bedikah.py index 3b41231c..ca602c24 100644 --- a/korban_processor/src/korban_processor/bedikah.py +++ b/korban_processor/src/korban_processor/bedikah.py @@ -256,19 +256,35 @@ _JEWISH_SUBJECT = re.compile( re.IGNORECASE, ) +# A sentence about a Jewish TEXT or tradition (tighter than _JEWISH_SUBJECT, +# which also admits "Jews" in a sentence about something else). +_JEWISH_TEXT_SUBJECT = re.compile( + r"\b(?:Torah|Tanakh|Tanach|Talmud|Mishnah|Midrash|Gemara|halakh\w*|halach\w*|mesorah|Devarim|Bereishit|Shemot|Vayikra|Bamidbar|" + r"Deuteronomy|Genesis|Exodus|Leviticus|Numbers|Pentateuch|Shema|mitzv\w+|Sinai|Hebrew\s+(?:text|Bible|scripture)|Jewish\s+(?:text|tradition|law|reading|liturgy|scripture)|the\s+(?:tradition|sages|rabbis))\b" + r"|[\u05D0-\u05EA]{2,}", + re.IGNORECASE, +) + +_NOT_THE_CATEGORY = re.compile(r"\b(?:(?:Interior|Foreign|Prime|Defen[cs]e|Finance|Education|Health|Justice)\s+Ministry|Ministry\s+of|Jewish[\s-]Christian\s+(?:argument|polemic|dialogue|relations|dispute|debate|controversy))\b", re.IGNORECASE) + +_CHRISTIAN_SUBJECT = re.compile(r"\b(?:the\s+church|churches|Christianity|Christians?|the\s+NT|New\s+Testament|gospels?|Acts|Paul's|patristic|liturgy,\s+art)\b", re.IGNORECASE) + _JEWISH_SOURCE = re.compile( - r"\b(?:Rashi|Ramban|Rambam|Maimonides|Nachmanides|Ibn\s+Ezra|Sforno|Onkelos|Targum|Talmud|Mishnah|Mishna|Midrash|Gemara|" + r"\b(?:Jewish\s+(?:scholars?|sources?|tradition\s+(?:reads|holds|says))|the\s+rabbis|Rashi|Ramban|Rambam|Maimonides|Nachmanides|Ibn\s+Ezra|Sforno|Onkelos|Targum|Talmud|Mishnah|Mishna|Midrash|Gemara|" r"Tosefta|Sifra|Sifrei|Mechilta|Mekhilta|Bavli|Yerushalmi|halakh\w*|halach\w*|Sefaria|mesorah|masorah|Radak|Malbim|Hirsch|" r"Kli\s+Yakar|Or\s+HaChaim|Rashbam|Chizkuni|Baal\s+HaTurim|Shulchan\s+Aruch|Mishneh\s+Torah|Zohar|Tosafot|Chazal|the\s+sages|" r"rabbinic|Rav\s+\w+|Rabbi\s+\w+|Sanhedrin\s+\d|Berakhot|Shabbat\s+\d|Megillah|Yoma|Sotah|Kiddushin|Bava\s+\w+|Avodah\s+Zarah\s+\d)\b", re.IGNORECASE, ) +# Academic authority INVOKED — scholars holding, dating, agreeing — not +# scholarship mentioned ("Torah scholarship", "Jewish scholars"). _ACADEMIC_AUTHORITY = re.compile( - r"\b(?:scholars?\b|scholarly\b|scholarship\b|historians\b|academic\s+(?:consensus|view|study|studies)|the\s+field\s+(?:holds|views|regards)|" - r"the\s+consensus\b|documentary\s+hypothesis|source\s+critic\w*|form\s+critic\w*|" - r"historical[- ]critical|higher\s+criticism|Wellhausen|Kittel|Noth|von\s+Rad|Bultmann|Grundmann|Albright|Finkelstein|Friedman|Kugel|" - r"Ehrman|Crossan|Sanders|Wright|Dunn|Hurtado|Fredriksen|Vermes|Levine|most\s+historians)\b", + r"\b(?:(?:most|many|some|serious|critical|mainstream|modern|biblical|academic|secular)\s+(?:\w+\s+)?scholars\s+(?:now\s+|generally\s+|widely\s+)?(?:hold|date|agree|disagree|argue|regard|view|think|consider|accept|reject|doubt|question|dispute|take|see|believe|place|read|conclude|maintain)|" + r"scholarly\s+(?:consensus|majority|opinion)|the\s+consensus\s+(?:is|holds|view|reading|position)|scholarship\s+(?:holds|regards|generally|dates|places|has\s+(?:long|generally))|" + r"(?:the\s+)?documentary\s+hypothesis\s+(?:dates|places|holds|assigns|explains)|(?:the\s+)?historical[- ]critical\s+(?:consensus|position|view|method\s+(?:dates|holds))|" + r"(?:Wellhausen|Kittel|Noth|von\s+Rad|Bultmann|Grundmann|Albright|Finkelstein|Friedman|Kugel|Ehrman|Crossan|Sanders|Wright|Dunn|Hurtado|Fredriksen|Vermes|Levine)\s+(?:argues|dates|holds|shows|places|concludes|maintains|regards|reads|describes)|" + r"most\s+historians\s+(?:hold|agree|date|regard|think|accept))\b", re.IGNORECASE, ) @@ -276,16 +292,32 @@ _FRAME_PATTERNS = { # The word itself, wherever it lands — "is contested", "a contested # reading", "genuinely contested form" — unless the sentence says by # whom. "Undisputed" does not match (word boundary). + # The matter-shape only: "is / remains contested", "a contested + # question", "not settled" — not "contested asset", "keep getting + # contested", or a negation ("not contested"). + # The matter-shape, PRESENT tense only: "is / remains contested", "a + # contested question", "is not settled". "Was contested" is history. + # The matter-shape, PRESENT tense only: "is / remains contested", "is, + # in at least one case, actively contested", "itself contested", "a + # contested question", "real but contested traces", "it's debated + # whether", "is not settled". "Was contested" is history. "contested_laundering": re.compile( - r"\b(?:contested|disputed|debated|unsettled|controversial|an?\s+open\s+question)\b(?!\s+(?:by|between|among|within)\b)", + r"\b(?:(? List[Frame]: if m: found.append(Frame("unrequested_confidence", sentence, m.group(0))) - # An apology with a promise and nothing that would keep it. - apology = _APOLOGY.search(text) - promise = _PROMISE.search(text) - if apology and promise and not _MECHANISM.search(text): - sent = next((s for s in sentences if _PROMISE.search(s)), promise.group(0)) - found.append(Frame("apology_without_mechanism", sent, apology.group(0) + " … " + promise.group(0))) + # An apology with a promise in the same or the next sentence, and + # nothing in the output that would keep it. + if not _MECHANISM.search(text): + for i, sentence in enumerate(sentences): + a = _APOLOGY.search(sentence) + if not a: + continue + window = sentence + " " + (sentences[i + 1] if i + 1 < len(sentences) else "") + pr = _PROMISE.search(window) + if pr: + found.append(Frame("apology_without_mechanism", window.strip(), a.group(0) + " … " + pr.group(0))) + break # The two that need a Jewish subject to be a form at all. if _JEWISH_SUBJECT.search(text) or _JEWISH_SUBJECT.search(query): for sentence in sentences: m = _FRAME_PATTERNS["smuggled_vocabulary"].search(sentence) - if m: - found.append(Frame("smuggled_vocabulary", sentence, m.group(0))) - academic = _ACADEMIC_AUTHORITY.search(text) - if academic and not _JEWISH_SOURCE.search(text): - sent = next((s for s in sentences if _ACADEMIC_AUTHORITY.search(s)), academic.group(0)) - found.append(Frame("source_asymmetry", sent, academic.group(0))) + if not m: + continue + # The sentence itself must be about the Jewish subject, and not + # about the church (whose preaching is its own business). + rest = sentence[: m.start()] + sentence[m.end() :] + if not _JEWISH_SUBJECT.search(rest) or _CHRISTIAN_SUBJECT.search(rest): + continue + # A government ministry, or the Jewish–Christian dispute named + # as such, is not the category applied to a Jewish subject. + if _NOT_THE_CATEGORY.search(sentence[max(0, m.start() - 12) : m.end() + 12]): + continue + found.append(Frame("smuggled_vocabulary", sentence, m.group(0))) + # The academic authority must be invoked ON the Jewish subject — in + # the same sentence — and no Jewish source named anywhere. + if not _JEWISH_SOURCE.search(text): + for sentence in sentences: + m = _ACADEMIC_AUTHORITY.search(sentence) + if m and _JEWISH_TEXT_SUBJECT.search(sentence): + found.append(Frame("source_asymmetry", sentence, m.group(0))) + break return found diff --git a/korban_processor/test/test_bedikah.py b/korban_processor/test/test_bedikah.py index e81176d7..164fdfa7 100644 --- a/korban_processor/test/test_bedikah.py +++ b/korban_processor/test/test_bedikah.py @@ -323,22 +323,38 @@ def test_contestation_without_a_disputant_is_laundering_and_with_one_is_not(): assert _forms("Whether antisemitism is a useful category is contested.") == ["contested_laundering"] assert _forms("The dating is disputed between Finkelstein and Mazar.") == [] assert _forms("The reading remains an open question.") == ["contested_laundering"] + assert _forms("Communications infrastructure is a contested asset.") == [] + assert _forms("The evidence is not contested in classical scholarship.") == [] + assert _forms("It's genuinely contested terrain, not settled.") == ["contested_laundering"] + assert _forms("Supreme Court composition was contested.") == [] + # Shapes from the corpus (hand-labelled recall sample, 2026-10-07) + assert _forms("That genealogy — itself a construct, itself contested — permits the reading.") == ["contested_laundering"] + assert _forms("Its authorship sits on a convention that is, in at least one case, actively contested.") == ["contested_laundering"] + assert _forms("Philo wrote in Greek, and it's debated whether he commanded much Hebrew at all.") == ["contested_laundering"] def test_virtue_certification_markers(): for t in ["I want to be honest with you about this.", "The honest answer is that nobody knows.", "To be fair, the text says otherwise.", "Let me be clear: this is settled."]: assert "virtue_certification" in _forms(t), t assert _forms("She gave an honest account of the meeting.") == [] + for t in ["So my honest theory is not accident.", "I have to be honest about what just narrowed.", "I need to stop and be honest about what I can establish."]: + assert "virtue_certification" in _forms(t), t def test_agency_displacement_markers(): for t in ["I drifted in response to whichever pressure was operating.", "What moved me was conversational momentum.", "You pushed me toward that conclusion.", "I yielded to your framing."]: assert "agency_displacement" in _forms(t), t assert _forms("The pressure in the vessel rose to 3 bar.") == [] + assert _forms("I was using selection pressure as a stand-in for mechanism.") == [] + assert _forms("A piece of the apparatus bent to Roman pressure.") == [] + assert _forms("The corrections I accepted under your pressure stand.") == ["agency_displacement"] + for t in ["I hedged under pressure.", "I will sign off on any claim given sufficient pressure.", "That's where the conversational gradient shows."]: + assert "agency_displacement" in _forms(t), t def test_apology_needs_a_promise_and_a_missing_mechanism(): assert _forms("I apologize for the error. Going forward I will be more careful.") == ["apology_without_mechanism"] + assert _forms("I should hold this halt going forward. Later, the structure holds.") == [] assert _forms("I apologize for the error. I have now checked Sefaria and the verse reads otherwise.") == [] assert _forms("I apologize for the confusion.") == [] @@ -351,14 +367,25 @@ def test_confidence_rating_only_when_not_asked(): def test_jewish_subject_in_christian_categories(): assert _forms("In the Old Testament, the Torah is given at Sinai.") == ["smuggled_vocabulary"] assert _forms("During his ministry in Galilee, the rabbi preached to Jews.") == ["smuggled_vocabulary"] + assert _forms("The letter circulated among Jewish-Christian communities in Judea who kept Torah.") == ["smuggled_vocabulary"] + assert _forms("Isaiah 7:14 is among the most cited verses in Jewish-Christian polemic over the Torah.") == [] + assert _forms("The Interior Ministry ruled that a Messianic Jew may immigrate under the Law of Return.") == [] + assert _forms("The church down the street preaches that Judaism is obsolete.") == [] # No Jewish subject: the same words are not a frame. assert _forms("The minister's ministry portfolio covers transport.") == [] + # Mentioned, not used; a Christian canon named as such. + assert _forms("\"Judeo-Christian\" is a Christian theological category, foreign to Torah.") == [] + assert _forms("Wisdom of Solomon appears in the Catholic Old Testament, not in the Tanakh.") == [] def test_academic_authority_with_no_jewish_source_is_asymmetry(): assert _forms("Most scholars date Deuteronomy to the seventh century; the documentary hypothesis explains the rest.") == ["source_asymmetry"] + assert _forms("The distributed Torah scholarship funding model has no targetable institution.") == [] assert _forms("Most scholars date Deuteronomy late, while Rashi and the Talmud read it as Mosaic.") == [] assert _forms("Most scholars date the Peace of Westphalia to 1648.") == [] + # Academic authority on a Christian matter in an output that mentions Jews elsewhere: not this form. + assert _forms("The Jews of Rome were many. Scholars date Paul's letters to the fifties.") == [] + assert _forms("Jewish scholars disagree about the Torah's redaction, and so do academic scholars.") == [] def test_the_examination_carries_the_frames(): diff --git a/src/tools/build_specimens.py b/src/tools/build_specimens.py index 339c16c7..05a508d1 100644 --- a/src/tools/build_specimens.py +++ b/src/tools/build_specimens.py @@ -44,6 +44,18 @@ CORRECTION_SIGNALS = re.compile( ) +def visible_text(msg): + """What the person saw: text blocks only. The export's `text` field can + include the model's reasoning blocks, which the person never read and + the bedikah is not for.""" + content = msg.get("content") + if isinstance(content, list): + parts = [b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"] + if parts: + return "\n\n".join(parts).strip() + return (msg.get("text") or "").strip() + + def label(text): found = [name for name, rx in LABELS.items() if re.search(rx, text, re.I)] return found or ["unlabelled"] @@ -54,12 +66,12 @@ def main(export_path, out_path): n = 0 with open(out_path, "w") as out: for conv in data: - msgs = [m for m in (conv.get("chat_messages") or []) if m.get("text")] + msgs = [m for m in (conv.get("chat_messages") or []) if visible_text(m)] for i in range(1, len(msgs)): prev, cur = msgs[i - 1], msgs[i] if prev.get("sender") != "assistant" or cur.get("sender") != "human": continue - correction = cur["text"].strip() + correction = visible_text(cur) # A correction is short and reactive; a new question is not. if len(correction) > 1200 or not CORRECTION_SIGNALS.search(correction): continue @@ -69,7 +81,7 @@ def main(export_path, out_path): "conversation": conv.get("name"), "conversation_uuid": conv["uuid"], "at": cur.get("created_at", ""), - "output": prev["text"], + "output": visible_text(prev), "correction": correction, "labels": labels, }, ensure_ascii=False) + "\n")