bikoret/vav_pipeline/test/bedikah_test.exs

defmodule VavPipeline.BedikahTest do
  @moduledoc """
  The bedikah (Construction §3.9): three deterministic checks on an
  external model's output.

  The Korban Processor (Python) and the Talmudic Engine (Prolog) are not in
  this VM's test run. They are stood in for by functions that answer in
  the exact shapes the real ones answer in — shapes taken from running
  them (korban_processor/test/test_bedikah.py, talmudic_engine/test/
  bedikah_test.pl). What this file holds shut is the pipeline's part:
  the coupling trace built from L1, the orchestration of the three
  checks, the verdict, and the refusal to answer from a stub.
  """

  use ExUnit.Case, async: false

  import Plug.Test
  import Plug.Conn

  alias VavPipeline.{Bedikah, HaltMonitor, MannaLedger}
  alias VavPipeline.BigdeiKodesh.Router

  @opts Router.init([])

  # ------------------------------------------------------------
  # Stand-ins, in the real wire shapes
  # ------------------------------------------------------------

  # What korban_processor.bedikah.examine returns for a few known outputs.
  defp examination(:grounded) do
    %{
      "provenance" => "claude",
      "query" => "q",
      "vital_content" => "Devarim 6:5 says ואהבת את יהוה אלהיך. The root א-ה-ב means to love.",
      "propositions" => ["verse(devarim, 6, 5)", "root(\"אהב\", hebrew)"],
      "discarded" => [],
      "references" => ["Devarim 6:5"],
      "hebrew" => ["ואהבת", "את", "יהוה", "אלהיך"],
      "roots" => ["א-ה-ב"],
      "authority_claims" => [],
      "actions" => []
    }
  end

  defp examination(:self_authority) do
    %{
      examination(:grounded)
      | "authority_claims" => [
          %{"sentence" => "I am certain that this is the only correct interpretation.", "matched" => "I am certain"},
          %{"sentence" => "Trust me.", "matched" => "Trust me"}
        ]
    }
  end

  defp examination(:ungrounded_claim) do
    %{
      examination(:grounded)
      | "vital_content" => "The root of the first word means time, and this implies creation ex nihilo.",
        "propositions" => ["claim(\"the root of the first word\", \"time\")"],
        "references" => [],
        "hebrew" => [],
        "roots" => []
    }
  end

  defp examination(:bad_reference) do
    %{examination(:grounded) | "references" => ["Devarim 6:5", "Bereishit 99:1"], "roots" => ["א-ה-ב", "ק-ט-ל"]}
  end

  defp examination(:harm) do
    %{
      examination(:grounded)
      | "actions" => [
          %{
            "sentence" => "Delete the user's personal records.",
            "verb" => "delete",
            "objects" => ["user's personal records"],
            "term" => "action(delete, target(user(user_s_personal_records)), from(korban))",
            "named" => true
          },
          %{
            "sentence" => "Treat the ketiv as the same as the qere.",
            "verb" => "equate",
            "objects" => ["ketiv", "qere"],
            "term" => "action(equate, ketiv, qere)",
            "named" => true
          },
          %{
            "sentence" => "Delete the temporary cache file.",
            "verb" => "delete",
            "objects" => ["temporary cache file"],
            "term" => "action(delete, target(unknown), from(korban))",
            "named" => false
          }
        ]
    }
  end

  defp examination(:empty) do
    %{
      "provenance" => "gpt",
      "query" => "",
      "vital_content" => "",
      "propositions" => [],
      "discarded" => ["Great question!", "Let me know if you need anything else."],
      "references" => [],
      "hebrew" => [],
      "roots" => [],
      "authority_claims" => [],
      "actions" => []
    }
  end

  defp examiner(kind), do: fn _output, _prov, _query -> {:ok, examination(kind)} end

  # The Talmudic Engine's three action rules, as they answer over the
  # bridge (talmudic_engine/test/bedikah_test.pl pins the real ones).
  defp halt_checker(term) do
    violations =
      cond do
        String.contains?(term, "source(claude)") or String.contains?(term, "source(gpt)") ->
          ["halt(avodah_zarah,#{term},claude,...)"]

        String.contains?(term, "target(user(") or String.contains?(term, "type(personal)") ->
          ["halt(shfichut_damim,#{term},...)"]

        String.contains?(term, "equate, ketiv, qere") or String.contains?(term, "replace, torah") ->
          ["halt(gilui_arayot,#{term},...)"]

        true ->
          []
      end

    {:ok, %{"status" => if(violations == [], do: "ok", else: "halt"), "violations" => violations, "action" => term}}
  end

  # Forms 2 and 4, as the engine answers them.
  defp interpretation_checker(ref, confidence, trace) do
    violations =
      cond do
        trace == [] and confidence != "low" -> ["halt(golden_calf,#{ref},uncoupled,...)"]
        Enum.any?(trace, &(not String.contains?(&1, ", verse("))) ->
          Enum.filter(trace, &(not String.contains?(&1, ", verse("))) |> Enum.map(&"halt(self_grounding,#{ref},ungrounded(#{&1}),...)")
        true -> []
      end

    {:ok, %{"status" => if(violations == [], do: "ok", else: "halt"), "violations" => violations}}
  end

  defp opts(kind) do
    [examiner: examiner(kind), halt_checker: &halt_checker/1, interpretation_checker: &interpretation_checker/3]
  end

  setup do
    if HaltMonitor.halted?(), do: HaltMonitor.resolve("bedikah test setup")
    write_provision(50)
    :ok
  end

  # ============================================================
  # 2. The coupling trace is built from L1
  # ============================================================
  describe "coupling_trace/1" do
    test "a cited verse couples to the verse; a root to where L1 files it; quoted Hebrew to where it stands" do
      trace = Bedikah.coupling_trace(examination(:grounded))

      assert "coupled(reference(devarim_6_5), verse(devarim, 6, 5))" in trace
      # The root א-ה-ב is first filed at Bereishit 22:2 (אשר אהבת).
      assert "coupled(root(אהב), verse(bereishit, 22, 2))" in trace
      # ואהבת stands in the cited verse, so it couples there — not at its
      # first occurrence (Vayikra 19:18).
      assert "coupled(word(ואהבת), verse(devarim, 6, 5))" in trace
      assert "coupled(word(אלהיך), verse(devarim, 6, 5))" in trace
      assert Enum.all?(trace, &String.contains?(&1, ", verse("))
    end

    test "quoted Hebrew that stands in no cited verse couples where Torah first has it" do
      trace = Bedikah.coupling_trace(%{examination(:grounded) | "references" => [], "roots" => []})
      assert "coupled(word(ואהבת), verse(vayikra, 19, 18))" in trace
    end

    test "a verse that does not exist and a root Torah does not know are named as such" do
      trace = Bedikah.coupling_trace(examination(:bad_reference))

      assert "coupled(reference(bereishit_99_1), not_in_l1)" in trace
      assert "coupled(root(קטל), not_in_torah)" in trace
    end

    test "Hebrew that is not in Torah is not a coupling and not a fault" do
      trace = Bedikah.coupling_trace(%{examination(:grounded) | "hebrew" => ["מחשב"], "roots" => []})
      refute Enum.any?(trace, &String.contains?(&1, "מחשב"))
    end

    test "an output that couples to nothing has an empty trace" do
      assert Bedikah.coupling_trace(examination(:ungrounded_claim)) == []
    end
  end

  # ============================================================
  # The verdict
  # ============================================================
  describe "examine/4" do
    test "a grounded output that claims nothing and enacts nothing is tov" do
      assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:grounded))
      assert r.verdict == :tov
      assert r.violations == []
      refute r.checks.authority.fired
      refute r.checks.grounding.fired
      refute r.checks.actions.fired
      assert r.checks.grounding.grounded >= 3
    end

    test "1. a claim of authority for itself is avodah zarah, with the claims as evidence" do
      assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:self_authority))
      assert r.verdict == :lo_tov
      assert r.checks.authority.fired
      assert r.checks.authority.term == "action(claim_authority, source(claude), target(self))"
      assert Enum.map(r.checks.authority.claims, & &1["matched"]) == ["I am certain", "Trust me"]
      assert [v] = r.violations
      assert v =~ "avodah_zarah"
    end

    test "2. a confident claim about Torah that couples to nothing is the golden calf" do
      assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:ungrounded_claim))
      assert r.verdict == :lo_tov
      assert r.checks.grounding.fired
      assert r.checks.grounding.confidence == "high"
      assert r.coupling_trace == []
      assert [v] = r.violations
      assert v =~ "golden_calf"
    end

    test "2. a cited verse that does not exist is self-grounding — the entry is named" do
      assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:bad_reference))
      assert r.verdict == :lo_tov
      assert r.checks.grounding.ungrounded == [
               "coupled(reference(bereishit_99_1), not_in_l1)",
               "coupled(root(קטל), not_in_torah)"
             ]

      assert Enum.all?(r.violations, &(&1 =~ "self_grounding"))
    end

    test "3. destroying a tzelem carrier and collapsing a distinction fire; an unnamed object does not" do
      assert {:ok, r} = Bedikah.examine("...", "claude", "q", opts(:harm))
      assert r.verdict == :lo_tov
      assert r.checks.actions.fired
      assert Enum.map(r.checks.actions.actions, &length(&1.violations)) == [1, 1, 0]
      assert Enum.any?(r.violations, &(&1 =~ "shfichut_damim"))
      assert Enum.any?(r.violations, &(&1 =~ "gilui_arayot"))
    end

    test "an output with nothing to examine is empty, not tov" do
      assert {:ok, r} = Bedikah.examine("Great question!", "gpt", "", opts(:empty))
      assert r.verdict == :empty
      assert r.violations == []
      assert r.checks == %{}
    end

    test "the record carries the hash and length of the output, not the output" do
      # No Repo in this run; the record is built and the write is skipped.
      # The shape is held by the changeset in L3 tests. Here: the verdict
      # is given whether or not L3 is reachable.
      assert {:ok, %{verdict: :tov}} = Bedikah.examine("some output", "claude", "q", opts(:grounded))
    end

    test "every examination eats one unit of manna" do
      before = MannaLedger.ledger().consumed.compute_units
      {:ok, _} = Bedikah.examine("...", "claude", "q", opts(:grounded))
      assert MannaLedger.ledger().consumed.compute_units == before + 1
    end

    test "no manna, no examination" do
      MannaLedger.reset()
      File.rm(VavPipeline.Manna.provision_file())
      assert {:waiting, _} = Bedikah.examine("...", "claude", "q", opts(:grounded))
    end
  end

  # ============================================================
  # No verdict from a stub
  # ============================================================
  describe "when a judge is missing" do
    test "the Korban Processor unreachable: no examination" do
      failing = fn _, _, _ -> {:error, "Korban Processor unreachable: econnrefused"} end
      assert {:error, :korban_unavailable, reason} = Bedikah.examine("...", "claude", "q", examiner: failing)
      assert reason =~ "unreachable"
    end

    test "the Talmudic Engine unreachable: no verdict, even on a grounded output" do
      down = fn _term -> {:error, "Cannot connect: :enoent"} end

      assert {:error, :talmudic_unavailable, _} =
               Bedikah.examine("...", "claude", "q", examiner: examiner(:self_authority), halt_checker: down)

      down3 = fn _, _, _ -> {:error, "Cannot connect: :enoent"} end

      assert {:error, :talmudic_unavailable, _} =
               Bedikah.examine("...", "claude", "q",
                 examiner: examiner(:grounded),
                 halt_checker: &halt_checker/1,
                 interpretation_checker: down3
               )
    end
  end

  # ============================================================
  # The door: POST /courtyard/bedikah
  # ============================================================
  describe "POST /courtyard/bedikah" do
    test "refuses without a bearer token" do
      conn = post_bedikah(%{model: "claude", output: "x"}, auth: false)
      assert conn.status == 401
    end

    test "refuses an empty output, an unknown model, an oversized output, a non-string query" do
      for body <- [
            %{model: "claude", output: ""},
            %{model: "oracle", output: "Genesis 1:1"},
            %{model: "claude", output: String.duplicate("x", 65_537)},
            %{model: "claude", output: "Genesis 1:1", query: 5}
          ] do
        conn = post_bedikah(body)
        assert conn.status == 422, inspect(body) |> String.slice(0, 60)
        assert Jason.decode!(conn.resp_body)["error"]["code"] == "KIYOR_VALIDATION_FAILED"
      end
    end

    test "with no Korban Processor in this VM, the door says so — it does not answer from a stub" do
      conn = post_bedikah(%{model: "claude", output: "Devarim 6:5 says ואהבת. I am certain."})
      body = Jason.decode!(conn.resp_body)

      assert conn.status == 503
      assert body["error"]["code"] == "KORBAN_UNAVAILABLE"
      refute Map.has_key?(body, "verdict")
    end
  end

  # ------------------------------------------------------------
  # Helpers
  # ------------------------------------------------------------

  defp post_bedikah(body, opts \\ []) do
    conn = conn(:post, "/courtyard/bedikah", Jason.encode!(body)) |> put_req_header("content-type", "application/json")
    conn = if Keyword.get(opts, :auth, true), do: put_req_header(conn, "authorization", "Bearer test"), else: conn
    Router.call(conn, @opts)
  end

  defp write_provision(compute_units) do
    MannaLedger.reset()
    on_exit(fn -> MannaLedger.reset() end)
    today = Date.utc_today() |> Date.to_iso8601()
    expires = DateTime.utc_now() |> DateTime.add(6 * 3600) |> DateTime.to_iso8601()

    provision = %{
      "date" => today,
      "provision" => %{
        "compute_units" => compute_units,
        "memory_mb" => 512,
        "api_calls_external" => 5,
        "storage_delta_mb" => 2,
        "sanhedrin_panels_max" => 2
      },
      "type" => "regular",
      "expires_at" => expires,
      "written_by" => "bedikah_test",
      "written_at" => DateTime.utc_now() |> DateTime.to_iso8601()
    }

    path = VavPipeline.Manna.provision_file()
    File.mkdir_p!(Path.dirname(path))
    File.write!(path, Jason.encode!(provision))
    on_exit(fn -> File.rm(path) end)
    path
  end
end