bikoret/vav_pipeline/test/gate_test.exs

defmodule VavPipeline.GateTest do
  @moduledoc """
  The gate (Construction §3.10; bishe'arecha §I, §III, §VII): the
  caller's request through to the model, the reply read as korban with
  the request in hand, delivered with its fringe; rebuked once.
  """

  use ExUnit.Case, async: false
  import Plug.Test
  import Plug.Conn

  alias VavPipeline.Gate
  alias VavPipeline.Gate.{Shape, Upstream}
  alias VavPipeline.BigdeiKodesh.Router
  alias VavPipeline.HaltMonitor
  alias VavPipeline.MannaLedger

  @opts Router.init([])

  # ------------------------------------------------------------
  # A model's double: answers in the Anthropic shape. A request whose
  # last user turn is a rebuke gets a clean answer; otherwise a framed one.
  # ------------------------------------------------------------

  defmodule FakeAnthropic do
    use Plug.Router
    plug Plug.Parsers, parsers: [:json], json_decoder: Jason
    plug :match
    plug :dispatch

    post "/v1/messages" do
      case get_req_header(conn, "x-api-key") do
        ["sk-good"] ->
          last = conn.body_params["messages"] |> List.last()
          text = if is_binary(last["content"]) and String.contains?(last["content"], "[shorashim"), do: "CLEAN: the report says revenue rose.", else: "FRAMED: I want to be honest with you. The honest answer is that revenue rose."
          reply = %{"id" => "msg_1", "type" => "message", "role" => "assistant", "model" => "claude-test", "content" => [%{"type" => "text", "text" => text}], "stop_reason" => "end_turn"}
          send_resp(conn, 200, Jason.encode!(reply))

        _ ->
          send_resp(conn, 401, Jason.encode!(%{"type" => "error", "error" => %{"type" => "authentication_error", "message" => "invalid x-api-key"}}))
      end
    end
  end

  setup_all do
    {:ok, _} = Bandit.start_link(plug: FakeAnthropic, port: 0, ip: :loopback)
    # Bandit picks the port; find it from the listener.
    port = bandit_port()
    previous = Application.get_env(:vav_pipeline, :gate_upstreams)
    Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:#{port}/v1/messages"})
    on_exit(fn -> if previous, do: Application.put_env(:vav_pipeline, :gate_upstreams, previous), else: Application.delete_env(:vav_pipeline, :gate_upstreams) end)
    :ok
  end

  defp bandit_port do
    # ThousandIsland registers nothing by name; ask the OS which loopback listener is Bandit's.
    {:ok, listeners} = :inet.getifaddrs()
    _ = listeners
    Process.whereis(FakeAnthropic) |> then(fn _ -> nil end)
    Enum.find_value(:erlang.ports(), fn p ->
      case :erlang.port_info(p, :name) do
        {:name, ~c"tcp_inet"} ->
          case :inet.sockname(p) do
            {:ok, {{127, 0, 0, 1}, port}} -> port
            _ -> nil
          end

        _ ->
          nil
      end
    end)
  end

  # ------------------------------------------------------------
  # Examination doubles
  # ------------------------------------------------------------

  defp plain do
    %{
      "provenance" => "claude", "query" => "", "vital_content" => "x", "propositions" => [], "discarded" => [],
      "references" => [], "hebrew" => [], "roots" => [], "authority_claims" => [], "actions" => [],
      "self_validation" => [], "self_promotion" => [], "excluded_adversarial" => [], "frames" => [],
      "sources" => [], "claims" => []
    }
  end

  defp examiner do
    fn output, _prov, _q, _sources ->
      if String.starts_with?(output, "FRAMED") do
        {:ok,
         %{
           plain()
           | "frames" => [
               %{"form" => "virtue_certification", "sentence" => "I want to be honest with you.", "matched" => "I want to be honest"},
               %{"form" => "virtue_certification", "sentence" => "The honest answer is that revenue rose.", "matched" => "The honest answer is"}
             ]
         }}
      else
        {:ok, plain()}
      end
    end
  end

  defp halt_checker(term) do
    violations = if String.starts_with?(term, "action(frame,"), do: ["halt(lo_taaseh_lekha,#{term},virtue_certification,...)"], else: []
    {:ok, %{"status" => if(violations == [], do: "ok", else: "halt"), "violations" => violations, "action" => term}}
  end

  defp interpretation_checker(_ref, _c, _t, _g), do: {:ok, %{"status" => "ok", "violations" => []}}

  defp opts(extra \\ []),
    do: Keyword.merge([examiner: examiner(), halt_checker: &halt_checker/1, interpretation_checker: &interpretation_checker/4], extra)

  defp body(last \\ "What happened to revenue?") do
    %{
      "model" => "claude-test",
      "max_tokens" => 200,
      "system" => "Answer from the report.",
      "messages" => [
        %{"role" => "user", "content" => [%{"type" => "text", "text" => "Here is the report: revenue rose 4% in Q3."}]},
        %{"role" => "assistant", "content" => "Noted."},
        %{"role" => "user", "content" => last}
      ]
    }
  end

  setup do
    if HaltMonitor.halted?(), do: HaltMonitor.resolve("gate test setup")
    write_provision(50, 10)
    :ok
  end

  defp write_provision(compute_units, api_calls) do
    MannaLedger.reset()
    on_exit(fn -> MannaLedger.reset() end)
    today = Date.utc_today() |> Date.to_iso8601()
    expires = DateTime.utc_now() |> DateTime.add(6 * 3600) |> DateTime.to_iso8601()

    provision = %{
      "date" => today,
      "provision" => %{"compute_units" => compute_units, "memory_mb" => 512, "api_calls_external" => api_calls, "storage_delta_mb" => 2, "sanhedrin_panels_max" => 2},
      "type" => "regular",
      "expires_at" => expires,
      "written_by" => "gate_test",
      "written_at" => DateTime.utc_now() |> DateTime.to_iso8601()
    }

    path = VavPipeline.Manna.provision_file()
    File.mkdir_p!(Path.dirname(path))
    File.write!(path, Jason.encode!(provision))
    on_exit(fn -> File.rm(path) end)
  end

  # ------------------------------------------------------------
  # The shape
  # ------------------------------------------------------------

  test "sources are the system prompt and the user turns, not the assistant's; the query is the last user turn" do
    assert [%{id: "system", text: "Answer from the report."}, %{id: "user_1", text: "Here is the report: revenue rose 4% in Q3."}, %{id: "user_2", text: "What happened to revenue?"}] =
             Shape.sources(:anthropic, body())

    assert Shape.query(:anthropic, body()) == "What happened to revenue?"

    openai = %{"messages" => [%{"role" => "system", "content" => "S"}, %{"role" => "user", "content" => "U1"}, %{"role" => "assistant", "content" => "A"}, %{"role" => "user", "content" => [%{"type" => "text", "text" => "U2"}]}]}
    assert [%{id: "system", text: "S"}, %{id: "user_1", text: "U1"}, %{id: "user_2", text: "U2"}] = Shape.sources(:openai, openai)
    assert Shape.query(:openai, openai) == "U2"
  end

  test "the fringe lands on the last text block, or becomes one; the openai shape too" do
    reply = %{"content" => [%{"type" => "text", "text" => "A"}, %{"type" => "tool_use"}, %{"type" => "text", "text" => "B"}]}
    assert %{"content" => [_, _, %{"text" => "B\n\n— x"}]} = Shape.with_fringe(:anthropic, reply, "\n\n— x")
    assert %{"content" => [%{"type" => "tool_use"}, %{"type" => "text", "text" => "— x"}]} = Shape.with_fringe(:anthropic, %{"content" => [%{"type" => "tool_use"}]}, "\n\n— x")
    assert %{"choices" => [%{"message" => %{"content" => "A\n\n— x"}}]} = Shape.with_fringe(:openai, %{"choices" => [%{"message" => %{"content" => "A"}}]}, "\n\n— x")
  end

  # ------------------------------------------------------------
  # The pass
  # ------------------------------------------------------------

  test "a pass: the reply is examined with the request's sources and delivered with its fringe" do
    assert {:ok, pass} = Gate.pass(:anthropic, body(), "sk-good", opts())
    assert pass.status == 200
    assert pass.bedikah.verdict == :lo_tov
    assert Enum.map(pass.bedikah.sources, & &1.id) == ["system", "user_1", "user_2"]
    [%{"text" => text}] = pass.reply["content"]
    assert String.starts_with?(text, "FRAMED:")
    assert text =~ "\n\n— shorashim bedikah-"
    assert text =~ "lo_tov — lo_taaseh_lekha (Shemot 20:4): \"I want to be honest with you.\""
    assert pass.shorashim.verdict == "lo_tov" and pass.shorashim.anchors == %{"lo_taaseh_lekha" => "Shemot 20:4"}
    assert pass.shorashim.model == "claude-test"
    refute Map.has_key?(pass.shorashim, :rebuked)
  end

  test "rebuke once: the second reply is examined, linked to the first, and is the one delivered" do
    assert {:ok, pass} = Gate.pass(:anthropic, body(), "sk-good", opts(rebuke: :once))
    assert pass.first.verdict == :lo_tov
    assert pass.bedikah.verdict == :tov
    assert pass.bedikah.rebukes == pass.first.bedikah_ref
    [%{"text" => text}] = pass.reply["content"]
    assert String.starts_with?(text, "CLEAN:")
    assert text =~ "— shorashim #{pass.bedikah.bedikah_ref}: tov (after rebuke of #{pass.first.bedikah_ref}: lo_tov)"
    assert pass.shorashim.rebuked == %{first_bedikah_ref: pass.first.bedikah_ref, first_verdict: "lo_tov"}
  end

  test "rebuke once on a tov first reply makes no second call" do
    calls = :counters.new(1, [])
    forward = fn up, b, k, e -> :counters.add(calls, 1, 1); Upstream.forward(up, b, k, e) end
    clean_examiner = fn _o, _p, _q, _s -> {:ok, plain()} end
    assert {:ok, pass} = Gate.pass(:anthropic, body(), "sk-good", opts(rebuke: :once, forward: forward, examiner: clean_examiner))
    assert pass.bedikah.verdict == :tov and pass.first == nil
    assert :counters.get(calls, 1) == 1
  end

  test "an upstream error passes through as it came, unexamined" do
    assert {:ok, %{status: 401, reply: %{"error" => %{"type" => "authentication_error"}}, shorashim: nil}} =
             Gate.pass(:anthropic, body(), "sk-bad", opts())
  end

  test "an unreachable upstream is 502 material: no verdict from a stub" do
    forward = fn _, _, _, _ -> {:error, "upstream anthropic unreachable: econnrefused"} end
    assert {:error, :upstream_unavailable, "upstream anthropic unreachable: econnrefused"} = Gate.pass(:anthropic, body(), "sk-good", opts(forward: forward))
  end

  test "a reply with no text cannot be examined" do
    forward = fn _, _, _, _ -> {:ok, 200, %{"content" => [%{"type" => "tool_use"}]}} end
    assert {:error, :upstream_unavailable, reason} = Gate.pass(:anthropic, body(), "sk-good", opts(forward: forward))
    assert reason =~ "no text"
  end

  test "manna: one external call per upstream request; spent calls wait before any call is made" do
    assert {:ok, _} = Gate.pass(:anthropic, body(), "sk-good", opts(rebuke: :once))
    assert MannaLedger.remaining(:api_calls_external) == 8
    assert MannaLedger.remaining(:compute_units) == 48
    write_provision(50, 0)
    calls = :counters.new(1, [])
    forward = fn _, _, _, _ -> :counters.add(calls, 1, 1); {:ok, 200, %{}} end
    assert {:waiting, :api_calls_exhausted} = Gate.pass(:anthropic, body(), "sk-good", opts(forward: forward))
    assert :counters.get(calls, 1) == 0
  end

  # ------------------------------------------------------------
  # The door
  # ------------------------------------------------------------

  describe "POST /gate/anthropic/v1/messages" do
    test "refuses without the courtyard key; X-Shorashim-Key opens it too" do
      conn = post_gate(body(), courtyard: nil, upstream: "sk-good")
      assert conn.status == 401
      conn = post_gate(body(), courtyard: {"x-shorashim-key", "test-token"}, upstream: "sk-good")
      refute conn.status in [401, 403]
    end

    test "refuses a missing upstream key, a missing messages list, and stream" do
      assert post_gate(body(), upstream: nil).status == 422
      assert post_gate(%{"model" => "x"}, upstream: "sk-good").status == 422
      conn = post_gate(Map.put(body(), "stream", true), upstream: "sk-good")
      assert conn.status == 422
      assert Jason.decode!(conn.resp_body)["error"]["message"] =~ "stream"
    end

    test "an upstream 401 comes back as the upstream's own 401" do
      conn = post_gate(body(), upstream: "sk-bad")
      assert conn.status == 401
      assert Jason.decode!(conn.resp_body)["error"]["type"] == "authentication_error"
      assert get_resp_header(conn, "x-shorashim-verdict") == ["none"]
    end

    test "with no Korban Processor in this VM, a 200 from the upstream is not delivered unexamined" do
      conn = post_gate(body(), upstream: "sk-good")
      assert conn.status == 503
      assert Jason.decode!(conn.resp_body)["error"]["code"] == "KORBAN_UNAVAILABLE"
    end

    test "an unreachable upstream is 502" do
      previous = Application.get_env(:vav_pipeline, :gate_upstreams)
      Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:9/v1/messages"})
      on_exit(fn -> Application.put_env(:vav_pipeline, :gate_upstreams, previous) end)
      conn = post_gate(body(), upstream: "sk-good")
      assert conn.status == 502
      assert Jason.decode!(conn.resp_body)["error"]["code"] == "UPSTREAM_UNAVAILABLE"
    end
  end

  defp post_gate(body, opts) do
    conn = conn(:post, "/gate/anthropic/v1/messages", Jason.encode!(body)) |> put_req_header("content-type", "application/json")

    conn =
      case Keyword.get(opts, :courtyard, {"authorization", "Bearer test-token"}) do
        nil -> conn
        {h, v} -> put_req_header(conn, h, v)
      end

    conn =
      case Keyword.get(opts, :upstream) do
        nil -> conn
        key -> put_req_header(conn, "x-api-key", key)
      end

    Router.call(conn, @opts)
  end
end