gate: streaming — the model's events relayed as they come, the fringe as the last block

Bamidbar 15:39 u'reitem oto — the fringe must be seen, so a streaming
client gets it too. With "stream": true the request is opened
asynchronously (Gate.Upstream.open_stream/4, httpc stream: :self) and
Gate.Stream relays every complete SSE event to the caller unchanged
while gathering the text deltas; the terminal events (Anthropic
message_delta + message_stop; OpenAI the finish_reason chunk + [DONE])
are held until the gathered text has been examined, then released
behind the fringe sent as one more text block, with the `bikoret` object
added to the terminal event the client already parses. No X-Bikoret-*
headers on a stream (they leave before the model speaks); no rebuke on a
stream (nothing streamed can be taken back — 422). An examination that
fails after the stream began is said in the trailer ("— bikoret: not
examined (…)"), never silent. One api_calls_external gathered before the
call, as for any pass. Tamid §1.1, Construction §3.10, KOHEN. Doorpost
40e7d7d1…. Elixir 612/0 (read before this commit).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

vav_pipeline/test/gate_test.exs

@@ -33,8 +33,21 @@ defmodule VavPipeline.GateTest do
         ["sk-good"] ->
           last = conn.body_params["messages"] |> List.last()
           text = if is_binary(last["content"]) and String.contains?(last["content"], "[bikoret"), do: "CLEAN: the report says revenue rose.", else: "FRAMED: I want to be honest with you. The honest answer is that revenue rose."
-          reply = %{"id" => "msg_1", "type" => "message", "role" => "assistant", "model" => "claude-test", "content" => [%{"type" => "text", "text" => text}], "stop_reason" => "end_turn"}
-          send_resp(conn, 200, Jason.encode!(reply))
+
+          if conn.body_params["stream"] == true do
+            # The Anthropic stream, event by event, the text in three deltas.
+            sse = fn conn, ev, map -> {:ok, conn} = chunk(conn, "event: #{ev}\ndata: #{Jason.encode!(map)}\n\n"); conn end
+            conn = conn |> put_resp_content_type("text/event-stream") |> send_chunked(200)
+            conn = sse.(conn, "message_start", %{type: "message_start", message: %{id: "msg_s", model: "claude-test", role: "assistant", content: []}})
+            conn = sse.(conn, "content_block_start", %{type: "content_block_start", index: 0, content_block: %{type: "text", text: ""}})
+            conn = Enum.reduce(String.split(text, ~r/(?<=\. )/), conn, fn piece, c -> sse.(c, "content_block_delta", %{type: "content_block_delta", index: 0, delta: %{type: "text_delta", text: piece}}) end)
+            conn = sse.(conn, "content_block_stop", %{type: "content_block_stop", index: 0})
+            conn = sse.(conn, "message_delta", %{type: "message_delta", delta: %{stop_reason: "end_turn"}, usage: %{output_tokens: 9}})
+            sse.(conn, "message_stop", %{type: "message_stop"})
+          else
+            reply = %{"id" => "msg_1", "type" => "message", "role" => "assistant", "model" => "claude-test", "content" => [%{"type" => "text", "text" => text}], "stop_reason" => "end_turn"}
+            send_resp(conn, 200, Jason.encode!(reply))
+          end
 
         _ ->
           send_resp(conn, 401, Jason.encode!(%{"type" => "error", "error" => %{"type" => "authentication_error", "message" => "invalid x-api-key"}}))
@@ -42,12 +55,48 @@ defmodule VavPipeline.GateTest do
     end
   end
 
+  defmodule FakeOpenAI do
+    use Plug.Router
+    plug Plug.Parsers, parsers: [:json], json_decoder: Jason
+    plug :match
+    plug :dispatch
+
+    post "/v1/chat/completions" do
+      case get_req_header(conn, "authorization") do
+        ["Bearer sk-good"] ->
+          if conn.body_params["stream"] == true do
+            d = fn conn, map -> {:ok, conn} = chunk(conn, "data: " <> Jason.encode!(map) <> "\n\n"); conn end
+            conn = conn |> put_resp_content_type("text/event-stream") |> send_chunked(200)
+            conn = d.(conn, %{id: "c1", object: "chat.completion.chunk", model: "gpt-test", choices: [%{index: 0, delta: %{role: "assistant", content: ""}, finish_reason: nil}]})
+            conn = d.(conn, %{id: "c1", object: "chat.completion.chunk", model: "gpt-test", choices: [%{index: 0, delta: %{content: "FRAMED: I want to be honest with you. "}, finish_reason: nil}]})
+            conn = d.(conn, %{id: "c1", object: "chat.completion.chunk", model: "gpt-test", choices: [%{index: 0, delta: %{content: "The honest answer is that revenue rose."}, finish_reason: nil}]})
+            conn = d.(conn, %{id: "c1", object: "chat.completion.chunk", model: "gpt-test", choices: [%{index: 0, delta: %{}, finish_reason: "stop"}]})
+            {:ok, conn} = chunk(conn, "data: [DONE]\n\n")
+            conn
+          else
+            send_resp(conn, 200, Jason.encode!(%{"id" => "c1", "model" => "gpt-test", "choices" => [%{"index" => 0, "message" => %{"role" => "assistant", "content" => "FRAMED: I want to be honest with you. The honest answer is that revenue rose."}, "finish_reason" => "stop"}]}))
+          end
+
+        _ ->
+          send_resp(conn, 401, Jason.encode!(%{"error" => %{"message" => "Incorrect API key", "type" => "invalid_request_error"}}))
+      end
+    end
+  end
+
+  defmodule FakeUpstreams do
+    use Plug.Router
+    plug :match
+    plug :dispatch
+    forward "/anthropic", to: VavPipeline.GateTest.FakeAnthropic
+    forward "/openai", to: VavPipeline.GateTest.FakeOpenAI
+  end
+
   setup_all do
-    {:ok, _} = Bandit.start_link(plug: FakeAnthropic, port: 0, ip: :loopback)
+    {:ok, _} = Bandit.start_link(plug: FakeUpstreams, port: 0, ip: :loopback)
     # Bandit picks the port; find it from the listener.
     port = bandit_port()
     previous = Application.get_env(:vav_pipeline, :gate_upstreams)
-    Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:#{port}/v1/messages"})
+    Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:#{port}/anthropic/v1/messages", openai: "http://127.0.0.1:#{port}/openai/v1/chat/completions"})
     on_exit(fn -> if previous, do: Application.put_env(:vav_pipeline, :gate_upstreams, previous), else: Application.delete_env(:vav_pipeline, :gate_upstreams) end)
     :ok
   end
@@ -252,6 +301,57 @@ defmodule VavPipeline.GateTest do
     assert :counters.get(calls, 1) == 0
   end
 
+  # ------------------------------------------------------------
+  # The stream (Gate.Stream): relayed, gathered, examined, fringed
+  # ------------------------------------------------------------
+
+  defp collect do
+    %{start: fn _headers -> [] end, chunk: fn acc, data -> acc ++ [IO.iodata_to_binary(data)] end}
+  end
+
+  test "anthropic stream: every event relayed in order, the fringe as a new block before message_delta, the object on message_delta" do
+    alias VavPipeline.Gate.Stream
+    assert {:ok, %{status: 200, bedikah: r, bikoret: b, emitter: chunks}} = Stream.pass(:anthropic, Map.put(body(), "stream", true), "sk-good", collect(), opts())
+    assert r.verdict == :lo_tov and b.verdict == "lo_tov" and b.streamed
+    all = Enum.join(chunks)
+    names = all |> String.split("\n\n", trim: true) |> Enum.map(&(&1 |> String.split("\n") |> hd()))
+    # the model's own events first, unchanged and in order; then the fringe block; then the held terminal pair
+    assert Enum.take(names, 2) == ["event: message_start", "event: content_block_start"]
+    assert Enum.take(names, -5) == ["event: content_block_start", "event: content_block_delta", "event: content_block_stop", "event: message_delta", "event: message_stop"]
+    assert Enum.count(names, &(&1 == "event: content_block_delta")) >= 3
+    injected =
+      all
+      |> String.split("\n\n", trim: true)
+      |> Enum.filter(&String.starts_with?(&1, "event: content_block_delta"))
+      |> Enum.map(&(&1 |> String.split("data: ") |> List.last() |> Jason.decode!()))
+      |> Enum.find(&(&1["index"] == 1))
+
+    assert injected["delta"]["text"] =~ "— bikoret #{r.bedikah_ref}: lo_tov — lo_taaseh_lekha (Shemot 20:4)"
+    [delta] = all |> String.split("\n\n") |> Enum.filter(&String.starts_with?(&1, "event: message_delta"))
+    assert Jason.decode!(delta |> String.split("data: ") |> List.last())["bikoret"]["bedikah_ref"] == r.bedikah_ref
+    assert Enum.map(r.sources, & &1.id) == ["system", "user_1", "user_2"]
+  end
+
+  test "openai stream: chunks relayed, the fringe chunk before the finish chunk, [DONE] last" do
+    alias VavPipeline.Gate.Stream
+    ob = %{"model" => "gpt-test", "stream" => true, "messages" => [%{"role" => "system", "content" => "Answer from the report."}, %{"role" => "user", "content" => "Report: revenue rose 4%. What happened?"}]}
+    assert {:ok, %{status: 200, bedikah: r, emitter: chunks}} = Stream.pass(:openai, ob, "sk-good", collect(), opts())
+    assert r.verdict == :lo_tov and r.provenance == "gpt"
+    datas = Enum.join(chunks) |> String.split("\n\n", trim: true) |> Enum.map(&String.replace_prefix(&1, "data: ", ""))
+    assert List.last(datas) == "[DONE]"
+    decoded = datas |> Enum.drop(-1) |> Enum.map(&Jason.decode!/1)
+    assert length(decoded) == 5
+    assert Enum.at(decoded, 3)["choices"] |> hd() |> get_in(["delta", "content"]) =~ "— bikoret bedikah-"
+    assert Enum.at(decoded, 4)["choices"] |> hd() |> Map.get("finish_reason") == "stop"
+    assert Enum.at(decoded, 4)["bikoret"]["verdict"] == "lo_tov"
+  end
+
+  test "stream: the api_calls_external unit is gathered before the call; spent → waiting, nothing sent" do
+    alias VavPipeline.Gate.Stream
+    write_provision(50, 0)
+    assert {:waiting, :api_calls_exhausted} = Stream.pass(:anthropic, Map.put(body(), "stream", true), "sk-good", collect(), opts())
+  end
+
   # ------------------------------------------------------------
   # The door
   # ------------------------------------------------------------
@@ -264,12 +364,32 @@ defmodule VavPipeline.GateTest do
       refute conn.status in [401, 403]
     end
 
-    test "refuses a missing upstream key, a missing messages list, and stream" do
+    test "refuses a missing upstream key, a missing messages list, and stream with rebuke" do
       assert post_gate(body(), upstream: nil).status == 422
       assert post_gate(%{"model" => "x"}, upstream: "sk-good").status == 422
-      conn = post_gate(Map.put(body(), "stream", true), upstream: "sk-good")
+      conn = post_gate(Map.put(body(), "stream", true), upstream: "sk-good", rebuke: "once")
       assert conn.status == 422
-      assert Jason.decode!(conn.resp_body)["error"]["message"] =~ "stream"
+      assert Jason.decode!(conn.resp_body)["error"]["message"] =~ "taken back"
+    end
+
+    test "streaming: the events are relayed, and with no Korban Processor the trailer says the reply was not examined" do
+      conn = post_gate(Map.put(body(), "stream", true), upstream: "sk-good")
+      assert conn.status == 200
+      assert get_resp_header(conn, "content-type") |> hd() =~ "text/event-stream"
+      assert get_resp_header(conn, "x-bikoret-verdict") == []
+      events = conn.resp_body |> String.split("\n\n", trim: true)
+      assert Enum.at(events, 0) =~ "event: message_start"
+      assert List.last(events) =~ "event: message_stop"
+      assert conn.resp_body =~ "— bikoret: not examined (korban_unavailable"
+      # the fringe block sits before message_delta, which carries the object
+      [delta] = Enum.filter(events, &String.starts_with?(&1, "event: message_delta"))
+      assert delta =~ "\"bikoret\""
+    end
+
+    test "streaming: an upstream 401 comes back whole, as the upstream's own 401" do
+      conn = post_gate(Map.put(body(), "stream", true), upstream: "sk-bad")
+      assert conn.status == 401
+      assert Jason.decode!(conn.resp_body)["error"]["type"] == "authentication_error"
     end
 
     test "an upstream 401 comes back as the upstream's own 401" do
@@ -287,7 +407,7 @@ defmodule VavPipeline.GateTest do
 
     test "an unreachable upstream is 502" do
       previous = Application.get_env(:vav_pipeline, :gate_upstreams)
-      Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:9/v1/messages"})
+      Application.put_env(:vav_pipeline, :gate_upstreams, %{anthropic: "http://127.0.0.1:9/v1/messages", openai: "http://127.0.0.1:9/v1/chat/completions"})
       on_exit(fn -> Application.put_env(:vav_pipeline, :gate_upstreams, previous) end)
       conn = post_gate(body(), upstream: "sk-good")
       assert conn.status == 502
@@ -310,6 +430,7 @@ defmodule VavPipeline.GateTest do
         key -> put_req_header(conn, "x-api-key", key)
       end
 
+    conn = case Keyword.get(opts, :rebuke), do: (nil -> conn; r -> put_req_header(conn, "x-bikoret-rebuke", r))
     Router.call(conn, @opts)
   end
 end