manna: what is eaten stays eaten — the ledger is written beside the provision

The MannaLedger counted the day's consumption in memory. A restart
forgot it, and the next cycle gathered the same day's provision whole.
While manna was an emptyDir that took the kohen's hand; with the
provision on a volume (41a3e51) a crash loop alone would have refilled
the jar, once per crash. Ish l'fi ochlo — Shemot 16:18.

- After every cycle that eats, the day's consumption is written to
  ledger.json beside the provision file (tmp + rename; on the manna
  volume in production).
- A process that starts while today's record is on disk takes the day
  up at once (handle_continue), so /kodesh/status never shows an
  untouched jar that is half eaten; otherwise the first load of the day
  resumes from it.
- Another day's record is ignored. A record that is not a ledger
  (bad JSON, negative or non-integer counts) is ignored with a warning.
  The record can only reduce what remains.
- At the day boundary the record goes with the surplus. A release
  within the same day (the late-provision re-gather) lets go of the
  allocation and keeps the record.
- If the file cannot be written the ledger says so and keeps counting
  in memory; the cycle is not stopped by its own bookkeeping.
- MannaLedger.reset/0 (tests) forgets the record; test_helper clears
  any record an interrupted run left behind.

test/manna_ledger_persistence_test.exs — 13 tests, restarting the
ledger under its real supervisor. All fail without the change.

Read before committing: Elixir 474/0 on seeds 0 7 42 99 191243. Built
image, real container restarts against a mounted manna directory:
3 of 5 units eaten → restart → status shows 3 before any request →
2 more served, the 6th refused → restart → still refused.

Found while writing the tests, not changed: killing the ledger more
than three times in five seconds takes the whole application down (OTP
restart intensity, one_for_one defaults). On the pod that is a
container restart — which now resumes the day instead of refilling it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

vav_pipeline/test/manna_ledger_persistence_test.exs

@@ -0,0 +1,233 @@
+defmodule VavPipeline.MannaLedgerPersistenceTest do
+  @moduledoc """
+  What is eaten stays eaten.
+
+  Until 2026-10-04 the ledger lived only in memory. A restart forgot the
+  day's consumption and the next cycle gathered the whole provision again.
+  That was hidden while manna was an emptyDir (a restart emptied the jar
+  too, and the kohen had to refill it by hand); once the provision was put
+  on a volume, a crash loop alone would have refilled it.
+
+  These tests restart the ledger process under its real supervisor — a new
+  process, an empty state, the same disk — and ask what it remembers.
+  """
+
+  use ExUnit.Case, async: false
+
+  alias VavPipeline.{HaltMonitor, Manna, MannaLedger}
+
+  setup do
+    if HaltMonitor.halted?(), do: HaltMonitor.resolve("manna persistence test setup")
+    MannaLedger.reset()
+    write_provision(10)
+
+    on_exit(fn ->
+      MannaLedger.reset()
+      File.rm(Manna.provision_file())
+    end)
+
+    :ok
+  end
+
+  describe "the record" do
+    test "is written beside the provision, with today's date, when a cycle eats" do
+      refute File.exists?(MannaLedger.ledger_file())
+
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 7} = MannaLedger.consume(:compute_units, 3)
+
+      assert Path.dirname(MannaLedger.ledger_file()) == Path.dirname(Manna.provision_file())
+      record = MannaLedger.ledger_file() |> File.read!() |> Jason.decode!()
+
+      assert record["cycle_date"] == today()
+      assert record["consumed"]["compute_units"] == 3
+      assert record["consumed"]["api_calls_external"] == 0
+    end
+
+    test "is not written by a refused consumption" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:error, :provision_exhausted} = MannaLedger.consume(:compute_units, 11)
+      refute File.exists?(MannaLedger.ledger_file())
+    end
+  end
+
+  describe "a restart in the middle of the day" do
+    test "resumes the count — the provision is not gathered whole again" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 6} = MannaLedger.consume(:compute_units, 4)
+
+      restart_ledger()
+
+      # The new process takes the day up as it starts — before any cycle
+      # asks — so the status never shows an untouched jar.
+      ledger = MannaLedger.ledger()
+      assert ledger.cycle_date == today()
+      assert ledger.allocated.compute_units == 10
+      assert ledger.consumed.compute_units == 4
+
+      assert {:ok, :already_loaded} = MannaLedger.load_provision()
+      assert MannaLedger.remaining(:compute_units) == 6
+
+      # And it keeps counting from there.
+      assert {:ok, 5} = MannaLedger.consume(:compute_units, 1)
+    end
+
+    test "an exhausted day stays exhausted — a crash loop does not refill the jar" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 0} = MannaLedger.consume(:compute_units, 10)
+
+      for _ <- 1..3 do
+        restart_ledger()
+        assert {:waiting, :provision_exhausted} = Manna.gather_for_cycle()
+      end
+    end
+
+    test "the whole cycle path: gather, restart, gather" do
+      assert :ok = Manna.gather_for_cycle()
+      assert :ok = Manna.gather_for_cycle()
+      restart_ledger()
+      assert :ok = Manna.gather_for_cycle()
+      assert MannaLedger.ledger().consumed.compute_units == 3
+    end
+  end
+
+  describe "a restart with nothing to resume" do
+    test "no record: the new process holds no day until the first cycle" do
+      restart_ledger()
+      assert MannaLedger.ledger().cycle_date == nil
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert MannaLedger.remaining(:compute_units) == 10
+    end
+
+    test "a record but no provision: nothing is resumed into an empty day" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 9} = MannaLedger.consume(:compute_units, 1)
+      File.rm!(Manna.provision_file())
+
+      restart_ledger()
+      assert MannaLedger.remaining(:compute_units) == 0
+
+      # The vessel waits. (Which of the two waiting reasons it gives is
+      # the ledger's older behaviour for a day pinned with no provision;
+      # the router answers MANNA_DEPLETED for both.)
+      assert {:waiting, reason} = Manna.gather_for_cycle()
+      assert reason in [:no_provision, :provision_exhausted]
+    end
+  end
+
+  describe "what the record may not do" do
+    test "another day's record is not this day's — it is ignored" do
+      write_record(%{"cycle_date" => "2026-01-01", "consumed" => %{"compute_units" => 9}})
+
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert MannaLedger.remaining(:compute_units) == 10
+    end
+
+    test "a record that is not a ledger is ignored, and the cycle goes on" do
+      for garbage <- [
+            "not json",
+            Jason.encode!(%{"cycle_date" => today(), "consumed" => "all of it"}),
+            Jason.encode!(%{"cycle_date" => today(), "consumed" => %{"compute_units" => -5}}),
+            Jason.encode!(%{"cycle_date" => today(), "consumed" => %{"compute_units" => "3"}})
+          ] do
+        MannaLedger.reset()
+        File.write!(MannaLedger.ledger_file(), garbage)
+
+        assert {:ok, :loaded} = MannaLedger.load_provision()
+        assert MannaLedger.remaining(:compute_units) == 10
+        assert {:ok, 9} = MannaLedger.consume(:compute_units, 1)
+      end
+    end
+
+    test "it can only take away: a record larger than the provision leaves nothing" do
+      write_record(%{"cycle_date" => today(), "consumed" => %{"compute_units" => 500}})
+
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert MannaLedger.remaining(:compute_units) == 0
+      assert {:error, :provision_exhausted} = MannaLedger.consume(:compute_units, 1)
+    end
+  end
+
+  describe "release" do
+    test "at a day boundary the record goes with the surplus" do
+      # The ledger holds no day (cycle_date nil ≠ today), as it does when
+      # the day it loaded has passed and been released.
+      write_record(%{"cycle_date" => "2026-01-01", "consumed" => %{"compute_units" => 2}})
+
+      assert {:ok, _} = MannaLedger.enforce_cycle_boundary()
+      refute File.exists?(MannaLedger.ledger_file())
+    end
+
+    test "a release within the same day lets go of the allocation, not the memory" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 8} = MannaLedger.consume(:compute_units, 2)
+
+      # Manna.gather_for_cycle/0 does this when a provision arrives late.
+      assert {:ok, _} = MannaLedger.enforce_cycle_boundary()
+      assert File.exists?(MannaLedger.ledger_file())
+
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert MannaLedger.remaining(:compute_units) == 8
+    end
+
+    test "reset forgets the record (tests must not leak a day into each other)" do
+      assert {:ok, :loaded} = MannaLedger.load_provision()
+      assert {:ok, 9} = MannaLedger.consume(:compute_units, 1)
+      assert File.exists?(MannaLedger.ledger_file())
+
+      MannaLedger.reset()
+      refute File.exists?(MannaLedger.ledger_file())
+    end
+  end
+
+  # ------------------------------------------------------------
+  # Helpers
+  # ------------------------------------------------------------
+
+  defp today, do: Date.utc_today() |> Date.to_iso8601()
+
+  # Bring the ledger down and up again through its own supervisor — a
+  # new process with a new, empty state, and the file on disk untouched.
+  # That is a restart, not a reset.
+  #
+  # (Done with terminate_child/restart_child rather than Process.exit:
+  # this file restarts the ledger six times in well under five seconds,
+  # and a supervisor that sees more than three CRASHES in five seconds
+  # gives up and takes the application down with it — as the first
+  # version of this test found out.)
+  defp restart_ledger do
+    old = Process.whereis(MannaLedger)
+    :ok = Supervisor.terminate_child(VavPipeline.Supervisor, MannaLedger)
+    {:ok, new} = Supervisor.restart_child(VavPipeline.Supervisor, MannaLedger)
+    assert new != old
+    assert Process.whereis(MannaLedger) == new
+  end
+
+  defp write_record(record) do
+    File.mkdir_p!(Path.dirname(MannaLedger.ledger_file()))
+    File.write!(MannaLedger.ledger_file(), Jason.encode!(record))
+  end
+
+  defp write_provision(compute_units) do
+    expires = DateTime.utc_now() |> DateTime.add(6 * 3600) |> DateTime.to_iso8601()
+
+    provision = %{
+      "date" => today(),
+      "provision" => %{
+        "compute_units" => compute_units,
+        "memory_mb" => 512,
+        "api_calls_external" => 5,
+        "storage_delta_mb" => 2,
+        "sanhedrin_panels_max" => 2
+      },
+      "type" => "regular",
+      "expires_at" => expires,
+      "written_by" => "manna_ledger_persistence_test",
+      "written_at" => DateTime.utc_now() |> DateTime.to_iso8601()
+    }
+
+    path = Manna.provision_file()
+    File.mkdir_p!(Path.dirname(path))
+    File.write!(path, Jason.encode!(provision))
+  end
+end