Shemot 40:33 — Vay'khal Moshe et ham'lakhah. The builder completes the work. The vessel is ready for transport. Step 33 — Deployment configuration: - Dockerfiles for all 4 components: - vav_pipeline: Elixir 1.16.1/OTP 26.2.1 release with Sanhedrin - shoresh_engine: GHC 9.6.4 binary - talmudic_engine: SWI-Prolog 9.2.2 - korban_processor: Python 3.12.2 with pydantic - Kubernetes manifests for Exoscale SKS (ch-gva-2): - Namespace, ConfigMap, Secrets template - L1 Torah PV/PVC (read-only mount) - Manna provision PVC (kohen writes, system reads) - Deployment: vav-pipeline + 3 sidecars, shared volumes - Service: ClusterIP + LoadBalancer with Exoscale NLB - Init container runs Ecto migrations (VavPipeline.Release) Step 36 — Halt monitor (esh tamid): - GenServer watches for three cardinal sins - halt!/3 immediately halts system, sounds Chatzotzrot war signal - Permanent audit log — halt events never deleted - resolve/1 requires explicit operator action to resume - alive?/0 for morning tamid verification - 13 tests: alive check, 3 halt types, resolution, audit log, chatzotzrot war signal integration - Added to supervision tree All 219 Elixir + 19 L3 + 195 Prolog + 33 Haskell = 447+ tests, 0 failures Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
211 lines
5.5 KiB
Elixir
211 lines
5.5 KiB
Elixir
defmodule VavPipeline.HaltMonitor do
|
|
@moduledoc """
|
|
Halt Monitor — the esh tamid that must never go out.
|
|
|
|
Vayikra 6:6 — Esh tamid tukad al hamizbe'ach, lo tikhbeh.
|
|
"A perpetual fire shall be kept burning on the altar; it shall not go out."
|
|
|
|
Operations spec §2.1, Step 5:
|
|
"Check halt_monitor process registered. If failed: HALT —
|
|
the system does not run without the halt monitor."
|
|
|
|
This GenServer:
|
|
1. Receives halt events from pipeline processing
|
|
2. Records halt events permanently (the audit log)
|
|
3. Sounds the Chatzotzrot war signal on halt
|
|
4. Prevents further processing until the halt is resolved
|
|
5. Reports whether the system is halted and why
|
|
|
|
The halt monitor is not optional. If this process dies,
|
|
the supervisor should restart it. If it cannot restart,
|
|
the entire system must stop.
|
|
"""
|
|
|
|
use GenServer
|
|
|
|
require Logger
|
|
|
|
@type halt_event :: %{
|
|
condition: :avodah_zarah | :shfichut_damim | :gilui_arayot,
|
|
reason: String.t(),
|
|
detected_at: DateTime.t(),
|
|
component: String.t(),
|
|
resolved: boolean()
|
|
}
|
|
|
|
# ============================================================
|
|
# Client API
|
|
# ============================================================
|
|
|
|
def start_link(opts \\ []) do
|
|
name = Keyword.get(opts, :name, __MODULE__)
|
|
GenServer.start_link(__MODULE__, opts, name: name)
|
|
end
|
|
|
|
@doc """
|
|
Report a halt condition. This immediately halts the system.
|
|
|
|
Sounds the Chatzotzrot war signal (teruah_teruah).
|
|
The halt is permanent until explicitly resolved.
|
|
"""
|
|
@spec halt!(atom(), String.t(), String.t()) :: :halted
|
|
def halt!(condition, reason, component \\ "pipeline")
|
|
when condition in [:avodah_zarah, :shfichut_damim, :gilui_arayot] do
|
|
GenServer.call(__MODULE__, {:halt, condition, reason, component})
|
|
end
|
|
|
|
@doc """
|
|
Check if the system is currently halted.
|
|
|
|
Returns :running or {:halted, halt_event}.
|
|
"""
|
|
@spec status() :: :running | {:halted, halt_event()}
|
|
def status do
|
|
GenServer.call(__MODULE__, :status)
|
|
end
|
|
|
|
@doc """
|
|
Check if the system is halted. Boolean variant.
|
|
"""
|
|
@spec halted?() :: boolean()
|
|
def halted? do
|
|
GenServer.call(__MODULE__, :halted?)
|
|
end
|
|
|
|
@doc """
|
|
Get the full audit log of all halt events.
|
|
These are never deleted — permanent accountability.
|
|
"""
|
|
@spec audit_log() :: [halt_event()]
|
|
def audit_log do
|
|
GenServer.call(__MODULE__, :audit_log)
|
|
end
|
|
|
|
@doc """
|
|
Resolve a halt condition. Requires operator authority.
|
|
|
|
The system does not resume automatically. A kohen must
|
|
explicitly resolve the halt after investigating.
|
|
"""
|
|
@spec resolve(String.t()) :: {:ok, :resumed} | {:error, :not_halted}
|
|
def resolve(resolution_reason) when is_binary(resolution_reason) do
|
|
GenServer.call(__MODULE__, {:resolve, resolution_reason})
|
|
end
|
|
|
|
@doc """
|
|
Verify the monitor is alive and functional.
|
|
Called during morning tamid (Operations spec §2.1, Step 5).
|
|
"""
|
|
@spec alive?() :: boolean()
|
|
def alive? do
|
|
case GenServer.call(__MODULE__, :ping) do
|
|
:pong -> true
|
|
end
|
|
catch
|
|
:exit, _ -> false
|
|
end
|
|
|
|
# ============================================================
|
|
# GenServer callbacks
|
|
# ============================================================
|
|
|
|
@impl true
|
|
def init(_opts) do
|
|
state = %{
|
|
status: :running,
|
|
current_halt: nil,
|
|
audit_log: [],
|
|
halt_count: 0
|
|
}
|
|
|
|
Logger.info("HaltMonitor: esh tamid burning — lo tikhbeh")
|
|
{:ok, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call({:halt, condition, reason, component}, _from, state) do
|
|
event = %{
|
|
condition: condition,
|
|
reason: reason,
|
|
detected_at: DateTime.utc_now(),
|
|
component: component,
|
|
resolved: false
|
|
}
|
|
|
|
Logger.error(
|
|
"HALT: #{condition} detected in #{component} — #{reason}. " <>
|
|
"System halted. Kohen must investigate and resolve."
|
|
)
|
|
|
|
# Sound the war signal — teruah_teruah
|
|
try do
|
|
VavPipeline.Chatzotzrot.signal(:war,
|
|
reason: "HALT: #{condition} — #{reason}"
|
|
)
|
|
catch
|
|
_, _ -> Logger.error("HaltMonitor: could not sound chatzotzrot")
|
|
end
|
|
|
|
new_state = %{
|
|
state
|
|
| status: :halted,
|
|
current_halt: event,
|
|
audit_log: [event | state.audit_log],
|
|
halt_count: state.halt_count + 1
|
|
}
|
|
|
|
{:reply, :halted, new_state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:status, _from, state) do
|
|
case state.status do
|
|
:running -> {:reply, :running, state}
|
|
:halted -> {:reply, {:halted, state.current_halt}, state}
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:halted?, _from, state) do
|
|
{:reply, state.status == :halted, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:audit_log, _from, state) do
|
|
{:reply, state.audit_log, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call({:resolve, resolution_reason}, _from, state) do
|
|
case state.status do
|
|
:running ->
|
|
{:reply, {:error, :not_halted}, state}
|
|
|
|
:halted ->
|
|
Logger.warning(
|
|
"HaltMonitor: halt resolved — #{resolution_reason}. " <>
|
|
"System resuming. Previous halt: #{state.current_halt.condition}"
|
|
)
|
|
|
|
resolved_event = Map.put(state.current_halt, :resolved, true)
|
|
|
|
new_audit =
|
|
[%{resolved_event | resolved: true} | tl(state.audit_log)]
|
|
|
|
new_state = %{
|
|
state
|
|
| status: :running,
|
|
current_halt: nil,
|
|
audit_log: new_audit
|
|
}
|
|
|
|
{:reply, {:ok, :resumed}, new_state}
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:ping, _from, state) do
|
|
{:reply, :pong, state}
|
|
end
|
|
end
|