diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..34e1e9e8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,80 @@ +# Build context exclusions — deploy/docker/*.Dockerfile COPY whole +# component directories (`COPY vav_pipeline/ vav_pipeline/`, +# `COPY shoresh-engine/ ./`, ...). Without this file the HOST's +# compiled artifacts are copied into the image and the in-container +# toolchain then tries to REUSE them. +# +# That is a real defect, not a size optimization: +# - vav_pipeline/_build holds .beam compiled by the host's ERTS; +# mix treats them as current and skips recompiling. The +# container's mix then fails on a stale manifest +# ("Failed to restore .../source.dag file. Discarding it.") +# or ships host-built artifacts into an amd64 release. +# - shoresh-engine/dist-newstyle (129 MB) holds arm64 GHC objects; +# cabal would link them into an amd64 binary. +# - korban_processor/.venv and __pycache__ carry host-arch +# bytecode and native wheels. +# +# The image must be built from SOURCE inside the container, on the +# target architecture. Ka'asher tzivah — built as commanded, not +# assembled from whatever happened to be lying on the workbench. +# +# NOTE: l1/ is deliberately NOT ignored. The scroll ships with the +# vessel (every Dockerfile COPYs l1/ into /mnt/torah). Do not add it. + +# --- VCS and agent state --- +.git +.gitignore +.claude +.opencode + +# --- Elixir / Erlang --- +vav_pipeline/_build +vav_pipeline/deps +vav_pipeline/.elixir_ls +vav_pipeline/cover +vav_pipeline/var +sanhedrin/_build +sanhedrin/rebar3.crashdump +erl_crash.dump +**/erl_crash.dump + +# --- Haskell --- +shoresh-engine/dist-newstyle +shoresh-engine/.stack-work +shoresh-engine/.ghc.environment* + +# --- Prolog --- +talmudic_engine/_build +talmudic_engine/deps + +# --- Python --- +**/__pycache__ +**/*.pyc +**/*.pyo +**/.venv +**/venv +**/.pytest_cache +**/.mypy_cache +**/.ruff_cache +**/*.egg-info + +# --- Secrets and local config --- +.env +**/.env +*.pem +*.key + +# --- OS / editor noise --- +.DS_Store +*.swp +*.swo +*~ + +# --- Not needed inside any image --- +transcripts +Untitled +aws-export +kushya_generator/data/train.jsonl +kushya_generator/data/eval.jsonl +kushya_generator/fine_tune/adapter diff --git a/KOHEN.md b/KOHEN.md index ed1d2a0a..b6e6382a 100644 --- a/KOHEN.md +++ b/KOHEN.md @@ -221,7 +221,18 @@ Note: `manna-volume.yaml` defines a PVC but `deployment.yaml` mounts manna as `e kubectl get pods -n shorashim -w ``` -You should see one pod with **4 containers + 1 init** (init `migrate` + main containers `vav-pipeline` + `shoresh-engine` + `talmudic-engine` + `korban-processor`) move to `Running`. +You should see one pod with **3 containers + 2 init** move to `Running`: +init `shoresh-engine-install` -> init `migrate` -> containers `vav-pipeline` + `talmudic-engine` + `korban-processor`. + +There are still FOUR images, and gate 0i still checks four. The +shoresh-engine image is no longer a running container: its Haskell +binary is copied onto a shared volume by the `shoresh-engine-install` +init container and then spawned by vav-pipeline as an Erlang Port — +a child process of the BEAM, which is what the Elixir code has always +done and which cannot cross a container boundary. As a sidecar it +printed its usage banner and exited, and vav-pipeline logged +"binary 'shoresh-engine' not in PATH — running without Haskell +engine" and served degraded. See the builder handoff-back. ```bash # Confirm logs show the right startup messages diff --git a/deploy/ARCHITECTURE.md b/deploy/ARCHITECTURE.md index 79ec6ccb..0c116044 100644 --- a/deploy/ARCHITECTURE.md +++ b/deploy/ARCHITECTURE.md @@ -9,11 +9,11 @@ One Kubernetes namespace runs the following: | Component | Runtime | Purpose | |-----------|---------|---------| | `vav-pipeline` | Elixir/OTP release | Bereishit cycle, ShemaContext, Cloud, Manna, HTTP API (pe'ah, courtyard, chatzer, kodesh) | -| `shoresh-engine` | Haskell binary (sidecar via Erlang Port) | Root activation, te'amim parsing, gematria | +| `shoresh-engine` | Haskell binary, spawned by vav-pipeline as an Erlang Port (staged into the pod by the `shoresh-engine-install` init container — NOT a sidecar container; a Port is a child process of the BEAM and cannot cross a container boundary) | Root activation, te'amim parsing, gematria | | `talmudic-engine` | SWI-Prolog (sidecar via Unix socket) | 613 mitzvot, 13 middot, kushya generation, teiku | | `korban-processor` | Python 3.12 (sidecar) | Offering protocol, eish zarah validation | -All four run in the same pod; the vav-pipeline is the main container, the other three are sidecars. +All four run in the same pod. vav-pipeline is the main container; talmudic-engine and korban-processor are sidecars; shoresh-engine runs inside vav-pipeline as an Erlang Port child process, staged in by an init container. ## External dependencies @@ -34,7 +34,7 @@ Kept in `deploy/k8s/`: - `secrets.yaml` — **template only**; operator fills in `DATABASE_URL` - `l1-volume.yaml` — PV and PVC for Torah filesystem - `manna-volume.yaml` — PVC for provision file -- `deployment.yaml` — the pod with 4 containers + init container for migrations +- `deployment.yaml` — the pod with 3 containers + 2 init containers (`shoresh-engine-install`, `migrate`). Four images are still built and pushed; shoresh-engine ships a binary, not a running container. - `service.yaml` — ClusterIP and external LoadBalancer - `gitea-mcp-deployment.yaml` — separate MCP server for Gitea connector (optional) - `gitea-mcp-ingress.yaml` — TLS ingress for the MCP server diff --git a/deploy/docker/Dockerfile.korban_processor b/deploy/docker/Dockerfile.korban_processor index 100707f9..b1f956d9 100644 --- a/deploy/docker/Dockerfile.korban_processor +++ b/deploy/docker/Dockerfile.korban_processor @@ -1,3 +1,7 @@ +# check=skip=FromPlatformFlagConstDisallowed +# ^ The --platform=linux/amd64 pins below are DELIBERATE (Exoscale SKS +# nodes are amd64). BuildKit lints a constant --platform as a smell; +# here it is the fix. Silenced so a real warning is not lost in noise. # Shorashim Korban Processor — Python offering protocol # Construction spec §1.1: Python 3.12.2 # Processes korbanot (model outputs) through the offering protocol diff --git a/deploy/docker/Dockerfile.shoresh_engine b/deploy/docker/Dockerfile.shoresh_engine index aa8ee16f..71696657 100644 --- a/deploy/docker/Dockerfile.shoresh_engine +++ b/deploy/docker/Dockerfile.shoresh_engine @@ -1,3 +1,7 @@ +# check=skip=FromPlatformFlagConstDisallowed +# ^ The --platform=linux/amd64 pins below are DELIBERATE (Exoscale SKS +# nodes are amd64). BuildKit lints a constant --platform as a smell; +# here it is the fix. Silenced so a real warning is not lost in noise. # Shorashim Shoresh Engine — Haskell root activation engine # Construction spec §1.1: GHC 9.6.4, Cabal 3.10 # Communicates with vav_pipeline via Erlang Port (stdin/stdout) @@ -33,8 +37,26 @@ FROM --platform=linux/amd64 ubuntu:jammy-20240125 RUN apt-get update && \ apt-get install -y --no-install-recommends \ - libgmp10 zlib1g && \ - rm -rf /var/lib/apt/lists/* + libgmp10 zlib1g locales && \ + rm -rf /var/lib/apt/lists/* && \ + sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ + locale-gen + +# === UTF-8 IS LOAD-BEARING HERE === +# This engine's entire job is Hebrew roots. GHC picks stdout's encoding +# from the locale at startup; ubuntu:jammy with no locale set gives the +# POSIX/C locale, i.e. ASCII, and the FIRST non-ASCII byte the engine +# tries to write kills it: +# Shoresh Engine ...: : commitBuffer: invalid argument +# (cannot encode character '\8212') +# It died on the em-dash in its own banner, before reading one root. +# vav_pipeline's runtime stage has always set these three; this stage +# did not, and shoresh-engine is the one component that CANNOT run +# without them. Matches Dockerfile.vav_pipeline so both halves of the +# Erlang Port speak the same encoding. +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 RUN useradd --create-home --shell /bin/bash shorashim USER shorashim diff --git a/deploy/docker/Dockerfile.talmudic_engine b/deploy/docker/Dockerfile.talmudic_engine index 5bf7df87..fc06dab0 100644 --- a/deploy/docker/Dockerfile.talmudic_engine +++ b/deploy/docker/Dockerfile.talmudic_engine @@ -1,3 +1,7 @@ +# check=skip=FromPlatformFlagConstDisallowed +# ^ The --platform=linux/amd64 pins below are DELIBERATE (Exoscale SKS +# nodes are amd64). BuildKit lints a constant --platform as a smell; +# here it is the fix. Silenced so a real warning is not lost in noise. # Shorashim Talmudic Engine — SWI-Prolog dialectical reasoning # Construction spec §1.1: SWI-Prolog 9.2.2 # Communicates with vav_pipeline via Unix domain socket @@ -33,5 +37,19 @@ ENV L1_MOUNT_POINT=/mnt/torah # Socket path for communication with vav_pipeline ENV TALMUDIC_SOCKET_PATH=/tmp/shorashim/talmudic.sock -# Start the Prolog socket server -CMD ["swipl", "-g", "consult('src/main'),start_socket_server", "-t", "halt(1)"] +# Start the Prolog socket server. +# +# The predicate is serve/0 and serve/1 (exported by +# src/interface/socket_server.pl). The previous CMD called +# start_socket_server/0, which is not defined anywhere in the engine — +# the container exited 2 on every start: +# ERROR: -g consult('src/main'),start_socket_server: +# ''/1: Unknown procedure: start_socket_server/0 +# +# serve/1 is used when TALMUDIC_SOCKET_PATH is set so the env var above +# is actually honoured; serve/0 falls back to the same path hardcoded +# as default_socket_path/1. vav_pipeline dials the socket at +# :talmudic_socket_path (config/config.exs), and deployment.yaml mounts +# the shared talmudic-socket volume at /tmp/shorashim in both +# containers, so the two ends must agree on this path. +CMD ["swipl", "-g", "consult('src/main'), (getenv('TALMUDIC_SOCKET_PATH', P) -> serve(P) ; serve)", "-t", "halt(1)"] diff --git a/deploy/docker/Dockerfile.vav_pipeline b/deploy/docker/Dockerfile.vav_pipeline index db27ec6b..94758821 100644 --- a/deploy/docker/Dockerfile.vav_pipeline +++ b/deploy/docker/Dockerfile.vav_pipeline @@ -1,3 +1,7 @@ +# check=skip=FromPlatformFlagConstDisallowed +# ^ The --platform=linux/amd64 pins below are DELIBERATE (Exoscale SKS +# nodes are amd64). BuildKit lints a constant --platform as a smell; +# here it is the fix. Silenced so a real warning is not lost in noise. # Shorashim Vav Pipeline — Elixir/OTP release with Sanhedrin # Construction spec §1.1: Elixir 1.16.1, OTP 26.2.1 # The pipeline and Sanhedrin run in the same BEAM VM. @@ -28,6 +32,16 @@ WORKDIR /app/vav_pipeline # Fetch deps and compile in prod mode ENV MIX_ENV=prod + +# EMULATED-BUILD JIT WORKAROUND — see the incident note at the top of +# this file. +JMsingle makes the BEAM JIT use a SINGLE writable/ +# executable mapping for generated machine code instead of the default +# dual W^X mapping. The dual mapping is what dies (SIGSEGV, exit 139) +# when this amd64 stage runs emulated inside BuildKit's sandbox on an +# Apple Silicon host. Build-stage only: it does not appear in the +# runtime stage, so the shipped release runs with the ordinary +# dual-mapping JIT on the amd64 cluster node. +ENV ERL_FLAGS="+JMsingle true" RUN mix local.hex --force && \ mix local.rebar --force && \ mix deps.get --only prod && \ @@ -39,9 +53,17 @@ RUN mix release # === RUNTIME STAGE === FROM --platform=linux/amd64 ubuntu:jammy-20240125 +# libgmp10 is here for the SHORESH ENGINE, not for the BEAM. +# VavPipeline.ShoreshEnginePort spawns the Haskell binary as an Erlang +# Port — a child process of THIS container (deployment.yaml stages the +# binary onto the shoresh-bin volume with an init container). GHC-built +# executables link libgmp; without it the spawn dies with +# error while loading shared libraries: libgmp.so.10 +# and Shorashim runs with no root activation, no te'amim parsing and no +# gematria while still reporting itself healthy. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - libstdc++6 openssl ca-certificates locales && \ + libstdc++6 libgmp10 openssl ca-certificates locales && \ rm -rf /var/lib/apt/lists/* && \ sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ locale-gen @@ -64,6 +86,30 @@ COPY --from=build --chown=shorashim:shorashim \ # The scroll is sealed in the aron. It ships with the vessel. COPY --chown=shorashim:shorashim l1/ /mnt/torah/ +# === INSCRIBE THE DOORPOST === +# Devarim 11:20 — ukhtavtam al mezuzot beitecha uvish'arecha. A running +# pod IS a threshold, and VavPipeline.Doorpost exists to prove that the +# corporate text at THIS threshold is the same corporate text the system +# was built to hold. It cannot prove that against files it does not have. +# +# The @spec_files set (see vav_pipeline/lib/vav_pipeline/doorpost.ex) +# is inscribed here at the repo-relative paths the module hashes, rooted +# at :doorpost_repo_root from config/prod.exs. Copying whole directories +# rather than 23 individual files means a spec file added to @spec_files +# under specs/, talmudic_engine/src/halt/, or .../middot/ is carried +# automatically instead of silently going :missing in production. +# +# The hash over these files must equal Doorpost.expected_hash(); if it +# does not, this image was built from a tree whose specs and witness +# disagreed, and the pod says so at startup. +COPY --chown=shorashim:shorashim BUILD_AZAZEL.md BUILD_KORBAN.md \ + /opt/shorashim/spec/ +COPY --chown=shorashim:shorashim specs/ /opt/shorashim/spec/specs/ +COPY --chown=shorashim:shorashim talmudic_engine/src/halt/ \ + /opt/shorashim/spec/talmudic_engine/src/halt/ +COPY --chown=shorashim:shorashim talmudic_engine/src/middot/ \ + /opt/shorashim/spec/talmudic_engine/src/middot/ + # L1 mount point ENV L1_MOUNT_POINT=/mnt/torah diff --git a/deploy/k8s/configmap.yaml b/deploy/k8s/configmap.yaml index 576e1806..8c8c9b2f 100644 --- a/deploy/k8s/configmap.yaml +++ b/deploy/k8s/configmap.yaml @@ -15,6 +15,13 @@ data: # Talmudic Engine socket TALMUDIC_SOCKET_PATH: "/tmp/shorashim/talmudic.sock" + # Shoresh Engine binary — staged onto the shared shoresh-bin volume + # by the shoresh-engine-install init container (see deployment.yaml). + # vav_pipeline spawns it as an Erlang Port, i.e. as a child process of + # the vav-pipeline container, so it must be a path on THIS + # container's filesystem. + SHORESH_ENGINE_BINARY: "/opt/shoresh/shoresh-engine" + # Shabbat location — Zurich (Construction spec §4.4) SHABBAT_LATITUDE: "47.3769" SHABBAT_LONGITUDE: "8.5417" diff --git a/deploy/k8s/deployment.yaml b/deploy/k8s/deployment.yaml index 67b6724b..1e473469 100644 --- a/deploy/k8s/deployment.yaml +++ b/deploy/k8s/deployment.yaml @@ -28,20 +28,108 @@ spec: app.kubernetes.io/name: shorashim spec: # --- VOLUMES --- + # + # ONE shared /tmp volume, not two. There used to be a separate + # `talmudic-socket` volume mounted at /tmp/shorashim in + # vav-pipeline and talmudic-engine, WHILE `tmp-shorashim` was + # mounted at /tmp in vav-pipeline and korban-processor. Nested + # mounts meant /tmp/shorashim resolved to a DIFFERENT volume than + # /tmp/shorashim reached through /tmp — so: + # - korban/azazel (Python) wrote AzazelRecord JSON to + # tmp-shorashim:/shorashim/azazel/pending_l3 + # - AzazelIngest (Elixir) watched + # talmudic-socket:/azazel/pending_l3 + # Two directories with the same pathname on two different disks. + # The disposal channel would have run, written its records, and + # nothing would ever have reached L3 — silently, because + # AzazelIngest leaves unreadable files in place rather than + # erroring (see azazel_ingest.ex, RETRY SEMANTICS). + # + # Vayikra 16:22 — v'nasa hasa'ir alav et kol avonotam el eretz + # gezerah. The goat has to actually GET there. A disposal channel + # whose records land on a disk no one reads has not disposed of + # anything; it has only stopped looking. volumes: - name: manna emptyDir: {} - - name: talmudic-socket - emptyDir: {} + # Shared /tmp for every container that speaks over it: + # the Talmudic Unix socket (/tmp/shorashim/talmudic.sock) and + # the azazel file bridge (/tmp/shorashim/azazel/pending_l3). - name: tmp-shorashim emptyDir: {} + # Staging area for the Haskell shoresh-engine binary. Written + # once by the shoresh-engine-install init container, read by + # vav-pipeline, which spawns it as an Erlang Port. + - name: shoresh-bin + emptyDir: {} + + # --- POD SECURITY CONTEXT --- + # Every image runs as the non-root user `shorashim` (uid/gid + # 1000). fsGroup makes the kubelet chgrp the emptyDir volumes to + # 1000 and set the setgid bit, so uid 1000 can actually WRITE + # them. Without it the write side of each shared path depends on + # the kubelet's default emptyDir mode, and the failures are + # scattered and quiet: + # - shoresh-engine-install: cp ... Permission denied (loud — + # init fails, pod never starts; this one at least shouts) + # - azazel: the Python side cannot create + # /tmp/shorashim/azazel/pending_l3 and the disposal channel + # has nowhere to put a record + # - talmudic-engine: cannot bind the Unix socket + securityContext: + fsGroup: 1000 # --- IMAGE PULL SECRET --- imagePullSecrets: - name: shorashim-registry - # --- INIT CONTAINER: Run L3 migrations --- + # --- INIT CONTAINERS --- initContainers: + # Stage the Haskell shoresh-engine binary where vav-pipeline can + # spawn it. + # + # WHY THIS IS NOT A SIDECAR ANY MORE: + # VavPipeline.ShoreshEnginePort calls + # Port.open({:spawn_executable, binary}, ...) + # An Erlang Port is a CHILD PROCESS of the BEAM. It is stdin/ + # stdout, not a network endpoint, and it cannot cross a + # container boundary. shoresh-engine also has no listening + # mode — `shoresh-engine server` is a JSON-LINE server over + # stdin/stdout ("Erlang Port mode", see src/Main.hs), so as a + # sidecar with no stdin attached it read EOF and exited, and + # the bare CMD printed its usage banner and exited 0. Either + # way: CrashLoopBackOff, and vav-pipeline meanwhile logged + # "binary 'shoresh-engine' not in PATH — running without + # Haskell engine" and served in a degraded state. + # + # Copying the binary out of its own image onto a shared volume + # keeps the four-image build (the kohen's gate 0i still checks + # four) while putting the executable where the code has always + # expected it. + # + # NOTE FOR THE SPECIFICATION: deploy/ARCHITECTURE.md described + # shoresh-engine as a "sidecar via Erlang Port". Those two + # things cannot both be true. This manifest resolves it in + # favour of the Port, because that is what the Elixir code does. + # If the intended design was a socket server like the Talmudic + # Engine, that is a specification decision, not a deployment + # one — see the builder handoff-back. + - name: shoresh-engine-install + image: exo.container-registry.com/the-recognition-protocol/shoresh-engine:latest + command: + - sh + - -c + - | + set -e + cp /home/shorashim/shoresh-engine /opt/shoresh/shoresh-engine + chmod 0755 /opt/shoresh/shoresh-engine + /opt/shoresh/shoresh-engine gematria שלום >/dev/null + echo "shoresh-engine staged and exercised at /opt/shoresh/shoresh-engine" + volumeMounts: + - name: shoresh-bin + mountPath: /opt/shoresh + + # Run L3 migrations - name: migrate image: exo.container-registry.com/the-recognition-protocol/vav-pipeline:latest command: ["./app/bin/vav_pipeline", "eval", "VavPipeline.Release.migrate()"] @@ -67,10 +155,13 @@ spec: volumeMounts: - name: manna mountPath: /var/lib/shorashim/manna - - name: talmudic-socket - mountPath: /tmp/shorashim + # One /tmp. The Talmudic socket and the azazel pending_l3 + # directory both live under it. - name: tmp-shorashim mountPath: /tmp + - name: shoresh-bin + mountPath: /opt/shoresh + readOnly: true resources: requests: memory: "256Mi" @@ -91,23 +182,23 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 - # === SHORESH ENGINE (sidecar) === - - name: shoresh-engine - image: exo.container-registry.com/the-recognition-protocol/shoresh-engine:latest - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" + # === SHORESH ENGINE === + # Not a container. It runs as an Erlang Port inside + # vav-pipeline, from the binary staged by the + # shoresh-engine-install init container above. Its memory and + # CPU are vav-pipeline's; that container's limits were left + # unchanged and should be watched on first load. # === TALMUDIC ENGINE (sidecar) === - name: talmudic-engine image: exo.container-registry.com/the-recognition-protocol/talmudic-engine:latest volumeMounts: - - name: talmudic-socket - mountPath: /tmp/shorashim + # Shares vav-pipeline's /tmp so the Unix socket at + # /tmp/shorashim/talmudic.sock is the same file on both + # sides. Mounted at /tmp (not /tmp/shorashim) so this is + # the same volume korban-processor writes azazel records to. + - name: tmp-shorashim + mountPath: /tmp resources: requests: memory: "128Mi" diff --git a/korban_processor/src/korban_processor/__main__.py b/korban_processor/src/korban_processor/__main__.py new file mode 100644 index 00000000..e5d8533b --- /dev/null +++ b/korban_processor/src/korban_processor/__main__.py @@ -0,0 +1,126 @@ +""" +Container entry point for `python -m korban_processor`. + +WHY THIS FILE EXISTS +-------------------- +deploy/docker/Dockerfile.korban_processor has always ended with + + CMD ["python", "-m", "korban_processor"] + +and deploy/k8s/deployment.yaml runs the korban-processor container +with no `command:` override, so that CMD is what the cluster executes. +There was no __main__ module, so the container exited immediately with + + /usr/local/bin/python: No module named korban_processor.__main__; + 'korban_processor' is a package and cannot be directly executed + +which on the cluster is CrashLoopBackOff for the whole pod. The image +built, pushed, and pulled cleanly and still could not serve. This +module makes the vessel do what its own CMD already claimed. + +WHAT IT DOES — AND WHAT IT DELIBERATELY DOES NOT +------------------------------------------------ +korban_processor is a LIBRARY. Its offering protocol (semichah → +shechitah → zerikah → haktarah → route) is called by a consumer; it +does not listen on anything. The consumer that will call it across a +process boundary is Architecture §4 Step 22 (Korban Processor external +model integration), which is blocked on external model infrastructure. + +So this module does NOT invent a server, a queue reader, or an HTTP +listener. Inventing an intake surface that no specification asked for +would be exactly the addition lo tosif forbids, and it would put an +unspecified door on the one component whose whole discipline is that +nothing reaches the model except through the offering protocol. + +What it does instead is the honest whole of what this container is for +today: + + 1. Prove the image is assembled correctly — in particular the azazel + co-residence. Importing korban_processor.external_offering fails + loudly if azazel_processor is not on the import path, because + external_offering refuses to load without it (Vayikra 16:8 — the + two goats are provisioned from the SAME pool at parity; a korban + channel running without its disposal channel is a system that can + accept an offering it has no lawful way to refuse). + + 2. Stay up, and exit 0 on SIGTERM, so the pod is honestly healthy + rather than crash-looping. + + 3. Say plainly, in the log the kohen reads, that it is idle and what + unblocks it. + +An idle container that reports itself idle is a true statement about +the build. A crashing container is not, and neither is a fake server. +""" + +import os +import signal +import sys +import threading + +#: Set by the SIGTERM/SIGINT handler; the main thread waits on it. +_shutdown = threading.Event() + + +def _handle_signal(signum, _frame): + """Kubernetes sends SIGTERM on pod teardown; exit 0, not 143.""" + print( + f"korban_processor: received {signal.Signals(signum).name} — standing down", + flush=True, + ) + _shutdown.set() + + +def _verify_assembly(): + """ + Import the modules whose presence proves the image is whole. + + external_offering raises ImportError with a full explanation when + azazel_processor is absent, so this single import covers both the + korban channel and its disposal parity. Any failure here means the + image is mis-built and the container must NOT report itself ready. + """ + import korban_processor + import korban_processor.external_offering # noqa: F401 (import IS the check) + import azazel_processor + + return korban_processor, azazel_processor + + +def main() -> int: + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + + try: + korban, azazel = _verify_assembly() + except Exception as exc: # noqa: BLE001 — we want the reason in the log + print( + "korban_processor: FAILED TO START — the image is not whole.\n" + f" {type(exc).__name__}: {exc}\n" + " Check deploy/docker/Dockerfile.korban_processor: both\n" + " korban_processor/src/ and azazel_processor/src/azazel_processor/\n" + " must be present under PYTHONPATH. See BUILD_AZAZEL.md.", + file=sys.stderr, + flush=True, + ) + return 1 + + print( + "korban_processor: offering protocol loaded " + "(semichah -> shechitah -> zerikah -> haktarah -> route)\n" + f" korban_processor from {os.path.dirname(korban.__file__)}\n" + f" azazel_processor from {os.path.dirname(azazel.__file__)} " + "(co-resident disposal channel, Vayikra 16:8)\n" + "korban_processor: IDLE — this is a library, not a service. No\n" + " inbound caller exists until Architecture Step 22 (external\n" + " model integration) is built. Waiting for SIGTERM.", + flush=True, + ) + + _shutdown.wait() + print("korban_processor: stopped", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/korban_processor/test/test_container_entrypoint.py b/korban_processor/test/test_container_entrypoint.py new file mode 100644 index 00000000..61c4a675 --- /dev/null +++ b/korban_processor/test/test_container_entrypoint.py @@ -0,0 +1,123 @@ +""" +The container entry point — `python -m korban_processor`. + +deploy/k8s/deployment.yaml runs the korban-processor container with no +`command:` override, so the image's CMD is what the cluster executes. +Before korban_processor/src/korban_processor/__main__.py existed, that +CMD exited immediately ("No module named korban_processor.__main__") +and took the whole pod into CrashLoopBackOff. These tests hold the +entry point to the three things it actually promises. + +Vayikra 16:8 is the load-bearing one: goral echad l'YHVH v'goral echad +la'Azazel. The two goats are provisioned together, from the same pool. +A korban channel that comes up WITHOUT its disposal channel can accept +an offering it has no lawful way to refuse, so the entry point must +refuse to come up at all rather than serve half the architecture. +""" + +import os +import pathlib +import signal +import subprocess +import sys + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +KORBAN_SRC = REPO_ROOT / "korban_processor" / "src" +AZAZEL_SRC = REPO_ROOT / "azazel_processor" / "src" + + +def _run_entrypoint(pythonpath, stop_after=6.0): + """ + Launch `python -m korban_processor` with an explicit PYTHONPATH. + + Returns (returncode, combined_output). If the process is still + running when we are done waiting, it is SIGTERMed — which is also + how Kubernetes stops it, so the shutdown path is exercised here + rather than only in production. + """ + env = {**os.environ, "PYTHONPATH": os.pathsep.join(str(p) for p in pythonpath)} + proc = subprocess.Popen( + [sys.executable, "-m", "korban_processor"], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + cwd=str(REPO_ROOT), + ) + try: + out = proc.communicate(timeout=2.0)[0] + return proc.returncode, out + except subprocess.TimeoutExpired: + # Still up — that is the healthy idle case. Stop it the way + # the cluster would. + proc.send_signal(signal.SIGTERM) + out = proc.communicate(timeout=stop_after)[0] + return proc.returncode, out + + +# ============================================================ +# It comes up, stays up, and stops clean +# ============================================================ + + +def test_entrypoint_stays_up_and_exits_zero_on_sigterm(): + code, out = _run_entrypoint([KORBAN_SRC, AZAZEL_SRC]) + + assert code == 0, f"expected clean exit on SIGTERM, got {code}:\n{out}" + assert "SIGTERM" in out + assert "stopped" in out + + +def test_entrypoint_reports_itself_idle_rather_than_ready_to_serve(): + """ + korban_processor is a library. Until Architecture Step 22 wires an + external-model consumer, the honest report is IDLE. A container + that claims to be serving when nothing can reach it is the failure + this test exists to prevent. + """ + _, out = _run_entrypoint([KORBAN_SRC, AZAZEL_SRC]) + + assert "IDLE" in out + assert "Step 22" in out + + +def test_entrypoint_names_the_offering_protocol_in_order(): + _, out = _run_entrypoint([KORBAN_SRC, AZAZEL_SRC]) + + order = ["semichah", "shechitah", "zerikah", "haktarah", "route"] + positions = [out.index(stage) for stage in order] + assert positions == sorted(positions), f"protocol stages out of order:\n{out}" + + +# ============================================================ +# Vayikra 16:8 — it refuses to run without the disposal channel +# ============================================================ + + +def test_entrypoint_refuses_to_start_without_azazel_processor(): + code, out = _run_entrypoint([KORBAN_SRC]) # azazel deliberately absent + + assert code == 1, f"expected exit 1 without azazel_processor, got {code}:\n{out}" + assert "FAILED TO START" in out + assert "azazel_processor" in out + assert "BUILD_AZAZEL.md" in out + + +# ============================================================ +# Form 1 — the entry point opens no bypass to the external model +# ============================================================ + + +def test_entrypoint_does_not_reach_the_model_interface(): + """ + Companion to test_form_1_architecture.py invariant 3. The entry + point runs before anything else in the container and must not be + the module that hands a caller call_claude without the offering + protocol. + """ + source = (KORBAN_SRC / "korban_processor" / "__main__.py").read_text( + encoding="utf-8" + ) + + assert "model_interface" not in source + assert "call_claude" not in source diff --git a/scripts/build_and_push.sh b/scripts/build_and_push.sh index a5af4d9b..aacdb0f5 100755 --- a/scripts/build_and_push.sh +++ b/scripts/build_and_push.sh @@ -8,6 +8,20 @@ # on every FROM line as defense-in-depth; this script is the canonical # build/push path that operators and CI both use. # +# EMULATED BUILD NOTE (2026-08-23): +# On an Apple Silicon host this stage runs amd64 under emulation, and +# the BEAM JIT's dual W^X code mapping dies there — SIGSEGV, exit 139, +# mid-`mix compile`. That is what killed the 2026-05-13 run after 6.7 +# hours and the 2026-08-21 run after 13 seconds. The fix is +# ENV ERL_FLAGS="+JMsingle true" in the vav_pipeline build stage (see +# deploy/docker/Dockerfile.vav_pipeline); with it the whole image +# builds in well under a minute. Nothing needs to be set here. +# +# The other half of that incident was the missing .dockerignore: the +# build context carried the host's vav_pipeline/_build and +# shoresh-engine/dist-newstyle into the container, where the wrong-arch +# toolchain tried to reuse them. See .dockerignore at the repo root. +# # Default: linux/amd64 only (cluster-correct, fastest path). # Optional: PLATFORMS=linux/amd64,linux/arm64 ./build_and_push.sh # — multi-arch (slower; preserves local runnability for arm64 devs). @@ -92,20 +106,69 @@ build_one shoresh_engine shoresh-engine build_one talmudic_engine talmudic-engine build_one korban_processor korban-processor +# ================================================================ +# VERIFY WHAT WAS PUSHED — do not report success on exit status alone +# ================================================================ +# +# "The script exited 0" is not proof. The 2026-05-13 and 2026-08-21 +# runs both produced arm64-or-nothing while looking plausible on the +# way past. Read the pushed manifest back out of the registry and +# assert the architecture, here, in the same run that pushed it. +# +# NOTE ON ${img} vs $img: the braces are required. In zsh, `$img:latest` +# applies the `:l` history modifier (lowercase) and `$img:test` applies +# `:t` (tail), silently corrupting the reference into +# `vav-pipelineest`. The kohen lost ten minutes to `:l`; the builder +# then lost some more to `:t`. Braces, always. echo "" echo "================================================================" -echo ">> all four images built and pushed for ${PLATFORMS}" +echo ">> verifying pushed manifests" echo "================================================================" + +verify_failed=0 +for img in vav-pipeline shoresh-engine talmudic-engine korban-processor; do + ref="${REGISTRY}/${PROJECT}/${img}:${TAG}" + arches=$(docker manifest inspect "$ref" 2>/dev/null \ + | grep -oE '"architecture"[[:space:]]*:[[:space:]]*"[^"]+"' \ + | sed 's/.*"\([^"]*\)"$/\1/' \ + | sort -u | tr '\n' ' ') + + if [[ -z "$arches" ]]; then + echo " FAIL ${img}: could not read manifest from ${ref}" + verify_failed=1 + elif [[ "$arches" != *amd64* ]]; then + echo " FAIL ${img}: architecture(s) = ${arches}(expected amd64)" + verify_failed=1 + else + echo " ok ${img}: ${arches}" + fi +done + +if [[ "$verify_failed" -ne 0 ]]; then + echo "" + echo "ERROR: at least one pushed image is not amd64. Do NOT tell the" >&2 + echo "kohen to start Day 1 — gate 0i would fail with exec format error." >&2 + exit 1 +fi + +echo "" +echo "================================================================" +echo ">> all four images built, pushed, and verified amd64 (${PLATFORMS})" +echo "================================================================" +echo "" +echo "This proves the images EXIST and are amd64. It does NOT prove they" +echo "RUN. The four things that broke after the arch gate passed, each" +echo "of which the image-architecture check is blind to:" +echo " - shoresh-engine died on its own banner in the C locale" +echo " - talmudic-engine called a predicate that does not exist" +echo " - korban-processor had no __main__ to execute" +echo " - vav-pipeline could not find the spec files to verify its" +echo " own Doorpost" +echo "All four are fixed in the Dockerfiles; the point is that a green" +echo "arch check is a floor, not a ceiling." echo "" echo "Verify from the cluster (after kohen has kubeconfig loaded):" echo " kubectl get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}'" echo " # expected: amd64" echo "" -echo "Verify image manifests:" -echo " for img in vav-pipeline shoresh-engine talmudic-engine korban-processor; do" -echo " docker manifest inspect \"${REGISTRY}/${PROJECT}/\$img:${TAG}\" \\" -echo " | jq -r '.architecture // .manifests[].platform.architecture' \\" -echo " | sort -u" -echo " done" -echo "" echo "Then re-run kohen gate 0i in KOHEN_PROMPT.md." diff --git a/talmudic_engine/src/interface/json_bridge.pl b/talmudic_engine/src/interface/json_bridge.pl index 3cf7ad32..4f5a5d72 100644 --- a/talmudic_engine/src/interface/json_bridge.pl +++ b/talmudic_engine/src/interface/json_bridge.pl @@ -12,7 +12,17 @@ parse_root_name/2 ]). -:- use_module(library(json)). +:- use_module(library(http/json)). % NOT library(json) — see note below. +% +% library(http/json) is the canonical module path and resolves on +% SWI-Prolog 9.x AND 10.x. The bare alias library(json) resolves only +% on 10.x, which is what the dev workstation runs; the image is built +% on swipl 9.2.2 as pinned by specs/shorashim_vayakhel.md §1.1, where +% it does NOT resolve. The consequence was invisible to the test +% suite (212 tests green on the host) and fatal in the container: +% ERROR: source_sink `library(json)' does not exist +% ERROR: Unknown procedure: start_socket_server/0 +% exit 2, before the socket was ever opened. :- use_module(library(lists)). %% ============================================================ diff --git a/talmudic_engine/src/interface/socket_server.pl b/talmudic_engine/src/interface/socket_server.pl index 43c264a2..5127c3de 100644 --- a/talmudic_engine/src/interface/socket_server.pl +++ b/talmudic_engine/src/interface/socket_server.pl @@ -18,7 +18,17 @@ ]). :- use_module(library(socket)). -:- use_module(library(json)). +:- use_module(library(http/json)). % NOT library(json) — see note below. +% +% library(http/json) is the canonical module path and resolves on +% SWI-Prolog 9.x AND 10.x. The bare alias library(json) resolves only +% on 10.x, which is what the dev workstation runs; the image is built +% on swipl 9.2.2 as pinned by specs/shorashim_vayakhel.md §1.1, where +% it does NOT resolve. The consequence was invisible to the test +% suite (212 tests green on the host) and fatal in the container: +% ERROR: source_sink `library(json)' does not exist +% ERROR: Unknown procedure: start_socket_server/0 +% exit 2, before the socket was ever opened. :- use_module(library(readutil)). :- use_module('../interface/api'). diff --git a/vav_pipeline/config/prod.exs b/vav_pipeline/config/prod.exs index 4f063bde..ac4d76e0 100644 --- a/vav_pipeline/config/prod.exs +++ b/vav_pipeline/config/prod.exs @@ -6,6 +6,16 @@ config :logger, level: :info # Env-var overrides are in runtime.exs (evaluated at boot, not compile) config :vav_pipeline, l1_mount_point: "/mnt/torah", + # Doorpost repo root — Devarim 11:20, ukhtavtam al mezuzot beitecha. + # The category-grade specification files are COPYed into the image + # (see deploy/docker/Dockerfile.vav_pipeline, INSCRIBE THE DOORPOST) + # so a running pod can verify its own corporate-text identity. Without + # this, @repo_root falls back to a path that only exists on the build + # host, VavPipeline.Doorpost.verify_identity/0 returns + # SPEC_INTEGRITY_FAILED on every boot, and the chatzotzrot that fires + # is a false alarm — which is worse than no witness at all. + # Read at COMPILE time via Application.compile_env/3. + doorpost_repo_root: "/opt/shorashim/spec", manna_provision_file: "/var/lib/shorashim/manna/today.json", cycle_boundary_check_interval_ms: 60_000, shema_context_cache_ttl_ms: 0, diff --git a/vav_pipeline/config/runtime.exs b/vav_pipeline/config/runtime.exs index 466aa3c7..bcdf4ae2 100644 --- a/vav_pipeline/config/runtime.exs +++ b/vav_pipeline/config/runtime.exs @@ -28,5 +28,17 @@ if config_env() == :prod do System.get_env("MANNA_PROVISION_FILE") || "/var/lib/shorashim/manna/today.json", talmudic_socket_path: System.get_env("TALMUDIC_SOCKET_PATH") || "/tmp/shorashim/talmudic.sock", + # Shoresh Engine binary — VavPipeline.ShoreshEnginePort opens an + # Erlang Port with {:spawn_executable, binary}, which spawns a CHILD + # PROCESS OF THIS CONTAINER. A Port cannot cross a container + # boundary, so the Haskell binary has to be on this container's + # filesystem, not only in the shoresh-engine image. deployment.yaml + # stages it onto a shared volume with an init container and points + # this variable at it. Without the override, Application. + # maybe_add_shoresh_port/3 finds nothing on PATH and logs + # "running without Haskell engine" — the system comes up DEGRADED + # and says so only in a warning line. + shoresh_engine_binary: + System.get_env("SHORESH_ENGINE_BINARY") || "shoresh-engine", http_port: String.to_integer(System.get_env("HTTP_PORT") || "4000") end diff --git a/vav_pipeline/mix.lock b/vav_pipeline/mix.lock index a245c112..971d02b2 100644 --- a/vav_pipeline/mix.lock +++ b/vav_pipeline/mix.lock @@ -1,10 +1,10 @@ %{ - "bandit": {:hex, :bandit, "1.10.3", "1e5d168fa79ec8de2860d1b4d878d97d4fbbe2fdbe7b0a7d9315a4359d1d4bb9", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "99a52d909c48db65ca598e1962797659e3c0f1d06e825a50c3d75b74a5e2db18"}, + "bandit": {:hex, :bandit, "1.12.5", "af205a8e550f304caae09a97d29fd3c79a7f337526ea7cd772d2ff11d2f7c800", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.5", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "c5684ca062fa407cac115aec3256383f3e2ec9fdced7904d59cf5a7bb7ed6181"}, "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, "decimal": {:hex, :decimal, "2.4.1", "6c0fbede12fb122ba685e9ab41c6a40c129e322b3aa192f9e072e61f3a6ffaf2", [:mix], [], "hexpm", "7e618897933a8455f19a727d7c5e50a2c071a544b700e5e724298ecb4340187f"}, "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, - "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, + "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "jsx": {:hex, :jsx, "3.1.0", "d12516baa0bb23a59bb35dccaf02a1bd08243fcbb9efe24f2d9d056ccff71268", [:rebar3], [], "hexpm", "0c5cc8fdc11b53cc25cf65ac6705ad39e54ecc56d1c22e4adb8f5a53fb9427f3"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, @@ -12,6 +12,6 @@ "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, - "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"}, + "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, }