# Shorashim Deployment — the mishkan # # Shemot 40:33 — Vay'khal Mosheh et ham'lakhah. # # Architecture §5.1: All inter-component communication uses # JSON over Unix domain sockets or Erlang distribution protocol. # No HTTP between internal components. # # L1 Torah is baked into each container image (~29MB). # The scroll is sealed in the aron. It ships with the vessel. --- apiVersion: apps/v1 kind: Deployment metadata: name: shorashim namespace: shorashim labels: app.kubernetes.io/name: shorashim app.kubernetes.io/version: "0.1.0" spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: shorashim template: metadata: labels: app.kubernetes.io/name: shorashim spec: # --- VOLUMES --- # # ONE shared /tmp volume, not two. There used to be a separate # `talmudic-socket` volume mounted at /tmp/shorashim in # vav-pipeline and talmudic-engine, WHILE `tmp-shorashim` was # mounted at /tmp in vav-pipeline and korban-processor. Nested # mounts meant /tmp/shorashim resolved to a DIFFERENT volume than # /tmp/shorashim reached through /tmp — so: # - korban/azazel (Python) wrote AzazelRecord JSON to # tmp-shorashim:/shorashim/azazel/pending_l3 # - AzazelIngest (Elixir) watched # talmudic-socket:/azazel/pending_l3 # Two directories with the same pathname on two different disks. # The disposal channel would have run, written its records, and # nothing would ever have reached L3 — silently, because # AzazelIngest leaves unreadable files in place rather than # erroring (see azazel_ingest.ex, RETRY SEMANTICS). # # Vayikra 16:22 — v'nasa hasa'ir alav et kol avonotam el eretz # gezerah. The goat has to actually GET there. A disposal channel # whose records land on a disk no one reads has not disposed of # anything; it has only stopped looking. volumes: - name: manna emptyDir: {} # Shared /tmp for every container that speaks over it: # the Talmudic Unix socket (/tmp/shorashim/talmudic.sock) and # the azazel file bridge (/tmp/shorashim/azazel/pending_l3). - name: tmp-shorashim emptyDir: {} # Staging area for the Haskell shoresh-engine binary. Written # once by the shoresh-engine-install init container, read by # vav-pipeline, which spawns it as an Erlang Port. - name: shoresh-bin emptyDir: {} # --- POD SECURITY CONTEXT --- # Every image runs as the non-root user `shorashim` (uid/gid # 1000). fsGroup makes the kubelet chgrp the emptyDir volumes to # 1000 and set the setgid bit, so uid 1000 can actually WRITE # them. Without it the write side of each shared path depends on # the kubelet's default emptyDir mode, and the failures are # scattered and quiet: # - shoresh-engine-install: cp ... Permission denied (loud — # init fails, pod never starts; this one at least shouts) # - azazel: the Python side cannot create # /tmp/shorashim/azazel/pending_l3 and the disposal channel # has nowhere to put a record # - talmudic-engine: cannot bind the Unix socket securityContext: fsGroup: 1000 # --- IMAGE PULL SECRET --- imagePullSecrets: - name: shorashim-registry # --- INIT CONTAINERS --- initContainers: # Stage the Haskell shoresh-engine binary where vav-pipeline can # spawn it. # # WHY THIS IS NOT A SIDECAR ANY MORE: # VavPipeline.ShoreshEnginePort calls # Port.open({:spawn_executable, binary}, ...) # An Erlang Port is a CHILD PROCESS of the BEAM. It is stdin/ # stdout, not a network endpoint, and it cannot cross a # container boundary. shoresh-engine also has no listening # mode — `shoresh-engine server` is a JSON-LINE server over # stdin/stdout ("Erlang Port mode", see src/Main.hs), so as a # sidecar with no stdin attached it read EOF and exited, and # the bare CMD printed its usage banner and exited 0. Either # way: CrashLoopBackOff, and vav-pipeline meanwhile logged # "binary 'shoresh-engine' not in PATH — running without # Haskell engine" and served in a degraded state. # # Copying the binary out of its own image onto a shared volume # keeps the four-image build (the kohen's gate 0i still checks # four) while putting the executable where the code has always # expected it. # # NOTE FOR THE SPECIFICATION: deploy/ARCHITECTURE.md described # shoresh-engine as a "sidecar via Erlang Port". Those two # things cannot both be true. This manifest resolves it in # favour of the Port, because that is what the Elixir code does. # If the intended design was a socket server like the Talmudic # Engine, that is a specification decision, not a deployment # one — see the builder handoff-back. - name: shoresh-engine-install image: exo.container-registry.com/the-recognition-protocol/shoresh-engine:latest command: - sh - -c - | set -e cp /home/shorashim/shoresh-engine /opt/shoresh/shoresh-engine chmod 0755 /opt/shoresh/shoresh-engine /opt/shoresh/shoresh-engine gematria שלום >/dev/null echo "shoresh-engine staged and exercised at /opt/shoresh/shoresh-engine" volumeMounts: - name: shoresh-bin mountPath: /opt/shoresh # Run L3 migrations # # TWO ROLES, ONE IMAGE. KOHEN.md:184 — "the migration runs as # avnadmin; the application runs as shorashim_app." This init # container was inheriting the APPLICATION's DATABASE_URL and so # ran DDL as shorashim_app, which does not have CREATE on schema # public and never should: # ** (Postgrex.Error) ERROR 42501 (insufficient_privilege) # permission denied for schema public # Could not create schema migrations table. # # Ecto.Migrator.run/3 calls ensure_schema_migrations_table! # BEFORE it looks at what is pending, so this fired on every pod # start even with migrations already current — the pod could # never reach Running. See vav_pipeline/lib/vav_pipeline/ # release.ex. # # WHY NOT JUST GRANT shorashim_app CREATE ON SCHEMA public: # because that is the boundary, not an obstacle in front of it. # Append-only would technically survive (the L3 tables are # avnadmin-owned with REVOKEs), but the application would hold # DDL power over the schema it writes into, and the app/migration # distinction KOHEN.md draws on purpose would exist only as a # convention. Vayikra 18 — a required distinction dissolved for # convenience is the gilui arayot shape, and the halt rule this # system runs on names exactly that. The kohen refused it on # those grounds and was right to; recording the reasoning here so # the next person to hit 42501 does not quietly re-open it. # # DATABASE_URL is overridden as an explicit `env:` entry rather # than by swapping the secretRef. `env:` takes precedence over # every `envFrom` source, so exactly ONE variable changes role # and anything else that lands in shorashim-secrets later is # still inherited by the migrator. # # shorashim-migrate-secrets is NOT in this repo and must not be. # It carries the DDL-capable credential; see # deploy/k8s/secrets.yaml for the shape and the create command. # If it is absent the pod fails at CreateContainerConfigError — # loudly, which is correct. There is deliberately no fallback to # the application role. - name: migrate image: exo.container-registry.com/the-recognition-protocol/vav-pipeline:latest command: ["./app/bin/vav_pipeline", "eval", "VavPipeline.Release.migrate()"] envFrom: - configMapRef: name: shorashim-config - secretRef: name: shorashim-secrets env: # DDL role — avnadmin, not shorashim_app. Overrides the # DATABASE_URL that shorashim-secrets supplies above. - name: DATABASE_URL valueFrom: secretKeyRef: name: shorashim-migrate-secrets key: DATABASE_URL # --- CONTAINERS --- containers: # === VAV PIPELINE (main) === - name: vav-pipeline image: exo.container-registry.com/the-recognition-protocol/vav-pipeline:latest ports: - containerPort: 4000 name: http envFrom: - configMapRef: name: shorashim-config - secretRef: name: shorashim-secrets volumeMounts: - name: manna mountPath: /var/lib/shorashim/manna # One /tmp. The Talmudic socket and the azazel pending_l3 # directory both live under it. - name: tmp-shorashim mountPath: /tmp - name: shoresh-bin mountPath: /opt/shoresh readOnly: true resources: requests: memory: "256Mi" cpu: "250m" limits: memory: "1Gi" cpu: "1000m" livenessProbe: httpGet: path: /kodesh/health port: 4000 initialDelaySeconds: 30 periodSeconds: 60 readinessProbe: httpGet: path: /kodesh/health port: 4000 initialDelaySeconds: 10 periodSeconds: 10 # === SHORESH ENGINE === # Not a container. It runs as an Erlang Port inside # vav-pipeline, from the binary staged by the # shoresh-engine-install init container above. Its memory and # CPU are vav-pipeline's; that container's limits were left # unchanged and should be watched on first load. # === TALMUDIC ENGINE (sidecar) === - name: talmudic-engine image: exo.container-registry.com/the-recognition-protocol/talmudic-engine:latest volumeMounts: # Shares vav-pipeline's /tmp so the Unix socket at # /tmp/shorashim/talmudic.sock is the same file on both # sides. Mounted at /tmp (not /tmp/shorashim) so this is # the same volume korban-processor writes azazel records to. - name: tmp-shorashim mountPath: /tmp resources: requests: memory: "128Mi" cpu: "100m" limits: memory: "512Mi" cpu: "500m" # === KORBAN PROCESSOR (sidecar) === # Korban_processor imports azazel_processor (the disposal # channel) in the same Python process. The Python side writes # AzazelRecord JSON files to /tmp/shorashim/azazel/pending_l3/; # vav-pipeline's AzazelIngest GenServer watches that directory # and ingests into L3. Both containers must mount the shared # tmp-shorashim volume for the file bridge to work. - name: korban-processor image: exo.container-registry.com/the-recognition-protocol/korban-processor:latest volumeMounts: - name: tmp-shorashim mountPath: /tmp resources: requests: memory: "128Mi" cpu: "100m" limits: memory: "512Mi" cpu: "500m"