bikoret/scripts/build_and_push.sh

#!/usr/bin/env bash
#
# build_and_push.sh — canonical builder procedure for bikoret images.
#
# Pin platform to linux/amd64 (Exoscale SKS) so a build on an Apple
# Silicon workstation does not silently ship arm64 binaries that the
# cluster cannot exec. The Dockerfiles also pin --platform=linux/amd64
# on every FROM line as defense-in-depth; this script is the canonical
# build/push path that operators and CI both use.
#
# EMULATED BUILD NOTE (2026-08-23):
# On an Apple Silicon host this stage runs amd64 under emulation, and
# the BEAM JIT's dual W^X code mapping dies there — SIGSEGV, exit 139,
# mid-`mix compile`. That is what killed the 2026-05-13 run after 6.7
# hours and the 2026-08-21 run after 13 seconds. The fix is
# ENV ERL_FLAGS="+JMsingle true" in the vav_pipeline build stage (see
# deploy/docker/Dockerfile.vav_pipeline); with it the whole image
# builds in well under a minute. Nothing needs to be set here.
#
# The other half of that incident was the missing .dockerignore: the
# build context carried the host's vav_pipeline/_build and
# shoresh-engine/dist-newstyle into the container, where the wrong-arch
# toolchain tried to reuse them. See .dockerignore at the repo root.
#
# Default: linux/amd64 only (cluster-correct, fastest path).
# Optional: PLATFORMS=linux/amd64,linux/arm64 ./build_and_push.sh
#   — multi-arch (slower; preserves local runnability for arm64 devs).
#
# Prerequisites:
#   - Docker daemon running
#   - docker buildx available
#   - Logged in to exo.container-registry.com (CLI secret from
#     Harbor → User Profile → CLI Secret)
#   - Run from the repo root
#
# What gets pushed (four images):
#   - vav-pipeline
#   - shoresh-engine
#   - talmudic-engine
#   - korban-processor  (azazel_processor is baked into this image)

set -euo pipefail

REGISTRY="${REGISTRY:-exo.container-registry.com}"
PROJECT="${PROJECT:-the-recognition-protocol}"
TAG="${TAG:-latest}"
PLATFORMS="${PLATFORMS:-linux/amd64}"

# Sanity: must be run from repo root (where deploy/ and l1/ live)
if [[ ! -d "deploy/docker" || ! -d "l1/torah" ]]; then
  echo "ERROR: run from the bikoret repo root (where deploy/ and l1/ live)" >&2
  exit 1
fi

# Sanity: docker buildx
# The Doorpost, read before the build: the spec files the image will carry
# must hash to the pin baked into VavPipeline.Doorpost, or the pod drifts
# at startup, readiness answers 503, and the gate goes dark (tablet 54,
# 2026-10-15). scripts/doorpost_check.py computes the same hash the same
# way and names the drifted files. DOORPOST_SKIP=1 only for a deliberate,
# documented exception — never to ship a drift.
if [[ "${DOORPOST_SKIP:-0}" != "1" ]]; then
  python3 -I scripts/doorpost_check.py || exit 1
fi

# The engine's verse facts are derived from L1 (scripts/build_engine_verses.py);
# an image whose facts lag the scroll reasons over a scroll that is not
# the one the pipeline reads. Regenerate and commit, never skip.
python3 -I scripts/build_engine_verses.py --check || exit 1

if ! docker buildx version >/dev/null 2>&1; then
  echo "ERROR: docker buildx not available — install Docker Desktop or buildx plugin" >&2
  exit 1
fi

# Sanity: registry login. We probe with a HEAD against the registry
# v2 endpoint; if 401, login is missing.
if ! docker info 2>/dev/null | grep -q "$REGISTRY"; then
  echo "Reminder: run 'docker login $REGISTRY' first (CLI secret from Harbor User Profile)."
  echo "Continuing — push will fail clearly if not logged in."
fi

# Create or use the bikoret buildx builder
if ! docker buildx inspect bikoret-builder >/dev/null 2>&1; then
  echo ">> creating buildx builder 'bikoret-builder'"
  docker buildx create --name bikoret-builder --use
else
  docker buildx use bikoret-builder
fi
docker buildx inspect --bootstrap >/dev/null

# Source dir → image name → Dockerfile mapping.
# korban_processor's Dockerfile also COPYs azazel_processor (see
# Dockerfile.korban_processor — AZAZEL CO-RESIDENT section).
build_one() {
  local component="$1"
  local image="$2"
  local dockerfile="deploy/docker/Dockerfile.${component}"

  if [[ ! -f "$dockerfile" ]]; then
    echo "ERROR: $dockerfile not found" >&2
    return 1
  fi

  echo ""
  echo "================================================================"
  echo ">> building ${image}:${TAG} for ${PLATFORMS}"
  echo "   dockerfile: ${dockerfile}"
  echo "================================================================"

  docker buildx build \
    --platform "$PLATFORMS" \
    --file "$dockerfile" \
    --tag "${REGISTRY}/${PROJECT}/${image}:${TAG}" \
    --push \
    .
}

build_one vav_pipeline      vav-pipeline
build_one shoresh_engine    shoresh-engine
build_one talmudic_engine   talmudic-engine
build_one korban_processor  korban-processor

# ================================================================
# VERIFY WHAT WAS PUSHED — do not report success on exit status alone
# ================================================================
#
# "The script exited 0" is not proof. The 2026-05-13 and 2026-08-21
# runs both produced arm64-or-nothing while looking plausible on the
# way past. Read the pushed manifest back out of the registry and
# assert the architecture, here, in the same run that pushed it.
#
# NOTE ON ${img} vs $img: the braces are required. In zsh, `$img:latest`
# applies the `:l` history modifier (lowercase) and `$img:test` applies
# `:t` (tail), silently corrupting the reference into
# `vav-pipelineest`. The kohen lost ten minutes to `:l`; the builder
# then lost some more to `:t`. Braces, always.
echo ""
echo "================================================================"
echo ">> verifying pushed manifests"
echo "================================================================"

verify_failed=0
for img in vav-pipeline shoresh-engine talmudic-engine korban-processor; do
  ref="${REGISTRY}/${PROJECT}/${img}:${TAG}"
  arches=$(docker manifest inspect "$ref" 2>/dev/null \
    | grep -oE '"architecture"[[:space:]]*:[[:space:]]*"[^"]+"' \
    | sed 's/.*"\([^"]*\)"$/\1/' \
    | sort -u | tr '\n' ' ')

  if [[ -z "$arches" ]]; then
    echo "  FAIL  ${img}: could not read manifest from ${ref}"
    verify_failed=1
  elif [[ "$arches" != *amd64* ]]; then
    echo "  FAIL  ${img}: architecture(s) = ${arches}(expected amd64)"
    verify_failed=1
  else
    echo "  ok    ${img}: ${arches}"
  fi
done

if [[ "$verify_failed" -ne 0 ]]; then
  echo ""
  echo "ERROR: at least one pushed image is not amd64. Do NOT tell the" >&2
  echo "kohen to start Day 1 — gate 0i would fail with exec format error." >&2
  exit 1
fi

echo ""
echo "================================================================"
echo ">> all four images built, pushed, and verified amd64 (${PLATFORMS})"
echo "================================================================"
echo ""
echo "This proves the images EXIST and are amd64. It does NOT prove they"
echo "RUN. The four things that broke after the arch gate passed, each"
echo "of which the image-architecture check is blind to:"
echo "  - shoresh-engine died on its own banner in the C locale"
echo "  - talmudic-engine called a predicate that does not exist"
echo "  - korban-processor had no __main__ to execute"
echo "  - vav-pipeline could not find the spec files to verify its"
echo "    own Doorpost"
echo "All four are fixed in the Dockerfiles; the point is that a green"
echo "arch check is a floor, not a ceiling."
echo ""
echo "Verify from the cluster (after kohen has kubeconfig loaded):"
echo "  kubectl get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}'"
echo "  # expected: amd64"
echo ""
echo "Then re-run kohen gate 0i in KOHEN_PROMPT.md."