diff --git a/KOHEN.md b/KOHEN.md index 509b5110..25204754 100644 --- a/KOHEN.md +++ b/KOHEN.md @@ -599,7 +599,9 @@ aligned, not that the output is true. Read `limits` in the response. ### Rolling out — *Nesi'at haMishkan* `scripts/shorashim-rollout` (also installed at `~/.local/bin/`) restarts -the deployment, waits for 3/3, and takes two consecutive 200s from +the deployment, waits for both pods (two replicas since 2026-10-25, +surge rollout: one pod stays Ready throughout, so a rollout no longer +shows in the watchdog log), and takes two consecutive 200s from `/kodesh/health` before reporting — a rollout is not done at 3/3 (NLB re-admission lag). A new image must already be pushed as `:latest`. If the day's manna has not been written, the new pod hangs at 2/3 until it diff --git a/deploy/k8s/configmap.yaml b/deploy/k8s/configmap.yaml index 7a0a5a7a..dc37ca64 100644 --- a/deploy/k8s/configmap.yaml +++ b/deploy/k8s/configmap.yaml @@ -66,5 +66,5 @@ data: # a pool of 20 fills its connection ceiling by itself (2026-10-15: # refusals "remaining connection slots are reserved" for 24 h, one # pool worker looping, Aiven's monitor refused). Ten is ample for a - # single replica. - POOL_SIZE: "10" + # single replica; two replicas since 2026-10-25 → 5 each, 10 in all. + POOL_SIZE: "5" diff --git a/deploy/k8s/deployment.yaml b/deploy/k8s/deployment.yaml index 043400e2..917b10c5 100644 --- a/deploy/k8s/deployment.yaml +++ b/deploy/k8s/deployment.yaml @@ -18,18 +18,25 @@ metadata: app.kubernetes.io/name: shorashim app.kubernetes.io/version: "0.1.0" spec: - replicas: 1 - # One replica on one node with no room to surge. The default - # (maxSurge 25% → 1 extra pod) asks for 450m CPU the node does not - # have beside the running pod, and the rollout hangs Pending while the - # old image keeps serving (tablet 31, 2026-10-04). Replace in place. - # The new pod reads the day's manna from the manna-pvc volume; it is - # Ready again without the kohen, as long as today's portion was written. + # Two replicas (2026-10-25). A pod that is not ready took every caller + # behind the gate dark, and every rollout left a five-minute 503 in + # the watchdog log; with two pods and a surge rollout the Service keeps + # one Ready pod throughout. One node: this covers rollouts and pod + # crashes, not the node. The node has 1700m CPU allocatable and 1470m + # was requested (tablet 31, 2026-10-04: a surge pod at 450m hung + # Pending), so the requests below are sized to what the pod USES + # (vav 20m, sidecars 2m, measured) with headroom — 200m per pod, 400m + # for two — not to the old 450m. Limits are unchanged. Each pod reads + # the day's manna from the manna-pvc volume and is Ready without the + # kohen as long as today's portion was written; the database pool is + # 5 per pod (POOL_SIZE in the ConfigMap), 10 in all, under the + # hobbyist tier's ceiling. + replicas: 2 strategy: type: RollingUpdate rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 + maxSurge: 1 + maxUnavailable: 0 selector: matchLabels: app.kubernetes.io/name: shorashim @@ -237,7 +244,7 @@ spec: resources: requests: memory: "256Mi" - cpu: "250m" + cpu: "120m" limits: memory: "1Gi" cpu: "1000m" @@ -293,8 +300,8 @@ spec: mountPath: /tmp resources: requests: - memory: "128Mi" - cpu: "100m" + memory: "64Mi" + cpu: "40m" limits: memory: "512Mi" cpu: "500m" @@ -318,8 +325,8 @@ spec: mountPath: /tmp resources: requests: - memory: "128Mi" - cpu: "100m" + memory: "64Mi" + cpu: "40m" limits: memory: "512Mi" cpu: "500m"