diff --git a/KOHEN.md b/KOHEN.md index c85a8140..57d6fbd6 100644 --- a/KOHEN.md +++ b/KOHEN.md @@ -488,6 +488,29 @@ curl https://bikoret.ch/kodesh/status | jq .halt_status curl https://bikoret.ch/kodesh/status | jq '.refuge_components' ``` +**The watchdog — *shomer*.** `scripts/bikoret-watchdog` (installed at +`~/.local/bin/`, cron every 5 minutes on the dev VM) probes +`/kodesh/health` from outside the cluster. Two consecutive failures — +ten minutes, so a rollout's own blip does not alert — open ONE Gitea +issue in this repo titled `WATCHDOG: bikoret.ch down since …`, naming +the failed checks and what to read first; while down it comments only +when the failed set changes; on recovery it comments the duration and +closes the issue. Every transition goes to +`~/shorashim-operator-notes/watchdog.log`. It needs a Gitea token with +`write:issue` at `~/.config/bikoret/gitea_watchdog_token` (0600) — the +clone token in `~/.git-credentials` has only the code scopes and Gitea +answers 403 on an issue. The kohen issues it (Gitea → Settings → +Applications → token, scopes `read:issue` + `write:issue`); the builder +does not write credentials. Until it exists the watchdog still logs and +tracks recovery; it cannot open the issue. 2026-10-15: the site was 503 +for 13.5 h and the Doorpost outage was found by losing Claude — nothing +watched. + +```bash +tail ~/shorashim-operator-notes/watchdog.log # transitions only; silence is health +BIKORET_WATCHDOG_DRY=1 bikoret-watchdog # a probe that opens nothing +``` + ### Issuing a key — *Mishmeret HaSha'ar* The courtyard and the chatzer open only to a key you issued. Keys live diff --git a/scripts/bikoret-watchdog b/scripts/bikoret-watchdog new file mode 100755 index 00000000..948a9b03 --- /dev/null +++ b/scripts/bikoret-watchdog @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""bikoret-watchdog — the kohen's eye on /kodesh/health, from outside the cluster. + +Every run (cron, every 5 minutes) probes the health endpoint. A failure is +anything but 200 with overall "healthy" (a 503 at the day boundary, a +Doorpost drift, a hung rollout, the gate dark). After two consecutive +failures (10 minutes — a rollout's own blip does not alert) it opens ONE +Gitea issue in the repo naming the failed checks; while down it comments +only when the failed set changes; on recovery it comments the duration and +closes the issue. Every transition goes to the log. 2026-10-15: the site +was 503 for 13.5 h at a day boundary and the transmitter learned of the +Doorpost outage by losing Claude — nothing watched. + +Paths: + state ~/.local/state/bikoret-watchdog/state.json + log ~/shorashim-operator-notes/watchdog.log + token ~/.git-credentials (the Gitea token; read, never printed) + +Env overrides (for a test run): BIKORET_HEALTH_URL, BIKORET_WATCHDOG_STATE, +BIKORET_WATCHDOG_LOG, BIKORET_GITEA_API (default https://git.bikoret.ch/api/v1), +BIKORET_GITEA_REPO (default heaviness510/bikoret), BIKORET_WATCHDOG_DRY=1 (no issue). +""" +import json, os, re, sys, time, urllib.request, urllib.error +from datetime import datetime, timezone + +HOME = os.path.expanduser("~") +URL = os.environ.get("BIKORET_HEALTH_URL", "https://bikoret.ch/kodesh/health") +STATE = os.environ.get("BIKORET_WATCHDOG_STATE", f"{HOME}/.local/state/bikoret-watchdog/state.json") +LOG = os.environ.get("BIKORET_WATCHDOG_LOG", f"{HOME}/shorashim-operator-notes/watchdog.log") +API = os.environ.get("BIKORET_GITEA_API", "https://git.bikoret.ch/api/v1") +REPO = os.environ.get("BIKORET_GITEA_REPO", "heaviness510/bikoret") +DRY = os.environ.get("BIKORET_WATCHDOG_DRY") == "1" +THRESHOLD = 2 + + +def now(): + return datetime.now(timezone.utc) + + +def log(msg): + os.makedirs(os.path.dirname(LOG), exist_ok=True) + with open(LOG, "a") as f: + f.write(f"{now().strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}\n") + + +def probe(): + """(ok, summary, failed_checks)""" + try: + with urllib.request.urlopen(urllib.request.Request(URL, headers={"user-agent": "bikoret-watchdog"}), timeout=15) as r: + code, body = r.status, r.read() + except urllib.error.HTTPError as e: + code, body = e.code, e.read() + except Exception as e: # DNS, TLS, timeout, refused + return False, f"unreachable: {type(e).__name__}: {e}", ["unreachable"] + try: + d = json.loads(body) + checks = d.get("checks") or {} + failed = sorted(k for k, v in checks.items() if v != "pass") + overall = d.get("overall") + ok = code == 200 and overall == "healthy" + return ok, f"HTTP {code} overall={overall} failed={failed or 'none'}", failed or ([f"HTTP {code}"] if not ok else []) + except Exception: + return False, f"HTTP {code} non-JSON body ({len(body)} bytes)", [f"HTTP {code}"] + + +def token(): + """The Gitea token: ~/.config/bikoret/gitea_watchdog_token first (a token + the kohen issued with write:issue — the clone token in ~/.git-credentials + has only the code scopes and Gitea answers 403 on an issue), else that.""" + try: + t = open(f"{HOME}/.config/bikoret/gitea_watchdog_token").read().strip() + if t: + return t + except FileNotFoundError: + pass + try: + for line in open(f"{HOME}/.git-credentials"): + m = re.match(r"https?://([^:]+):([^@]+)@", line.strip()) + if m: + return m.group(2) + except FileNotFoundError: + pass + return None + + +def gitea(method, path, payload=None): + if DRY: + log(f"DRY {method} {path} {json.dumps(payload)[:120] if payload else ''}") + return {"number": -1, "html_url": "(dry)"} + tok = token() + if not tok: + log("no Gitea token in ~/.git-credentials — cannot open an issue") + return None + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(f"{API}/repos/{REPO}{path}", data=data, method=method, + headers={"authorization": f"token {tok}", "content-type": "application/json", "accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=20) as r: + return json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: + log(f"Gitea {method} {path} -> HTTP {e.code}: {e.read()[:120]!r}") + return None + except Exception as e: + log(f"Gitea {method} {path} failed: {type(e).__name__}: {e}") + return None + + +def load_state(): + try: + return json.load(open(STATE)) + except Exception: + return {"fails": 0, "down_since": None, "issue": None, "failed": []} + + +def save_state(s): + os.makedirs(os.path.dirname(STATE), exist_ok=True) + tmp = STATE + ".tmp" + json.dump(s, open(tmp, "w")) + os.replace(tmp, STATE) + + +def main(): + s = load_state() + ok, summary, failed = probe() + t = now().strftime("%Y-%m-%d %H:%M:%SZ") + + if ok: + if s["issue"] is not None or s["down_since"]: + since = s["down_since"] or t + mins = int((now() - datetime.fromisoformat(since.replace("Z", "+00:00").replace(" ", "T"))).total_seconds() // 60) + log(f"RECOVERED after {mins} min ({summary})") + if s["issue"] is not None: + gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"Recovered at {t} after {mins} min. `{summary}`"}) + gitea("PATCH", f"/issues/{s['issue']}", {"state": "closed"}) + elif s["fails"]: + log(f"ok again after {s['fails']} failed probe(s), no alert raised ({summary})") + save_state({"fails": 0, "down_since": None, "issue": None, "failed": []}) + return 0 + + s["fails"] += 1 + if not s["down_since"]: + s["down_since"] = t + log(f"FAIL #{s['fails']}: {summary}") + + if s["fails"] >= THRESHOLD and s["issue"] is None: + body = (f"bikoret.ch health has failed {s['fails']} consecutive probes since {s['down_since']}.\n\n" + f"Last probe: `{summary}`\n\nFailed checks: {', '.join(failed) or 'none named'}\n\n" + "What to look at first: the vav-pipeline startup log (`kubectl -n shorashim logs -c vav-pipeline | grep -v health`) — " + "a Doorpost drift, stale manna at the day boundary, and a missing anchor all look the same from outside. " + "Claude Code behind the gate is down with the site: `claude-direct` or `claude-key` (KOHEN.md, Fallback).\n\n" + "Opened by bikoret-watchdog on the dev VM; it will comment and close this issue on recovery.") + r = gitea("POST", "/issues", {"title": f"WATCHDOG: bikoret.ch down since {s['down_since']} ({', '.join(failed)[:60]})", "body": body, "labels": []}) + if r: + s["issue"] = r.get("number") + s["failed"] = failed + log(f"ALERT issue #{s['issue']} opened: {r.get('html_url')}") + elif s["issue"] is not None and failed != s.get("failed"): + gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"{t}: failed checks now {', '.join(failed)} — `{summary}`"}) + s["failed"] = failed + log(f"issue #{s['issue']} updated: failed={failed}") + + save_state(s) + return 1 + + +if __name__ == "__main__": + sys.exit(main())