kohen: the watchdog — /kodesh/health watched from outside, a Gitea issue on failure
2026-10-15: the site was 503 for 13.5 h across a day boundary and the Doorpost outage was found by losing Claude; 24 h of database refusals were found the same way. Nothing watched. scripts/bikoret-watchdog (installed at ~/.local/bin, cron every 5 min on the dev VM) probes /kodesh/health. Two consecutive failures open ONE Gitea issue in this repo naming the failed checks and what to read first; while down it comments only when the failed set changes; on recovery it comments the duration and closes the issue. Transitions go to ~/shorashim-operator-notes/watchdog.log; silence is health. Proven: healthy → exit 0, no log; dead URL ×3 → FAIL #1, #2, one ALERT (not two), #3 no second issue; recovery → comment + close. The real Gitea call was refused 403: the clone token in ~/.git-credentials has only the code scopes. The script prefers ~/.config/bikoret/gitea_watchdog_token (write:issue), which the kohen issues; until then it logs and tracks recovery but cannot open the issue. KOHEN.md "Every morning" carries it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Scott KarpandClaude Fable 5.1 committed
1 parent
02f71dab49
commit
489cf11416
2 files changed
+190
No files matched your search
Executable
+167
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bikoret-watchdog — the kohen's eye on /kodesh/health, from outside the cluster.
|
||||
|
||||
Every run (cron, every 5 minutes) probes the health endpoint. A failure is
|
||||
anything but 200 with overall "healthy" (a 503 at the day boundary, a
|
||||
Doorpost drift, a hung rollout, the gate dark). After two consecutive
|
||||
failures (10 minutes — a rollout's own blip does not alert) it opens ONE
|
||||
Gitea issue in the repo naming the failed checks; while down it comments
|
||||
only when the failed set changes; on recovery it comments the duration and
|
||||
closes the issue. Every transition goes to the log. 2026-10-15: the site
|
||||
was 503 for 13.5 h at a day boundary and the transmitter learned of the
|
||||
Doorpost outage by losing Claude — nothing watched.
|
||||
|
||||
Paths:
|
||||
state ~/.local/state/bikoret-watchdog/state.json
|
||||
log ~/shorashim-operator-notes/watchdog.log
|
||||
token ~/.git-credentials (the Gitea token; read, never printed)
|
||||
|
||||
Env overrides (for a test run): BIKORET_HEALTH_URL, BIKORET_WATCHDOG_STATE,
|
||||
BIKORET_WATCHDOG_LOG, BIKORET_GITEA_API (default https://git.bikoret.ch/api/v1),
|
||||
BIKORET_GITEA_REPO (default heaviness510/bikoret), BIKORET_WATCHDOG_DRY=1 (no issue).
|
||||
"""
|
||||
import json, os, re, sys, time, urllib.request, urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HOME = os.path.expanduser("~")
|
||||
URL = os.environ.get("BIKORET_HEALTH_URL", "https://bikoret.ch/kodesh/health")
|
||||
STATE = os.environ.get("BIKORET_WATCHDOG_STATE", f"{HOME}/.local/state/bikoret-watchdog/state.json")
|
||||
LOG = os.environ.get("BIKORET_WATCHDOG_LOG", f"{HOME}/shorashim-operator-notes/watchdog.log")
|
||||
API = os.environ.get("BIKORET_GITEA_API", "https://git.bikoret.ch/api/v1")
|
||||
REPO = os.environ.get("BIKORET_GITEA_REPO", "heaviness510/bikoret")
|
||||
DRY = os.environ.get("BIKORET_WATCHDOG_DRY") == "1"
|
||||
THRESHOLD = 2
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def log(msg):
|
||||
os.makedirs(os.path.dirname(LOG), exist_ok=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(f"{now().strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}\n")
|
||||
|
||||
|
||||
def probe():
|
||||
"""(ok, summary, failed_checks)"""
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(URL, headers={"user-agent": "bikoret-watchdog"}), timeout=15) as r:
|
||||
code, body = r.status, r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
code, body = e.code, e.read()
|
||||
except Exception as e: # DNS, TLS, timeout, refused
|
||||
return False, f"unreachable: {type(e).__name__}: {e}", ["unreachable"]
|
||||
try:
|
||||
d = json.loads(body)
|
||||
checks = d.get("checks") or {}
|
||||
failed = sorted(k for k, v in checks.items() if v != "pass")
|
||||
overall = d.get("overall")
|
||||
ok = code == 200 and overall == "healthy"
|
||||
return ok, f"HTTP {code} overall={overall} failed={failed or 'none'}", failed or ([f"HTTP {code}"] if not ok else [])
|
||||
except Exception:
|
||||
return False, f"HTTP {code} non-JSON body ({len(body)} bytes)", [f"HTTP {code}"]
|
||||
|
||||
|
||||
def token():
|
||||
"""The Gitea token: ~/.config/bikoret/gitea_watchdog_token first (a token
|
||||
the kohen issued with write:issue — the clone token in ~/.git-credentials
|
||||
has only the code scopes and Gitea answers 403 on an issue), else that."""
|
||||
try:
|
||||
t = open(f"{HOME}/.config/bikoret/gitea_watchdog_token").read().strip()
|
||||
if t:
|
||||
return t
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
for line in open(f"{HOME}/.git-credentials"):
|
||||
m = re.match(r"https?://([^:]+):([^@]+)@", line.strip())
|
||||
if m:
|
||||
return m.group(2)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def gitea(method, path, payload=None):
|
||||
if DRY:
|
||||
log(f"DRY {method} {path} {json.dumps(payload)[:120] if payload else ''}")
|
||||
return {"number": -1, "html_url": "(dry)"}
|
||||
tok = token()
|
||||
if not tok:
|
||||
log("no Gitea token in ~/.git-credentials — cannot open an issue")
|
||||
return None
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
req = urllib.request.Request(f"{API}/repos/{REPO}{path}", data=data, method=method,
|
||||
headers={"authorization": f"token {tok}", "content-type": "application/json", "accept": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
except urllib.error.HTTPError as e:
|
||||
log(f"Gitea {method} {path} -> HTTP {e.code}: {e.read()[:120]!r}")
|
||||
return None
|
||||
except Exception as e:
|
||||
log(f"Gitea {method} {path} failed: {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def load_state():
|
||||
try:
|
||||
return json.load(open(STATE))
|
||||
except Exception:
|
||||
return {"fails": 0, "down_since": None, "issue": None, "failed": []}
|
||||
|
||||
|
||||
def save_state(s):
|
||||
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
||||
tmp = STATE + ".tmp"
|
||||
json.dump(s, open(tmp, "w"))
|
||||
os.replace(tmp, STATE)
|
||||
|
||||
|
||||
def main():
|
||||
s = load_state()
|
||||
ok, summary, failed = probe()
|
||||
t = now().strftime("%Y-%m-%d %H:%M:%SZ")
|
||||
|
||||
if ok:
|
||||
if s["issue"] is not None or s["down_since"]:
|
||||
since = s["down_since"] or t
|
||||
mins = int((now() - datetime.fromisoformat(since.replace("Z", "+00:00").replace(" ", "T"))).total_seconds() // 60)
|
||||
log(f"RECOVERED after {mins} min ({summary})")
|
||||
if s["issue"] is not None:
|
||||
gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"Recovered at {t} after {mins} min. `{summary}`"})
|
||||
gitea("PATCH", f"/issues/{s['issue']}", {"state": "closed"})
|
||||
elif s["fails"]:
|
||||
log(f"ok again after {s['fails']} failed probe(s), no alert raised ({summary})")
|
||||
save_state({"fails": 0, "down_since": None, "issue": None, "failed": []})
|
||||
return 0
|
||||
|
||||
s["fails"] += 1
|
||||
if not s["down_since"]:
|
||||
s["down_since"] = t
|
||||
log(f"FAIL #{s['fails']}: {summary}")
|
||||
|
||||
if s["fails"] >= THRESHOLD and s["issue"] is None:
|
||||
body = (f"bikoret.ch health has failed {s['fails']} consecutive probes since {s['down_since']}.\n\n"
|
||||
f"Last probe: `{summary}`\n\nFailed checks: {', '.join(failed) or 'none named'}\n\n"
|
||||
"What to look at first: the vav-pipeline startup log (`kubectl -n shorashim logs <pod> -c vav-pipeline | grep -v health`) — "
|
||||
"a Doorpost drift, stale manna at the day boundary, and a missing anchor all look the same from outside. "
|
||||
"Claude Code behind the gate is down with the site: `claude-direct` or `claude-key` (KOHEN.md, Fallback).\n\n"
|
||||
"Opened by bikoret-watchdog on the dev VM; it will comment and close this issue on recovery.")
|
||||
r = gitea("POST", "/issues", {"title": f"WATCHDOG: bikoret.ch down since {s['down_since']} ({', '.join(failed)[:60]})", "body": body, "labels": []})
|
||||
if r:
|
||||
s["issue"] = r.get("number")
|
||||
s["failed"] = failed
|
||||
log(f"ALERT issue #{s['issue']} opened: {r.get('html_url')}")
|
||||
elif s["issue"] is not None and failed != s.get("failed"):
|
||||
gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"{t}: failed checks now {', '.join(failed)} — `{summary}`"})
|
||||
s["failed"] = failed
|
||||
log(f"issue #{s['issue']} updated: failed={failed}")
|
||||
|
||||
save_state(s)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user