bikoret/scripts/bikoret-watchdog

#!/usr/bin/env python3
"""bikoret-watchdog — the kohen's eye on /kodesh/health, from outside the cluster.

Every run (cron, every 5 minutes) probes the health endpoint. A failure is
anything but 200 with overall "healthy" (a 503 at the day boundary, a
Doorpost drift, a hung rollout, the gate dark). After two consecutive
failures (10 minutes — a rollout's own blip does not alert) it opens ONE
Gitea issue in the repo naming the failed checks; while down it comments
only when the failed set changes; on recovery it comments the duration and
closes the issue. Every transition goes to the log. 2026-10-15: the site
was 503 for 13.5 h at a day boundary and the transmitter learned of the
Doorpost outage by losing Claude — nothing watched.

Paths:
  state   ~/.local/state/bikoret-watchdog/state.json
  log     ~/shorashim-operator-notes/watchdog.log
  token   ~/.git-credentials (the Gitea token; read, never printed)

Env overrides (for a test run): BIKORET_HEALTH_URL, BIKORET_WATCHDOG_STATE,
BIKORET_WATCHDOG_LOG, BIKORET_GITEA_API (default https://git.bikoret.ch/api/v1),
BIKORET_GITEA_REPO (default heaviness510/bikoret), BIKORET_WATCHDOG_DRY=1 (no issue).
"""
import json, os, re, sys, time, urllib.request, urllib.error
from datetime import datetime, timezone

HOME = os.path.expanduser("~")
URL = os.environ.get("BIKORET_HEALTH_URL", "https://bikoret.ch/kodesh/health")
STATE = os.environ.get("BIKORET_WATCHDOG_STATE", f"{HOME}/.local/state/bikoret-watchdog/state.json")
LOG = os.environ.get("BIKORET_WATCHDOG_LOG", f"{HOME}/shorashim-operator-notes/watchdog.log")
API = os.environ.get("BIKORET_GITEA_API", "https://git.bikoret.ch/api/v1")
REPO = os.environ.get("BIKORET_GITEA_REPO", "heaviness510/bikoret")
DRY = os.environ.get("BIKORET_WATCHDOG_DRY") == "1"
THRESHOLD = 2


def now():
    return datetime.now(timezone.utc)


def log(msg):
    os.makedirs(os.path.dirname(LOG), exist_ok=True)
    with open(LOG, "a") as f:
        f.write(f"{now().strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}\n")


def probe():
    """(ok, summary, failed_checks)"""
    try:
        with urllib.request.urlopen(urllib.request.Request(URL, headers={"user-agent": "bikoret-watchdog"}), timeout=15) as r:
            code, body = r.status, r.read()
    except urllib.error.HTTPError as e:
        code, body = e.code, e.read()
    except Exception as e:  # DNS, TLS, timeout, refused
        return False, f"unreachable: {type(e).__name__}: {e}", ["unreachable"]
    try:
        d = json.loads(body)
        checks = d.get("checks") or {}
        failed = sorted(k for k, v in checks.items() if v != "pass")
        overall = d.get("overall")
        ok = code == 200 and overall == "healthy"
        return ok, f"HTTP {code} overall={overall} failed={failed or 'none'}", failed or ([f"HTTP {code}"] if not ok else [])
    except Exception:
        return False, f"HTTP {code} non-JSON body ({len(body)} bytes)", [f"HTTP {code}"]


def token():
    """The Gitea token: ~/.config/bikoret/gitea_watchdog_token first (a token
    the kohen issued with write:issue — the clone token in ~/.git-credentials
    has only the code scopes and Gitea answers 403 on an issue), else that."""
    try:
        t = open(f"{HOME}/.config/bikoret/gitea_watchdog_token").read().strip()
        if t:
            return t
    except FileNotFoundError:
        pass
    try:
        for line in open(f"{HOME}/.git-credentials"):
            m = re.match(r"https?://([^:]+):([^@]+)@", line.strip())
            if m:
                return m.group(2)
    except FileNotFoundError:
        pass
    return None


def gitea(method, path, payload=None):
    if DRY:
        log(f"DRY {method} {path} {json.dumps(payload)[:120] if payload else ''}")
        return {"number": -1, "html_url": "(dry)"}
    tok = token()
    if not tok:
        log("no Gitea token in ~/.git-credentials — cannot open an issue")
        return None
    data = json.dumps(payload).encode() if payload is not None else None
    req = urllib.request.Request(f"{API}/repos/{REPO}{path}", data=data, method=method,
                                 headers={"authorization": f"token {tok}", "content-type": "application/json", "accept": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            return json.loads(r.read() or b"{}")
    except urllib.error.HTTPError as e:
        log(f"Gitea {method} {path} -> HTTP {e.code}: {e.read()[:120]!r}")
        return None
    except Exception as e:
        log(f"Gitea {method} {path} failed: {type(e).__name__}: {e}")
        return None


def load_state():
    try:
        return json.load(open(STATE))
    except Exception:
        return {"fails": 0, "down_since": None, "issue": None, "failed": []}


def save_state(s):
    os.makedirs(os.path.dirname(STATE), exist_ok=True)
    tmp = STATE + ".tmp"
    json.dump(s, open(tmp, "w"))
    os.replace(tmp, STATE)


def main():
    s = load_state()
    ok, summary, failed = probe()
    t = now().strftime("%Y-%m-%d %H:%M:%SZ")

    if ok:
        if s["issue"] is not None or s["down_since"]:
            since = s["down_since"] or t
            mins = int((now() - datetime.fromisoformat(since.replace("Z", "+00:00").replace(" ", "T"))).total_seconds() // 60)
            log(f"RECOVERED after {mins} min ({summary})")
            if s["issue"] is not None:
                gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"Recovered at {t} after {mins} min. `{summary}`"})
                gitea("PATCH", f"/issues/{s['issue']}", {"state": "closed"})
        elif s["fails"]:
            log(f"ok again after {s['fails']} failed probe(s), no alert raised ({summary})")
        save_state({"fails": 0, "down_since": None, "issue": None, "failed": []})
        return 0

    s["fails"] += 1
    if not s["down_since"]:
        s["down_since"] = t
    log(f"FAIL #{s['fails']}: {summary}")

    if s["fails"] >= THRESHOLD and s["issue"] is None:
        body = (f"bikoret.ch health has failed {s['fails']} consecutive probes since {s['down_since']}.\n\n"
                f"Last probe: `{summary}`\n\nFailed checks: {', '.join(failed) or 'none named'}\n\n"
                "What to look at first: the vav-pipeline startup log (`kubectl -n shorashim logs <pod> -c vav-pipeline | grep -v health`) — "
                "a Doorpost drift, stale manna at the day boundary, and a missing anchor all look the same from outside. "
                "Claude Code behind the gate is down with the site: `claude-direct` or `claude-key` (KOHEN.md, Fallback).\n\n"
                "Opened by bikoret-watchdog on the dev VM; it will comment and close this issue on recovery.")
        r = gitea("POST", "/issues", {"title": f"WATCHDOG: bikoret.ch down since {s['down_since']} ({', '.join(failed)[:60]})", "body": body, "labels": []})
        if r:
            s["issue"] = r.get("number")
            s["failed"] = failed
            log(f"ALERT issue #{s['issue']} opened: {r.get('html_url')}")
    elif s["issue"] is not None and failed != s.get("failed"):
        gitea("POST", f"/issues/{s['issue']}/comments", {"body": f"{t}: failed checks now {', '.join(failed)} — `{summary}`"})
        s["failed"] = failed
        log(f"issue #{s['issue']} updated: failed={failed}")

    save_state(s)
    return 1


if __name__ == "__main__":
    sys.exit(main())