#!/usr/bin/env python3
"""Every scheduled job's failure reaches a person, not just a log.

Why (11-12, twice in one day): a step crashed on every X round for two days and only runs.log knew,
so each round worked from a 72-hour-old list and engagement fell from 77 likes a day to 5; the round
agents even wrote "needs a look" into that log, which nobody reads. Hours earlier a lane sat idle with
its result in. Same class: a failure that only reaches a log. These checks need no change to any job:

  exit     every launchd job under --prefix whose last run exited with a code that job does not
           declare as fine (`exit_ok`), and that is not running now
  fresh    every required output in the registry (`fresh`) that is missing or older than its limit:
           a job that loses its input and keeps running empty rounds still stops refreshing what it makes
  x-engage queued engagement without executed actions, or repeated same-reason post refusals
  net-failover more than 60 minutes continuously on the backup network (opt-in state source)
  release-protection registry differs from the deploy CLI's own protected set for over 1 hour (opt-in)
  lan-proxy m5.local resolves off the home LAN, or hub HTTP fails through the M4 relay (opt-in)

The problems are written to $RSTACK_STATE_DIR/asks-waiting/<host>-jobs.tsv, so the conductor's
asks-waiting hook shows them on every turn while they last, and each NEW problem runs --alert once
(with a one-line summary as its last argument); a cleared one is logged as recovered.

Registry (default ~/.config/rstack/job-health.json):
    {"fresh": [{"path": "~/x/BOOKMARKS_FRESH.md", "max_hours": 6, "check_hours": [9, 22], "why": "X rounds read it"}],
     "exit_ok": {"com.example.lane-yield": {"codes": [1], "why": "exit 1 is its FAIL verdict, alerted on its own"}},
     "disk": [{"path": "/", "min_free_gib": 25, "why": "deploys refuse under 15 GiB"}],
     "x_engage": {"path": "~/.local/share/x-engage/log.jsonl", "host": "m4"},
     "net_failover": {"path": "~/.local/state/net-failover/state.json", "label": "M4"},
     "release_protection": [{"deploy_root": "~/.local/share/family-games"}]}

    x_engage is opt-in: omit host for a local path; SSH is used only with a configured host.
    --x-engage-log FILE overrides the registry source with a local file. Skips are logged to stderr.
    net_failover is opt-in too, with the same explicit host/local behavior. It requires the
    fail-back script's current_net and network_since (Unix seconds); legacy state is skipped.
    lan_proxy is local to the M4, opt-in, and probes /api/config through 127.0.0.1:14325.
    Keys: target, proxy_port, http_host (a name the hub accepts), timeout, label. See lan-proxy.md.

    job-health [--prefix com.example.] [--registry FILE] [--alert CMD] [--push HOST] [--dry]
"""
import argparse
from collections import Counter
from datetime import datetime, timezone
from fnmatch import fnmatchcase
import glob
import http.client
import json
import math
import os
from pathlib import Path
import shlex
import socket
import subprocess
import sys
import time

sys.path.insert(0, str(Path(__file__).resolve().parent))
from _lan import home_lan_addresses, resolve_ipv4



def host_label():
    """A stable short host label. 11-17: the m5's HostName was unset, so the network named it mac.lan and
    every per-host file switched from m5-* to mac-*; the conductor saw a 15-hour-old m5 list and thought the
    scan was dead. RSTACK_HOST wins; on macOS the LocalHostName (set by the owner, not the network); else
    the short hostname."""
    import os, socket, subprocess, sys
    value = os.environ.get('RSTACK_HOST', '').strip()
    if value:
        return value
    if sys.platform == 'darwin':
        try:
            r = subprocess.run(['/usr/sbin/scutil', '--get', 'LocalHostName'], capture_output=True, text=True, timeout=5)
            if r.returncode == 0 and r.stdout.strip():
                return r.stdout.strip()
        except (OSError, subprocess.SubprocessError):
            pass
    return socket.gethostname().split('.')[0]

def state_dir():
    base = os.environ.get('RSTACK_STATE_DIR') or os.path.join(
        os.environ.get('XDG_STATE_HOME') or os.path.expanduser('~/.local/state'), 'rstack')
    return Path(os.path.expanduser(base))


def launchd_jobs(prefix, listing=None):
    """{label: (pid or None, last exit status)} for jobs under prefix."""
    if listing is None:
        listing = subprocess.run(['launchctl', 'list'], capture_output=True, text=True, timeout=30).stdout
    out = {}
    for line in listing.splitlines()[1:]:
        parts = line.split('\t') if '\t' in line else line.split()
        # 11-13: a bare run took every com.apple.* agent macOS kills (121 "exited -9" rows) into the conductor's
        # prompt. The operating system's own agents are never ours to report.
        if len(parts) < 3 or not parts[2].startswith(prefix) or parts[2].startswith('com.apple.'):
            continue
        pid = None if parts[0] == '-' else parts[0]
        try:
            status = int(parts[1])
        except ValueError:
            continue
        out[parts[2]] = (pid, status)
    return out


def exit_problems(jobs, exit_ok):
    rows = []
    for label, (pid, status) in sorted(jobs.items()):
        ok = set((exit_ok.get(label) or {}).get('codes', [])) | {0}
        if pid is None and status not in ok:
            rows.append(('exit', label, 'last run exited {}'.format(status)))
    return rows


def fresh_problems(entries, now):
    rows = []
    for e in entries:
        # check_hours [start, end): local hours when the producer runs. 11-13: x-engage rebuilds the bookmark
        # list 08:15-20:15 only, so every night the list is 12 h old and a flat 6 h limit woke general for nothing.
        window = e.get('check_hours')
        if window and not (int(window[0]) <= time.localtime(now).tm_hour < int(window[1])):
            continue
        paths = glob.glob(os.path.expanduser(e['path']))
        limit = float(e['max_hours'])
        if not paths:
            rows.append(('fresh', e['path'], 'missing (needed every {:g} h: {})'.format(limit, e.get('why', ''))))
            continue
        age = (now - max(os.path.getmtime(p) for p in paths)) / 3600
        if age > limit:
            rows.append(('fresh', e['path'], '{:.1f} h old, limit {:g} h ({})'.format(age, limit, e.get('why', ''))))
    return rows


def report(rows, tool, stem, label, alert='', push=None):
    """Show (kind, what, detail) rows to the conductor while they last, and alert once per new one.

    Writes $RSTACK_STATE_DIR/asks-waiting/<host>-<stem>.tsv (`label<TAB>what<TAB>kind: detail`), which
    the asks-waiting hook prints on every conductor turn; runs `alert` once for each (kind, what) not
    seen on the last run, with a one-line summary as its last argument; logs NEW and RECOVERED lines to
    $RSTACK_STATE_DIR/<tool>/log. repo-freshness reuses this, so both report the same way."""
    host = host_label()
    out = state_dir() / 'asks-waiting' / '{}-{}.tsv'.format(host, stem)
    seen_file = state_dir() / tool / 'seen.json'
    out.parent.mkdir(parents=True, exist_ok=True)
    seen_file.parent.mkdir(parents=True, exist_ok=True)
    try:
        seen = set(json.loads(seen_file.read_text()))
    except (OSError, ValueError):
        seen = set()
    keys = {'{}\t{}'.format(k, w) for k, w, _ in rows}
    new = [r for r in rows if '{}\t{}'.format(r[0], r[1]) not in seen]
    gone = seen - keys
    with open(state_dir() / tool / 'log', 'a') as fh:
        for k, w, d in new:
            fh.write('{} NEW {} {} {}\n'.format(time.strftime('%F %T'), k, w, d))
        for key in sorted(gone):
            fh.write('{} RECOVERED {}\n'.format(time.strftime('%F %T'), key.replace('\t', ' ')))
    if new and alert:
        summary = '{} on {}: {}'.format(len(new), host, '; '.join('{} {} ({})'.format(k, w, d) for k, w, d in new))[:900]
        try:
            result = subprocess.run(shlex.split(alert) + [tool + ': ' + summary], capture_output=True, timeout=60)
            delivered = result.returncode == 0
        except (OSError, subprocess.SubprocessError):
            delivered = False
        if not delivered:
            print(tool + ': alert delivery failed; new findings will be retried', file=sys.stderr)
            keys -= {'{}\t{}'.format(k, w) for k, w, _ in new}
    tmp = out.with_suffix('.tmp')
    tmp.write_text(''.join('{}\t{}\t{}\n'.format(label, w, '{}: {}'.format(k, d)) for k, w, d in rows))
    os.replace(tmp, out)
    seen_file.write_text(json.dumps(sorted(keys)))
    if push:
        subprocess.run(['ssh', '-o', 'BatchMode=yes', push, 'mkdir -p ' + str(out.parent)], capture_output=True, timeout=60)
        subprocess.run(['scp', '-q', str(out), '{}:{}'.format(push, out)], capture_output=True, timeout=60)


def disk_problems(entries, usage=None):
    """A disk under its floor (11-14: the M4 at 97% full, under the 15 GiB floor, blocked games deploys; the
    first anyone heard was a failed deploy). Set the floor above the point where jobs start refusing."""
    import shutil
    rows = []
    for e in entries:
        try:
            free = (usage or shutil.disk_usage)(os.path.expanduser(e['path'])).free / 2 ** 30
        except OSError as exc:
            rows.append(('disk', e['path'], 'unreadable: {}'.format(type(exc).__name__)))
            continue
        if free < float(e['min_free_gib']):
            rows.append(('disk', e['path'], '{:.1f} GiB free, floor {:g} GiB ({})'.format(
                free, float(e['min_free_gib']), e.get('why', ''))))
    return rows


def output_heartbeat(rows, queued_kinds, executed_kinds, windows, now):
    """Return (stall reasons, detail) from output events, independent of their producer.

    Rows have kind, ISO-8601 utc and optional reason. Queued-kind patterns accept shell wildcards.
    Each window has hours and either queued/executed kind lists, or refused/count.
    queued_hours defaults to hours; None includes all queued rows, even older drafts.
    To add PR follow-ups or outreach, normalize their ledger rows to this shape, supply
    queue/success kinds and windows, then pass a stalled result to report(), as x-engage does.
    Only explicit success kinds count: rounds, drafts and attempts are never output.
    """
    events, last = [], dict.fromkeys(executed_kinds)
    for row in rows:
        try:
            stamp = datetime.fromisoformat(row['utc'].replace('Z', '+00:00'))
            if stamp.tzinfo is None:
                continue
            at = stamp.timestamp()
        except (KeyError, ValueError, TypeError, AttributeError):
            continue
        if at > now:
            continue
        kind = row.get('kind', '')
        events.append((at, kind, row.get('reason') or 'unspecified'))
        if kind in last and (last[kind] is None or at > last[kind]):
            last[kind] = at

    def matches(kind, patterns):
        return any(fnmatchcase(kind, pattern) for pattern in patterns)

    reasons, details = [], []
    for window in windows:
        hours = window['hours']
        recent = [event for event in events if event[0] >= now - hours * 3600]
        if 'refused' in window:
            counts = Counter(reason for _, kind, reason in recent if kind == window['refused'])
            top = counts.most_common(1)
            details.append('top refusal/{:g}h: {}'.format(
                hours, '{} ({}x)'.format(*top[0]) if top else 'none'))
            if top and top[0][1] >= window['count']:
                reasons.append('{} same-reason refusals in {:g}h'.format(top[0][1], hours))
            continue
        queue_hours = window.get('queued_hours', hours)
        queued = events if queue_hours is None else [
            event for event in events if event[0] >= now - queue_hours * 3600]
        counts = Counter(kind for _, kind, _ in queued
                         if matches(kind, queued_kinds) and matches(kind, window['queued']))
        details.append('pending/{}: {}'.format(
            'all' if queue_hours is None else '{:g}h'.format(queue_hours),
            ', '.join('{}={}'.format(k, n) for k, n in sorted(counts.items())) or 'none'))
        success = [last[k] for k in window['executed'] if last.get(k) is not None]
        if counts and (not success or max(success) < now - hours * 3600):
            reasons.append('no {} executed in {:g}h'.format('/'.join(window['executed']), hours))
    details.append('last executed: ' + ', '.join('{}={}'.format(
        kind, datetime.fromtimestamp(at, timezone.utc).isoformat().replace('+00:00', 'Z')
        if at is not None else 'never') for kind, at in last.items()))
    return reasons, '; '.join(reasons + details)


X_QUEUED = ('*_pending', 'draft_reply')
X_REPLIES = ('reply', 'reply_posted', 'self_reply')
X_EXECUTED = ('like', 'follow') + X_REPLIES
X_WINDOWS = (
    dict(hours=12, queued=X_QUEUED, executed=('like', 'follow')),
    dict(hours=48, queued=('draft_reply',), executed=X_REPLIES, queued_hours=None),
    dict(hours=24, refused='post_refused', count=3),
)


def x_engage_problems(source, now):
    """Read only the explicitly configured local log or SSH source; skip loudly if unavailable."""
    try:
        if not source or not source.get('path'):
            raise ValueError('no log configured (set x_engage.path or --x-engage-log)')
        path = source['path']
        if source.get('host'):
            # Expand ~ on the producer, not on this machine; quote all configured path bytes.
            remote = '"$HOME"/' + shlex.quote(path[2:]) if path.startswith('~/') else shlex.quote(path)
            result = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10',
                                     source['host'], 'cat -- ' + remote],
                                    capture_output=True, text=True, timeout=30)
            if result.returncode:
                raise OSError('SSH log read failed (exit {})'.format(result.returncode))
            lines = result.stdout.splitlines()
        else:
            lines = Path(path).expanduser().read_text().splitlines()
    except (OSError, ValueError, subprocess.SubprocessError) as exc:
        print('job-health: x-engage skipped: {}'.format(exc), file=sys.stderr)
        return []
    rows, invalid = [], 0
    for line in lines:
        try:
            row = json.loads(line)
            if not isinstance(row, dict):
                raise ValueError('not an object')
        except ValueError:
            invalid += 1
            continue
        rows.append(dict(kind=row.get('action', row.get('type', '')),
                         utc=row.get('utc', row.get('ts')),
                         reason=row.get('reason') or (row.get('text') if row.get('action') == 'post_refused' else None)))
    if invalid:
        print('job-health: x-engage ignored {} malformed log rows'.format(invalid), file=sys.stderr)
    reasons, detail = output_heartbeat(rows, X_QUEUED, X_EXECUTED, X_WINDOWS, now)
    return [('x-engage', 'X engagement stalled', detail)] if reasons else []


def net_failover_problems(source, now):
    """Read the configured fail-back state; never probe or switch a network."""
    if source is None:
        return []
    try:
        if not isinstance(source, dict) or not source.get('path'):
            raise ValueError('set net_failover.path to the fail-back state file')
        path = source['path']
        if source.get('host'):
            remote = '"$HOME"/' + shlex.quote(path[2:]) if path.startswith('~/') else shlex.quote(path)
            result = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10',
                                     source['host'], 'cat -- ' + remote],
                                    capture_output=True, text=True, timeout=30)
            if result.returncode:
                raise OSError('SSH state read failed (exit {})'.format(result.returncode))
            contents = result.stdout
        else:
            contents = Path(path).expanduser().read_text()
        state = json.loads(contents)
        if not isinstance(state, dict) or 'current_net' not in state:
            raise ValueError('state lacks current_net; install the fail-back script first')
        if state['current_net'] != 'starlink':
            return []
        since = state.get('network_since')
        if (isinstance(since, bool) or not isinstance(since, (int, float))
                or not math.isfinite(since) or not 0 <= since <= now):
            raise ValueError('backup state lacks a valid network_since Unix timestamp')
    except (OSError, ValueError, TypeError, subprocess.SubprocessError) as exc:
        print('job-health: net-failover skipped: {}'.format(exc), file=sys.stderr)
        return []
    minutes = (now - since) / 60
    if minutes <= 60:
        return []
    label = source.get('label') or source.get('host') or host_label()
    return [('net-failover', label, '{} on backup network for {:.0f} min (starlink)'.format(label, minutes))]


def lan_proxy_problems(source, resolver=None, connection=None):
    """Opt-in M4 check: m5 resolves on the home LAN AND HTTP answers through the relay."""
    if source is None:
        return []
    rows = []
    label = 'm5.local via LAN proxy'
    try:
        if not isinstance(source, dict):
            raise ValueError('lan_proxy must be an object')
        host = source.get('target', 'm5.local')
        label = source.get('label') or host + ' via LAN proxy'
        addresses = (resolver or resolve_ipv4)(host)
        if not addresses or set(home_lan_addresses(addresses)) != set(addresses):
            raise ValueError('{} resolves outside 192.168.86.x: {}'.format(host, ','.join(addresses)))
    except (OSError, ValueError, TypeError, subprocess.SubprocessError) as exc:
        rows.append(('lan-proxy-dns', label, str(exc)))
    conn = None
    try:
        if not isinstance(source, dict):
            raise ValueError('lan_proxy must be an object')
        port = int(source.get('proxy_port', 14325))
        timeout = float(source.get('timeout', 8))
        if not 1 <= port <= 65535 or not 0 < timeout <= 30:
            raise ValueError('invalid proxy_port or timeout')
        conn = (connection or http.client.HTTPConnection)('127.0.0.1', port, timeout=timeout)
        conn.request('GET', '/api/config', headers={
            'Host': source.get('http_host') or 'localhost', 'Connection': 'close'})
        response = conn.getresponse()
        if response.status != 200 or not response.read(1):
            raise ValueError('hub /api/config returned HTTP {} or empty body'.format(response.status))
    except (OSError, ValueError, TypeError, http.client.HTTPException) as exc:
        rows.append(('lan-proxy-hub', label, 'hub not answering through proxy: ' + str(exc)))
    finally:
        if conn is not None:
            conn.close()
    return rows


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
    ap.add_argument('--prefix', default=os.environ.get('JOB_HEALTH_PREFIX', ''),
                    help="your jobs' launchd label prefix, e.g. com.example. Without one the run only prints: "
                         'it writes no list and sends no alert, so an ad-hoc run cannot flood the conductor')
    ap.add_argument('--registry', default=os.environ.get('JOB_HEALTH_REGISTRY',
                                                         os.path.expanduser('~/.config/rstack/job-health.json')))
    ap.add_argument('--alert', default=os.environ.get('JOB_HEALTH_ALERT', ''),
                    help='command run once per new problem, with a one-line summary as its last argument')
    ap.add_argument('--push', help='copy this host\'s list to the same path on HOST (the conductor\'s Mac)')
    ap.add_argument('--x-engage-log', help='local x-engage JSONL log (overrides the registry source)')
    ap.add_argument('--dry', action='store_true')
    a = ap.parse_args(argv)
    bad = []
    try:
        reg = json.loads(Path(a.registry).read_text())
    except FileNotFoundError:
        reg = {}
    except ValueError as exc:   # a registry that does not parse is a problem of its own, not an all-clear
        reg, bad = {}, [('registry', a.registry, 'unreadable: {}'.format(exc))]
    now = time.time()
    x_source = dict(path=a.x_engage_log) if a.x_engage_log is not None else reg.get('x_engage')
    try:
        sys.path.insert(0, str(Path(__file__).resolve().parent))
        from _codex_budget import health_problems
        codex_rows = health_problems(reg.get('codex'), now, repair=bool(a.prefix) and not a.dry)
    except (OSError, ValueError, TypeError, KeyError, ImportError) as exc:
        codex_rows = [('codex-read', 'Codex spend backstop', 'unavailable: ' + type(exc).__name__)]
    release_rows = []
    if reg.get('release_protection'):
        try:
            sys.path.insert(0, str(Path(__file__).resolve().parent))
            from release_sync import health_problems as release_health
            release_rows = release_health(reg['release_protection'], now,
                state_dir() / 'job-health/release-protection.json', write=bool(a.prefix) and not a.dry)
        except (OSError, ValueError, TypeError, KeyError, ImportError) as exc:
            release_rows = [('release-protection', 'registry comparison', 'unavailable: ' + str(exc))]
    rows = (bad + codex_rows + release_rows + exit_problems(launchd_jobs(a.prefix), reg.get('exit_ok') or {})
            + fresh_problems(reg.get('fresh') or [], now) + disk_problems(reg.get('disk') or [])
            + x_engage_problems(x_source, now) + net_failover_problems(reg.get('net_failover'), now)
            + lan_proxy_problems(reg.get('lan_proxy')))
    lines = ['{}\t{}\t{}'.format(kind, what, detail) for kind, what, detail in rows]
    for l in lines:
        print(l)
    if not a.prefix and not a.dry:
        print('job-health: no --prefix (or JOB_HEALTH_PREFIX), so this run only printed; nothing was written or '
              'alerted', file=sys.stderr)
        return 0
    if a.dry:
        return 1 if rows else 0
    if codex_rows and not a.alert:
        a.alert = str(Path.home() / '.local/bin/tell-general') + ' decision'
    report(rows, 'job-health', 'jobs', 'job', a.alert, a.push)
    # Problems are reported above, not through this exit code: a scheduled run that found problems
    # exiting 1 would list job-health itself as a failing job (11-12, its own first day).
    return 0


if __name__ == '__main__':
    sys.exit(main())
