#!/usr/bin/env python3
"""The nightly check for clone drift, public releases and stale backups on every configured host.

Why (11-13): an audit found 8 stale public repos, 7 stale or missing retros, and live code that
differed from the repo it came from. Nothing had failed; each had simply aged, and nobody runs an
audit by hand every night. This lists every public repo, says how it reaches the public (its kind),
and checks each kind the way it can drift:

  pushed     the public repo is the working clone. Drift includes a local branch that tracks
             the public default branch but carries commits not on it, so a plain push publishes them
             (`private-commits`).
  published  the public repo is a release of a private working repo through a PUBLISH whitelist
             (publish-public). Drift: what a publish would export now (the whitelisted files committed
             at the source's HEAD, agent working files dropped) differs from the public default
             branch (`unpublished`). The private repo being ahead of or behind the public one is not
             drift; only the export is compared.
  archived   read-only on GitHub; listed, not checked.
  declared   a public repo with no clone on purpose (config `declared`, with a reason).
  unknown    a public repo that is none of these: a FAILURE. Every public repo must have one way in.

Clones with an owned remote (public or private), plus explicit `local_only` paths, are checked for
`behind`, `no-upstream`, stale `dirty` files and aged `unpushed` commits. Without configured owners,
any remote qualifies. Dirty counts tracked changes at all depths and top-level untracked files.
Fetch is opt-in; otherwise cached tracking refs are used. A fetch failure suppresses stale-ref
comparisons. Detached HEADs must be declared in `detached` to suppress no-upstream.
Also, on whichever host each entry names: README auto-blocks that differ from a fresh render
(`readme-blocks check`, in every clone or source whose README has `<!-- auto:` markers), installed files
whose hash differs from their repo copy (`installed`), a retro older than a lane result that has sat
out of it for more than `max_days` (`retro`), and a backup command that no launchd job or crontab
line runs on a schedule, or a backup clone whose newest commit is stale (`backup`). A host that cannot be reached, a source that cannot be read, or a
GitHub call that fails is a problem too: the check fails closed.

Problems go to $RSTACK_STATE_DIR/asks-waiting/<host>-repos.tsv (the conductor's asks-waiting hook
prints it every turn). --alert sends counts by kind and the first five findings, at most once per
`alert_hours` (default 24), with its last successful send recorded under RSTACK_STATE_DIR. A run that
reported exits 0; only --dry (print, write nothing) exits 1 when it finds a problem.

    repo-freshness [--config FILE] [--alert CMD] [--push HOST] [--dry]

Config (default ~/.config/rstack/repo-freshness.json). Every entry may name a `host` (an ssh alias);
without one it is read on the machine running the check. Paths may start with ~ (that host's home).

    {"owners": ["example-user"],                          GitHub owners whose public repos are listed
     "roots": [{"path": "~/dev", "depth": 4},              where clones and PUBLISH files are found
               {"path": "~/dev", "depth": 4, "host": "other-mac"}],
     "exclude": ["~/dev/scratch/*"],                       clone or source paths (globs) never counted
     "fetch": false,                                     true: one fetch --all per clone, timeout 30 s
     "dirty_max_hours": 24, "unpushed_max_hours": 24,      age limits for every clone, including private ones
     "detached": [{"path": "~/dev/pinned", "host": "other-mac"}], intentional detached HEADs (exact paths)
     "alert_hours": 24,                                  minimum hours between summary alerts (0: every run)
     "declared": {"example-user/old-site": {"kind": "frozen", "why": "kept for links; no clone"}},
     "agent_files": "~/.config/git/agent-files.gitignore", dropped from a publish, as publish-public does
     "readme_blocks": "readme-blocks",                     the render/check command on each host
     "installed": [{"live": "~/.local/bin", "repo": "~/dev/tools/bin", "names": ["a-tool", "b-tool"]},
                   {"live": "~/.claude/hooks/gate.py", "repo": "~/dev/tools/hooks/gate.py"}],
     "retros": [{"lane": "demo", "retro": "~/dev/demo/RETRO.md",
                 "results": "~/.local/share/research-harness/lanes/demo/RESULTS.jsonl", "max_days": 7}],
     "backups": [{"name": "notes", "match": "backup-notes"}, string a scheduled job's command contains
                 {"name": "results", "repo": "~/dev/results-backup", "host": "other-mac", "max_hours": 48}]}

Backup `repo` checks the newest commit reachable from HEAD or any local/fetched ref, with the same
fetch setting and cache as discovered clones. `max_hours` defaults to 48. Both `repo` and `match`
may be supplied to check age and scheduling. --dry writes no state and sends no alerts; use
`fetch: false` as well for a strictly read-only scan (fetch updates Git refs even with --dry).
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import fnmatch
import importlib.machinery
import importlib.util
import json
import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
import time

HERE = Path(__file__).resolve().parent
TIMEOUT = 120
LOCAL = ''


def load_tool(name):
    loader = importlib.machinery.SourceFileLoader(name.replace('-', '_'), str(HERE / name))
    spec = importlib.util.spec_from_loader(loader.name, loader)
    mod = importlib.util.module_from_spec(spec)
    loader.exec_module(mod)
    return mod


class Host:
    """Run a command here, or on an ssh alias over one shared connection."""

    def __init__(self, name):
        self.name = name or LOCAL
        self._home = None
        self.error = None

    def label(self, path=''):
        return '{}:{}'.format(self.name, path) if self.name else path

    def run(self, argv, timeout=TIMEOUT, stdin=None):
        if self.name:
            argv = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'ControlMaster=auto',
                    '-o', 'ControlPath=/tmp/repo-freshness-%C', '-o', 'ControlPersist=60', self.name,
                    ' '.join(shlex.quote(a) for a in argv)]
        try:
            r = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, input=stdin, errors='replace')
        except subprocess.TimeoutExpired:
            return 124, '', 'timed out after {} s'.format(timeout)
        except OSError as exc:
            return 127, '', str(exc)
        return r.returncode, r.stdout, r.stderr

    def sh(self, script, *args, timeout=TIMEOUT):
        return self.run(['sh', '-c', script, 'sh', *args], timeout=timeout)

    @property
    def home(self):
        if self._home is None:
            if not self.name:
                self._home = os.path.expanduser('~')
            else:
                rc, out, err = self.run(['sh', '-c', 'echo "$HOME"'], timeout=30)
                if rc != 0 or not out.strip():
                    self.error = (err.strip() or 'exit {}'.format(rc))[:200]
                    self._home = ''
                else:
                    self._home = out.strip()
        return self._home

    def path(self, p):
        p = str(p)
        return self.home + p[1:] if p == '~' or p.startswith('~/') else p


def repo_id(url):
    """git@github.com:o/r.git, https://github.com/o/r, ssh://... -> github.com/o/r (egress-gate's rule)."""
    u = re.sub(r'\.git/?$', '', url.strip().rstrip('/')).lower()
    m = re.match(r'^(?:[a-z+]+://)?(?:[^@/]+@)?([^:/]+)[:/](.+)$', u)
    return '{}/{}'.format(m.group(1), m.group(2)) if m else u


def glob_rx(g):
    """A PUBLISH line as publish-public reads it: ** any depth, * within a folder, a trailing / the folder."""
    g = g.strip().lstrip('/')
    if g.endswith('/'):
        g += '**'
    out, i = '', 0
    while i < len(g):
        if g.startswith('**/', i):
            out, i = out + '(?:.*/)?', i + 3
        elif g.startswith('**', i):
            out, i = out + '.*', i + 2
        elif g[i] == '*':
            out, i = out + '[^/]*', i + 1
        elif g[i] == '?':
            out, i = out + '[^/]', i + 1
        else:
            out, i = out + re.escape(g[i]), i + 1
    return re.compile(out + r'\Z')


def parse_publish(text):
    remote, inc, exc = None, [], []
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        if line.startswith('remote '):
            remote = line.split(None, 1)[1]
        elif line.startswith('!'):
            exc.append(glob_rx(line[1:]))
        else:
            inc.append(glob_rx(line))
    return remote, inc, exc


# ---------------------------------------------------------------- inventory

SCAN = r'''
root=$1; depth=$2
[ -d "$root" ] || { echo "@@missing $root"; exit 0; }
find "$root" -maxdepth "$depth" \( -name node_modules -o -name .venv -o -name .Trash \) -prune -o \
     -name .git -print -prune 2>/dev/null | while read -r g; do
  d=$(dirname "$g")
  top=$(git -C "$d" rev-parse --show-toplevel 2>/dev/null)
  if [ -n "$top" ] && [ "$(cd "$d" && pwd -P)" != "$(cd "$top" && pwd -P)" ]; then
    continue  # Git fell back to a parent checkout; this is not another clone.
  fi
  printf '@@clone\t%s\n' "$d"
  git -C "$d" remote -v 2>/dev/null | awk '$3 == "(fetch)" {print "remote\t" $1 "\t" $2}'
done
find "$root" -maxdepth "$((depth + 1))" \( -name node_modules -o -name .venv -o -name .Trash -o -name .git \) -prune -o \
     -name PUBLISH -type f -print 2>/dev/null | while read -r f; do
  printf '@@publish\t%s\t%s\n' "$(dirname "$f")" "$(sed -n 's/^[[:space:]]*remote[[:space:]][[:space:]]*//p' "$f" | head -1)"
done
'''


def scan_root(host, root, depth, problems):
    """[(host, path, {remote name: repo id})] and [(host, dir, remote id)] under one root."""
    rc, out, err = host.sh(SCAN, host.path(root), str(int(depth)), timeout=600)
    if rc != 0:
        problems.append(('host', host.label(root), 'scan failed: {}'.format((err.strip() or 'exit {}'.format(rc))[:200])))
        return [], []
    clones, sources = [], []
    for line in out.splitlines():
        parts = line.split('\t')
        if line.startswith('@@missing'):
            problems.append(('config', host.label(root), 'scan root does not exist'))
        elif parts[0] == '@@clone' and len(parts) == 2:
            clones.append((host, parts[1], {}))
        elif parts[0] == 'remote' and len(parts) == 3 and clones:
            clones[-1][2][parts[1]] = repo_id(parts[2])
        elif parts[0] == '@@publish' and len(parts) == 3:
            sources.append((host, parts[1], repo_id(parts[2]) if parts[2] else ''))
    return clones, sources


def public_repos(owners, gh):
    """{repo id: {name, archived, branch}}; raises RuntimeError when GitHub cannot be read."""
    out = {}
    for owner in owners:
        r = subprocess.run(shlex.split(gh) + ['repo', 'list', owner, '--visibility', 'public', '--limit', '1000',
                                              '--json', 'nameWithOwner,isArchived,defaultBranchRef'],
                           capture_output=True, text=True, timeout=TIMEOUT)
        if r.returncode != 0:
            raise RuntimeError('gh repo list {} failed: {}'.format(owner, r.stderr.strip()[:200]))
        for repo in json.loads(r.stdout or '[]'):
            name = repo['nameWithOwner']
            out['github.com/' + name.lower()] = dict(name=name, archived=bool(repo.get('isArchived')),
                                                     branch=(repo.get('defaultBranchRef') or {}).get('name') or 'main')
        # `gh repo list` exits 0 on an empty owner; an owner with nothing public is fine, a typo is caught by
        # nothing being listed for it, so say so.
    return out


def excluded(host, path, globs):
    return any(fnmatch.fnmatch(path, host.path(g)) for g in globs)


def classify(public, clones, sources, declared):
    """{repo id: (kind, detail, [clones], source or None)} and the problems the inventory itself finds."""
    problems, inv = [], {}
    by_remote = {}
    for c in clones:
        for rid in set(c[2].values()):
            by_remote.setdefault(rid, []).append(c)
    pub_src = {}
    for s in sources:
        pub_src.setdefault(s[2], []).append(s)
    for rid, meta in sorted(public.items(), key=lambda kv: kv[1]['name'].lower()):
        name = meta['name']
        srcs = pub_src.get(rid, [])
        src_paths = {(s[0].name, s[1]) for s in srcs}
        others = [c for c in by_remote.get(rid, []) if (c[0].name, c[1]) not in src_paths]
        if meta['archived']:
            inv[rid] = ('archived', 'read-only on GitHub', [], None)
        elif srcs:
            if len(srcs) > 1:
                problems.append(('inventory', name, 'published from {} sources: {}'.format(
                    len(srcs), ', '.join(s[0].label(s[1]) for s in srcs))))
            inv[rid] = ('published', 'source ' + srcs[0][0].label(srcs[0][1]), others, srcs[0])
        elif others:
            inv[rid] = ('pushed', '{} clone(s): {}'.format(len(others), ', '.join(
                c[0].label(c[1]) for c in others[:3]) + (', ...' if len(others) > 3 else '')), others, None)
        elif name in declared or name.lower() in {k.lower() for k in declared}:
            d = declared.get(name) or next(v for k, v in declared.items() if k.lower() == name.lower())
            inv[rid] = ('declared:' + str(d.get('kind', 'other')), d.get('why', ''), [], None)
            if not d.get('why'):
                problems.append(('inventory', name, 'declared without a "why"'))
        else:
            inv[rid] = ('unknown', 'no clone, no PUBLISH source, not declared', [], None)
            problems.append(('unknown', name, 'public repo that is neither pushed (a clone under the scan roots) '
                                              'nor published (a PUBLISH file naming it); give it one way in, '
                                              'archive it, or declare it with a reason'))
    for s in sources:
        if s[2] and s[2] not in public and s[2].startswith('github.com/'):
            problems.append(('inventory', s[0].label(s[1]), 'PUBLISH names {} which is not a public repo of '
                                                             'the listed owners'.format(s[2])))
    return inv, problems


# ---------------------------------------------------------------- checks

CLONE = r'''
cd "$1" || exit 3
export GIT_OPTIONAL_LOCKS=0
r=$2; b=$3
c=$(git rev-parse --git-common-dir 2>/dev/null) && echo "common $(cd "$c" && pwd -P)"
echo "head $(git symbolic-ref -q --short HEAD)"
git for-each-ref --format='%(refname:short) %(upstream:short)' refs/heads | while read -r br up; do
  [ "$up" = "$r/$b" ] && echo "track $br $(git rev-list --count "$br" --not "$up" 2>/dev/null)"
done
[ -d .git ] && grep -qE '<!--[[:space:]]*auto:' README.md 2>/dev/null && echo "readme README.md"
exit 0
'''


def clone_state(host, path, remote, branch):
    rc, out, err = host.sh(CLONE, path, remote, branch)
    if rc != 0:
        return None, (err.strip() or 'exit {}'.format(rc))[:200]
    st = dict(dirty=0, common=path, head='', unpushed=0, track=[], readme=None)
    for line in out.splitlines():
        k, _, v = line.partition(' ')
        if k in ('dirty', 'unpushed'):
            st[k] = int(v) if v.strip().isdigit() else 0
        elif k in ('common', 'head'):
            st[k] = v.strip()
        elif k == 'track':
            br, _, n = v.rpartition(' ')
            st['track'].append((br, int(n) if n.isdigit() else 0))
        elif k == 'readme':
            st['readme'] = v.strip()
    return st, None


# A single remote invocation per clone, using NUL-delimited status for spaces, newlines and renames.
# Optional locks are disabled so a scan without fetch does not refresh the clone's index.
FRESHNESS = r'''
import json, os, subprocess, sys
path, fetch = sys.argv[1], json.loads(sys.argv[2])
env = dict(os.environ, GIT_OPTIONAL_LOCKS='0', GIT_TERMINAL_PROMPT='0')
def git(*args, timeout=60, required=True):
    p = subprocess.run(['git', '-C', path, *args], capture_output=True, env=env, timeout=timeout)
    if required and p.returncode:
        raise RuntimeError(os.fsdecode(p.stderr).strip() or 'git exited %s' % p.returncode)
    return p
st = {}
try:
    top = os.fsdecode(git('rev-parse', '--show-toplevel').stdout).strip()
    if os.path.realpath(top) != os.path.realpath(path):
        raise RuntimeError('not a checkout root; Git resolves to ' + top)
    if fetch:
        try:
            p = git('-c', 'maintenance.auto=false', 'fetch', '--all', '--no-recurse-submodules',
                    '--quiet', timeout=30, required=False)
            if p.returncode:
                st['fetch_error'] = os.fsdecode(p.stderr).strip() or 'exit %s' % p.returncode
        except subprocess.TimeoutExpired:
            st['fetch_error'] = 'timed out after 30 s'
    st['detached'] = git('symbolic-ref', '-q', 'HEAD', required=False).returncode != 0
    upstream = git('rev-parse', '--verify', '@{u}', required=False)
    st['upstream'] = upstream.returncode == 0
    if st['upstream'] and not st.get('fetch_error'):
        st['behind'], st['unpushed'] = map(int, git('rev-list', '--left-right', '--count',
                                                  '@{u}...HEAD').stdout.split())
        times = git('log', '--format=%ct', '@{u}..HEAD').stdout.split()
        st['unpushed_oldest'] = min(map(int, times)) if times else None
    latest = git('log', '-1', '--format=%ct', '--all', 'HEAD', required=False)
    st['latest'] = int(latest.stdout.strip()) if latest.returncode == 0 and latest.stdout.strip() else None
    fields = git('status', '--porcelain=v1', '-z', '--untracked-files=normal', timeout=120).stdout.split(b'\0')
    mtimes, count, i = [], 0, 0
    while i < len(fields) and fields[i]:
        entry = fields[i]
        i += 1
        if entry[:2] == b'??' and b'/' in entry[3:]:
            continue  # Untracked job directories belong to their lanes, not this checkout.
        if b'R' in entry[:2] or b'C' in entry[:2]:
            i += 1  # the original path follows a rename/copy; count the destination once
        count += 1
        filename = os.path.join(path, os.fsdecode(entry[3:]))
        # A deletion has no file mtime: use its nearest surviving parent directory's mtime.
        while not os.path.lexists(filename):
            filename = os.path.dirname(filename)
        mtimes.append(os.lstat(filename).st_mtime)
    st['dirty'] = count
    st['dirty_oldest'] = min(mtimes) if mtimes else None
except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
    st['error'] = str(exc)[:300]
print(json.dumps(st))
'''


def freshness_state(host, path, fetch=False):
    rc, out, err = host.run(['python3', '-c', FRESHNESS, path, json.dumps(bool(fetch))], timeout=180)
    if rc != 0:
        return {'error': (err.strip() or 'exit {}'.format(rc))[:300]}
    try:
        return json.loads(out)
    except ValueError:
        return {'error': 'invalid clone state: ' + (err or out).strip()[:200]}


def clone_problems(host, path, st, cfg, now):
    where = '{}:{}'.format(host.name or 'local', path)
    rows = []
    if st.get('fetch_error'):
        rows.append(('fetch', where, 'fetch failed: ' + st['fetch_error'][:300]))
    if st.get('error'):
        rows.append(('host', where, 'cannot read clone: ' + st['error']))
    intentional = any((e.get('host') or LOCAL) == host.name and host.path(e['path']) == path
                      for e in cfg.get('detached', []))
    if 'upstream' in st and not st['upstream'] and not (st['detached'] and intentional):
        rows.append(('no-upstream', where, 'detached HEAD; no upstream' if st['detached']
                     else 'current branch has no upstream'))
    if st.get('behind'):
        rows.append(('behind', where, 'behind {}'.format(st['behind'])))
    if st.get('dirty_oldest') is not None:
        age = (now - st['dirty_oldest']) / 3600
        limit = float(cfg.get('dirty_max_hours', 24))
        if age > limit:
            rows.append(('dirty', where, 'dirty files {} oldest {:.1f}h (limit {:g}h)'.format(st['dirty'], age, limit)))
    if st.get('unpushed_oldest') is not None:
        age = (now - st['unpushed_oldest']) / 3600
        limit = float(cfg.get('unpushed_max_hours', 24))
        if age > limit:
            rows.append(('unpushed', where, 'unpushed {} oldest {:.1f}h (limit {:g}h)'.format(
                st['unpushed'], age, limit)))
    return rows


def pushed_problems(name, branch, clones, states):
    """Public-default branch safety, aggregated across worktrees; freshness is checked per clone.

    One row per (kind, repo), not per clone: one repo had 51 worktrees, and a row each would flood the
    conductor's prompt the way 121 job rows did on 11-13. The row names the count and the worst three."""
    found, checked_out, seen = {'private-commits': []}, {}, set()
    for (host, path, remotes), st in zip(clones, states):
        if st is not None:
            checked_out.setdefault((host.name, st['common']), set()).add(st['head'])
    for (host, path, remotes), st in zip(clones, states):
        if st is None:
            continue
        where = host.label(path)
        for br, n in st['track']:
            key = (host.name, st['common'], br)
            if n and br not in checked_out.get(key[:2], ()) and key not in seen:
                seen.add(key)
                found['private-commits'].append((n, '{} {}'.format(where, br)))
    what = {'private-commits': 'local branch(es) tracking public {} with commits not on it; a plain push '
                               'publishes them'.format(branch)}
    rows = []
    for kind, items in found.items():
        if items:
            items.sort(key=lambda x: -x[0])
            worst = ', '.join('{} ({})'.format(w, n) for n, w in items[:3]) + (', ...' if len(items) > 3 else '')
            rows.append((kind, name, '{} {}: {}'.format(len(items), what[kind], worst)))
    return rows


def published_problems(name, branch, source, agent_globs, gh, gate):
    host, src = source[0], source[1]
    rc, text, err = host.run(['cat', src + '/PUBLISH'])
    if rc != 0:
        return [('unpublished', name, 'cannot read {}: {}'.format(host.label(src + '/PUBLISH'), err.strip()[:200]))]
    _remote, inc, exc = parse_publish(text)
    rc, tree, err = host.run(['git', '-C', src, 'ls-tree', '-r', '-z', 'HEAD'])
    if rc != 0:
        return [('unpublished', name, 'cannot list {} HEAD: {}'.format(host.label(src), err.strip()[:200]))]
    export = {}
    for entry in filter(None, tree.split('\0')):
        meta, _, path = entry.partition('\t')
        parts = meta.split()
        if len(parts) != 3 or parts[1] != 'blob':
            continue
        if any(r.match(path) for r in inc) and not any(r.match(path) for r in exc) \
                and not gate.is_agent_file(path, agent_globs):
            export[path] = parts[2]
    r = subprocess.run(shlex.split(gh) + ['api', 'repos/{}/git/trees/{}?recursive=1'.format(name, branch)],
                       capture_output=True, text=True, timeout=TIMEOUT)
    if r.returncode != 0:
        return [('unpublished', name, 'cannot read the public {} tree: {}'.format(branch, (r.stderr or r.stdout).strip()[:200]))]
    data = json.loads(r.stdout)
    if data.get('truncated'):
        return [('unpublished', name, 'public tree listing was truncated; cannot compare')]
    public = {t['path']: t['sha'] for t in data.get('tree', []) if t.get('type') == 'blob'}
    changed = sorted(p for p in export if public.get(p) != export[p])
    removed = sorted(p for p in public if p not in export)
    if not changed and not removed:
        return []
    shown = ', '.join((changed + removed)[:5]) + (', ...' if len(changed) + len(removed) > 5 else '')
    return [('unpublished', name, 'a publish from {} would change {} file(s) and remove {} on public {}: {}'.format(
        host.label(src), len(changed), len(removed), branch, shown))]


def readme_problem(host, path, readme, cmd):
    rc, out, err = host.run(shlex.split(cmd) + ['check', path + '/' + readme])
    if rc == 0:
        return []
    if rc == 1:
        stale = re.findall(r'block "([^"]+)"', out)
        return [('readme', host.label(path + '/' + readme), 'auto-blocks differ from a fresh render: {}'.format(
            ', '.join(stale) or out.strip()[:200]))]
    return [('readme', host.label(path + '/' + readme), 'readme-blocks could not check it (exit {}): {}'.format(
        rc, (err or out).strip()[:200]))]


# Each side as a git blob id: the repo side as committed at HEAD when it is in a checkout (a working-tree
# edit is not what was installed from), the file itself otherwise.
HASH = r'''live=$1; repo=$2
if [ -f "$live" ]; then git hash-object "$live"; else echo missing; fi
if [ ! -f "$repo" ]; then echo missing
else git -C "$(dirname "$repo")" rev-parse -q --verify "HEAD:./$(basename "$repo")" 2>/dev/null || git hash-object "$repo"; fi'''


def installed_problems(entries, host_of):
    rows = []
    for e in entries:
        host = host_of(e.get('host'))
        if e.get('names'):
            pairs = [(e['live'].rstrip('/') + '/' + n, e['repo'].rstrip('/') + '/' + n) for n in e['names']]
        else:
            pairs = [(e['live'], e['repo'])]
        for live, repo in pairs:
            rc, out, err = host.sh(HASH, host.path(live), host.path(repo))
            got = out.split()
            if rc != 0 or len(got) != 2:
                rows.append(('installed', host.label(live), 'cannot hash: {}'.format((err.strip() or 'exit {}'.format(rc))[:200])))
            elif 'missing' in got:
                which = live if got[0] == 'missing' else repo
                rows.append(('installed', host.label(live), '{} is missing'.format(which)))
            elif got[0] != got[1]:
                rows.append(('installed', host.label(live), 'differs from {} as committed: live code is not the '
                             'repo\'s'.format(repo)))
    return rows


RETRO = r'''f=$1
[ -f "$f" ] || { echo none; exit 0; }
t=$(git -C "$(dirname "$f")" log -1 --format=%ct -- "$(basename "$f")" 2>/dev/null)
[ -n "$t" ] && echo "$t" || date -r "$f" +%s'''


def stamp(row):
    for k in ('utc', 'ts', 'date'):
        v = row.get(k)
        if isinstance(v, str):
            try:
                d = datetime.fromisoformat(v.replace('Z', '+00:00'))
                return d.timestamp() if d.tzinfo else time.mktime(d.timetuple())
            except ValueError:
                continue
    return None


def retro_problems(entries, host_of, now):
    rows = []
    for e in entries:
        host = host_of(e.get('host'))
        lane, max_days = e.get('lane', e.get('retro')), float(e.get('max_days', 7))
        rc, out, err = host.run(['cat', host.path(e['results'])])
        if rc != 0:
            rows.append(('retro', lane, 'cannot read results {}: {}'.format(e['results'], err.strip()[:200])))
            continue
        times = []
        for line in out.splitlines():
            try:
                t = stamp(json.loads(line))
            except ValueError:
                continue
            if t is not None:
                times.append(t)
        rc, out, err = host.sh(RETRO, host.path(e['retro']))
        if rc != 0:
            rows.append(('retro', lane, 'cannot date {}: {}'.format(e['retro'], err.strip()[:200])))
            continue
        retro_t = None if out.strip() in ('', 'none') else float(out.split()[0])
        after = [t for t in times if retro_t is None or t > retro_t]
        if not after:
            continue
        waiting = (now - min(after)) / 86400
        if waiting <= max_days:
            continue
        if retro_t is None:
            rows.append(('retro', lane, 'no retro at {}; {} result(s), the oldest {:.0f} days old'.format(
                e['retro'], len(after), waiting)))
        else:
            rows.append(('retro', lane, '{} is from {}; {} newer result(s), the oldest {:.0f} days out of it '
                         '(limit {:g})'.format(e['retro'], time.strftime('%Y-%m-%d', time.localtime(retro_t)),
                                               len(after), waiting, max_days)))
    return rows


BACKUP = r'''m=$1
crontab -l 2>/dev/null | grep -v '^[[:space:]]*#' | grep -qF -- "$m" && { echo cron; exit 0; }
for p in "$HOME"/Library/LaunchAgents/*.plist /Library/LaunchAgents/*.plist /Library/LaunchDaemons/*.plist; do
  [ -f "$p" ] || continue
  grep -qF -- "$m" "$p" || continue
  l=$(plutil -extract Label raw "$p" 2>/dev/null)
  s=no; grep -qE 'StartCalendarInterval|StartInterval' "$p" && s=yes
  ld=no; [ -n "$l" ] && launchctl list "$l" >/dev/null 2>&1 && ld=yes
  echo "plist $l $s $ld"
done
exit 0'''


def backup_problems(entries, host_of, states=None, now=None):
    now = time.time() if now is None else now
    rows = []
    for e in entries:
        host = host_of(e.get('host'))
        name = e.get('name') or e.get('repo') or e.get('match')
        if e.get('repo'):
            path = host.path(e['repo'])
            where = '{}:{} ({})'.format(host.name or 'local', path, name)
            st = states[(host.name, path)] if states is not None else freshness_state(host, path)
            if st.get('error') and 'latest' not in st:
                rows.append(('backup', where, 'cannot read newest commit: ' + st['error']))
            elif st.get('latest') is None:
                rows.append(('backup', where, 'no commits in backup clone'))
            else:
                age, limit = (now - st['latest']) / 3600, float(e.get('max_hours', 48))
                if age > limit:
                    rows.append(('backup', where, 'newest commit {:.1f}h old (limit {:g}h)'.format(age, limit)))
        if not e.get('match'):
            continue
        rc, out, err = host.sh(BACKUP, e['match'])
        if rc != 0:
            rows.append(('backup', name, 'cannot read the schedules: {}'.format(err.strip()[:200])))
            continue
        lines = out.split('\n')
        if 'cron' in lines:
            continue
        plists = [l.split() for l in lines if l.startswith('plist ')]
        if any(p[-2:] == ['yes', 'yes'] for p in plists):
            continue
        if not plists:
            detail = 'no launchd job or crontab line runs "{}"'.format(e['match'])
        else:
            detail = 'launchd job(s) {} run it but {}'.format(', '.join(p[1] for p in plists if len(p) > 3), '; '.join(
                'not on a schedule' if p[-2] != 'yes' else 'not loaded' for p in plists))
        rows.append(('backup', host.label(name), detail))
    return rows


# ---------------------------------------------------------------- main

def check(cfg, gh='gh', now=None, workers=6):
    """(inventory lines, problem rows)."""
    now = time.time() if now is None else now
    gate = load_tool('egress-gate')
    hosts = {}

    def host_of(name):
        name = name or LOCAL
        if name not in hosts:
            hosts[name] = Host(name)
        return hosts[name]

    problems = []
    for name in {r.get('host') or LOCAL for r in cfg.get('roots', [])} | \
            {e.get('host') or LOCAL for k in ('installed', 'retros', 'backups') for e in cfg.get(k, [])}:
        h = host_of(name)
        if not h.home:
            problems.append(('host', name, 'unreachable: {}'.format(h.error)))
    live = lambda e: not host_of(e.get('host')).error
    clones, sources = [], []
    excl = cfg.get('exclude', [])
    for r in cfg.get('roots', []):
        if not live(r):
            continue
        h = host_of(r.get('host'))
        c, s = scan_root(h, r['path'], r.get('depth', 4), problems)
        clones += [x for x in c if not excluded(h, x[1], excl)]
        sources += [x for x in s if not excluded(h, x[1], excl)]
    # Overlapping roots must not fetch or report the same checkout twice. Explicit backup clones
    # outside the roots share this cache, too, including multiple backup entries for one path.
    clones = list({(h.name, p): (h, p, remotes) for h, p, remotes in clones}.values())
    sources = list({(h.name, p): (h, p, rid) for h, p, rid in sources}.values())
    targets = {(h.name, p): (h, p) for h, p, _ in clones}
    for e in cfg.get('backups', []):
        if live(e) and e.get('repo'):
            h = host_of(e.get('host'))
            p = h.path(e['repo'])
            targets[(h.name, p)] = (h, p)
    def inspect(target):
        h, p = target
        return (h.name, p), freshness_state(h, p, cfg.get('fetch', False))
    with ThreadPoolExecutor(max_workers=workers) as pool:
        fresh_states = dict(pool.map(inspect, targets.values()))
    discovered = {(h.name, p) for h, p, _ in clones}
    # 11-14 live run: 131 of 153 no-upstream rows were job folders with no remote at all (each job runs
    # `git init`). A clone with no remote cannot fall behind or go unpushed; it is checked only if listed
    # in `local_only` (paths that must still have no stale uncommitted work).
    # Friends job folders also hold clones of OTHER people's repos (detached, patched copies): their
    # drift is not ours. So the clone checks cover clones with a remote under one of `owners` (or
    # `clone_owners` when set), plus anything listed in `local_only`.
    mine = [o.lower() for o in (cfg.get('clone_owners') or cfg.get('owners') or [])]
    has_remote = {(h.name, p): any(str(r).lower().startswith(tuple('github.com/{}/'.format(o) for o in mine))
                                   for r in remotes.values()) if mine else bool(remotes)
                  for h, p, remotes in clones}
    local_only = {((e.get('host') or LOCAL), host_of(e.get('host')).path(e['path'])) for e in cfg.get('local_only', [])}
    for key, (h, p) in targets.items():
        if key in discovered and (has_remote.get(key, True) or (h.name, p) in local_only):
            problems += clone_problems(h, p, fresh_states[key], cfg, now)
        elif fresh_states[key].get('fetch_error'):
            problems.append(('fetch', '{}:{}'.format(h.name or 'local', p),
                             'fetch failed: ' + fresh_states[key]['fetch_error'][:300]))
    inventory_ok = bool(cfg.get('owners'))
    try:
        public = public_repos(cfg.get('owners', []), gh)
    except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as exc:
        public = {}
        inventory_ok = False
        problems.append(('inventory', 'github', str(exc)[:300]))
    if not public and cfg.get('owners') and not any(k == 'inventory' for k, _, _ in problems):
        problems.append(('inventory', 'github', 'no public repos listed for owners {}'.format(cfg.get('owners'))))
    inv, more = classify(public, clones, sources, cfg.get('declared') or {}) if inventory_ok else ({}, [])
    problems += more
    if any(h.error for h in hosts.values()):
        problems = [(k, w, d + ' (a host was unreachable, so its clones were not seen)' if k == 'unknown' else d)
                    for k, w, d in problems]
    lines = ['{}\t{}\t{}'.format(kind, public[rid]['name'], detail) for rid, (kind, detail, _, _) in inv.items()]
    rb = cfg.get('readme_blocks', 'readme-blocks')
    agent_globs_by_host = {}

    def agent_globs(h):
        if h.name not in agent_globs_by_host:
            rc, out, _ = h.run(['cat', h.path(cfg.get('agent_files', '~/.config/git/agent-files.gitignore'))])
            agent_globs_by_host[h.name] = [l.strip().lower() for l in out.splitlines()
                                           if rc == 0 and l.strip() and not l.lstrip().startswith('#')]
        return agent_globs_by_host[h.name]

    jobs = []
    for rid, (kind, _, cl, src) in inv.items():
        meta = public[rid]
        if kind == 'pushed':
            for c in cl:
                remote = next(n for n, v in c[2].items() if v == rid)
                jobs.append(('clone', rid, c, remote, meta['branch']))
        elif kind == 'published':
            jobs.append(('published', rid, src, None, meta['branch']))
            for c in cl:   # another clone of a published repo still must not hold unpushed work for it
                remote = next(n for n, v in c[2].items() if v == rid)
                jobs.append(('clone', rid, c, remote, meta['branch']))

    def do(job):
        what, rid, c, remote, branch = job
        if what == 'published':
            h = c[0]
            out = published_problems(public[rid]['name'], branch, c, agent_globs(h), gh, gate)
            rc, txt, _ = h.sh('[ -d "$1/.git" ] && grep -qE "<!--[[:space:]]*auto:" "$1/README.md" 2>/dev/null && echo y',
                              c[1])
            if txt.strip() == 'y':
                out += readme_problem(h, c[1], 'README.md', rb)
            return job, None, out
        st, err = clone_state(c[0], c[1], remote, branch)
        if st and fresh_states[(c[0].name, c[1])].get('fetch_error'):
            st['track'] = []  # Public branch comparisons would also use stale fetched refs.
        out = [('host', c[0].label(c[1]), 'cannot read clone: ' + err)] if err else []
        if st and st['readme']:
            out += readme_problem(c[0], c[1], st['readme'], rb)
        return job, st, out

    with ThreadPoolExecutor(max_workers=workers) as pool:
        results = list(pool.map(do, jobs))
    by_repo = {}
    for job, st, rows in results:
        problems += rows
        if job[0] == 'clone':
            by_repo.setdefault(job[1], ([], []))
            by_repo[job[1]][0].append(job[2])
            by_repo[job[1]][1].append(st)
    for rid, (cl, states) in by_repo.items():
        problems += pushed_problems(public[rid]['name'], public[rid]['branch'], cl, states)
    problems += installed_problems([e for e in cfg.get('installed', []) if live(e)], host_of)
    problems += retro_problems([e for e in cfg.get('retros', []) if live(e)], host_of, now)
    problems += backup_problems([e for e in cfg.get('backups', []) if live(e)], host_of, fresh_states, now)
    return lines, problems


def alert_summary(rows):
    counts = ', '.join('{}={}'.format(k, n) for k, n in sorted(Counter(k for k, _, _ in rows).items()))
    first = '; '.join('{} {} ({})'.format(k, w, d) for k, w, d in rows[:5])
    return 'repo-freshness: {} finding(s): {}; {}'.format(len(rows), counts, first).replace('\n', ' ').replace('\r', ' ')


def alert_problems(rows, cmd, hours=24, now=None):
    if not rows or not cmd:
        return
    now = time.time() if now is None else now
    path = load_tool('job-health').state_dir() / 'repo-freshness' / 'last-alert.json'
    try:
        last = float(json.loads(path.read_text())['sent_at'])
    except (OSError, ValueError, KeyError, TypeError):
        last = None
    if last is not None and now - last < float(hours) * 3600:
        return
    try:
        result = subprocess.run(shlex.split(cmd) + [alert_summary(rows)], capture_output=True, timeout=60)
    except (OSError, subprocess.TimeoutExpired) as exc:
        print('repo-freshness: alert failed: {}'.format(exc), file=sys.stderr)
        return
    if result.returncode != 0:
        print('repo-freshness: alert failed (exit {})'.format(result.returncode), file=sys.stderr)
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps({'sent_at': now}))


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
    ap.add_argument('--config', default=os.environ.get('REPO_FRESHNESS_CONFIG',
                                                       os.path.expanduser('~/.config/rstack/repo-freshness.json')))
    ap.add_argument('--alert', default=os.environ.get('REPO_FRESHNESS_ALERT', ''),
                    help='command receiving counts and the first five findings, at most once per alert_hours (default 24)')
    ap.add_argument('--push', help="copy this host's list to the same path on HOST (the conductor's Mac)")
    ap.add_argument('--dry', action='store_true', help='print only; no state or alerts (fetch still runs if configured)')
    a = ap.parse_args(argv)
    gh = os.environ.get('REPO_FRESHNESS_GH', 'gh')
    cfg = {}
    try:
        cfg = json.loads(Path(a.config).expanduser().read_text())
        lines, rows = check(cfg, gh)
    except FileNotFoundError:
        print('repo-freshness: no config at {} (see --help for its shape); nothing was checked'.format(a.config),
              file=sys.stderr)
        return 2
    except (ValueError, KeyError, TypeError, AttributeError) as exc:   # a broken config is a problem, not an all-clear
        lines, rows = [], [('config', a.config, 'unreadable: {}'.format(exc))]
    print('# inventory: kind\trepo\thow it reaches the public')
    for l in lines:
        print(l)
    print('# problems: {}'.format(len(rows)))
    for k, w, d in rows:
        print('{}\t{}\t{}'.format(k, w, d))
    if a.dry:
        return 1 if rows else 0
    load_tool('job-health').report(rows, 'repo-freshness', 'repos', 'repo', '', a.push)
    alert_problems(rows, a.alert, cfg.get('alert_hours', 24))
    return 0


if __name__ == '__main__':
    sys.exit(main())
