#!/usr/bin/env bash
# run.sh: scheduled agent run -> digest -> GUARANTEED delivery. Lab-agnostic, two-phase.
#
# A SKELETON with working control flow. What the agent should DO lives in prompt-ingest.md and
# prompt-write.md (marked YOUR SPEC); how the digest reaches you lives in notify.sh.
#
# Flow:
# dated branch + throwaway sparse worktree (idempotent per day)
# -> self-heal: land any stranded branch from an earlier day FIRST
# -> probe primary backend through the SAME adapter + flags as the real run; failover if it fails
# -> PHASE 1 ingest (orchestrator + cheap sub-agents) writes a compact brief, capped so the writer
# starts by DIGEST_WRITER_START no matter what ingest is doing
# -> PHASE 2 write (best writer, fresh session, reads only the brief); next backend if no digest
# -> COMPILE: scripts/digest-lint.py; on failure ONE repair pass with the lint report; lint again
# -> consume the intraday ledger (shell-side, only if the digest blended every entry in)
# -> no digest? deterministic fallback (scripts/digest-fallback.py)
# -> deliver (notify.sh digest); delivery failed? deliver a failure notice
# -> land (merge main into the branch, fast-forward main), guarded; blocked? notice
# An EXIT trap sends a notice if the script dies anywhere before delivery. Silence is not a
# permitted outcome of this script.
#
# Test without delivering or landing: NIGHTLY_TEST=1 scripts/nightly/run.sh
#
# This runs agents with permission prompts disabled (unattended). That is a real trade-off, accepted
# because the run is confined to a throwaway worktree, raw/ is the quarantine layer, and the prompt
# scopes the job. Know what you are agreeing to before you schedule it.
set -euo pipefail
export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
export KB_NIGHTLY=1 # the scheduled run CONSUMES the intraday ledger: exempt from pre-commit check 4
KB="${KB_ROOT:-$HOME/kb}" # the MAIN tree
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # prompts/adapter next to THIS script, so a worktree copy can be test-run
. "$HERE/models.env"
AR="$HERE/agent-run.sh"
NOTIFY="$HERE/notify.sh"
DAY="$(date '+%Y-%m-%d')"
YESTERDAY="$(date -v-1d '+%Y-%m-%d' 2>/dev/null || date -d yesterday '+%Y-%m-%d')"
TEST="${NIGHTLY_TEST:-}"
BR="nightly-$(date '+%Y%m%d')"; [ -n "$TEST" ] && BR="nightlytest-$(date '+%Y%m%d%H%M')"
WTBASE="$HOME/.worktrees/$(basename "$KB")"
WT="$WTBASE/$BR"
LOG="${NIGHTLY_LOG:-$HOME/.local/state/agent-kb/nightly.log}"
mkdir -p "$(dirname "$LOG")" "$WTBASE"
exec >>"$LOG" 2>&1
echo "=== $(date '+%F %T %Z') scheduled run ${TEST:+(TEST MODE)} ==="
notice() { # notice SUBJECT BODY
[ -n "$TEST" ] && { echo "(test mode: notice not sent) $1"; return 0; }
"$NOTIFY" notice "$1" "$2" || echo "could not send notice: $1"
}
# ── DEATH IS NEVER SILENT. Every "guaranteed delivery" layer used to sit AFTER the model phases, so a
# crash between the digest commit and the send produced nothing at all. This fires on any non-zero exit
# before delivery and sends the tail of the log.
DELIVERED=0
on_exit() {
local rc=$?
[ "$rc" -eq 0 ] || [ "$DELIVERED" -eq 1 ] && return 0
notice "KB scheduled run $DAY: DIED (exit $rc) before delivery" \
"Branch $BR / worktree $WT may hold a written digest. Last log lines:
$(tail -n 25 "$LOG" 2>/dev/null)"
}
trap on_exit EXIT
land_branch() { # land_branch BRANCH: merge main into it, ff main, clean up. Non-zero + untouched if unsafe.
local br="$1" wt="$WTBASE/$1"
if [ -d "$wt" ] && [ -n "$(git -C "$wt" status --porcelain --untracked-files=no)" ]; then
echo "LAND $br: its worktree has uncommitted tracked changes; left in place"; return 1; fi
if [ -n "$(git -C "$KB" status --porcelain --untracked-files=no)" ]; then
echo "LAND $br: the MAIN tree has uncommitted tracked changes, so main cannot fast-forward:"
git -C "$KB" status --short --untracked-files=no | sed 's/^/ /'; return 1; fi
if [ -d "$wt" ]; then
git -C "$wt" merge --no-edit main || { git -C "$wt" merge --abort 2>/dev/null; echo "LAND $br: merging main failed"; return 1; }
fi
git -C "$KB" merge --ff-only "$br" || { echo "LAND $br: fast-forward failed"; return 1; }
[ -d "$wt" ] && git -C "$KB" worktree remove --force "$wt"
git -C "$KB" branch -D "$br"; echo "landed + cleaned up $br"
}
model_for() { # model_for BACKEND ROLE (INGEST | SUBAGENT | WRITER | PROBE)
local v; v="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')_$2_MODEL"; printf '%s' "${!v:-}"
}
probe_backend() { # one tiny call through the SAME adapter and flags as the real run
local b="$1" pf tl out attempt
pf="$(mktemp -t probe.XXXXXX)"; tl="$(mktemp -t probe-log.XXXXXX)"; echo "Reply with exactly: OK" >"$pf"
for attempt in 1 2 3; do
if out="$("$AR" "$b" "$(model_for "$b" PROBE)" low 240 "$WT" "$pf" "$tl" 2>&1)" && grep -q "OK" <<<"$out"; then
echo "probe $b: OK (attempt $attempt)"; rm -f "$pf" "$tl"; return 0; fi
echo "probe $b attempt $attempt FAILED: $(printf '%s' "$out" | tail -c 300) | transcript: $(tail -c 400 "$tl")"
[ "$attempt" -lt 3 ] && sleep 20
done
rm -f "$pf" "$tl"; return 1
}
build_prompt() { # build_prompt TEMPLATE OUT BACKEND (substitution in Python: no shell quoting hazards)
NB_BACKEND="$3" NB_SUBAGENT="$(model_for "$3" SUBAGENT)" python3 "$HERE/build_prompt.py" "$1" "$2"
}
PHASE_RC=0
run_phase() { # run_phase NAME BACKEND ROLE EFFORT TIMEOUT PROMPT_FILE; never aborts the script
local model; model="$(model_for "$2" "$3")"
echo "-- phase $1 on $2/$model started $(date '+%T')"
"$AR" "$2" "$model" "$4" "$5" "$WT" "$6" "$(dirname "$LOG")/nightly-$1-$2.log" && PHASE_RC=0 || PHASE_RC=$?
echo "-- phase $1 on $2/$model finished $(date '+%T') rc=$PHASE_RC"
return 0
}
# ── Idempotence: a branch for today means today already ran (or a failed run awaits inspection).
if git -C "$KB" show-ref --quiet "refs/heads/$BR"; then echo "branch $BR exists; skipping. Inspect $WT"; exit 0; fi
# ── Self-heal: one blocked landing must never snowball into a week of unlanded work.
if [ -z "$TEST" ]; then
for b in $(git -C "$KB" for-each-ref --format='%(refname:short)' 'refs/heads/nightly-*'); do
[ "$b" = "$BR" ] && continue
echo "stale branch $b found; landing it first"; land_branch "$b" || echo "stale branch $b could not land; left for attention"
done
fi
# ── Sparse worktree (skip big untracked-by-design dirs). Never write into the live main tree.
git -C "$KB" worktree add --no-checkout "$WT" -b "$BR" main
git -C "$WT" sparse-checkout set --no-cone '/*' '!/inbox/'
git -C "$WT" checkout
cd "$WT"
BASE_SHA="$(git rev-parse HEAD)"
# ── Optional preflight: health checks that raise alerts (scripts/alert.py) the SAME night.
[ -n "${PREFLIGHT_CMD:-}" ] && { $PREFLIGHT_CMD || echo "preflight reported a problem (alert raised)"; }
# ── Backends: probe primary, failover if needed; pick the writer order.
PRIMARY="$NIGHTLY_BACKEND"; FAILOVER="$NIGHTLY_FAILOVER"
if ! probe_backend "$PRIMARY"; then
if [ -n "$FAILOVER" ] && [ "$FAILOVER" != "$PRIMARY" ] && probe_backend "$FAILOVER"; then
echo "primary $PRIMARY unavailable; running on $FAILOVER"; PRIMARY="$FAILOVER"; FAILOVER=""
else
echo "no backend passed the probe; skipping model phases (the fallback digest will be delivered)"; PRIMARY=""
fi
fi
[ "$FAILOVER" = "$PRIMARY" ] && FAILOVER=""
WRITER_ORDER="$PRIMARY $FAILOVER"
if [ -n "$PRIMARY" ] && [ -n "$NIGHTLY_WRITER" ] && [ "$NIGHTLY_WRITER" != "$PRIMARY" ] && probe_backend "$NIGHTLY_WRITER"; then
WRITER_ORDER="$NIGHTLY_WRITER $PRIMARY"
fi
echo "backends: primary=${PRIMARY:-none} writer order: ${WRITER_ORDER:-none}"
DIGEST="$WT/outbox/Digest - $DAY.html"
BRIEF="$WT/raw/_index/digest-brief-$DAY.md"
PENDING_N="$(KB_ROOT="$WT" python3 "$WT/scripts/day-ledger.py" count 2>/dev/null || echo 0)"
LEDGER_PENDING="$(mktemp -t ledger-pending.XXXXXX)"
KB_ROOT="$WT" python3 "$WT/scripts/day-ledger.py" pending --json >"$LEDGER_PENDING" 2>/dev/null || echo "[]" >"$LEDGER_PENDING"
echo "intraday ledger: $PENDING_N pending"
DELIVERY_STEP="DELIVERY: commit the digest file, then stop. The wrapper delivers it. Put the subject line in the HTML as : the actual read in under 90 characters, never a generic label."
export NB_DAY="$DAY" NB_YESTERDAY="$YESTERDAY" NB_PENDING_N="$PENDING_N" NB_DELIVERY="$DELIVERY_STEP" NB_TEST="$TEST"
WRITER_BACKEND=""
if [ -n "$PRIMARY" ]; then
# PHASE 1: ingest, capped so the writer starts on time (ingest is optional work; the digest is the deliverable)
P1="$(mktemp -t p1.XXXXXX)"
if build_prompt "$HERE/prompt-ingest.md" "$P1" "$PRIMARY"; then
now_s=$(date +%s)
start_s=$(date -j -f "%Y-%m-%d %H:%M" "$DAY $DIGEST_WRITER_START" +%s 2>/dev/null || date -d "$DAY $DIGEST_WRITER_START" +%s 2>/dev/null || echo $((now_s + INGEST_TIMEOUT)))
cap=$(( start_s - now_s )); [ "$cap" -lt 600 ] && cap=600
[ "$cap" -lt "$INGEST_TIMEOUT" ] && ING_T="$cap" || ING_T="$INGEST_TIMEOUT"
echo "ingest budget: ${ING_T}s (writer must start by $DIGEST_WRITER_START)"
run_phase ingest "$PRIMARY" INGEST "$INGEST_EFFORT" "$ING_T" "$P1"
else
echo "ingest prompt failed to build; skipping phase 1"
fi
[ -s "$BRIEF" ] && echo "brief: $(wc -w <"$BRIEF" | tr -d ' ') words" || echo "PHASE 1 WROTE NO BRIEF; the writer reconstructs from git log"
# Corrections still unpropagated after phase 1 go into the brief, and the digest must mention them.
CORR_OUT="$(KB_ROOT="$WT" python3 "$WT/scripts/coherence.py" --corrections-only 2>&1)" && CORR_RC=0 || CORR_RC=$?
if [ "$CORR_RC" -eq 1 ]; then
{ echo; echo "## Corrections not yet propagated (coherence check 5): the digest must list these"; echo "$CORR_OUT"; } >>"$BRIEF"
fi
# PHASE 2: write, failing over to the next backend if no digest appears
for WB in $WRITER_ORDER; do
P2="$(mktemp -t p2.XXXXXX)"
build_prompt "$HERE/prompt-write.md" "$P2" "$WB" || { echo "write prompt failed to build for $WB"; continue; }
run_phase write "$WB" WRITER "$WRITER_EFFORT" "$WRITER_TIMEOUT" "$P2"
if [ -s "$DIGEST" ]; then WRITER_BACKEND="$WB"; break; fi
echo "phase write on $WB produced no digest"
done
fi
# ── COMPILE: lint, ONE repair pass, lint again. Delivered even if it still fails (never silence).
LINT_EXTRA=()
[ "${CORR_RC:-0}" -eq 1 ] && LINT_EXTRA+=(--must-mention '[Cc]orrection')
# Add deterministic must-mentions here, e.g. "a source you always want summarized posted tonight":
# if ; then LINT_EXTRA+=(--must-mention 'Their Name'); fi
if [ -s "$DIGEST" ]; then
LINT_OUT="$(python3 "$WT/scripts/digest-lint.py" --ledger-pending "$LEDGER_PENDING" ${LINT_EXTRA[@]+"${LINT_EXTRA[@]}"} "$DIGEST" 2>&1)" && LINT_RC=0 || LINT_RC=$?
echo "digest-lint pass 1: rc=$LINT_RC"; echo "$LINT_OUT"
if [ "$LINT_RC" -ne 0 ] && [ -n "${WRITER_BACKEND:-$PRIMARY}" ]; then
RP="$(mktemp -t repair.XXXXXX)"
cat >"$RP" <&1)" && LINT_RC=0 || LINT_RC=$?
echo "digest-lint after repair: rc=$LINT_RC"; echo "$LINT_OUT"
[ "$LINT_RC" -ne 0 ] && echo "DIGEST STILL FAILS LINT; delivering anyway. Tighten the prompt or the repair."
fi
git add "outbox/Digest - $DAY.html" && git commit -qm "nightly: digest $DAY (post-lint)" || true
fi
# ── Consume the intraday ledger: shell-side, only if every entry made it in. Otherwise they roll over.
if [ "${PENDING_N:-0}" -gt 0 ] && [ -s "$DIGEST" ]; then
OK="$(python3 "$WT/scripts/digest-lint.py" --json --ledger-pending "$LEDGER_PENDING" "$DIGEST" 2>/dev/null \
| python3 -c 'import json,sys; print(int(json.load(sys.stdin).get("ledger_ok", False)))' 2>/dev/null || echo 0)"
if [ "$OK" = 1 ]; then
KB_ROOT="$WT" python3 "$WT/scripts/day-ledger.py" consume --digest "$DAY"
git add raw/_index/day-ledger.jsonl && git commit -qm "nightly: consumed $PENDING_N intraday entries into digest $DAY" || true
else
echo "intraday ledger NOT consumed: some entries missing from the digest; they roll to tomorrow"
fi
fi
# ── GUARANTEED DELIVERY. Every branch delivers something, or delivers a notice that it could not.
if [ ! -s "$DIGEST" ]; then
echo "model produced NO digest; building the deterministic fallback"
KB_ROOT="$WT" python3 "$WT/scripts/digest-fallback.py" --date "$DAY" --out "$DIGEST" || echo "fallback builder ALSO failed"
[ -s "$DIGEST" ] && git add "outbox/Digest - $DAY.html" && git commit -qm "nightly: fallback digest $DAY" || true
fi
SUBJECT="$(sed -n 's/.*/dev/null | head -1)"
SUBJECT="${SUBJECT:-KB digest $DAY}"
if [ -n "$TEST" ]; then
echo "test mode: not delivering; digest at $DIGEST"
elif [ -s "$DIGEST" ]; then
"$NOTIFY" digest "$DIGEST" "$SUBJECT" && echo "digest delivered" \
|| notice "KB digest $DAY: DELIVERY FAILED" "The digest was written ($DIGEST) but delivery failed. See $LOG."
else
notice "KB digest $DAY: NO DIGEST PRODUCED" "Neither the model nor the fallback produced a digest. See $LOG."
fi
DELIVERED=1 # from here on, failures get their own notices; the death trap stands down
# ── Safety net: commit leftovers by explicit path, then land (guarded).
if [ -n "$(git status --porcelain)" ]; then
echo "leftovers found; safety-committing"
git add outbox raw wiki log.md state.md index.md 2>/dev/null || true
git commit -qm "nightly: safety-commit of leftovers" || true
fi
if [ -n "$TEST" ]; then
echo "test mode: not landing; inspect $WT (branch $BR)"
elif ! land_branch "$BR"; then
notice "KB scheduled run $DAY: LANDING BLOCKED" "The digest was delivered, but $BR could not land on main (reason in $LOG). Tomorrow's run retries it first."
fi
echo "=== done $(date '+%T') ==="