", " ", fragment, flags=re.S)
t = html.unescape(re.sub(r"<[^>]+>", " ", t))
return re.sub(r"\s+", " ", t).strip()
def words(t):
return len(t.split())
def kind_of(name):
n = name.lower().strip()
for kind, pat in KIND_RULES:
if re.search(pat, n):
return kind
return "other"
def sections(src, is_html):
"""[(name, body)] split on headings."""
if is_html:
heads = list(re.finditer(r"]*>(.*?)", src, flags=re.S | re.I))
else:
heads = list(re.finditer(r"^#{2,3}\s+(.*)$", src, flags=re.M))
out = []
for i, m in enumerate(heads):
end = heads[i + 1].start() if i + 1 < len(heads) else len(src)
out.append((text_of(m.group(1)), src[m.end():end]))
return out
def rows(body, is_html):
if is_html:
parts = re.findall(r"<(?:p|li|td)[^>]*>(.*?)(?:p|li|td)>", body, flags=re.S)
return [text_of(p) for p in parts if "| MAX_WORDS:
hard.append(f"L1 total {total} words > {MAX_WORDS}")
for leak in LEAKS:
if leak in raw:
hard.append(f"S1 tool-leak fingerprint present: {leak!r}")
secs = sections(src, is_html)
rep["sections"] = {}
for name, body in secs:
w = words(text_of(body) if is_html else body)
rep["sections"][name] = w
if w > MAX_SECTION_WORDS:
hard.append(f"L3 section '{name}' {w} words > {MAX_SECTION_WORDS}")
for r in rows(body, is_html):
if words(r) > MAX_ROW_WORDS:
hard.append(f"L4 row {words(r)} words > {MAX_ROW_WORDS}: \"{r[:70]}...\"")
# L7 order
want = [k.strip() for k in order.split(",") if k.strip()]
seen = [kind_of(n) for n, _ in secs]
rep["section_kinds"] = seen
rank = {k: i for i, k in enumerate(want)}
last = -1
for (name, _), k in zip(secs, seen):
r = rank.get(k, rank.get("other", len(want)))
if r < last:
hard.append(f"L7 section '{name}' ({k}) is out of order; expected order {' > '.join(want)}")
last = max(last, r)
# L6 must-mention
for pat in must_mention:
if not re.search(pat, full, re.I):
hard.append(f"L6 required mention missing: /{pat}/")
# L10 permission slips in Asks
rep["permission_slips"] = []
for name, body in secs:
if kind_of(name) != "asks":
continue
for r in rows(body, is_html):
if AUTO_ALLOWED.search(r):
rep["permission_slips"].append(r[:120])
hard.append(f"L10 Ask requests permission already given (do it, then delete the ask): \"{r[:100]}\"")
# L5 intraday ledger blended in
entries, missing = [], []
if ledger_pending:
try:
entries = json.load(open(ledger_pending, encoding="utf-8"))
except Exception as e:
hard.append(f"L5 could not read ledger file {ledger_pending}: {e}")
hay = (full + " " + html.unescape(urllib.parse.unquote(src))).lower()
for e in entries:
keys = [k for k in (e.get("pages") or []) + (e.get("authors") or []) if k]
keys += [k.split(" - ")[0].strip() for k in keys if " - " in k and len(k.split(" - ")[0].strip()) >= 3]
if (e.get("pins") or "").strip():
keys.append(e["pins"].strip()[:40])
if keys and not any(k.lower() in hay for k in keys):
missing.append({"ts": e.get("ts"), "source": e.get("source"), "keys": keys[:4]})
rep["ledger_entries"], rep["ledger_missing"] = len(entries), missing
rep["ledger_ok"] = not missing
for m in missing:
hard.append(f"L5 intraday entry not blended in ({m['source']} {str(m['ts'])[:16]}): mention one of {m['keys']}")
# soft style report
per_k = 1000 / max(total, 1)
lex, hits = 0, {}
for wgt, pat in TIER.items():
found = [m.group(0).lower() for m in re.finditer(pat, full, flags=re.I)]
if found:
hits[str(wgt)] = sorted(set(found))[:12]
lex += wgt * len(found)
sents = [x for x in re.split(r"(?<=[.!?])\s+", full) if words(x) > 2]
lens = [words(x) for x in sents] or [0]
rep["style"] = {
"wordlist_version": WORDLIST_VERSION,
"lexical_score_per_1k": round(lex * per_k, 1), "lexical_hits": hits,
"negative_parallelism": len(re.findall(NEG_PARALLEL, full, flags=re.I)),
"participle_tails": len(re.findall(PARTICIPLE_TAIL, full, flags=re.I)),
"tricolons": len(re.findall(TRICOLON, full)),
"em_dashes_per_1k": round(full.count("—") * per_k, 1),
"sentences": len(sents), "sentence_len_mean": round(statistics.mean(lens), 1),
"sentence_len_stdev": round(statistics.pstdev(lens), 1) if len(lens) > 1 else 0,
}
rep["hard_fails"] = hard
return rep
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("file")
ap.add_argument("--json", action="store_true")
ap.add_argument("--must-mention", action="append", default=[], help="regex that must appear (repeatable)")
ap.add_argument("--ledger-pending", help="JSON from `day-ledger.py pending --json` (L5)")
ap.add_argument("--order", default=os.environ.get("DIGEST_ORDER", "today,read,other,asks"))
a = ap.parse_args()
r = lint(a.file, a.must_mention, a.ledger_pending, a.order)
if a.json:
print(json.dumps(r, indent=2, ensure_ascii=False))
else:
print(f"digest-lint: {r['words']} words (cap {MAX_WORDS}); kinds {' > '.join(r['section_kinds']) or '-'}")
for k, v in r["sections"].items():
print(f" section {k!r}: {v} words" + (" (over cap)" if v > MAX_SECTION_WORDS else ""))
if r["ledger_entries"]:
print(f" ledger: {r['ledger_entries']} intraday entries, {len(r['ledger_missing'])} not blended in")
st = r["style"]
print(f" style: lexical {st['lexical_score_per_1k']}/1k {st['lexical_hits']} - neg-parallel "
f"{st['negative_parallelism']} - participle-tails {st['participle_tails']} - tricolons {st['tricolons']} - "
f"em-dash {st['em_dashes_per_1k']}/1k - sentences {st['sentences']} mean {st['sentence_len_mean']} "
f"sd {st['sentence_len_stdev']}")
if r["hard_fails"]:
print(" HARD FAILS:")
for h in r["hard_fails"]:
print(" -", h)
else:
print(" passes the hard rules")
sys.exit(2 if r["hard_fails"] else 0)
if __name__ == "__main__":
main()
|