src/stale-canon.ts

Functions

Imports

It imports src/canon.ts, src/env.ts, src/github/client.ts, src/github/pr-files.ts, src/log.ts, src/provenance.ts, src/time.ts, src/watcher-detect.ts. Imported by src/watcher-gather.ts.

Source

1import type { Env } from "./env";
2import { isCanonPath } from "./canon";
3import { ghFetch, resolveRepo } from "./github/client";
4import { readPrFiles, type PrFileRow } from "./github/pr-files";
5import { logEvent } from "./log";
6import { externalFence } from "./provenance";
7import type { Finding } from "./watcher-detect";
8import { DAY_MS } from "./time";
9
10// STALE CANON AFTER A MERGE (Capsid improvement list, item 23: Foxhound's ROUTES.md and
11// DESIGN.md still said the admin reused the app shell after a pull request changed it).
12// Each watcher pass:
13//
14//   1. lists each mapped namespace's newest closed pull requests (its primary repo, one
15//      read each) and reads the changed files of up to MAX_PR_READS_PER_PASS that merged
16//      into the default branch and are not yet remembered. What each one removed, renamed
17//      and modified is kept in APP_KV under STALE_CANON_KEY for RETAIN_DAYS.
18//   2. matches the remembered merges against each namespace's canon documents
19//      (staleCanonDoc below) and its typed edges, and raises one digest finding per
20//      document and pull request:
21//        - a document that cites, in its body or by an edge to `<namespace>/<path>`, a
22//          file the pull request deleted or renamed (and no later merge brought back);
23//        - a document with an edge to a file the pull request modified.
24//   A document written after the merge is not flagged: its writer had the chance to read
25//   the change. So the finding clears when the document is next written, when the
26//   citation or edge goes away, or when the merge leaves the window.
27//
28// Nothing here writes a document. The finding is the whole act.
29//
30// Out of scope for this version: a document that describes the old behaviour in words
31// without naming a path (Foxhound's own case), edges from another namespace's documents,
32// and a namespace's repos other than its primary. Those need a reader of meaning, not of
33// paths.
34
35const STALE_CANON_KEY = "stale-canon:merges";
36const RETAIN_DAYS = 30;
37const RETAIN_MS = RETAIN_DAYS * DAY_MS;
38// Reads per pass: one pull request list per namespace (at most MAX_NAMESPACES), the
39// files of at most MAX_PR_READS_PER_PASS pull requests, and two D1 reads per namespace
40// with a remembered merge. A burst of merges is caught up over the next passes.
41export const MAX_NAMESPACES = 20;
42const MAX_PR_READS_PER_PASS = 5;
43const PULLS_LISTED = 20;
44// Paths kept per list per pull request. A larger change is kept cut, and says so.
45export const MAX_PATHS_PER_PR = 300;
46const MAX_DOCS_PER_NAMESPACE = 200;
47const MAX_EDGES_PER_NAMESPACE = 2000;
48// Findings per pass. Sorted first, so the same ones are shown pass after pass.
49export const MAX_FLAGS = 40;
50const CITED_SHOWN = 6;
51
52// What a merge did to the repo's files, as remembered.
53export interface MergedChange {
54  number: number;
55  url: string;
56  merged_at: string;
57  // Deleted (to: null) or renamed away (to: the new path).
58  removed: Array<{ path: string; to: string | null }>;
59  // Modified in place.
60  changed: string[];
61  // Added, or the new name of a rename: a later merge that brings a path back.
62  added: string[];
63  // Why the lists are not the whole pull request, or null.
64  partial: string | null;
65}
66
67interface NamespaceMerges {
68  repo: string;
69  prs: MergedChange[];
70}
71
72export type MergeMemory = Record<string, NamespaceMerges>;
73
74/** What one pull request's file rows remove, rename, modify and add. Pure. */
75export function changeFromFiles(rows: readonly PrFileRow[]): Pick<MergedChange, "removed" | "changed" | "added" | "partial"> {
76  const removed: MergedChange["removed"] = [];
77  const changed: string[] = [];
78  const added: string[] = [];
79  for (const f of rows) {
80    if (f.previous_filename && f.previous_filename !== f.filename) {
81      removed.push({ path: f.previous_filename, to: f.filename });
82      added.push(f.filename);
83    } else if (f.status === "removed") removed.push({ path: f.filename, to: null });
84    else if (f.status === "added" || f.status === "copied") added.push(f.filename);
85    // modified, changed, and a rename to the same name: the file is still there.
86    else changed.push(f.filename);
87  }
88  const cut = removed.length > MAX_PATHS_PER_PR || changed.length > MAX_PATHS_PER_PR || added.length > MAX_PATHS_PER_PR;
89  return {
90    removed: removed.slice(0, MAX_PATHS_PER_PR),
91    changed: changed.slice(0, MAX_PATHS_PER_PR),
92    added: added.slice(0, MAX_PATHS_PER_PR),
93    partial: cut ? `more than ${MAX_PATHS_PER_PR} paths in one list; the first ${MAX_PATHS_PER_PR} were kept` : null,
94  };
95}
96
97const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
98
99/** Whether `body` names `path` as a whole path: "src/a.ts" in "see src/a.ts." but not in
100 *  "src/a.tsx", "src/a.ts.bak" or "lib/src/a.ts". A path with neither a slash nor a dot
101 *  ("Makefile") is an ordinary word in prose and is never matched by body; an edge to it
102 *  still counts. Pure. */
103export function citesPath(body: string, path: string): boolean {
104  if (!path.includes("/") && !path.includes(".")) return false;
105  return new RegExp(`(?<![\\w./-])${escapeRe(path)}(?![\\w/-]|\\.\\w)`).test(body);
106}
107
108// The types that say how an app works, beside the canon paths (src/canon.ts).
109const CANON_TYPES: ReadonlySet<string> = new Set(["core", "spec", "protocol", "procedural", "reference"]);
110// Dated records say what was true when written (src/rule-docs.ts); an old path there is history.
111const DATED_PREFIXES = ["archive/", "research/", "reports/", "audits/", "jobs/"];
112const RETIRED_STATUSES: ReadonlySet<string> = new Set(["superseded", "closed"]);
113
114export interface CanonDoc {
115  path: string;
116  type: string | null;
117  status: string | null;
118  updated_at: string;
119  body: string | null;
120}
121
122/** Whether a document is one this check holds to the code: a canon path or a canon type,
123 *  not a dated record, not retired. Pure. */
124export function staleCanonDoc(namespace: string, doc: Pick<CanonDoc, "path" | "type" | "status">): boolean {
125  if (DATED_PREFIXES.some((p) => doc.path.startsWith(p))) return false;
126  if (doc.status && RETIRED_STATUSES.has(doc.status)) return false;
127  return isCanonPath(namespace, doc.path) || (doc.type !== null && CANON_TYPES.has(doc.type));
128}
129
130/** A D1 datetime ("2026-10-18 09:00:00", UTC) or an ISO string, as milliseconds. */
131export function instant(at: string): number {
132  const iso = /[zZ]$|[+-]\d\d:?\d\d$/.test(at) ? at : `${at.replace(" ", "T")}Z`;
133  return Date.parse(iso);
134}
135
136export interface Edge {
137  from_path: string;
138  type: string;
139  to_path: string;
140}
141
142interface Cited {
143  path: string;
144  how: "deleted" | "renamed" | "changed";
145  to: string | null;
146  via: "body" | "edge";
147}
148
149export interface StaleFlag {
150  namespace: string;
151  doc: string;
152  pr: number;
153  url: string;
154  merged_at: string;
155  cited: Cited[];
156  partial: string | null;
157}
158
159/** The documents each remembered merge leaves stale, per the rules at the top of this
160 *  file. One flag per document and pull request, naming every path it cites. Pure. */
161export function staleCanonFlags(namespace: string, merges: readonly MergedChange[], docs: readonly CanonDoc[], edges: readonly Edge[]): StaleFlag[] {
162  const ordered = [...merges].sort((a, b) => instant(a.merged_at) - instant(b.merged_at));
163  const edgesFrom = new Map<string, Set<string>>();
164  for (const e of edges) edgesFrom.set(e.from_path, (edgesFrom.get(e.from_path) ?? new Set()).add(e.to_path));
165  const flags: StaleFlag[] = [];
166  ordered.forEach((pr, i) => {
167    const merged = instant(pr.merged_at);
168    if (Number.isNaN(merged)) return;
169    // A path a later merge added back, or modified, exists again.
170    const later = new Set(ordered.slice(i + 1).flatMap((p) => [...p.added, ...p.changed]));
171    for (const doc of docs) {
172      if (!staleCanonDoc(namespace, doc)) continue;
173      const written = instant(doc.updated_at);
174      // Written since the merge, or a stamp that does not parse: not judged stale.
175      if (Number.isNaN(written) || written >= merged) continue;
176      const linked = edgesFrom.get(doc.path) ?? new Set<string>();
177      const cited: Cited[] = [];
178      for (const r of pr.removed) {
179        if (later.has(r.path)) continue;
180        const how = r.to === null ? "deleted" : "renamed";
181        if (linked.has(r.path)) cited.push({ path: r.path, how, to: r.to, via: "edge" });
182        else if (citesPath(doc.body ?? "", r.path)) cited.push({ path: r.path, how, to: r.to, via: "body" });
183      }
184      for (const path of pr.changed) {
185        if (linked.has(path)) cited.push({ path, how: "changed", to: null, via: "edge" });
186      }
187      if (cited.length > 0) flags.push({ namespace, doc: doc.path, pr: pr.number, url: pr.url, merged_at: pr.merged_at, cited, partial: pr.partial });
188    }
189  });
190  return flags;
191}
192
193// FNV-1a, 32 bits: a short stable tag for a document path inside a fingerprint, which
194// must hold no "]" (the title carries it in brackets).
195function tag(text: string): string {
196  let h = 0x811c9dc5;
197  for (let i = 0; i < text.length; i++) {
198    h ^= text.charCodeAt(i);
199    h = Math.imul(h, 0x01000193) >>> 0;
200  }
201  return h.toString(16).padStart(8, "0");
202}
203
204const STALE_CANON_PREFIX = "stale-canon-";
205
206/** One fingerprint per document and pull request: stable while both stand. */
207export function staleCanonFingerprint(flag: Pick<StaleFlag, "namespace" | "doc" | "pr">): string {
208  return `${STALE_CANON_PREFIX}${flag.namespace}-${flag.pr}-${tag(flag.doc)}`;
209}
210
211const describe = (c: Cited): string =>
212  `${c.path} (${c.how === "renamed" ? `renamed to ${c.to}` : c.how}; cited by ${c.via === "edge" ? "a typed edge" : "path in the body"})`;
213
214/** The flags as watcher findings, sorted and capped at MAX_FLAGS. Pure. */
215export function staleCanonFindings(flags: readonly StaleFlag[]): Finding[] {
216  const sorted = [...flags].sort((a, b) => a.namespace.localeCompare(b.namespace) || b.pr - a.pr || a.doc.localeCompare(b.doc));
217  return sorted.slice(0, MAX_FLAGS).map((f) => {
218    const fingerprint = staleCanonFingerprint(f);
219    const first = f.cited[0]!;
220    const more = f.cited.length > 1 ? ` and ${f.cited.length - 1} more` : "";
221    const verb = first.how === "renamed" ? "renamed" : first.how;
222    const headline = `${f.namespace}/${f.doc} may be stale after PR #${f.pr}: it cites ${first.path}${more}, which the PR ${verb}`;
223    const evidence = [
224      `document: ${f.namespace}/${f.doc}`,
225      `pull request: ${f.url}, merged ${f.merged_at}`,
226      ...f.cited.slice(0, CITED_SHOWN).map((c) => `cites ${describe(c)}`),
227      ...(f.cited.length > CITED_SHOWN ? [`and ${f.cited.length - CITED_SHOWN} more cited path(s)`] : []),
228      ...(f.partial ? [`the pull request's file list was kept cut: ${f.partial}`] : []),
229      "Read the document against the pull request and rewrite what no longer holds. Writing it clears this; nothing was rewritten.",
230    ];
231    return {
232      fingerprint,
233      namespace: f.namespace,
234      title: `Watcher: ${headline} [${fingerprint}]`,
235      body: [
236        "A canon document cites a file a merged pull request removed, renamed or changed (src/stale-canon.ts).",
237        "",
238        "## Evidence",
239        "",
240        externalFence("watcher-evidence", fingerprint, evidence.map((line) => `- ${line}`).join("\n")),
241      ].join("\n"),
242      evidence,
243    };
244  });
245}
246
247// ---------------------------------------------------------------------------
248// The memory, and the merges still to read.
249
250/** The stored memory, or null with the reason when it does not parse. Absent is empty. */
251export function parseMemory(raw: string | null): { memory: MergeMemory } | { problem: string } {
252  if (raw === null) return { memory: {} };
253  try {
254    const parsed: unknown = JSON.parse(raw);
255    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return { problem: "not an object" };
256    for (const v of Object.values(parsed as Record<string, unknown>)) {
257      const ns = v as Partial<NamespaceMerges> | null;
258      if (!ns || typeof ns.repo !== "string" || !Array.isArray(ns.prs)) return { problem: "a namespace entry has no repo or prs" };
259    }
260    return { memory: parsed as MergeMemory };
261  } catch (err) {
262    return { problem: err instanceof Error ? err.message : String(err) };
263  }
264}
265
266/** The memory without the merges older than RETAIN_DAYS. Pure. */
267export function pruneMemory(memory: MergeMemory, now: Date): MergeMemory {
268  const out: MergeMemory = {};
269  for (const [ns, entry] of Object.entries(memory)) {
270    const prs = entry.prs.filter((p) => now.getTime() - instant(p.merged_at) <= RETAIN_MS);
271    if (prs.length > 0) out[ns] = { repo: entry.repo, prs };
272  }
273  return out;
274}
275
276export interface ListedPull {
277  number: number;
278  html_url: string;
279  merged_at: string | null;
280  base?: { ref?: string; repo?: { default_branch?: string } };
281}
282
283/** The listed pull requests that merged into the default branch inside the window and
284 *  are not remembered yet, oldest first. Pure. */
285export function newMerges(listed: readonly ListedPull[], known: ReadonlySet<number>, now: Date): ListedPull[] {
286  return listed
287    .filter((p) => p.merged_at !== null && !known.has(p.number))
288    .filter((p) => p.base?.ref !== undefined && p.base.ref === p.base.repo?.default_branch)
289    .filter((p) => now.getTime() - instant(p.merged_at!) <= RETAIN_MS)
290    .sort((a, b) => instant(a.merged_at!) - instant(b.merged_at!));
291}
292
293/** The namespaces this pass reads, from a list read with one row past the cap, and the
294 *  memory with every namespace no longer mapped forgotten. Past the cap the list is not
295 *  the whole mapping, so a namespace missing from it may still be mapped: nothing is
296 *  forgotten then, and the pass is not complete. Mutates `memory`. */
297export function namespacesThisPass(rows: readonly string[], memory: MergeMemory): { listed: string[]; capped: boolean } {
298  const capped = rows.length > MAX_NAMESPACES;
299  const listed = rows.slice(0, MAX_NAMESPACES);
300  if (!capped) {
301    const names = new Set(listed);
302    for (const ns of Object.keys(memory)) if (!names.has(ns)) delete memory[ns];
303  }
304  return { listed, capped };
305}
306
307// ---------------------------------------------------------------------------
308// The pass.
309
310export interface StaleCanonRead {
311  findings: Finding[];
312  // The memory and every namespace's documents were read, so a finding not raised has
313  // cleared. A GitHub read that failed only delays a merge, so it does not count here.
314  complete: boolean;
315  note: string;
316}
317
318export async function readStaleCanon(env: Env, now: Date): Promise<StaleCanonRead> {
319  let complete = true;
320  const parsed = parseMemory(await env.APP_KV.get(STALE_CANON_KEY));
321  if ("problem" in parsed) {
322    // Logged and started again, never silently dropped. The findings it held are not
323    // cleared on this pass: the check does not count as run.
324    logEvent("error", "STALE_CANON_MEMORY_UNREADABLE", { message: `STALE_CANON_MEMORY_UNREADABLE: ${parsed.problem}; starting it again` });
325    complete = false;
326  }
327  const before = "memory" in parsed ? JSON.stringify(parsed.memory) : null;
328  const memory = pruneMemory("memory" in parsed ? parsed.memory : {}, now);
329
330  const { results: mapped } = await env.DB.prepare(
331    "SELECT namespace FROM namespaces WHERE repos IS NOT NULL AND repos NOT IN ('', '[]') ORDER BY namespace LIMIT ?1"
332  )
333    .bind(MAX_NAMESPACES + 1)
334    .all<{ namespace: string }>();
335
336  // Every namespace's new merges, then the oldest MAX_PR_READS_PER_PASS of them read.
337  // A namespace no longer mapped is forgotten, unless the list hit MAX_NAMESPACES.
338  const { listed, capped } = namespacesThisPass((mapped ?? []).map((m) => m.namespace), memory);
339  if (capped) complete = false;
340  const pending: Array<{ namespace: string; owner: string; repo: string; pull: ListedPull }> = [];
341  const unread: string[] = [];
342  for (const namespace of listed) {
343    try {
344      const { owner, repo, full } = await resolveRepo(env, namespace);
345      // A namespace remapped to another repo starts again: its paths were another repo's.
346      if (memory[namespace] && memory[namespace].repo !== full) delete memory[namespace];
347      const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/pulls?state=closed&sort=updated&direction=desc&per_page=${PULLS_LISTED}`);
348      if (!resp.ok) throw new Error(`the pull request list returned ${resp.status}`);
349      const listed = (await resp.json()) as ListedPull[];
350      const known = new Set((memory[namespace]?.prs ?? []).map((p) => p.number));
351      for (const pull of newMerges(Array.isArray(listed) ? listed : [], known, now)) pending.push({ namespace, owner, repo, pull });
352      memory[namespace] ??= { repo: full, prs: [] };
353    } catch (err) {
354      unread.push(`${namespace} (${err instanceof Error ? err.message : String(err)})`);
355    }
356  }
357  pending.sort((a, b) => instant(a.pull.merged_at!) - instant(b.pull.merged_at!));
358  let read = 0;
359  for (const p of pending.slice(0, MAX_PR_READS_PER_PASS)) {
360    try {
361      const files = await readPrFiles(env, p.owner, p.repo, p.pull.number);
362      const change = changeFromFiles(files.rows);
363      memory[p.namespace]!.prs.push({
364        number: p.pull.number,
365        url: p.pull.html_url,
366        merged_at: p.pull.merged_at!,
367        ...change,
368        partial: [files.problem, change.partial].filter(Boolean).join("; ") || null,
369      });
370      read++;
371    } catch (err) {
372      unread.push(`${p.namespace} #${p.pull.number} (${err instanceof Error ? err.message : String(err)})`);
373    }
374  }
375  const kept = pruneMemory(memory, now);
376  if (JSON.stringify(kept) !== before) await env.APP_KV.put(STALE_CANON_KEY, JSON.stringify(kept));
377
378  const flags: StaleFlag[] = [];
379  for (const [namespace, entry] of Object.entries(kept)) {
380    // The candidates in SQL; staleCanonDoc decides.
381    const [docs, edges] = await env.DB.batch([
382      env.DB.prepare(
383        `SELECT path, type, status, updated_at, body FROM documents
384         WHERE namespace = ?1 AND path NOT LIKE 'archive/%'
385           AND (type IN (SELECT value FROM json_each(?2)) OR path = 'core.md' OR path = 'decisions.md' OR path LIKE 'conventions%')
386         ORDER BY path LIMIT ?3`
387      ).bind(namespace, JSON.stringify([...CANON_TYPES]), MAX_DOCS_PER_NAMESPACE),
388      env.DB.prepare("SELECT from_path, type, to_path FROM document_links WHERE from_ns = ?1 AND to_ns = ?1 LIMIT ?2").bind(namespace, MAX_EDGES_PER_NAMESPACE),
389    ]);
390    flags.push(...staleCanonFlags(namespace, entry.prs, (docs?.results ?? []) as CanonDoc[], (edges?.results ?? []) as Edge[]));
391  }
392  const findings = staleCanonFindings(flags);
393  const remembered = Object.values(kept).reduce((n, e) => n + e.prs.length, 0);
394  const note =
395    `${listed.length} namespace(s) listed${capped ? ` (more than ${MAX_NAMESPACES} are mapped; the rest were not read)` : ""}, ${read} of ${pending.length} new merge(s) read, ${remembered} remembered, ${flags.length} stale document flag(s)` +
396    (flags.length > MAX_FLAGS ? `, the first ${MAX_FLAGS} raised` : "") +
397    (unread.length ? `; not read: ${unread.join(", ")}` : "");
398  return { findings, complete, note };
399}