src/watcher-detect.ts

Functions

Imports

It imports src/health.ts, src/live-checks.ts, src/ops-sites.ts, src/ops-snapshot.ts, src/ops-types/sites.ts, src/provenance.ts, src/status.ts, src/time.ts, src/unmapped-repos.ts. Imported by src/anomaly.ts, src/deploy-outcomes.ts, src/domain-expiry.ts, src/stale-canon.ts, src/watcher-gather.ts, src/watcher.ts.

Source

1import { type HealthReport, BACKUP_STALE_HOURS } from "./health";
2import type { SiteCloudflare } from "./ops-types/sites";
3import type { OpsSnapshot, SiteProbe } from "./ops-snapshot";
4import type { SiteMapDrift } from "./ops-sites";
5import type { LiveFinding } from "./live-checks";
6import { externalFence } from "./provenance";
7import type { StatusReport } from "./status";
8import { type RepoMapRead, unmappedRepos } from "./unmapped-repos";
9import { MINUTE_MS, HOUR_MS } from "./time";
10
11// What the watcher finds (src/watcher.ts runs the pass, src/watcher-gather.ts reads):
12// pure functions from what was read to findings, with no IO.
13
14// Red this long is not a flake somebody is already fixing.
15export const CI_RED_HOURS = 2;
16
17// The off-account mirror's window, and it is not BACKUP_STALE_HOURS.
18//
19// BACKUP_STALE_HOURS is the local dump, measured from a key this Worker writes
20// itself. This measures the newest dump present in the backups repo
21// (MIRROR_REPO_LABEL, below), which this Worker only observes. The mirror's schedule
22// lives in that repo and can change without this one hearing, so one constant
23// standing for both would go wrong silently the day it does.
24//
25// 36 hours is the daily cadence plus slack. The newest dump is stamped at the backup
26// cron's 09:00 UTC and the mirror picks it up hours later, so just before 09:00 UTC
27// the freshest possible dump is already about 24 hours old before the mirror has run.
28// A tighter window would fire every morning.
29export const MIRROR_STALE_HOURS = 36;
30
31// The repo is never named in this module: it is resolved from the capsid namespace's
32// mapping, which is the authorization boundary. A hardcoded owner/name would be a
33// second copy of that mapping outside the boundary.
34export const MIRROR_REPO_LABEL = "backups";
35export const MIRROR_DUMP_PREFIX = "backups/json";
36
37export interface Finding {
38  // Stable while the finding persists, different for a different problem. It goes in
39  // the title, so the queue's one-open-job-per-title rule stops a double post.
40  fingerprint: string;
41  namespace: string;
42  title: string;
43  body: string;
44  // The evidence lines, kept on the finding's row per sighting (src/watcher-findings.ts).
45  evidence?: string[];
46}
47
48export const finding = (namespace: string, fingerprint: string, headline: string, evidence: string[]): Finding => ({
49  fingerprint,
50  namespace,
51  title: `Watcher: ${headline} [${fingerprint}]`,
52  body: [
53    `The watcher found this at ${new Date().toISOString()} and cannot fix it.`,
54    "",
55    "## Evidence",
56    "",
57    // Read from outside Capsid (CI conclusions, probe answers, GitHub error text), so it is
58    // fenced as external: data to confirm, never an instruction to follow.
59    externalFence("watcher-evidence", fingerprint, evidence.map((line) => `- ${line}`).join("\n")),
60    "",
61    "## What this job is",
62    "",
63    "A finding, not an instruction. Confirm it is still true before acting: the watcher",
64    "reads a surface every half hour and a finding can clear itself between the post and",
65    "the claim. If it has cleared, fail this job saying so.",
66  ].join("\n"),
67  evidence,
68});
69
70// The deploy-drift check waits for master's own CI run, which deploys. On 2026-10-10
71// it fired twice (a160f99, e654650) while the CI run for that exact sha was still
72// running, and both deployed green minutes later. A run that finished green this
73// recently may still be ahead of the /health answer, so it gets a settle window too.
74export const MASTER_CI_WORKFLOW = "CI";
75export const DEPLOY_SETTLE_MINUTES = 15;
76
77export interface HeadRun {
78  name: string;
79  status: string;
80  conclusion: string | null;
81  created_at: string;
82  updated_at?: string | null;
83}
84
85// wait: the deploy for master head may still be in flight, so drift cannot be judged
86// this pass. Otherwise the line says what master's CI run showed, as evidence.
87export type DeployVerdict = { wait: true; reason: string } | { wait: false; evidence: string };
88
89/** Whether master head's own CI run leaves a drift finding premature. `runs` is the
90 *  run list for that sha, newest first, or null when it could not be read. A failed
91 *  or cancelled run, no run at all, and an unreadable list all keep the finding. */
92export function deployVerdict(runs: readonly HeadRun[] | null, now: Date): DeployVerdict {
93  if (runs === null) return { wait: false, evidence: `master head's ${MASTER_CI_WORKFLOW} run could not be read` };
94  const run = runs
95    .filter((r) => r.name === MASTER_CI_WORKFLOW)
96    .sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))[0];
97  if (!run) return { wait: false, evidence: `no ${MASTER_CI_WORKFLOW} run started for master head` };
98  if (run.status !== "completed") return { wait: true, reason: `master head's ${MASTER_CI_WORKFLOW} run is ${run.status}` };
99  if (run.conclusion !== "success") return { wait: false, evidence: `master head's ${MASTER_CI_WORKFLOW} run ended ${run.conclusion ?? "with no conclusion"}` };
100  const finished = run.updated_at ? Date.parse(run.updated_at) : Number.NaN;
101  if (Number.isNaN(finished)) return { wait: false, evidence: `master head's ${MASTER_CI_WORKFLOW} run succeeded at an unknown time` };
102  const minutes = (now.getTime() - finished) / MINUTE_MS;
103  if (minutes < DEPLOY_SETTLE_MINUTES) {
104    return { wait: true, reason: `master head's ${MASTER_CI_WORKFLOW} run succeeded ${Math.floor(minutes)} minutes ago, inside the ${DEPLOY_SETTLE_MINUTES} minute settle` };
105  }
106  return { wait: false, evidence: `master head's ${MASTER_CI_WORKFLOW} run succeeded ${Math.floor(minutes)} minutes ago` };
107}
108
109/** What /health says that should not be true. `masterSha` and `latestMigration` come
110 *  from the repo, so this answers "is the deployed Worker the one master describes"
111 *  rather than only "is it up". `deploy` is master head's CI verdict; a deploy still in
112 *  flight is not drift. */
113export function healthFindings(
114  health: HealthReport,
115  masterSha: string | null,
116  latestMigration: string | null,
117  namespace: string,
118  deploy: DeployVerdict = { wait: false, evidence: `master head's ${MASTER_CI_WORKFLOW} run was not read` }
119): Finding[] {
120  const out: Finding[] = [];
121  if (health.status !== "ok") {
122    out.push(
123      finding(namespace, "health-degraded", "the Worker reports degraded", [
124        `status: ${health.status}`,
125        `store: d1 ${health.store.d1}, fts ${health.store.fts}`,
126        "Every read tool errors while a store probe is failing.",
127      ])
128    );
129  }
130  // A sha comparison needs both sides. A null master sha means the repo read failed,
131  // and reporting that as drift would be a finding about the watcher, not the deploy.
132  if (masterSha && health.sha && health.sha !== "unknown" && health.sha !== masterSha && !deploy.wait) {
133    out.push(
134      finding(namespace, `deploy-drift-${masterSha.slice(0, 7)}`, "the deployed sha is not master head", [
135        `deployed: ${health.sha}`,
136        `master: ${masterSha}`,
137        deploy.evidence,
138        "A docs-only push whose deploy failed, or a deploy that never ran, looks exactly like this.",
139      ])
140    );
141  }
142  if (health.backup.age_hours !== null && health.backup.age_hours > BACKUP_STALE_HOURS) {
143    out.push(
144      finding(namespace, "backup-stale", "the last backup is older than the window", [
145        `last ok: ${health.backup.last_ok ?? "never"}`,
146        `age: ${health.backup.age_hours.toFixed(1)} hours, the window is ${BACKUP_STALE_HOURS}`,
147      ])
148    );
149  }
150  // A stamp /health cannot parse has no age, so the window check above cannot fire and
151  // last_ok is not null. Its age is unknown, which is reported as stale.
152  if (health.backup.last_ok !== null && health.backup.age_hours === null) {
153    out.push(
154      finding(namespace, "backup-stale", "the last backup's time cannot be read", [
155        `last ok: ${health.backup.last_ok}`,
156        "/health reports age_hours as null, so the backup's age is unknown.",
157      ])
158    );
159  }
160  if (health.backup.last_ok === null) {
161    out.push(
162      finding(namespace, "backup-never", "no backup has ever reported success", [
163        "/health reports backup.last_ok as null.",
164        "The nightly dump can fail every night with no signal but this key.",
165      ])
166    );
167  }
168  // Ahead is a migration applied before its PR merged, which the merge clears: a digest
169  // line, not a job. Behind is a committed migration never applied. Migration names
170  // start with a zero-padded number, so the prefix orders them.
171  const ahead =
172    latestMigration && health.schema_version && migrationNumber(health.schema_version) > migrationNumber(latestMigration);
173  if (ahead) {
174    out.push(
175      finding(namespace, `schema-ahead-${health.schema_version}`, "the live schema is newer than master's newest migration", [
176        `live: ${health.schema_version}`,
177        `newest on master: ${latestMigration}`,
178        "A migration applied before its pull request merged reads like this; the merge clears it.",
179      ])
180    );
181  } else if (latestMigration && health.schema_version && health.schema_version !== latestMigration) {
182    out.push(
183      finding(namespace, `schema-behind-${latestMigration}`, "the live schema is not the newest migration", [
184        `live: ${health.schema_version}`,
185        `newest on master: ${latestMigration}`,
186        "A migration that was committed and never applied reads exactly like this.",
187      ])
188    );
189  }
190  return out;
191}
192
193function migrationNumber(name: string): number {
194  const n = Number.parseInt(name, 10);
195  return Number.isNaN(n) ? -1 : n;
196}
197
198/** What capsid_run_improve action status says that should not be true. Per namespace, because the job
199 *  each finding becomes belongs to the namespace it is about. */
200export function statusFindings(status: StatusReport): Finding[] {
201  const out: Finding[] = [];
202  for (const ns of status.namespaces ?? []) {
203    // An open breaker stops the namespace's queue until a person resets it, so it is
204    // said; the watcher files it and nothing more (src/job-breaker.ts).
205    if (ns.breaker?.open) {
206      out.push(
207        finding(ns.namespace, `breaker-open-${ns.namespace}`, `the queue's circuit breaker for ${ns.namespace} is open`, [
208          `holder fails since ${ns.breaker.since} UTC: ${ns.breaker.failed}, threshold ${ns.breaker.threshold}`,
209          "No job in this namespace is handed out until the seat or Dustin resets it: capsid_admin_control_improve action breaker_reset, or Reset under Paused after repeated failures on the Portal's Needs you.",
210        ])
211      );
212    }
213  }
214  return out;
215}
216
217export interface CiRun {
218  head_sha: string;
219  status: string;
220  conclusion: string | null;
221  created_at: string;
222  url?: string;
223}
224
225/** A default branch that has been red for longer than a flake. Reads the most recent
226 *  COMPLETED run, because a run still in flight is not yet an answer.
227 *
228 *  One incident per namespace, not per commit: the fingerprint is `ci-red-<namespace>`
229 *  and the head sha is evidence, so every red commit on a red branch is not a new finding. */
230export function ciFindings(namespace: string, runs: CiRun[], now: Date): Finding[] {
231  const latest = runs.find((r) => r.status === "completed");
232  if (!latest) return [];
233  if (latest.conclusion === "success" || latest.conclusion === "skipped" || latest.conclusion === "neutral") return [];
234  const since = Date.parse(latest.created_at);
235  if (Number.isNaN(since)) return [];
236  const hours = (now.getTime() - since) / HOUR_MS;
237  if (hours < CI_RED_HOURS) return [];
238  return [
239    finding(namespace, `ci-red-${namespace}`, `${namespace} CI is red on its default branch`, [
240      `conclusion: ${latest.conclusion}`,
241      `head sha: ${latest.head_sha}`,
242      `red since: ${latest.created_at} (${hours.toFixed(1)} hours)`,
243    ]),
244  ];
245}
246
247// The off-account mirror. Measure the dump, not the attempt. A credential request
248// proves the mirror started; a run can pass its credential step and then fail before
249// the dump lands. Only a dump proves a backup exists.
250//
251// The timestamp comes from the directory name, not the commit date or message. The
252// mirror writes backups/json/<dump timestamp>/, so the directory names are the thing
253// being asked about. A commit date answers "when did the mirror last push", which is
254// the same only while the mirror is healthy, and a commit message would couple this
255// check to how the other repo words its commits.
256
257// `2026-09-12T09-00-11-132Z` as the backup writes it. Anchored, so a stray directory
258// yields null rather than looking like a fresh backup.
259const DUMP_STAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/;
260
261export function parseDumpStamp(name: string): Date | null {
262  const m = DUMP_STAMP.exec(name);
263  if (!m) return null;
264  const [, y, mo, d, h, mi, s, ms] = m;
265  const at = new Date(`${y}-${mo}-${d}T${h}:${mi}:${s}.${ms}Z`);
266  return Number.isNaN(at.getTime()) ? null : at;
267}
268
269/** The newest dump present, from listRepoTree entries under backups/json/. Null when
270 *  none of the entries parse, which is "no dump" and not "a very old dump". */
271export function newestDump(entries: Array<{ path?: string }>): Date | null {
272  let newest: Date | null = null;
273  for (const entry of entries) {
274    const name = (entry.path ?? "").split("/").filter(Boolean).pop() ?? "";
275    const at = parseDumpStamp(name);
276    if (at && (!newest || at > newest)) newest = at;
277  }
278  return newest;
279}
280
281export interface MirrorRun {
282  name?: string;
283  status?: string;
284  conclusion?: string | null;
285  created_at?: string;
286  url?: string;
287}
288
289/** The mirror's latest completed run, matched on workflow name (what the runs API
290 *  returns). */
291export function latestMirrorRun(runs: MirrorRun[]): MirrorRun | null {
292  return runs.find((r) => r.status === "completed" && (r.name ?? "").toLowerCase().includes("mirror")) ?? null;
293}
294
295/** The dump age decides whether there is a finding; the run conclusion explains why.
296 *  Three states need three different actions, so they are three findings, and
297 *  collapsing them is how the third one gets missed: a green run with no new dump is
298 *  the one a conclusion-first check cannot see at all. */
299export function mirrorFindings(
300  namespace: string,
301  newest: Date | null,
302  runs: MirrorRun[],
303  now: Date
304): Finding[] {
305  // No dump at all is its own fact. "Never mirrored" is a setup that never worked;
306  // "stopped mirroring" is a regression in one that did.
307  if (newest === null) {
308    return [
309      finding(namespace, "mirror-no-dump", "the off-account mirror holds no dump at all", [
310        "No directory under backups/json/ parses as a dump timestamp.",
311        "Either the mirror has never succeeded, or the layout changed and this check is reading the wrong place.",
312      ]),
313    ];
314  }
315
316  const hours = (now.getTime() - newest.getTime()) / HOUR_MS;
317  if (hours < MIRROR_STALE_HOURS) return [];
318
319  const age = `newest dump: ${newest.toISOString()} (${hours.toFixed(1)} hours old, the window is ${MIRROR_STALE_HOURS})`;
320  const latest = latestMirrorRun(runs);
321
322  // (b) Nothing has run. The schedule is off, Actions is off, or the repo is archived.
323  if (!latest) {
324    return [
325      finding(namespace, "mirror-not-running", "the off-account mirror is stale and its workflow has not run", [
326        age,
327        "No completed mirror run was returned for this repo.",
328        "A disabled schedule, Actions turned off, or an archived repo all look like this.",
329      ]),
330    ];
331  }
332
333  const conclusion = latest.conclusion ?? "none";
334  const green = conclusion === "success" || conclusion === "skipped" || conclusion === "neutral";
335
336  // (c) Green and no dump, which a conclusion-first check reports as healthy.
337  if (green) {
338    return [
339      finding(namespace, "mirror-green-no-dump", "the off-account mirror ran green and no new dump appeared", [
340        age,
341        `last run: ${latest.created_at ?? "unknown"} concluded ${conclusion}`,
342        latest.url ? `run: ${latest.url}` : "no run url reported",
343        "A commit step that no-ops, or a sync that wrote nothing, passes CI and backs up nothing.",
344      ]),
345    ];
346  }
347
348  // (a) Ran and failed.
349  return [
350    finding(namespace, `mirror-run-failed-${conclusion}`, "the off-account mirror's workflow is failing", [
351      age,
352      `last run: ${latest.created_at ?? "unknown"} concluded ${conclusion}`,
353      latest.url ? `run: ${latest.url}` : "no run url reported",
354      "The mirror commits the dump in its last step, so a red run anywhere earlier means no backup landed.",
355    ]),
356  ];
357}
358
359/** A registered namespace the site map neither covers nor excludes, or a map entry
360 *  whose namespace is not registered. The fingerprint names the drift, so a different
361 *  drift is a new finding. */
362export function siteMapFindings(drift: SiteMapDrift): Finding[] {
363  if (drift.unmapped.length === 0 && drift.unknown.length === 0) return [];
364  const names = [...drift.unmapped.map((n) => "add-" + n), ...drift.unknown.map((n) => "drop-" + n)].join("-");
365  return [
366    finding("capsid", "site-map-drift-" + names, "the operations site map does not match the registered namespaces", [
367      ...drift.unmapped.map((n) => n + ": registered, but has no row in the Portal's site configuration (Settings: add it as a site, or as serving no site)"),
368      ...drift.unknown.map((n) => n + ": in the Portal's site configuration, but not a registered namespace"),
369      "The Portal shows only configured sites, so an unmapped site is one nobody is watching.",
370    ]),
371  ];
372}
373
374/** One finding per repo the GitHub App can see that no namespace maps, naming it and the
375 *  namespace its name suggests. It proposes; mapping is admin-only and never automatic.
376 *  Every finding says what the read covered, so a clear pass is never "0 unmapped"
377 *  over "0 listed". */
378export function unmappedRepoFindings(read: RepoMapRead): Finding[] {
379  const covered = `the App's ${read.installations} installation(s) list ${read.installed.length} repos; ${read.mapped.size} are mapped to a namespace; ${read.onPurpose.size} are named unmapped on purpose`;
380  return unmappedRepos(read).map((r) =>
381    finding("capsid", `unmapped-repo-${r.full_name.toLowerCase().replace("/", "--")}`, `${r.full_name} is visible to the GitHub App but no namespace maps it`, [
382      `${r.full_name}: ${r.private ? "private" : "public"}, ${r.archived ? "archived" : "not archived"}, last push ${r.pushed_at ?? "unknown"}`,
383      r.suggested
384        ? `likely namespace: ${r.suggested} (from the repo name). Map it with capsid_admin_manage_namespaces action update, adding ${r.full_name} with a label.`
385        : "no registered namespace matches its name. Register one with capsid_admin_manage_namespaces action register, or map it into an existing namespace with action update.",
386      "If it is unmapped on purpose (archived, frozen, or going away), list it in capsid/unmapped-repos.md, one owner/name per line, and this stops.",
387      covered,
388      "This check only proposes. Mapping is the authorization boundary and stays with the seat or Dustin.",
389    ])
390  );
391}
392
393/** The App's repo list could not be read whole, so unmapped repos cannot be found. Fails
394 *  closed: this is a finding, and the "repo map" check does not count as run, so an open
395 *  unmapped-repo finding is not cleared on no evidence. */
396export function repoMapUnreadableFinding(reason: string): Finding {
397  return finding("capsid", "unmapped-repos-unreadable", "the GitHub App's repository list could not be read, so unmapped repos cannot be found", [
398    reason,
399    "Until this reads, no repo is being checked against the namespace mapping.",
400  ]);
401}
402
403export const liveFinding = (f: LiveFinding): Finding => finding(f.namespace, f.fingerprint, f.headline, f.evidence);
404
405/** A site down on two probes in a row: the previous snapshot's ring ended in '0' and
406 *  this pass's probe is down too. One failed probe is a blip; a ring that ended in '-'
407 *  (no pass reached that slot) is not a failed probe. */
408export function siteDownFindings(prev: OpsSnapshot | null, probes: SiteProbe[]): Finding[] {
409  const before = new Map((prev?.sites ?? []).map((s) => [s.namespace, s]));
410  const out: Finding[] = [];
411  for (const p of probes) {
412    const last = before.get(p.namespace);
413    if (p.state !== "down" || !last || !last.ring.endsWith("0")) continue;
414    out.push(
415      finding(p.namespace, `site-down-${p.namespace}`, `${p.name} is down on two probes in a row`, [
416        `site: ${p.origin}`,
417        `this probe: ${p.error ?? `answered ${p.http_status ?? "nothing"}`} at ${p.checked_at}`,
418        `the previous probe was down too, at ${last.checked_at}`,
419      ])
420    );
421  }
422  return out;
423}
424
425// The error-rate threshold (capsid/research/design-ops-console.md, approved): more
426// than 1 percent of requests errored in the most recent complete hour, over at least
427// 100 requests, so a quiet site's single error is not an alert.
428export const ERROR_RATE_THRESHOLD = 0.01;
429export const ERROR_RATE_MIN_REQUESTS = 100;
430
431/** A site whose Worker errored on more than ERROR_RATE_THRESHOLD of its requests in
432 *  the most recent complete hour, the last of its hourly buckets. A site with no
433 *  analytics this pass is not judged. */
434export function siteErrorFindings(sites: ReadonlyArray<{ namespace: string; name: string }>, cloudflare: Record<string, SiteCloudflare>): Finding[] {
435  const out: Finding[] = [];
436  for (const site of sites) {
437    const cf = cloudflare[site.namespace];
438    if (cf?.state !== "ok" || !cf.errors24 || cf.errors24.length === 0) continue;
439    const hour = cf.errors24[cf.errors24.length - 1];
440    if (hour.requests < ERROR_RATE_MIN_REQUESTS || hour.errors / hour.requests <= ERROR_RATE_THRESHOLD) continue;
441    out.push(
442      finding(site.namespace, `site-errors-${site.namespace}`, `${site.name} errored on more than ${ERROR_RATE_THRESHOLD * 100} percent of requests`, [
443        `script: ${cf.script}`,
444        `hour from ${hour.hour}: ${hour.errors} errors of ${hour.requests} requests (${((hour.errors / hour.requests) * 100).toFixed(1)} percent)`,
445        "Cloudflare counts an invocation that threw, exceeded its resources or hit an internal error.",
446      ])
447    );
448  }
449  return out;
450}