src/maintenance.ts

Functions

Imports

It imports src/env.ts, src/maintenance-branches.ts, src/maintenance-deploys.ts, src/maintenance-prs.ts, src/maintenance-releases.ts, src/model-drift.ts, src/ops-types/views.ts, src/public-data.ts, src/roster.ts, src/rule-docs.ts, src/time.ts. Imported by src/maintenance-branches.ts, src/maintenance-deploys.ts, src/maintenance-prs.ts, src/maintenance-releases.ts, src/portal-maintenance.ts, src/status.ts, src/tick.ts.

Source

1import type { Env } from "./env";
2import type { OpsMaintenanceItem } from "./ops-types/views";
3import { ROSTER } from "./roster";
4import { gatherBranchItems, githubBranchReaders, type BranchReaders } from "./maintenance-branches";
5import { gatherPrItems, githubPrReaders, type PrReaders } from "./maintenance-prs";
6import { gatherDeployItems, githubDeployReaders, type DeployReaders } from "./maintenance-deploys";
7import { gatherReleaseItems, githubReleaseReaders, type ReleaseReaders } from "./maintenance-releases";
8import { gatherModelDriftItems } from "./model-drift";
9import { bundledModel } from "./public-data";
10import { gatherRuleDocItems } from "./rule-docs";
11import { DAY_MS } from "./time";
12
13// The actor the merge-resume step records its touches under. Spelled here, not imported, so this pass does not depend on
14// that step's pull request having landed: until it does, the query finds no rows.
15const MERGE_RESUME_ACTOR = "system:merge-resume";
16
17// THE DAILY MAINTENANCE PASS (job_549550d73d4e, Dustin 2026-10-07: routine maintenance
18// keeps not happening, and each stale thing was found only when he asked). One pass a day
19// on the five-minute tick, reusing its stamp pattern rather than adding a second scheduler.
20// Every finding either was acted on by a step that already audit-logs (the merge-resume
21// step) or lands in one short list, served by capsid_run_improve action status, that the seat reads at the
22// start of a session. This pass lists; it acts twice. It prunes merged branches, which is
23// off until the seat turns it on and audit-logs each delete (src/maintenance-branches.ts),
24// and it tags a shared package's merged version bump, which is on until the seat turns it
25// off and audit-logs each tag (src/maintenance-releases.ts).
26//
27// The job rules need only the database. The pull request rules (src/maintenance-prs.ts)
28// and the branch rules (src/maintenance-branches.ts) read GitHub for every roster
29// namespace. The undeployed-merge rule (src/maintenance-deploys.ts) reads the
30// watcher's snapshot and each site's default branch. The release-tag rule
31// (src/maintenance-releases.ts) reads each shared package's repo.
32//
33// The release tags ride this daily pass rather than the five-minute tick: a merged bump
34// is tagged within a day, which is soon enough for the apps' bump pull requests the tag
35// starts, and the pass already has the switch, the cap, the audit pattern and the list
36// the seat reads, where a version it would not tag (below the newest tag) belongs.
37
38const MAINTENANCE_KEY = "maintenance:list";
39export const MAINTENANCE_LAST_KEY = "maintenance:last";
40
41// The pass runs once per UTC day, after this hour, so the list is fresh when the seat
42// starts a session in the morning (06:00 in Chicago is 11:00 or 12:00 UTC).
43const DAILY_AFTER_UTC_HOUR = 11;
44const MAX_ITEMS_PER_RULE = 50;
45const FOLLOWUP_PROMISE = /follow(?:[- ]?ups?)?\b[^.]{0,60}\bseparate jobs?/i;
46const LATER_DATE = /\bLATER\b[^0-9]{0,40}(\d{4}-\d{2}-\d{2})/i;
47
48// The item is the Portal's contract too (GET /portal/api/maintenance), so it lives with the
49// other Portal types in src/ops-types/.
50/** The model drift check, injected so a test can run the pass without GitHub. */
51export type DriftReader = (env: Env) => Promise<{ items: MaintenanceItem[]; read: number }>;
52
53export type MaintenanceItem = OpsMaintenanceItem;
54
55interface MaintenanceList {
56  generated: string;
57  items: MaintenanceItem[];
58  /** Open pull requests read per namespace. A namespace missing here was not read, and
59   *  has a prs-not-checked item saying why. */
60  prs_read: Record<string, number>;
61  /** Branches read per namespace, the same way: missing means not read, and a
62   *  branches-not-checked item says why. */
63  branches_read: Record<string, number>;
64  /** Sites whose live deploy was compared with their default branch, per namespace. */
65  deploys_read: Record<string, number>;
66  /** Facts compared with system/model.json: mapped repos, App repos and watched sites. */
67  model_read: number;
68  /** Rule documents checked for their cap and for stale tool names (src/rule-docs.ts). */
69  rule_docs_read: number;
70  /** Shared packages whose version was compared with their tags (src/maintenance-releases.ts). */
71  releases_read: number;
72}
73
74type MaintenanceReport =
75  | { ran: false; note: string }
76  | { ran: true; note: string; list: MaintenanceList };
77
78/** A queued job whose title carries "LATER <date>" with the date already past. */
79export function laterPassed(jobs: Array<{ id: string; namespace: string; title: string }>, today: string): MaintenanceItem[] {
80  const out: MaintenanceItem[] = [];
81  for (const j of jobs) {
82    const date = LATER_DATE.exec(j.title)?.[1];
83    if (date && date < today) {
84      out.push({ rule: "later-passed", namespace: j.namespace, job: j.id, line: `${j.id} is queued as LATER ${date}, a date that has passed: re-title it or work it.` });
85    }
86  }
87  return out;
88}
89
90/** Whether `body` names the pull request `url` as a whole, not as the prefix of a longer
91 *  number (".../pull/1" must not match ".../pull/12"). */
92export function namesPullRequest(body: string, url: string): boolean {
93  let from = body.indexOf(url);
94  while (from !== -1) {
95    const next = body.charAt(from + url.length);
96    if (!/[0-9]/.test(next)) return true;
97    from = body.indexOf(url, from + 1);
98  }
99  return false;
100}
101
102/** A job that promised follow-up jobs ("follow as separate jobs") in its summary. */
103export function promisesFollowUps(summary: string | null): boolean {
104  return summary !== null && FOLLOWUP_PROMISE.test(summary);
105}
106
107export async function gatherMaintenance(
108  env: Env,
109  now: Date,
110  prReaders: PrReaders = githubPrReaders(env),
111  branchReaders: BranchReaders = githubBranchReaders(env),
112  deployReaders: DeployReaders = githubDeployReaders(env),
113  driftReader: DriftReader = (e) => gatherModelDriftItems(e, bundledModel()),
114  releaseReaders: ReleaseReaders = githubReleaseReaders(env)
115): Promise<MaintenanceList> {
116  const today = now.toISOString().slice(0, 10);
117  const items: MaintenanceItem[] = [];
118
119  const queuedLater = await env.DB.prepare("SELECT id, namespace, title FROM jobs WHERE status = 'queued' AND title LIKE '%LATER%' LIMIT ?1")
120    .bind(MAX_ITEMS_PER_RULE)
121    .all<{ id: string; namespace: string; title: string }>();
122  items.push(...laterPassed(queuedLater.results ?? [], today));
123
124  // A queued job whose body names a pull request that merged under ANOTHER job: the work
125  // it asks for has probably shipped. The instr() is a prefilter; namesPullRequest is the
126  // check, because a short number is the prefix of a longer one.
127  const shipped = await env.DB.prepare(
128    `SELECT j.id, j.namespace, j.body, p.pr_url, p.job_id AS shipped_by
129       FROM jobs j JOIN job_outcome_prs p ON p.merged = 1 AND p.job_id != j.id AND instr(j.body, p.pr_url) > 0
130      WHERE j.status = 'queued' LIMIT ?1`
131  )
132    .bind(MAX_ITEMS_PER_RULE * 4)
133    .all<{ id: string; namespace: string; body: string; pr_url: string; shipped_by: string }>();
134  const seen = new Set<string>();
135  for (const row of shipped.results ?? []) {
136    if (seen.has(row.id) || !namesPullRequest(row.body, row.pr_url)) continue;
137    seen.add(row.id);
138    items.push({
139      rule: "shipped-elsewhere",
140      namespace: row.namespace,
141      job: row.id,
142      line: `${row.id} is queued but names ${row.pr_url}, which merged under ${row.shipped_by}: supersede it if that was the work.`,
143    });
144  }
145
146  // A finished or blocked job that said follow-ups would be posted as separate jobs, with
147  // no job created in its namespace since it last moved. A heuristic that lists, never acts.
148  const promised = await env.DB.prepare(
149    "SELECT id, namespace, result_summary, updated_at FROM jobs WHERE status IN ('done', 'blocked') AND result_summary LIKE '%separate job%' LIMIT ?1"
150  )
151    .bind(MAX_ITEMS_PER_RULE)
152    .all<{ id: string; namespace: string; result_summary: string | null; updated_at: string }>();
153  for (const row of promised.results ?? []) {
154    if (!promisesFollowUps(row.result_summary)) continue;
155    const later = await env.DB.prepare("SELECT COUNT(*) AS n FROM jobs WHERE namespace = ?1 AND created_at > ?2").bind(row.namespace, row.updated_at).first<{ n: number }>();
156    if ((later?.n ?? 0) === 0) {
157      items.push({ rule: "followups-missing", namespace: row.namespace, job: row.id, line: `${row.id} promised follow-up jobs, and none was posted in ${row.namespace} since: post them or say they are not coming.` });
158    }
159  }
160
161  // What the merge-resume step did in the last day, so the seat sees it was done for them.
162  const since = new Date(now.getTime() - DAY_MS).toISOString();
163  const resumed = await env.DB.prepare(
164    `SELECT t.job_id AS job_id, t.namespace AS namespace FROM job_touches t JOIN jobs j ON j.id = t.job_id
165      WHERE t.actor = ?1 AND t.kind = 'resume' AND t.at >= ?2 LIMIT ?3`
166  )
167    .bind(MERGE_RESUME_ACTOR, since, MAX_ITEMS_PER_RULE)
168    .all<{ job_id: string; namespace: string }>();
169  for (const row of resumed.results ?? []) {
170    items.push({ rule: "auto-resumed", namespace: row.namespace, job: row.job_id, line: `${row.job_id} was resumed because its pull requests merged: its holder confirms the deploy and completes it.` });
171  }
172
173  // Green driver pull requests waiting for the seat, and pull requests red for days.
174  const prs = await gatherPrItems(ROSTER, prReaders, now);
175  items.push(...prs.items);
176
177  // Merged branches (listed, or pruned when the seat has turned that on), and old ones.
178  const branches = await gatherBranchItems(env, ROSTER, branchReaders, now);
179  items.push(...branches.items);
180
181  // A live Worker older than its default branch's last commit.
182  const deploys = await gatherDeployItems(deployReaders, now);
183  items.push(...deploys.items);
184
185  // system/model.json against what Capsid and GitHub know (src/model-drift.ts).
186  const drift = await driftReader(env);
187  items.push(...drift.items);
188
189  // A rule document over its cap or naming a tool the Worker does not serve.
190  const ruleDocs = await gatherRuleDocItems(env.DB);
191  items.push(...ruleDocs.items);
192
193  // A shared package's version with no tag: tagged (on by default), or listed.
194  const releases = await gatherReleaseItems(env, releaseReaders);
195  items.push(...releases.items);
196
197  return {
198    generated: now.toISOString(),
199    items,
200    prs_read: prs.read,
201    branches_read: branches.read,
202    deploys_read: deploys.read,
203    model_read: drift.read,
204    rule_docs_read: ruleDocs.read,
205    releases_read: releases.read,
206  };
207}
208
209/** The daily pass. Not due returns a note and records nothing; due gathers the list, keeps
210 *  it in KV for capsid_run_improve action status, and stamps the day. The stamp is written after the list,
211 *  so a pass that threw runs again on the next tick. */
212export async function maintenanceTick(
213  env: Env,
214  now: Date,
215  prReaders?: PrReaders,
216  branchReaders?: BranchReaders,
217  deployReaders?: DeployReaders,
218  driftReader?: DriftReader,
219  releaseReaders?: ReleaseReaders
220): Promise<MaintenanceReport> {
221  const today = now.toISOString().slice(0, 10);
222  if (now.getUTCHours() < DAILY_AFTER_UTC_HOUR) return { ran: false, note: `before ${DAILY_AFTER_UTC_HOUR}:00 UTC` };
223  const last = await env.APP_KV.get(MAINTENANCE_LAST_KEY).catch(() => null);
224  if (last && last.slice(0, 10) === today) return { ran: false, note: `already ran ${last}` };
225  const list = await gatherMaintenance(env, now, prReaders, branchReaders, deployReaders, driftReader, releaseReaders);
226  await env.APP_KV.put(MAINTENANCE_KEY, JSON.stringify(list));
227  await env.APP_KV.put(MAINTENANCE_LAST_KEY, now.toISOString());
228  const prsRead = Object.values(list.prs_read).reduce((a, b) => a + b, 0);
229  const reposRead = Object.keys(list.prs_read).length;
230  const branchesRead = Object.values(list.branches_read).reduce((a, b) => a + b, 0);
231  const branchRepos = Object.keys(list.branches_read).length;
232  const pruned = list.items.filter((i) => i.rule === "branch-pruned").length;
233  const deploySites = Object.keys(list.deploys_read).length;
234  const tagged = list.items.filter((i) => i.rule === "release-tagged").length;
235  return {
236    ran: true,
237    note: `${list.items.length} item(s) listed; ${prsRead} open pull request(s) read in ${reposRead} of ${ROSTER.length} repo(s); ${branchesRead} branch(es) read in ${branchRepos} of ${ROSTER.length} repo(s), ${pruned} pruned; ${deploySites} site deploy(s) compared; ${list.model_read} fact(s) compared with the system model; ${list.rule_docs_read} rule document(s) checked; ${list.releases_read} shared package version(s) compared with their tags, ${tagged} tagged`,
238    list,
239  };
240}
241
242/** The stored list, or null when none was ever written or it cannot be read. */
243export async function readMaintenance(env: Env): Promise<MaintenanceList | null> {
244  const raw = await env.APP_KV.get(MAINTENANCE_KEY).catch(() => null);
245  if (!raw) return null;
246  try {
247    const parsed = JSON.parse(raw) as MaintenanceList;
248    if (!Array.isArray(parsed.items) || typeof parsed.generated !== "string") return null;
249    // A list written before a rule existed read nothing for it.
250    const counts = (v: unknown): Record<string, number> => (v && typeof v === "object" ? (v as Record<string, number>) : {});
251    return {
252      ...parsed,
253      // The disk rule was removed; a list stored before then may still carry its items.
254      items: parsed.items.filter((i) => !i.rule.startsWith("disk-")),
255      prs_read: counts(parsed.prs_read),
256      branches_read: counts(parsed.branches_read),
257      deploys_read: counts(parsed.deploys_read),
258      model_read: typeof parsed.model_read === "number" ? parsed.model_read : 0,
259      rule_docs_read: typeof parsed.rule_docs_read === "number" ? parsed.rule_docs_read : 0,
260      releases_read: typeof parsed.releases_read === "number" ? parsed.releases_read : 0,
261    };
262  } catch {
263    return null;
264  }
265}