src/site-repair.ts

Functions

Imports

It imports src/env.ts, src/log.ts, src/ops-sites.ts, src/ops-types/sites.ts, src/site-operator.ts, src/store-guards.ts, src/task-runs.ts, src/time.ts. Imported by src/controls-perform.ts, src/controls-plan.ts, src/site-convergence.ts, src/site-watch.ts.

Source

1import type { Env } from "./env";
2import { logEvent } from "./log";
3import { readSiteRow } from "./ops-sites";
4import type { SiteOperator } from "./ops-types/sites";
5import { callLine, callOperator, toolRefusal, type FetchLike, type OperatorCall } from "./site-operator";
6import { auditStatement } from "./store-guards";
7import { taskRunStatements } from "./task-runs";
8import { MINUTE_MS, DAY_MS } from "./time";
9
10// ONE REPAIR, from the Portal, capsid_admin_run_controls or the watcher (job_09e5f6cbf782). The
11// site_repair control and the watcher's repair step both come here, so the allowlist, the
12// rate limit and the record are one implementation: an audit_log row (site-repair) and a
13// task_runs row (site-repair) for every call that left the Worker, and none for one
14// refused before it.
15//
16// The rate limit is per site and tool, in APP_KV, so a stuck check cannot loop:
17//   - any call waits MIN_GAP_MS after the last one, whoever made it (a double click, or the
18//     watcher and a person at once);
19//   - the watcher makes at most AUTO_PER_DAY calls per site and tool in a rolling day, and
20//     after that only a person can run it. The finding says so.
21
22export const REPAIR_AUDIT_ACTION = "site-repair";
23const LIMIT_PREFIX = "site-repair-limit:";
24const MIN_GAP_MS = 2 * MINUTE_MS;
25const AUTO_GAP_MS = 30 * MINUTE_MS;
26export const AUTO_PER_DAY = 4;
27
28interface LimitRecord {
29  last: string;
30  auto: string[];
31}
32
33const limitKey = (namespace: string, tool: string) => `${LIMIT_PREFIX}${namespace}:${tool}`;
34
35async function readLimit(env: Pick<Env, "APP_KV">, namespace: string, tool: string): Promise<LimitRecord | null> {
36  const raw = await env.APP_KV.get(limitKey(namespace, tool));
37  if (!raw) return null;
38  try {
39    const parsed = JSON.parse(raw) as LimitRecord;
40    return typeof parsed.last === "string" && Array.isArray(parsed.auto) ? parsed : null;
41  } catch {
42    // A record that does not parse is replaced by the next call's; it limits nothing until
43    // then, and that is logged so it is not silent.
44    logEvent("warn", "SITE_REPAIR_LIMIT_UNREADABLE", { message: `SITE_REPAIR_LIMIT_UNREADABLE ${namespace} ${tool}` });
45    return null;
46  }
47}
48
49/** Why this call must wait, or null. `auto` is the watcher. */
50export function limitRefusal(record: LimitRecord | null, now: Date, auto: boolean): string | null {
51  if (!record) return null;
52  const since = now.getTime() - Date.parse(record.last);
53  const gap = auto ? AUTO_GAP_MS : MIN_GAP_MS;
54  if (since < gap) return `it ran ${Math.round(since / MINUTE_MS)} minute(s) ago; the next call may go after ${Math.round(gap / MINUTE_MS)} minutes.`;
55  if (auto) {
56    const today = record.auto.filter((at) => now.getTime() - Date.parse(at) < DAY_MS);
57    if (today.length >= AUTO_PER_DAY) return `the watcher has run it ${today.length} times in the last day, its limit; a person can still run it from the Portal.`;
58  }
59  return null;
60}
61
62/** The rate limit for a person's call, read now: what a preview shows. */
63export async function limitRefusalFor(env: Pick<Env, "APP_KV">, namespace: string, tool: string, now: Date): Promise<string | null> {
64  return limitRefusal(await readLimit(env, namespace, tool), now, false);
65}
66
67async function writeLimit(env: Pick<Env, "APP_KV">, namespace: string, tool: string, before: LimitRecord | null, now: Date, auto: boolean): Promise<void> {
68  const kept = (before?.auto ?? []).filter((at) => now.getTime() - Date.parse(at) < DAY_MS);
69  const next: LimitRecord = { last: now.toISOString(), auto: auto ? [...kept, now.toISOString()] : kept };
70  await env.APP_KV.put(limitKey(namespace, tool), JSON.stringify(next), { expirationTtl: 2 * 86_400 });
71}
72
73export type RepairOutcome =
74  | { ran: false; refusal: string }
75  | { ran: true; call: OperatorCall; line: string; recorded: boolean };
76
77/** Run one repair on one site. Refused, with nothing sent, when the site has no usable
78 *  operator configuration, the tool is not allowed, or the rate limit holds. */
79export async function runSiteRepair(
80  env: Env,
81  args: { namespace: string; tool: string; actor: string; auto: boolean; via: string },
82  now: Date,
83  fetchImpl: FetchLike = fetch,
84  site?: { origin: string; operator: SiteOperator }
85): Promise<RepairOutcome> {
86  let target = site;
87  if (!target) {
88    const row = await readSiteRow(env.DB, args.namespace);
89    if (!row?.origin) return { ran: false, refusal: `${args.namespace} is not a configured site.` };
90    if (!row.operator) return { ran: false, refusal: row.operator_problem ?? `${args.namespace} has no operator API configured, so Capsid may call nothing on it.` };
91    target = { origin: row.origin, operator: row.operator };
92  }
93  const refusal = toolRefusal(target.operator, args.tool);
94  if (refusal) return { ran: false, refusal };
95  const before = await readLimit(env, args.namespace, args.tool);
96  const wait = limitRefusal(before, now, args.auto);
97  if (wait) return { ran: false, refusal: `${args.tool} on ${args.namespace} waits: ${wait}` };
98  // The limit is written before the call, which narrows a race between two callers but does
99  // not close it: KV has no compare-and-set, so two that read the same record both go. The
100  // operator API's own limits are the hard bound (Grok audit 2026-10-21, CA-7).
101  await writeLimit(env, args.namespace, args.tool, before, now, args.auto);
102  const started = new Date();
103  const call = await callOperator(env, target, args.tool, fetchImpl);
104  const line = callLine(args.tool, call);
105  const finished = new Date();
106  // The record of a call that left the Worker. A failed write is logged and said, and the
107  // call's own result still goes back: it happened.
108  let recorded = true;
109  try {
110    await env.DB.batch([
111      auditStatement(env.DB, args.actor, REPAIR_AUDIT_ACTION, args.namespace, null, {
112        tool: args.tool,
113        via: args.via,
114        status: call.status,
115        converged: call.converged,
116        expected: call.expected,
117        present: call.present,
118        error: call.error,
119      }),
120      ...taskRunStatements(env.DB, "site-repair", started, finished, call.ok && call.converged !== false ? "ok" : "refused", `${args.namespace}: ${line} (${args.via})`),
121    ]);
122  } catch (err) {
123    recorded = false;
124    logEvent("error", "SITE_REPAIR_NOT_RECORDED", { message: `SITE_REPAIR_NOT_RECORDED ${args.namespace} ${args.tool}: ${err instanceof Error ? err.message : String(err)}` });
125  }
126  return { ran: true, call, line, recorded };
127}