src/site-watch.ts

Functions

Imports

It imports src/env.ts, src/log.ts, src/ops-types/sites.ts, src/site-operator.ts, src/site-repair.ts, src/time.ts. Imported by src/watcher-gather.ts.

Source

1import type { Env } from "./env";
2import { logEvent } from "./log";
3import type { OpsSiteConfig, SiteOperator } from "./ops-types/sites";
4import { runSiteRepair, type RepairOutcome } from "./site-repair";
5import { tokenSet, type FetchLike } from "./site-operator";
6import { DAY_MS } from "./time";
7
8// THE WATCHER'S SITE REPAIRS (job_09e5f6cbf782), so a site's own watchdog Worker can be
9// retired later. On each watcher pass, for every site that opted in (ops_sites.operator):
10//   1. Read its health route. Healthy: nothing to do.
11//   2. Unhealthy: the failing checks' names. Repair only when EVERY failing check maps to
12//      an allowlisted repair (the site's own rule, app/lib/health/repair.mjs): an unmapped
13//      failure may share a cause with the rest, so nothing is repaired and it is reported.
14//   3. Each repair through runSiteRepair (src/site-repair.ts): allowlisted, rate-limited,
15//      audit-logged and one task_runs row each. Then the health is read again.
16//   4. Still unhealthy, or a repair failed: one finding, site-health-<namespace>, which the
17//      Portal shows under Incidents and which clears when a pass finds the site healthy.
18//   5. Its weekly tools (refresh_citations) once every WEEK_MS.
19// A red-to-green or green-to-red change mails Dustin once, where the seat has bound a
20// send_email binding; the finding opens either way.
21//
22// The watcher still decides nothing a person has not configured: the site names which
23// check maps to which tool, the Worker's ceiling bounds what any site may name, and the
24// rate limit stops a stuck check from looping.
25
26const HEALTH_TIMEOUT_MS = 10_000;
27const WEEK_MS = 7 * DAY_MS;
28const WEEKLY_PREFIX = "site-weekly:";
29const ALERT_PREFIX = "site-alert:";
30const TEXT_MAX = 300;
31
32/** A finding, in the watcher's terms, without importing the watcher (it imports this). */
33interface SiteFinding {
34  fingerprint: string;
35  namespace: string;
36  headline: string;
37  evidence: string[];
38}
39
40export interface SiteWatchReport {
41  findings: SiteFinding[];
42  // Every opted-in site's health was read, so a site with no finding is known healthy
43  // and its open finding may be cleared.
44  complete: boolean;
45  lines: string[];
46}
47
48interface Reading {
49  status: number;
50  body: unknown;
51  error: string | null;
52}
53
54const clip = (s: string) => (s.length > TEXT_MAX ? `${s.slice(0, TEXT_MAX - 1)}…` : s);
55
56async function readHealth(fetchImpl: FetchLike, url: string): Promise<Reading> {
57  try {
58    const res = await fetchImpl(url, { headers: { "user-agent": "capsid-watcher", "cache-control": "no-cache" }, redirect: "manual", signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) });
59    return { status: res.status, body: await res.json().catch(() => null), error: null };
60  } catch (err) {
61    return { status: 0, body: null, error: clip(err instanceof Error ? err.message : String(err)) };
62  }
63}
64
65const healthy = (r: Reading) => r.status === 200 && (r.body as { ok?: unknown } | null)?.ok === true;
66
67/** The failing checks' names. ok === false, not !ok, so a check missing the field is not
68 *  counted as failing (the site's own failingCheckNames). */
69export function failingChecks(body: unknown): string[] {
70  const checks = (body as { checks?: unknown } | null)?.checks;
71  if (!Array.isArray(checks)) return [];
72  return checks
73    .filter((c) => c && typeof c === "object" && (c as { ok?: unknown }).ok === false && typeof (c as { name?: unknown }).name === "string")
74    .map((c) => clip((c as { name: string }).name));
75}
76
77/** What to do about a failing health reading: the repairs, in the configuration's order,
78 *  or why nothing is repaired. */
79export function repairPlan(failing: string[], operator: SiteOperator): { tools: string[] } | { none: string } {
80  if (failing.length === 0) return { none: "the health route reported unhealthy without naming a failing check, so there is nothing to map to a repair" };
81  const unmapped = failing.filter((n) => !Object.hasOwn(operator.repairs, n));
82  if (unmapped.length > 0) return { none: `${unmapped.join(", ")} ${unmapped.length === 1 ? "maps" : "map"} to no repair, and a failure with an unmapped part may share its cause, so nothing was repaired` };
83  const order = Object.keys(operator.repairs);
84  return { tools: [...new Set(order.filter((check) => failing.includes(check)).map((check) => operator.repairs[check]))] };
85}
86
87const repairLine = (tool: string, o: RepairOutcome) => (o.ran ? o.line : `${tool} was not run: ${o.refusal}`);
88
89/** One site, one pass. */
90async function watchSite(env: Env, site: OpsSiteConfig & { origin: string; operator: SiteOperator }, actor: string, fetchImpl: FetchLike, now: Date): Promise<{ finding: SiteFinding | null; read: boolean; lines: string[]; weekly: SiteFinding[] }> {
91  const lines: string[] = [];
92  const weekly: SiteFinding[] = [];
93  const operator = site.operator;
94  const target = { origin: site.origin, operator };
95  const secretSet = tokenSet(env, operator);
96
97  // The weekly tools first and on their own: a weekly run is not a repair and does not
98  // wait for drift.
99  for (const tool of operator.weekly) {
100    const key = `${WEEKLY_PREFIX}${site.namespace}:${tool}`;
101    const last = await env.APP_KV.get(key);
102    if (last && now.getTime() - Date.parse(last) < WEEK_MS) continue;
103    if (!secretSet) {
104      weekly.push({ fingerprint: `site-weekly-${site.namespace}-${tool}`, namespace: site.namespace, headline: `${site.name}'s weekly ${tool} cannot run`, evidence: [`the Capsid Worker secret ${operator.auth_var} is not set, so nothing was called.`] });
105      continue;
106    }
107    const outcome = await runSiteRepair(env, { namespace: site.namespace, tool, actor, auto: true, via: "watcher weekly" }, now, fetchImpl, target);
108    lines.push(`${site.namespace}: ${repairLine(tool, outcome)}`);
109    if (outcome.ran && outcome.call.ok) await env.APP_KV.put(key, now.toISOString(), { expirationTtl: 30 * 86_400 });
110    else if (outcome.ran) weekly.push({ fingerprint: `site-weekly-${site.namespace}-${tool}`, namespace: site.namespace, headline: `${site.name}'s weekly ${tool} failed`, evidence: [outcome.line] });
111  }
112
113  if (!site.health_path) return { finding: null, read: true, lines, weekly };
114  const reading = await readHealth(fetchImpl, site.origin + site.health_path);
115  // A reading that never happened names no drift: the site-down probe covers an
116  // unreachable site, and this check does not count as run, so nothing is cleared.
117  if (reading.status === 0) return { finding: null, read: false, lines: [...lines, `${site.namespace}: health not read (${reading.error})`], weekly };
118  if (healthy(reading)) return { finding: null, read: true, lines, weekly };
119
120  const failing = failingChecks(reading.body);
121  const evidence = [`health: ${site.origin}${site.health_path} answered ${reading.status}; failing: ${failing.join(", ") || "(none named)"}`];
122  const plan = repairPlan(failing, operator);
123  if ("none" in plan) {
124    evidence.push(plan.none);
125  } else if (!secretSet) {
126    evidence.push(`the mapped repairs (${plan.tools.join(", ")}) were not run: the Capsid Worker secret ${operator.auth_var} is not set.`);
127  } else {
128    let missed = false;
129    for (const tool of plan.tools) {
130      const outcome = await runSiteRepair(env, { namespace: site.namespace, tool, actor, auto: true, via: "watcher" }, now, fetchImpl, target);
131      const line = repairLine(tool, outcome);
132      evidence.push(`repair: ${line}`);
133      lines.push(`${site.namespace}: ${line}`);
134      if (!outcome.ran || !outcome.call.ok || outcome.call.converged === false) missed = true;
135    }
136    const recheck = await readHealth(fetchImpl, site.origin + site.health_path);
137    if (!missed && healthy(recheck)) return { finding: null, read: true, lines, weekly };
138    evidence.push(
139      recheck.status === 0
140        ? `re-check: not read (${recheck.error}), so the repairs proved nothing`
141        : `re-check: answered ${recheck.status}; still failing: ${failingChecks(recheck.body).join(", ") || (healthy(recheck) ? "none" : "(none named)")}`
142    );
143  }
144  return {
145    finding: { fingerprint: `site-health-${site.namespace}`, namespace: site.namespace, headline: `${site.name} is unhealthy and self-repair did not settle it`, evidence },
146    read: true,
147    lines,
148    weekly,
149  };
150}
151
152/** Every opted-in site, one pass, then the red and green mail. */
153export async function watchSites(env: Env, config: readonly OpsSiteConfig[], actor: string, fetchImpl: FetchLike, now: Date): Promise<SiteWatchReport> {
154  const findings: SiteFinding[] = [];
155  const lines: string[] = [];
156  let complete = true;
157  for (const site of config) {
158    if (!site.origin || !site.operator) continue;
159    const r = await watchSite(env, site as OpsSiteConfig & { origin: string; operator: SiteOperator }, actor, fetchImpl, now);
160    findings.push(...r.weekly);
161    lines.push(...r.lines);
162    if (!r.read) {
163      complete = false;
164      continue;
165    }
166    if (r.finding) findings.push(r.finding);
167    lines.push(...(await mailOnChange(env, site.namespace, site.name, r.finding, now)));
168  }
169  return { findings, complete, lines };
170}
171
172interface AlertState {
173  red: boolean;
174  since: string;
175}
176
177/** Mails once on a change between healthy and not. The state moves only once the mail is
178 *  sent, or when there is nothing to send it with, so a failed send retries next pass. */
179async function mailOnChange(env: Env, namespace: string, name: string, finding: SiteFinding | null, now: Date): Promise<string[]> {
180  const key = `${ALERT_PREFIX}${namespace}`;
181  let before: AlertState | null = null;
182  try {
183    const raw = await env.APP_KV.get(key);
184    before = raw ? (JSON.parse(raw) as AlertState) : null;
185  } catch (err) {
186    logEvent("error", "SITE_ALERT_STATE_UNREADABLE", { message: `SITE_ALERT_STATE_UNREADABLE ${namespace}: ${err instanceof Error ? err.message : String(err)}` });
187  }
188  const red = finding !== null;
189  // No state yet and healthy: nothing changed that anyone needs to hear about.
190  if ((before?.red ?? false) === red) return [];
191  const subject = red ? `${name} is unhealthy and self-repair did not settle it` : `${name} recovered`;
192  const text = red
193    ? [`Changed to unhealthy at ${now.toISOString()}. You will not be mailed again until it recovers.`, "", ...(finding?.evidence ?? []), "", "The finding is open in Capsid Portal, Incidents."].join("\n")
194    : [`Healthy again at ${now.toISOString()}${before ? `, after being unhealthy since ${before.since}` : ""}.`].join("\n");
195  let note: string;
196  if (env.EMAIL && env.ALERT_EMAIL && env.ALERT_FROM) {
197    try {
198      await env.EMAIL.send({ from: env.ALERT_FROM, to: env.ALERT_EMAIL, subject, text });
199      note = `${namespace}: mailed "${subject}"`;
200    } catch (err) {
201      const message = err instanceof Error ? err.message : String(err);
202      logEvent("error", "SITE_ALERT_MAIL_FAILED", { message: `SITE_ALERT_MAIL_FAILED ${namespace}: ${message}` });
203      // The state stays, so the next pass sends it again.
204      return [`${namespace}: the "${subject}" mail did not send (${clip(message)}); it is retried next pass`];
205    }
206  } else {
207    note = `${namespace}: "${subject}" was not mailed: no send_email binding (EMAIL, ALERT_EMAIL, ALERT_FROM) is configured`;
208    logEvent("warn", "SITE_ALERT_NOT_MAILED", { message: note });
209  }
210  await env.APP_KV.put(key, JSON.stringify({ red, since: now.toISOString() } satisfies AlertState));
211  return [note];
212}