app/lib/health/repair.mjs

Functions

Imports

It imports app/lib/health/ask-converge.mjs. Imported by workers/watchdog.ts.

Source

1// Repairs only when EVERY failing check is a known drift class: an unknown one blocks all repair, since
2// a compound failure may share a root cause. No token still fails the run: alert-only, never silence.
3
4import { awaitAskConvergence } from "./ask-converge.mjs";
5
6/** Only idempotent repairs through the operator API, since this runs unattended. Map order is repair order. */
7const REPAIRABLE = /** @type {const} */ ({
8  // Content first: sync_ask reads search_docs, which sync_posts, sync_procedures, sync_dictionary,
9  // sync_pages and sync_publications rewrite, so the reverse uploads a stale corpus.
10  "content-drift": "sync_posts",
11  "procedures-drift": "sync_procedures",
12  // Before the pages: an entry write refreshes the page row it opens, and a page compile reads the entry from D1.
13  "dictionary-drift": "sync_dictionary",
14  // Beside the dictionary: the roster is drawn into the program page and the home page, which a page compile or a
15  // purge may re-open, so it converges before the pages do. It writes no search record.
16  "roster-drift": "sync_roster",
17  // Before the pages as well: the phage page is compiled from the phage rows, so a page compile reads them from D1,
18  // and a phage write refreshes the page row it is drawn into.
19  "phage-drift": "sync_phages",
20  // Beside the phages: nothing is drawn from a registry row into another store, so it converges on its own.
21  "registry-drift": "sync_registry",
22  "pages-drift": "sync_pages",
23  "publications-drift": "sync_publications",
24  // Reads nothing the Ask upload reads; listed with the file-derived stores.
25  "llms-drift": "sync_llms",
26  "cv-drift": "sync_cv",
27  // After sync_cv, which it follows: the PDF is drawn from the rows sync_cv converges. It reads nothing the Ask
28  // upload reads.
29  "cv-pdf-drift": "sync_cv_pdf",
30  "ask-index-drift": "sync_ask",
31  "media-index-drift": "sync_media",
32  // Order-independent. backup_media only copies and has no delete branch, which is why it may fire unattended.
33  "media-backup-drift": "backup_media",
34});
35
36/**
37 * @param {unknown} failingNames names of checks reporting ok:false
38 * @param {{ hasToken: boolean }} options
39 * @returns {{
40 *   repair: string[],
41 *   unknown: string[],
42 *   alertOnly: boolean,
43 *   reason: string,
44 * }}
45 */
46export function repairPlan(failingNames, { hasToken }) {
47  const names = Array.isArray(failingNames)
48    ? failingNames.filter((n) => typeof n === "string" && n.length > 0)
49    : [];
50
51  if (names.length === 0) {
52    return {
53      repair: [],
54      unknown: [],
55      alertOnly: true,
56      reason:
57        "no failing check was named, so there is nothing to repair. A run that reports " +
58        "unhealthy without naming a check is a fault in the endpoint, not a drift.",
59    };
60  }
61
62  const unknown = names.filter((n) => !(n in REPAIRABLE));
63
64  if (unknown.length > 0) {
65    return {
66      repair: [],
67      unknown,
68      alertOnly: true,
69      reason:
70        `no self-repair: ${unknown.join(", ")} ${unknown.length === 1 ? "is" : "are"} not a ` +
71        `known drift class. Repair is not attempted for the recognized classes either, ` +
72        `because a compound failure may share a root cause.`,
73    };
74  }
75
76  if (!hasToken) {
77    return {
78      repair: [],
79      unknown: [],
80      alertOnly: true,
81      reason:
82        "no self-repair: OPERATOR_TOKEN is not set on this watcher, so the repair door " +
83        "cannot be opened. The GitHub half takes it as a repository secret " +
84        "(`gh secret set OPERATOR_TOKEN`); the watchdog Worker takes it as a wrangler " +
85        "secret (`wrangler secret put OPERATOR_TOKEN -c wrangler.watchdog.jsonc`). " +
86        "Alerting only.",
87    };
88  }
89
90  // Ordered by the map and deduplicated, so one tool is never called twice and run logs compare.
91  const tools = Object.keys(REPAIRABLE)
92    .filter((name) => names.includes(name))
93    .map((name) => REPAIRABLE[/** @type {keyof typeof REPAIRABLE} */ (name)]);
94
95  return {
96    repair: [...new Set(tools)],
97    unknown: [],
98    alertOnly: false,
99    reason: `self-repair: ${names.join(", ")} -> ${[...new Set(tools)].join(", ")}`,
100  };
101}
102
103/**
104 * An unreadable body yields no names, so repairPlan alerts rather than writes. ok === false, not !ok,
105 * so a check missing the field is not counted as failing.
106 *
107 * @param {unknown} body
108 * @returns {string[]}
109 */
110export function failingCheckNames(body) {
111  const checks = /** @type {any} */ (body)?.checks;
112  if (!Array.isArray(checks)) return [];
113  return checks
114    .filter((c) => c && typeof c === "object" && c.ok === false && typeof c.name === "string")
115    .map((c) => c.name);
116}
117
118// The recheck is in the action list: a repair's own 200 proves one index, the recheck proves the
119// endpoint agrees, and a step living only in the caller is a step no test can name.
120
121/**
122 * @typedef {{ type: "repair", tool: string }} RepairAction
123 * @typedef {{ type: "recheck" }} RecheckAction
124 * @typedef {{ type: "notify", reason: string }} NotifyAction
125 * @typedef {RepairAction | RecheckAction | NotifyAction} WatchdogAction
126 */
127
128/**
129 * @typedef {{ status: number, body: unknown, error?: string }} HealthReading
130 */
131
132/**
133 * A 200 whose body says `ok: true`; anything else, a 503 included, is unhealthy.
134 *
135 * @param {HealthReading | null | undefined} reading
136 * @returns {boolean}
137 */
138function isHealthyReading(reading) {
139  const body = reading?.body;
140  return (
141    Number(reading?.status) === 200 &&
142    !!body &&
143    typeof body === "object" &&
144    /** @type {any} */ (body).ok === true
145  );
146}
147
148/**
149 * Anything but a healthy 200 takes the same path: a 503 is how this endpoint reports a failing check.
150 *
151 * @param {HealthReading} reading
152 * @param {{ hasToken: boolean }} options
153 * @returns {WatchdogAction[]}
154 */
155export function watchdogActions(reading, { hasToken }) {
156  const status = Number(reading?.status);
157  const body = reading?.body;
158
159  if (isHealthyReading(reading)) return [];
160
161  // A transport failure is reported with its cause: a timeout and a 1042 are different pages, and
162  // without the cause the mail says "no failing check was named" about a fetch that never happened.
163  if (status === 0) {
164    const cause = typeof reading?.error === "string" && reading.error ? reading.error : "";
165    return [
166      {
167        type: "notify",
168        reason:
169          `the health endpoint could not be reached at all${cause ? `: ${cause}` : `, and the ` +
170            `reading carried no cause, which is itself a defect in the reader`}. Nothing was ` +
171          `repaired, because a reading that never happened names no drift.`,
172      },
173    ];
174  }
175
176  const plan = repairPlan(failingCheckNames(body), { hasToken });
177
178  if (plan.alertOnly) return [{ type: "notify", reason: plan.reason }];
179
180  return [
181    ...plan.repair.map((tool) => /** @type {RepairAction} */ ({ type: "repair", tool })),
182    { type: "recheck" },
183  ];
184}
185
186/**
187 * A failed repair and a still-failing recheck are both reported, not just the first.
188 *
189 * @param {{ misses: string[], recheck: HealthReading | null }} outcome
190 * @returns {WatchdogAction[]}
191 */
192export function watchdogOutcome({ misses, recheck }) {
193  const failed = Array.isArray(misses) ? misses.filter((m) => typeof m === "string" && m) : [];
194
195  const reasons = [];
196  if (failed.length > 0) reasons.push(`self-repair FAILED: ${failed.join("; ")}. The drift stands.`);
197
198  if (recheck === null || recheck === undefined) {
199    reasons.push("the re-check did not complete, so the repair proved nothing.");
200  } else {
201    const stillFailing = failingWithCounts(recheck.body);
202    if (!isHealthyReading(recheck)) {
203      reasons.push(
204        `self-repair ran and the endpoint is STILL unhealthy (HTTP ${recheck.status}` +
205          `${stillFailing.length ? `, failing: ${stillFailing.join(", ")}` : ""}).`,
206      );
207    }
208  }
209
210  return reasons.length > 0 ? [{ type: "notify", reason: reasons.join(" ") }] : [];
211}
212
213/**
214 * Each failing check by name, with the counts the public body carries for a failing drift check.
215 *
216 * @param {unknown} body
217 * @returns {string[]}
218 */
219function failingWithCounts(body) {
220  const checks = /** @type {any} */ (body)?.checks;
221  if (!Array.isArray(checks)) return [];
222  return checks
223    .filter((c) => c && typeof c === "object" && c.ok === false && typeof c.name === "string")
224    .map((c) =>
225      typeof c.expected === "number" && typeof c.present === "number"
226        ? `${c.name} (expected ${c.expected}, present ${c.present})`
227        : c.name,
228    );
229}
230
231/**
232 * After the watchdog's own sync_ask, the Ask index can list short for up to about a minute: it is
233 * eventually consistent, and a full re-upload is followed by a short count (157 then 155 in deploy
234 * run 36274700184). So a re-check still failing only on that is re-read on ship's cadence and window
235 * (awaitAskConvergence) before it can alert. The last full reading is returned, so another failing
236 * check, or a count that never arrives, still alerts with its counts.
237 *
238 * @param {{
239 *   recheck: HealthReading | null,
240 *   attempted: string[],
241 *   reread: () => Promise<HealthReading>,
242 *   sleep: (ms: number) => Promise<unknown>,
243 *   now?: () => number,
244 * }} options
245 * @returns {Promise<{ reading: HealthReading | null, polls: number }>}
246 */
247export async function settleAskRecheck({ recheck, attempted, reread, sleep, now }) {
248  if (!recheck || !attempted.includes("sync_ask")) return { reading: recheck, polls: 0 };
249  if (!failingCheckNames(recheck.body).includes("ask-index-drift")) return { reading: recheck, polls: 0 };
250
251  let latest = recheck;
252  const { polls } = await awaitAskConvergence({
253    reading: async () => {
254      const reading = await reread();
255      // A reading that never arrived names no drift, so the last real one stands.
256      if (Number(reading?.status) === 0) return null;
257      latest = reading;
258      const checks = /** @type {any} */ (reading.body)?.checks;
259      return Array.isArray(checks) ? (checks.find((c) => c?.name === "ask-index-drift") ?? null) : null;
260    },
261    sleep,
262    now,
263  });
264  return { reading: latest, polls };
265}
266
267/**
268 * Reports the operator API's own error and detail, since a bare status discards the diagnosis. A 422
269 * is unrepairable: resending the same corpus cannot correct it; usually the repo is ahead of the deploy.
270 *
271 * @param {string} tool the operator tool that was called
272 * @param {number} status the HTTP status it answered
273 * @param {unknown} payload the parsed response body, or null
274 * @returns {{ miss: string, unrepairable: boolean }}
275 */
276export function refusalMiss(tool, status, payload) {
277  const body = payload && typeof payload === "object" ? /** @type {any} */ (payload) : null;
278  const said = typeof body?.error === "string" ? body.error.trim() : "";
279  const field = typeof body?.detail?.field === "string" && body.detail.field ? body.detail.field : "";
280  const line = typeof body?.detail?.line === "number" ? body.detail.line : null;
281
282  return {
283    miss:
284      `${tool} answered ${status}` +
285      (said ? `: ${said}` : "") +
286      (field ? ` [field ${field}]` : "") +
287      (line === null ? "" : ` [line ${line}]`),
288    unrepairable: status === 422,
289  };
290}