app/lib/health/verdicts.mjs

Functions

Imports

It imports app/lib/error-message.mjs. Imported by app/lib/health/checks.server.ts, app/lib/media/rebuild.server.ts, app/lib/operator/sync-tools.server.ts, app/lib/search/ask.server.ts, app/routes/api.health.ts.

Source

1import { errorMessage } from "../error-message.mjs";
2
3/**
4 * A derived index against its source: the sizes, what it lacks and what it holds beyond the source,
5 * each sorted. Kept apart so a missing key and an extra one cannot cancel out in a count.
6 *
7 * @param {Set<string>} expected
8 * @param {Set<string>} present
9 */
10export function setDrift(expected, present) {
11  return {
12    expected: expected.size,
13    present: present.size,
14    missing: [...expected].filter((k) => !present.has(k)).sort(),
15    extra: [...present].filter((k) => !expected.has(k)).sort(),
16  };
17}
18
19/**
20 * Both directions summed: a missing record and a stale one are each a wrong answer. Matches the admin badge.
21 *
22 * @param {{ expected: number, present: number, missing: string[], stale: string[] }} status
23 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
24 */
25export function askDriftVerdict(status) {
26  const drift = status.missing.length + status.stale.length;
27  if (drift === 0) {
28    return {
29      ok: true,
30      detail: `Ask index agrees with D1: ${status.expected} expected, ${status.present} present.`,
31    };
32  }
33  return {
34    ok: false,
35    detail:
36      `Ask index drift ${drift}: ${status.missing.length} missing, ` +
37      `${status.stale.length} stale, ${status.expected} expected, ` +
38      `${status.present} present. Repair with sync-ask on /admin/posts.`,
39    counts: { expected: status.expected, present: status.present },
40  };
41}
42
43/**
44 * Both directions summed, since they can cancel in a count comparison while both are true. Here R2 is
45 * the truth and D1 the projection; media-backup-drift is the other way round.
46 *
47 * @param {{ expected: number, present: number, missing: string[], extra: string[] }} status
48 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
49 */
50export function mediaDriftVerdict(status) {
51  const drift = status.missing.length + status.extra.length;
52  if (drift === 0) {
53    return {
54      ok: true,
55      detail: `Media index agrees with R2 and public/: ${status.expected} expected, ${status.present} present.`,
56    };
57  }
58  return {
59    ok: false,
60    detail:
61      `Media index drift ${drift}: ${status.missing.length} missing, ` +
62      `${status.extra.length} extra, ${status.expected} expected, ` +
63      `${status.present} present. Repair with sync_media on the operator API, ` +
64      `or the rebuild action on /admin/media.`,
65    counts: { expected: status.expected, present: status.present },
66  };
67}
68
69/**
70 * The counts travel so zero missing cannot mean zero examined. Mismatched is counted apart from missing:
71 * a wrong copy, not an absent one, means something rewrote the mirror.
72 *
73 * @param {{ objects: number, twins: number, missing: string[], mismatched: string[] }} status
74 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
75 */
76export function mediaBackupDriftVerdict(status) {
77  const missing = status.missing.length;
78  const mismatched = status.mismatched.length;
79
80  if (status.objects === 0) {
81    return {
82      ok: true,
83      detail:
84        "MEDIA bucket holds no objects, so the mirror has nothing to hold either. " +
85        "0 objects, 0 twins, 0 missing: examined nothing rather than verified anything.",
86    };
87  }
88
89  if (missing === 0 && mismatched === 0) {
90    return {
91      ok: true,
92      detail:
93        `Every MEDIA object has a byte-identical twin: ${status.objects} objects, ` +
94        `${status.twins} twins, 0 missing.`,
95    };
96  }
97
98  return {
99    ok: false,
100    detail:
101      `Media backup drift: ${status.objects} objects, ${status.twins} twins, ` +
102      `${missing} missing, ${mismatched} mismatched. ` +
103      `First missing: ${status.missing[0] ?? "none"}. ` +
104      `First mismatched: ${status.mismatched[0] ?? "none"}. ` +
105      `Repair with backup_media on the operator API, which only ever copies.`,
106    counts: { expected: status.objects, present: status.twins },
107  };
108}
109
110/**
111 * Shared with the sync_posts repair, so what the check calls drift and what the repair repairs agree.
112 *
113 *   changed  a slug present on both sides whose blob sha differs: the file
114 *            moved and the row has not followed (or a row's provenance was
115 *            corrupted, which repairs identically).
116 *   unrowed  a file with no row: a post committed from a clone that no sync
117 *            has landed yet, or a lost row.
118 *   unfiled  a row with no file: a post deleted from the repository whose
119 *            rows outlived it.
120 *
121 * @param {Array<{ slug: string, sha: string }>} files from the Contents
122 *   directory listing, blob sha per markdown file
123 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
124 * @returns {{ changed: string[], unrowed: string[], unfiled: string[] }}
125 */
126export function contentDriftCompare(files, rows) {
127  const rowBySlug = new Map(rows.map((r) => [r.slug, r]));
128  const fileSlugs = new Set(files.map((f) => f.slug));
129
130  const changed = files
131    .filter((f) => {
132      const row = rowBySlug.get(f.slug);
133      return row !== undefined && row.source_blob_sha !== f.sha;
134    })
135    .map((f) => f.slug)
136    .sort();
137  const unrowed = files
138    .filter((f) => !rowBySlug.has(f.slug))
139    .map((f) => f.slug)
140    .sort();
141  const unfiled = rows
142    .filter((r) => !fileSlugs.has(r.slug))
143    .map((r) => r.slug)
144    .sort();
145
146  return { changed, unrowed, unfiled };
147}
148
149/**
150 * present counts rows in full agreement, so equal totals with one sha changed still read as drift.
151 *
152 * @param {Array<{ slug: string, sha: string }>} files
153 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
154 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
155 */
156export function contentDriftVerdict(files, rows) {
157  return sourceDriftVerdict(files, rows, { label: "Content", noun: "post", tool: "sync_posts" });
158}
159
160/**
161 * The same comparison for the procedure files and the procedures table, so what counts as drift is one
162 * rule for both.
163 *
164 * @param {Array<{ slug: string, sha: string }>} files
165 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
166 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
167 */
168export function procedureDriftVerdict(files, rows) {
169  return sourceDriftVerdict(files, rows, {
170    label: "Procedure",
171    noun: "procedure",
172    tool: "sync_procedures",
173  });
174}
175
176/**
177 * The same comparison for the dictionary entry files and the dictionary_entries table.
178 *
179 * @param {Array<{ slug: string, sha: string }>} files
180 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
181 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
182 */
183export function dictionaryDriftVerdict(files, rows) {
184  return sourceDriftVerdict(files, rows, { label: "Dictionary", noun: "dictionary entry", tool: "sync_dictionary" });
185}
186
187/**
188 * The same comparison for the page files and the pages table.
189 *
190 * @param {Array<{ slug: string, sha: string }>} files
191 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
192 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
193 */
194export function pageDriftVerdict(files, rows) {
195  return sourceDriftVerdict(files, rows, { label: "Page", noun: "page", tool: "sync_pages" });
196}
197
198/**
199 * The same comparison for the CV files and the cv table.
200 *
201 * @param {Array<{ slug: string, sha: string }>} files
202 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
203 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
204 */
205export function cvDriftVerdict(files, rows) {
206  return sourceDriftVerdict(files, rows, { label: "CV", noun: "CV", tool: "sync_cv" });
207}
208
209/**
210 * The CV's PDF against the CV D1 holds now: the stored object's fingerprint (custom metadata) against the
211 * fingerprint of the data. `stored` is null when there is no object and "" when there is one that carries no
212 * fingerprint (neither is current). Derived store against its source, as the others, repaired by sync_cv_pdf.
213 *
214 * @param {{ stored: string | null, expected: string }} sides
215 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
216 */
217export function cvPdfDriftVerdict({ stored, expected }) {
218  if (stored === expected) {
219    return { ok: true, detail: `The stored CV PDF was drawn from the CV data D1 holds now (CV data ${expected}).` };
220  }
221  const what =
222    stored === null
223      ? "there is no stored CV PDF"
224      : stored === ""
225        ? "the stored CV PDF carries no fingerprint"
226        : `the stored CV PDF was drawn from CV data ${stored}`;
227  return {
228    ok: false,
229    detail: `CV PDF drift 1: ${what}, and D1's CV is CV data ${expected}. Repair with sync_cv_pdf.`,
230    counts: { expected: 1, present: 0 },
231  };
232}
233
234/**
235 * The same comparison for the publication files and the publications table.
236 *
237 * @param {Array<{ slug: string, sha: string }>} files
238 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
239 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
240 */
241export function publicationDriftVerdict(files, rows) {
242  return sourceDriftVerdict(files, rows, { label: "Publication", noun: "publication", tool: "sync_publications" });
243}
244
245/**
246 * The same comparison for the roster files and the roster table.
247 *
248 * @param {Array<{ slug: string, sha: string }>} files
249 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
250 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
251 */
252export function rosterDriftVerdict(files, rows) {
253  return sourceDriftVerdict(files, rows, { label: "Roster", noun: "cohort", tool: "sync_roster" });
254}
255
256/**
257 * The same comparison for the phage files and the phages table.
258 *
259 * @param {Array<{ slug: string, sha: string }>} files
260 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
261 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
262 */
263export function phageDriftVerdict(files, rows) {
264  return sourceDriftVerdict(files, rows, { label: "Phage", noun: "phage", tool: "sync_phages" });
265}
266
267/**
268 * The same comparison for the registry files and the registry table. An item is read by its address, `<kind>/<id>`.
269 *
270 * @param {Array<{ slug: string, sha: string }>} files
271 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
272 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
273 */
274export function registryDriftVerdict(files, rows) {
275  return sourceDriftVerdict(files, rows, { label: "Registry", noun: "registry item", tool: "sync_registry" });
276}
277
278/**
279 * The same comparison for content/llms.txt and the settings row /llms.txt is served from: one file, one row.
280 *
281 * @param {Array<{ slug: string, sha: string }>} files
282 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
283 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
284 */
285export function llmsDriftVerdict(files, rows) {
286  return sourceDriftVerdict(files, rows, { label: "llms.txt", noun: "llms.txt", tool: "sync_llms" });
287}
288
289/**
290 * @param {Array<{ slug: string, sha: string }>} files
291 * @param {Array<{ slug: string, source_blob_sha: string | null }>} rows
292 * @param {{ label: string, noun: string, tool: string }} words
293 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
294 */
295function sourceDriftVerdict(files, rows, { label, noun, tool }) {
296  const { changed, unrowed, unfiled } = contentDriftCompare(files, rows);
297  const drift = changed.length + unrowed.length + unfiled.length;
298  if (drift === 0) {
299    return {
300      ok: true,
301      detail: `D1 agrees with the repository: ${files.length} ${noun} file(s), every blob sha matched by its row.`,
302    };
303  }
304  return {
305    ok: false,
306    detail:
307      `${label} drift ${drift}: ${changed.length} sha-changed, ${unrowed.length} ` +
308      `file(s) with no row, ${unfiled.length} row(s) with no file, against ` +
309      `${files.length} ${noun} file(s). Repair with ${tool} on the operator API.`,
310    counts: { expected: files.length, present: files.length - changed.length - unrowed.length },
311  };
312}
313
314/**
315 * Counted on the _docsize shadows: COUNT(*) on an external-content fts5 table reads through to its
316 * content table and never fails. A count that cannot be read FAILS rather than comparing undefined.
317 *
318 * @param {Record<string, unknown>} counts
319 * @returns {{ ok: boolean, detail: string, counts?: { expected: number, present: number } }}
320 */
321export function ftsEqualityVerdict(counts) {
322  const NAMES = ["posts", "postsFts", "docs", "identity", "prose"];
323
324  const unreadable = NAMES.filter((n) => !Number.isInteger(counts[n]) || Number(counts[n]) < 0);
325  if (unreadable.length > 0) {
326    return {
327      ok: false,
328      detail:
329        `FTS counts unreadable: ${unreadable.join(", ")} ` +
330        `${unreadable.length === 1 ? "is" : "are"} missing or not a whole number. ` +
331        `A count that cannot be read is not a passing check.`,
332    };
333  }
334
335  const posts = Number(counts.posts);
336  const postsFts = Number(counts.postsFts);
337  const docs = Number(counts.docs);
338  const identity = Number(counts.identity);
339  const prose = Number(counts.prose);
340
341  const broken = [];
342  if (posts !== postsFts) broken.push(`posts=${posts} but posts_fts_docsize=${postsFts}`);
343  if (identity !== docs || prose !== docs) {
344    broken.push(
345      `search_docs=${docs} but search_identity_docsize=${identity} and ` +
346        `search_prose_docsize=${prose}`,
347    );
348  }
349
350  if (broken.length === 0) {
351    return {
352      ok: true,
353      detail:
354        `FTS indexes agree: posts=${posts}=posts_fts_docsize, ` +
355        `search_docs=${docs}=identity=prose.`,
356    };
357  }
358
359  return {
360    ok: false,
361    detail:
362      `FTS index drift: ${broken.join("; ")}. ` +
363      `Rebuild with INSERT INTO <index>(<index>) VALUES('rebuild'). ` +
364      `Do NOT DELETE FROM either index; that corrupts it further.`,
365    // The first disagreeing pair only; the full five are in detail, which stays off the wire.
366    counts:
367      posts !== postsFts
368        ? { expected: posts, present: postsFts }
369        : { expected: docs, present: identity !== docs ? identity : prose },
370  };
371}
372
373/** Measured: ask-index-drift maxed at 2332 ms over 12 samples, so 3 s clears it without false alarms. */
374export const CHECK_TIMEOUT_MS = 3000;
375
376/**
377 * A timeout is a failed check. It does not cancel the work: only the RESPONSE is bounded, and abandoned
378 * work keeps accumulating under sustained timeouts.
379 *
380 * @param {Promise<{ ok: boolean, detail: string, counts?: { expected: number, present: number } }>} promise
381 * @param {number} ms
382 * @param {string} name
383 * @returns {Promise<{ ok: boolean, detail: string, counts?: { expected: number, present: number } }>}
384 */
385export function withTimeout(promise, ms, name) {
386  return new Promise((resolve) => {
387    const timer = setTimeout(
388      () =>
389        resolve({
390          ok: false,
391          detail: `check ${name} did not answer within ${ms}ms and is reported failed`,
392        }),
393      ms,
394    );
395
396    promise.then(
397      (value) => {
398        clearTimeout(timer);
399        resolve(value);
400      },
401      (error) => {
402        clearTimeout(timer);
403        resolve({
404          ok: false,
405          detail: `check threw: ${errorMessage(error)}`,
406        });
407      },
408    );
409  });
410}
411
412/**
413 * Detail strings never reach this unauthenticated wire: they carry an R2 object key and shadow table
414 * names. A FAILING check carries its two counts, so a flap can be sized; a passing one, name and ok.
415 *
416 * @param {{ checks: Array<{ name: string, ok: boolean, counts?: { expected: number, present: number } }> }} run
417 * @returns {{ ok: boolean, checks: Array<{ name: string, ok: boolean, expected?: number, present?: number }> }}
418 */
419export function publicHealthBody(run) {
420  return {
421    ok: run.checks.every((c) => c.ok),
422    checks: run.checks.map((c) => {
423      // Field by field, not spread, so a field added to HealthCheck later cannot leak onto the wire.
424      if (c.ok || !c.counts) return { name: c.name, ok: c.ok };
425      return {
426        name: c.name,
427        ok: c.ok,
428        expected: c.counts.expected,
429        present: c.counts.present,
430      };
431    }),
432  };
433}
434