app/lib/health/checks.server.ts

Functions

Imports

It imports app/db/cv.ts, app/kb/procedures/parse.mjs, app/kb/registry/compile.mjs, app/kb/registry/kinds.mjs, app/lib/content/hashes.mjs, app/lib/cv/parse.mjs, app/lib/cv/pdf-html.mjs, app/lib/cv/pdf.server.ts, app/lib/dictionary/parse.mjs, app/lib/editor/github.server.ts, app/lib/health/verdicts.mjs, app/lib/llms/validate.mjs, app/lib/media/backup.server.ts, app/lib/media/rebuild.server.ts, app/lib/pages/compile.mjs, app/lib/phages/compile.mjs, app/lib/publications/parse.mjs, app/lib/roster/compile.mjs, app/lib/search/ask.server.ts, app/lib/timing.ts. Imported by app/lib/operator/sync-tools.server.ts, app/routes/admin._index.tsx, app/routes/api.health.ts.

Source

1// Every decision lives in verdicts.mjs: check:tests cannot reach a module that imports a binding.
2
3import {
4  CHECK_TIMEOUT_MS,
5  askDriftVerdict,
6  contentDriftVerdict,
7  procedureDriftVerdict,
8  cvDriftVerdict,
9  cvPdfDriftVerdict,
10  dictionaryDriftVerdict,
11  pageDriftVerdict,
12  publicationDriftVerdict,
13  llmsDriftVerdict,
14  rosterDriftVerdict,
15  phageDriftVerdict,
16  registryDriftVerdict,
17  ftsEqualityVerdict,
18  mediaDriftVerdict,
19  mediaBackupDriftVerdict,
20  withTimeout,
21} from "~/lib/health/verdicts.mjs";
22import { askIndexStatus, type AskIndexStatus } from "~/lib/search/ask.server";
23import { timed, type Timings } from "~/lib/timing";
24import { GitHubError, listDirectory, listPostFiles } from "~/lib/editor/github.server";
25import { REGISTRY_DIR } from "~/kb/registry/compile.mjs";
26import { KINDS } from "~/kb/registry/kinds.mjs";
27import { PROCEDURES_DIR } from "~/kb/procedures/parse.mjs";
28import { gitBlobSha } from "~/lib/content/hashes.mjs";
29import { LLMS_PATH, LLMS_SETTING_KEY } from "~/lib/llms/validate.mjs";
30import { readCv } from "~/db/cv";
31import { cvFingerprint } from "~/lib/cv/pdf-html.mjs";
32import { storedCvPdfFingerprint } from "~/lib/cv/pdf.server";
33import { CV_DIR } from "~/lib/cv/parse.mjs";
34import { DICTIONARY_DIR } from "~/lib/dictionary/parse.mjs";
35import { PAGES_DIR } from "~/lib/pages/compile.mjs";
36import { PUBLICATIONS_DIR } from "~/lib/publications/parse.mjs";
37import { PHAGES_DIR } from "~/lib/phages/compile.mjs";
38import { ROSTER_DIR } from "~/lib/roster/compile.mjs";
39import { mediaIndexStatus } from "~/lib/media/rebuild.server";
40import { backupStatus } from "~/lib/media/backup.server";
41
42/**
43 * The two sides the content-drift comparison reads: the repository's post files with their blob
44 * shas, and D1's rows with the sha each was rendered from. The health check and sync_posts both
45 * read them here, so they cannot compare different things.
46 */
47export async function readContentSides(env: Env & { GITHUB_TOKEN?: string }) {
48  const files = (await listPostFiles(env)).map((e) => ({ slug: e.slug, sha: e.sha, path: e.path }));
49  const rows = await env.DB.prepare(
50    "SELECT slug, source_blob_sha FROM posts WHERE source_path IS NOT NULL",
51  ).all<{ slug: string; source_blob_sha: string | null }>();
52  return { files, rows: rows.results ?? [] };
53}
54
55/**
56 * The same two sides for procedures: `content/procedures/*.md` with their blob shas, and the table's
57 * rows with the sha each was compiled from. The check and sync_procedures both read them here. A published
58 * version with no sealed frozen copy reads as having no sha, so it is drift and the sync that repairs it
59 * freezes the copy (drizzle/0027_procedure_versions.sql).
60 */
61export async function readProcedureSides(env: Env & { GITHUB_TOKEN?: string }) {
62  const files = await listMarkdownFiles(env, PROCEDURES_DIR);
63  const rows = await env.DB.prepare(
64    `SELECT p.slug, p.path,
65        CASE WHEN p.status = 'published' AND p.version IS NOT NULL AND p.version <> ''
66              AND NOT EXISTS (SELECT 1 FROM procedure_versions v WHERE v.slug = p.slug AND v.version = p.version AND v.sealed = 1)
67             THEN NULL ELSE p.source_blob_sha END AS source_blob_sha
68       FROM procedures p WHERE p.source_path IS NOT NULL`,
69  ).all<{ slug: string; path: string; source_blob_sha: string | null }>();
70  return { files, rows: rows.results ?? [] };
71}
72
73/** The markdown files of a directory, each with its blob sha, the slug being the file name without .md. */
74async function listMarkdownFiles(env: Env & { GITHUB_TOKEN?: string }, dir: string) {
75  return (await listDirectory(env, dir))
76    .filter((e) => e.type === "file" && e.name.endsWith(".md"))
77    .map((e) => ({ slug: e.name.slice(0, -".md".length), sha: e.sha, path: e.path }));
78}
79
80/** The same two sides for dictionary entries: `content/dictionary/*.md` and the dictionary_entries table's rows. */
81export async function readDictionarySides(env: Env & { GITHUB_TOKEN?: string }) {
82  const files = await listMarkdownFiles(env, DICTIONARY_DIR);
83  // `key` answers as `slug`, the name every kind's drift comparison and convergence loop reads a row by.
84  const rows = await env.DB.prepare(
85    "SELECT key AS slug, path, source_blob_sha FROM dictionary_entries",
86  ).all<{ slug: string; path: string; source_blob_sha: string | null }>();
87  return { files, rows: rows.results ?? [] };
88}
89
90/** The same two sides for pages: `content/pages/*.md` and the pages table's rows. */
91export async function readPageSides(env: Env & { GITHUB_TOKEN?: string }) {
92  const files = await listMarkdownFiles(env, PAGES_DIR);
93  const rows = await env.DB.prepare(
94    "SELECT slug, path, source_blob_sha FROM pages",
95  ).all<{ slug: string; path: string; source_blob_sha: string | null }>();
96  return { files, rows: rows.results ?? [] };
97}
98
99/** The same two sides for the CV: `content/cv/*.md` and the cv table's rows. */
100export async function readCvSides(env: Env & { GITHUB_TOKEN?: string }) {
101  const files = await listMarkdownFiles(env, CV_DIR);
102  const rows = await env.DB.prepare("SELECT slug, source_blob_sha FROM cv").all<{
103    slug: string;
104    source_blob_sha: string | null;
105  }>();
106  return { files, rows: rows.results ?? [] };
107}
108
109/** The same two sides for the roster: `content/roster/*.md` and the roster table's rows. */
110export async function readRosterSides(env: Env & { GITHUB_TOKEN?: string }) {
111  const files = await listMarkdownFiles(env, ROSTER_DIR);
112  const rows = await env.DB.prepare(
113    "SELECT slug, source_blob_sha FROM roster",
114  ).all<{ slug: string; source_blob_sha: string | null }>();
115  return { files, rows: rows.results ?? [] };
116}
117
118/** The same two sides for the phages: `content/phages/*.md` and the phages table's rows. */
119export async function readPhageSides(env: Env & { GITHUB_TOKEN?: string }) {
120  const files = await listMarkdownFiles(env, PHAGES_DIR);
121  const rows = await env.DB.prepare(
122    "SELECT slug, source_blob_sha FROM phages",
123  ).all<{ slug: string; source_blob_sha: string | null }>();
124  return { files, rows: rows.results ?? [] };
125}
126
127/**
128 * The same two sides for the registry: `content/registry/<kind>/<id>.md` for every kind, and the registry table's rows,
129 * each read by its address `<kind>/<id>`. A kind's directory that does not exist yet is an empty kind, not a fault: a
130 * kind is defined before its first file is written.
131 */
132export async function readRegistrySides(env: Env & { GITHUB_TOKEN?: string }) {
133  const files: Array<{ slug: string; sha: string; path: string }> = [];
134  for (const kind of Object.keys(KINDS)) {
135    try {
136      for (const file of await listMarkdownFiles(env, `${REGISTRY_DIR}/${kind}`)) {
137        files.push({ ...file, slug: `${kind}/${file.slug}` });
138      }
139    } catch (error) {
140      if (!(error instanceof GitHubError && error.status === 404)) throw error;
141    }
142  }
143  const rows = await env.DB.prepare(
144    "SELECT kind || '/' || id AS slug, source_blob_sha FROM registry",
145  ).all<{ slug: string; source_blob_sha: string | null }>();
146  return { files, rows: rows.results ?? [] };
147}
148
149/** The same two sides for publications: `content/publications/*.md` and the publications table's rows. */
150export async function readPublicationSides(env: Env & { GITHUB_TOKEN?: string }) {
151  const files = await listMarkdownFiles(env, PUBLICATIONS_DIR);
152  const rows = await env.DB.prepare(
153    "SELECT slug, doi_key, source_blob_sha FROM publications",
154  ).all<{ slug: string; doi_key: string | null; source_blob_sha: string | null }>();
155  return { files, rows: rows.results ?? [] };
156}
157
158/**
159 * The same two sides for llms.txt: the one file with its blob sha, and the settings row /llms.txt is served
160 * from, given the blob sha of the bytes it holds (the row has no column for one). The slug is a name for the
161 * pair, so the shared comparison applies.
162 */
163export async function readLlmsSides(env: Env & { GITHUB_TOKEN?: string }) {
164  const dir = LLMS_PATH.slice(0, LLMS_PATH.lastIndexOf("/"));
165  const file = (await listDirectory(env, dir)).find((e) => e.type === "file" && e.path === LLMS_PATH);
166  const row = await env.DB.prepare("SELECT value FROM settings WHERE key = ?1")
167    .bind(LLMS_SETTING_KEY)
168    .first<{ value: string }>();
169  return {
170    files: file ? [{ slug: "llms", sha: file.sha, path: file.path }] : [],
171    rows: row ? [{ slug: "llms", source_blob_sha: await gitBlobSha(row.value) }] : [],
172  };
173}
174
175/** @public The admin Overview's loader returns these, so the route's inferred type names it. */
176export interface HealthCheck {
177  name: string;
178  ok: boolean;
179  detail: string;
180  /** Only on failure: the one part of a failing check besides its name that reaches the wire. */
181  counts?: { expected: number; present: number };
182}
183
184/** What a caller may share with the run: the request's timing marks, and a memoized read of the Ask index. */
185export interface HealthOptions {
186  timings?: Timings;
187  askStatus?: () => Promise<AskIndexStatus | null>;
188}
189
190/** The Ask index's status: the caller's shared read when it has one, else a fresh listing. Unreadable is a failed check. */
191async function askStatusOf(env: Env, options: HealthOptions): Promise<AskIndexStatus> {
192  if (!options.askStatus) return askIndexStatus(env);
193  const status = await options.askStatus();
194  if (!status) throw new Error("the Ask index could not be listed");
195  return status;
196}
197
198interface HealthRun {
199  checks: HealthCheck[];
200  failed: HealthCheck[];
201}
202
203/** A check that throws is reported as failed, or one broken check would silence every other. */
204export async function runHealthChecks(env: Env, options: HealthOptions = {}): Promise<HealthRun> {
205  /* Every check is independent and each has its own timeout, so they run together: the run takes as long as its slowest
206     check, not the sum of all of them. The order of the answers is the order below. */
207  const pending: Array<Promise<HealthCheck>> = [];
208  const guard = (name: string, run: Parameters<typeof guardOne>[1]) =>
209    timed(options.timings, `health_${name}`, () => guardOne(name, run));
210
211  pending.push(
212    guard("ask-index-drift", async () => askDriftVerdict(await askStatusOf(env, options))),
213  );
214
215  // The INDEX; media-backup-drift below compares BYTES. Easy to confuse.
216  pending.push(
217    guard("media-index-drift", async () => mediaDriftVerdict(await mediaIndexStatus(env))),
218  );
219
220  pending.push(
221    guard("media-backup-drift", async () => {
222      // Both buckets listed in full: a truncated read would report a clean sweep of the part it saw.
223      return mediaBackupDriftVerdict(await backupStatus(env));
224    }),
225  );
226
227  pending.push(
228    guard("content-drift", async () => {
229      // An unreadable repository fails this check, which the repair plan refuses to act on alone.
230      const { files, rows } = await readContentSides(env);
231      return contentDriftVerdict(files, rows);
232    }),
233  );
234
235  pending.push(
236    guard("procedures-drift", async () => {
237      // A procedure edited, added or deleted through git, or a save whose D1 write failed, shows here.
238      const { files, rows } = await readProcedureSides(env);
239      return procedureDriftVerdict(files, rows);
240    }),
241  );
242
243  pending.push(
244    guard("dictionary-drift", async () => {
245      // An entry edited, added or deleted through git, or a save whose D1 write failed, shows here.
246      const { files, rows } = await readDictionarySides(env);
247      return dictionaryDriftVerdict(files, rows);
248    }),
249  );
250
251  pending.push(
252    guard("roster-drift", async () => {
253      // A cohort edited, added or deleted through git, or a save whose D1 write failed, shows here.
254      const { files, rows } = await readRosterSides(env);
255      return rosterDriftVerdict(files, rows);
256    }),
257  );
258
259  pending.push(
260    guard("phage-drift", async () => {
261      // A phage edited, added or deleted through git, or a save whose D1 write failed, shows here.
262      const { files, rows } = await readPhageSides(env);
263      return phageDriftVerdict(files, rows);
264    }),
265  );
266
267  pending.push(
268    guard("registry-drift", async () => {
269      // An item edited, added or deleted through git, or a save whose D1 write failed, shows here.
270      const { files, rows } = await readRegistrySides(env);
271      return registryDriftVerdict(files, rows);
272    }),
273  );
274
275  pending.push(
276    guard("pages-drift", async () => {
277      // A page edited, added or deleted through git, or a save whose D1 write failed, shows here.
278      const { files, rows } = await readPageSides(env);
279      return pageDriftVerdict(files, rows);
280    }),
281  );
282
283  pending.push(
284    guard("cv-drift", async () => {
285      // A CV file edited through git, or a save whose D1 write failed, shows here.
286      const { files, rows } = await readCvSides(env);
287      return cvDriftVerdict(files, rows);
288    }),
289  );
290
291  pending.push(
292    guard("cv-pdf-drift", async () => {
293      // The PDF is drawn from D1's CV, so it is compared with that and not with the repository: a stale CV row
294      // is cv-drift's finding, and sync_cv runs before sync_cv_pdf.
295      const [stored, cv] = await Promise.all([storedCvPdfFingerprint(env), readCv(env)]);
296      return cvPdfDriftVerdict({ stored, expected: await cvFingerprint(cv) });
297    }),
298  );
299
300  pending.push(
301    guard("publications-drift", async () => {
302      const { files, rows } = await readPublicationSides(env);
303      return publicationDriftVerdict(files, rows);
304    }),
305  );
306
307  pending.push(
308    guard("llms-drift", async () => {
309      // A file edited through git, or a save whose D1 write failed, shows here; the served row is derived.
310      const { files, rows } = await readLlmsSides(env);
311      return llmsDriftVerdict(files, rows);
312    }),
313  );
314
315  pending.push(
316    guard("fts-equality", async () => {
317      // Index counts come from the _docsize shadows: COUNT(*) on an external-content fts5 table reads
318      // through to its content table and can never disagree with it.
319      const row = await env.DB.prepare(
320        "SELECT (SELECT COUNT(*) FROM posts) AS posts, " +
321          "(SELECT COUNT(*) FROM posts_fts_docsize) AS postsFts, " +
322          "(SELECT COUNT(*) FROM search_docs) AS docs, " +
323          "(SELECT COUNT(*) FROM search_identity_docsize) AS identity, " +
324          "(SELECT COUNT(*) FROM search_prose_docsize) AS prose",
325      ).first<Record<string, unknown>>();
326
327      // A null row reaches the verdict as unreadable counts: a sentence, not a stack.
328      return ftsEqualityVerdict(row ?? {});
329    }),
330  );
331
332  const checks = await Promise.all(pending);
333  return { checks, failed: checks.filter((c) => !c.ok) };
334}
335
336async function guardOne(
337  name: string,
338  run: () => Promise<{ ok: boolean; detail: string; counts?: { expected: number; present: number } }>,
339): Promise<HealthCheck> {
340  // The async wrapper turns a SYNCHRONOUS throw in run into a rejection withTimeout can guard.
341  const started = (async () => run())();
342  const { ok, detail, counts } = await withTimeout(started, CHECK_TIMEOUT_MS, name);
343  return counts ? { name, ok, detail, counts } : { name, ok, detail };
344}
345