workers/watchdog.ts

Functions

Imports

It imports app/lib/error-message.mjs, app/lib/health/alert-state.mjs, app/lib/health/error-rate.mjs, app/lib/health/repair.mjs, app/lib/seo.ts. No module of this repo imports it.

Source

1// Relative imports on purpose: this Worker's esbuild bundle does not read the `~/` tsconfig mapping,
2// so `~/` here would typecheck and then fail at deploy.
3import {
4  failingCheckNames,
5  refusalMiss,
6  settleAskRecheck,
7  watchdogActions,
8  watchdogOutcome,
9} from "../app/lib/health/repair.mjs";
10import {
11  alertTransition,
12  deliverTransition,
13  formatDuration,
14} from "../app/lib/health/alert-state.mjs";
15import {
16  ERROR_WINDOW_MINUTES,
17  errorRateQuery,
18  errorRateVerdict,
19} from "../app/lib/health/error-rate.mjs";
20import { SITE_ORIGIN } from "../app/lib/seo";
21import { errorMessage } from "../app/lib/error-message.mjs";
22
23// A script name, not a hostname, so not derived from SITE_ORIGIN. Must equal the site config's `name`.
24const SITE_SCRIPT_NAME = "dustinedwards";
25
26/**
27 * A separate Worker because a watcher inside the site dies with it. It reaches the site only through
28 * the `SITE` service binding (error 1042 refuses same-zone Worker fetch), so it cannot prove the site
29 * is reachable from outside. It must never write the health snapshot, whose age is its liveness.
30 */
31
32// `ALERT_EMAIL` is a var, not a secret: an inbox address grants nothing.
33interface WatchdogEnv {
34  SITE: Fetcher;
35  EMAIL: SendEmail;
36  ALERT_EMAIL: string;
37  OPERATOR_TOKEN?: string;
38  APP_KV: KVNamespace;
39  // Optional, and absence is reported by name so losing sight of errors never looks like seeing none.
40  CLOUDFLARE_ACCOUNT_ID?: string;
41  CLOUDFLARE_API_TOKEN?: string;
42}
43
44// Namespaced because this KV is shared with the Ask cache and the site's other KV state.
45const STATE_KEY = "watchdog:alert-state";
46
47// Absent and unreadable differ: unreadable means the dedupe is blind, and it must not stay quiet.
48async function readState(env: WatchdogEnv): Promise<{ readable: boolean; stored: unknown }> {
49  try {
50    return { readable: true, stored: await env.APP_KV.get(STATE_KEY, "json") };
51  } catch (error) {
52    console.error(
53      JSON.stringify({
54        watchdog: "state-read-failed",
55        error: errorMessage(error),
56      }),
57    );
58    return { readable: false, stored: null };
59  }
60}
61
62// Never throws: a failed write costs at worst one duplicate mail, a throw would cost this alert.
63async function writeState(
64  env: WatchdogEnv,
65  state: { red: boolean; since: string; checks: string[] },
66): Promise<void> {
67  try {
68    await env.APP_KV.put(STATE_KEY, JSON.stringify(state));
69  } catch (error) {
70    console.error(
71      JSON.stringify({
72        alert: "watchdog-state-write-failed",
73        watchdog: "state-write-failed",
74        error: errorMessage(error),
75      }),
76    );
77  }
78}
79
80// Not derived from SITE_ORIGIN: that is `*.workers.dev`, which Email Sending would refuse as a sender.
81const ALERT_FROM = { email: "watchdog@dustinedwards.info", name: "dustinedwards.info watchdog" };
82
83type Reading = { status: number; body: unknown; error?: string };
84
85// Never throws, and keeps the cause: a timeout is a site problem, a 1042 is a misdirected binding.
86// Exported only so the worker test can drive the catch.
87export async function readHealth(env: WatchdogEnv): Promise<Reading> {
88  try {
89    const response = await env.SITE.fetch(`${SITE_ORIGIN}/api/health`, {
90      headers: { "user-agent": "watchdog", "cache-control": "no-cache" },
91    });
92    return { status: response.status, body: await response.json().catch(() => null) };
93  } catch (error) {
94    return {
95      status: 0,
96      body: null,
97      error: errorMessage(error),
98    };
99  }
100}
101
102// The verdict is read, never inferred from a 200. The token is never logged or put in the mail.
103async function repair(
104  env: WatchdogEnv,
105  token: string,
106  tool: string,
107): Promise<{ miss: string; unrepairable: boolean }> {
108  let response: Response;
109  let payload: unknown;
110  try {
111    response = await env.SITE.fetch(`${SITE_ORIGIN}/api/operator`, {
112      method: "POST",
113      headers: {
114        authorization: `Bearer ${token}`,
115        "content-type": "application/json",
116        "user-agent": "watchdog",
117      },
118      body: JSON.stringify({ tool, args: {} }),
119    });
120    payload = await response.json().catch(() => null);
121  } catch (error) {
122    return {
123      miss: `${tool} did not complete: ${errorMessage(error)}`,
124      unrepairable: false,
125    };
126  }
127
128  if (!response.ok) return refusalMiss(tool, response.status, payload);
129
130  const report = (
131    payload && typeof payload === "object"
132      ? ((payload as { data?: unknown }).data ?? payload)
133      : null
134  ) as { converged?: unknown; expected?: unknown; present?: unknown } | null;
135  if (!report || typeof report.converged !== "boolean") {
136    return {
137      miss: `${tool} answered without a converged verdict, so nothing was proven`,
138      unrepairable: false,
139    };
140  }
141  if (report.converged !== true) {
142    return {
143      miss: `${tool} ran and did not converge (${report.expected} expected, ${report.present} present)`,
144      unrepairable: false,
145    };
146  }
147  return { miss: "", unrepairable: false };
148}
149
150/**
151 * The one firing a week that refreshes the citation counts: Sunday, 03:00 to 03:14 UTC. The cron fires every
152 * fifteen minutes, so exactly one firing falls in that quarter hour, and the main Worker needs no cron of its
153 * own (wrangler.jsonc.example keeps `crons: []` on purpose). Exported only so the worker test can drive it.
154 */
155export function isCitationRefreshWindow(scheduledTime: number): boolean {
156  const at = new Date(scheduledTime);
157  return at.getUTCDay() === 0 && at.getUTCHours() === 3 && at.getUTCMinutes() < 15;
158}
159
160/**
161 * Calls the operator API's refresh_citations, which replaces each paper's count in D1 and keeps the old one
162 * where OpenAlex fails. Returns the problem, or null when nothing was wrong or it was not this firing's turn;
163 * the caller throws it after the health mail, so the invocation is recorded as failed and the cause is named.
164 * Exported only so the worker test can drive it.
165 */
166export async function refreshCitationsWeekly(
167  env: WatchdogEnv,
168  token: string,
169  scheduledTime: number,
170): Promise<string | null> {
171  if (!isCitationRefreshWindow(scheduledTime)) return null;
172  if (!token) return "the weekly citation refresh did not run: OPERATOR_TOKEN is not set on this Worker";
173  let response: Response;
174  let payload: unknown;
175  try {
176    response = await env.SITE.fetch(`${SITE_ORIGIN}/api/operator`, {
177      method: "POST",
178      headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "user-agent": "watchdog" },
179      body: JSON.stringify({ tool: "refresh_citations", args: {} }),
180    });
181    payload = await response.json().catch(() => null);
182  } catch (error) {
183    return `the weekly citation refresh did not complete: ${errorMessage(error)}`;
184  }
185  if (!response.ok) {
186    const body = payload as { error?: string; detail?: { failures?: Array<{ doi: string; stage: string }> } } | null;
187    const failed = (body?.detail?.failures ?? []).map((f) => `${f.doi} (${f.stage})`).join(", ");
188    return `the weekly citation refresh answered ${response.status}: ${body?.error ?? "(no error was given)"}${failed ? ` Not refreshed: ${failed}.` : ""}`;
189  }
190  console.log(JSON.stringify({ watchdog: "citations-refreshed", data: (payload as { data?: unknown } | null)?.data ?? null }));
191  return null;
192}
193
194/**
195 * The one firing a day that purges expired zero-result search queries: 04:00 to 04:14 UTC, a different
196 * quarter hour than the weekly citation refresh above so the two never share a firing. Replaces the admin
197 * Tools page's manual "remove expired queries" button, which this Worker has no cron of its own to drive
198 * (wrangler.jsonc.example keeps `crons: []` on purpose). Exported only so the worker test can drive it.
199 */
200export function isZeroResultPurgeWindow(scheduledTime: number): boolean {
201  const at = new Date(scheduledTime);
202  return at.getUTCHours() === 4 && at.getUTCMinutes() < 15;
203}
204
205/**
206 * Calls the operator API's purge_zero_results, which removes every zero-result query row past its
207 * retention cutoff (the same cutoff the removed button enforced). Returns the problem, or null when
208 * nothing was wrong or it was not this firing's turn; the caller throws it, so the invocation is
209 * recorded as failed and the cause is named. Exported only so the worker test can drive it.
210 */
211export async function purgeZeroResultsDaily(
212  env: WatchdogEnv,
213  token: string,
214  scheduledTime: number,
215): Promise<string | null> {
216  if (!isZeroResultPurgeWindow(scheduledTime)) return null;
217  if (!token) return "the daily zero-result purge did not run: OPERATOR_TOKEN is not set on this Worker";
218  let response: Response;
219  let payload: unknown;
220  try {
221    response = await env.SITE.fetch(`${SITE_ORIGIN}/api/operator`, {
222      method: "POST",
223      headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "user-agent": "watchdog" },
224      body: JSON.stringify({ tool: "purge_zero_results", args: {} }),
225    });
226    payload = await response.json().catch(() => null);
227  } catch (error) {
228    return `the daily zero-result purge did not complete: ${errorMessage(error)}`;
229  }
230  if (!response.ok) {
231    const body = payload as { error?: string } | null;
232    return `the daily zero-result purge answered ${response.status}: ${body?.error ?? "(no error was given)"}`;
233  }
234  console.log(
235    JSON.stringify({ watchdog: "zero-results-purged", data: (payload as { data?: unknown } | null)?.data ?? null }),
236  );
237  return null;
238}
239
240/**
241 * The one firing a day that takes the D1 dump: 05:00 to 05:14 UTC, its own quarter hour so it never
242 * shares a firing with the citation refresh (Sunday 03:00) or the zero-result purge (04:00).
243 * Exported only so the worker test can drive it.
244 */
245export function isD1BackupWindow(scheduledTime: number): boolean {
246  const at = new Date(scheduledTime);
247  return at.getUTCHours() === 5 && at.getUTCMinutes() < 15;
248}
249
250/**
251 * Calls the operator API's backup_d1, which dumps D1 to the BACKUPS bucket, checks the dump and prunes
252 * by retention. Returns the problem, or null when nothing was wrong or it was not this firing's turn;
253 * the caller throws it, so the invocation is recorded as failed and the cause is named. A failed run
254 * is retried by nothing until tomorrow, which is why the failure must be loud. Exported only so the
255 * worker test can drive it.
256 */
257export async function backupD1Daily(
258  env: WatchdogEnv,
259  token: string,
260  scheduledTime: number,
261): Promise<string | null> {
262  if (!isD1BackupWindow(scheduledTime)) return null;
263  if (!token) return "the daily D1 backup did not run: OPERATOR_TOKEN is not set on this Worker";
264  let response: Response;
265  let payload: unknown;
266  try {
267    response = await env.SITE.fetch(`${SITE_ORIGIN}/api/operator`, {
268      method: "POST",
269      headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "user-agent": "watchdog" },
270      body: JSON.stringify({ tool: "backup_d1", args: {} }),
271    });
272    payload = await response.json().catch(() => null);
273  } catch (error) {
274    return `the daily D1 backup did not complete: ${errorMessage(error)}`;
275  }
276  if (!response.ok) {
277    const body = payload as { error?: string } | null;
278    return `the daily D1 backup answered ${response.status}: ${body?.error ?? "(no error was given)"}`;
279  }
280  console.log(JSON.stringify({ watchdog: "d1-backed-up", data: (payload as { data?: unknown } | null)?.data ?? null }));
281  return null;
282}
283
284/**
285 * The one firing a day that checks the publications list against ORCID and Crossref: 06:00 to 06:14 UTC,
286 * its own quarter hour. Exported only so the worker test can drive it.
287 */
288export function isPublicationCheckWindow(scheduledTime: number): boolean {
289  const at = new Date(scheduledTime);
290  return at.getUTCHours() === 6 && at.getUTCMinutes() < 15;
291}
292
293// The DOIs already mailed about, so a paper waiting to be added through Carrel is mailed once, not daily.
294const PUBLICATIONS_MAILED_KEY = "watchdog:publications-mailed";
295
296type FoundWork = { doi: string; title: string; year: number | null };
297
298/**
299 * Calls the operator API's check_publications and mails each DOI it finds that no file carries, once.
300 * Returns the problem, or null when nothing was wrong or it was not this firing's turn; the caller throws
301 * it, so the invocation is recorded as failed and the cause is named. Exported only so the worker test
302 * can drive it.
303 */
304export async function checkPublicationsDaily(
305  env: WatchdogEnv,
306  token: string,
307  scheduledTime: number,
308): Promise<string | null> {
309  if (!isPublicationCheckWindow(scheduledTime)) return null;
310  if (!token) return "the daily publication check did not run: OPERATOR_TOKEN is not set on this Worker";
311  let response: Response;
312  let payload: unknown;
313  try {
314    response = await env.SITE.fetch(`${SITE_ORIGIN}/api/operator`, {
315      method: "POST",
316      headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "user-agent": "watchdog" },
317      body: JSON.stringify({ tool: "check_publications", args: {} }),
318    });
319    payload = await response.json().catch(() => null);
320  } catch (error) {
321    return `the daily publication check did not complete: ${errorMessage(error)}`;
322  }
323  if (!response.ok) {
324    const body = payload as { error?: string } | null;
325    return `the daily publication check answered ${response.status}: ${body?.error ?? "(no error was given)"}`;
326  }
327  const data = (payload as { data?: { notOnSite?: FoundWork[]; nameMatches?: FoundWork[] } } | null)?.data;
328  let mailed: Set<string>;
329  try {
330    mailed = new Set(((await env.APP_KV.get(PUBLICATIONS_MAILED_KEY, "json")) as string[] | null) ?? []);
331  } catch (error) {
332    // Unreadable is not empty: mailing every DOI again daily would bury the one that is new.
333    return `the daily publication check could not read which DOIs it already mailed: ${errorMessage(error)}`;
334  }
335  const fresh = (works: FoundWork[] = []) => works.filter((w) => !mailed.has(w.doi.toLowerCase()));
336  const tied = fresh(data?.notOnSite);
337  const byName = fresh(data?.nameMatches);
338  console.log(JSON.stringify({ watchdog: "publications-checked", new: tied.length + byName.length }));
339  if (tied.length + byName.length === 0) return null;
340
341  const line = (w: FoundWork) => `${w.title || "(no title)"}${w.year ? ` (${w.year})` : ""}: https://doi.org/${w.doi}`;
342  const sent = await alert(env, "New publication found for dustinedwards.info", [
343    ...(tied.length > 0 ? ["ORCID or Crossref lists these under your ORCID, and the site does not:", ...tied.map(line)] : []),
344    ...(tied.length > 0 && byName.length > 0 ? [""] : []),
345    ...(byName.length > 0 ? ["Crossref lists these under your name, without your ORCID. Check they are yours:", ...byName.map(line)] : []),
346    "",
347    "Add a paper through Carrel. Each DOI is mailed once.",
348  ]);
349  if (!sent) return "the daily publication check found new DOIs and the mail did not send; it will try again tomorrow";
350  for (const w of [...tied, ...byName]) mailed.add(w.doi.toLowerCase());
351  try {
352    await env.APP_KV.put(PUBLICATIONS_MAILED_KEY, JSON.stringify([...mailed]));
353  } catch (error) {
354    return `the daily publication check mailed its DOIs and could not record them, so tomorrow mails them again: ${errorMessage(error)}`;
355  }
356  return null;
357}
358
359async function alert(env: WatchdogEnv, subject: string, lines: string[]): Promise<boolean> {
360  try {
361    await env.EMAIL.send({
362      to: env.ALERT_EMAIL,
363      from: ALERT_FROM,
364      subject,
365      text: lines.join("\n"),
366    });
367    return true;
368  } catch (error) {
369    console.error(
370      JSON.stringify({
371        alert: "watchdog-notify-failed",
372        error: errorMessage(error),
373      }),
374    );
375    return false;
376  }
377}
378
379/**
380 * Not a health check: `ship` refuses unless `/api/health` is ok, so an error spike there would block
381 * deploying the fix. Never throws, and any failure to read the rate is reported as not ok.
382 * Exported only so the worker test can drive it against a stubbed analytics API.
383 */
384export async function readErrorRate(
385  env: WatchdogEnv,
386): Promise<{ ok: boolean; detail: string; configured: boolean }> {
387  const account = env.CLOUDFLARE_ACCOUNT_ID ?? "";
388  const token = env.CLOUDFLARE_API_TOKEN ?? "";
389  if (!account || !token) {
390    return {
391      ok: true,
392      configured: false,
393      detail:
394        "error rate NOT CHECKED: " +
395        `${!account ? "CLOUDFLARE_ACCOUNT_ID" : "CLOUDFLARE_API_TOKEN"} is not set on this ` +
396        "Worker. Set it with `wrangler secret put CLOUDFLARE_API_TOKEN -c " +
397        "wrangler.watchdog.jsonc` (Account / Account Analytics / Read).",
398    };
399  }
400
401  const until = new Date();
402  const since = new Date(until.getTime() - ERROR_WINDOW_MINUTES * 60 * 1000);
403  const body = errorRateQuery({
404    accountId: account,
405    scriptName: SITE_SCRIPT_NAME,
406    since: since.toISOString(),
407    until: until.toISOString(),
408  });
409
410  /** The GraphQL answer as far as it is read; every level is optional-chained below. */
411  type InvocationsAnswer = {
412    errors?: Array<{ message?: string } | null>;
413    data?: { viewer?: { accounts?: Array<{ workersInvocationsAdaptive?: Parameters<typeof errorRateVerdict>[0] }> } };
414  } | null;
415  let payload: InvocationsAnswer = null;
416  try {
417    const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
418      method: "POST",
419      headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
420      body: JSON.stringify(body),
421    });
422    payload = (await response.json().catch(() => null)) as InvocationsAnswer;
423    if (!response.ok) {
424      return {
425        ok: false,
426        configured: true,
427        detail: `the invocations query answered HTTP ${response.status}, so the error rate is unknown.`,
428      };
429    }
430  } catch (error) {
431    return {
432      ok: false,
433      configured: true,
434      detail:
435        "the invocations query did not complete, so the error rate is unknown: " +
436        `${errorMessage(error)}`,
437    };
438  }
439
440  // A GraphQL 200 can carry errors; an absent `data` would otherwise sum to zero errors.
441  if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
442    return {
443      ok: false,
444      configured: true,
445      detail:
446        "the invocations query returned GraphQL errors, so the error rate is unknown: " +
447        `${payload.errors.map((e) => e?.message).join("; ").slice(0, 200)}`,
448    };
449  }
450
451  const verdict = errorRateVerdict(
452    payload?.data?.viewer?.accounts?.[0]?.workersInvocationsAdaptive,
453  );
454  return { ok: verdict.ok, configured: true, detail: verdict.detail };
455}
456
457const renderBody = (reading: Reading | null): string =>
458  reading === null
459    ? "(no reading was taken)"
460    : reading.status === 0
461      ? // No body reached us, so print the cause rather than "null".
462        `HTTP 0 (the request did not complete)\n${reading.error ?? "(no cause was recorded)"}`
463      : `HTTP ${reading.status}\n${JSON.stringify(reading.body, null, 2)}`;
464
465type Action = ReturnType<typeof watchdogActions>[number];
466
467// Repairs run every firing and only the mail is deduplicated, so `alerting` is where it ended.
468type AlertDraft = { alerting: boolean; subject: string; lines: string[]; failing: string[] };
469
470const QUIET: AlertDraft = { alerting: false, subject: "", lines: [], failing: [] };
471
472// Called only when every action is a repair or a recheck; a notify action never reaches here.
473async function runRepairs(
474  env: WatchdogEnv,
475  token: string,
476  reading: Reading,
477  actions: Action[],
478): Promise<AlertDraft> {
479  const attempted: string[] = [];
480  const misses: string[] = [];
481  // A refusal abandons the rest: the Ask corpus is read from what content repair writes.
482  let refused = false;
483  for (const action of actions) {
484    if (action.type !== "repair") continue;
485    attempted.push(action.tool);
486    const outcome = await repair(env, token, action.tool);
487    if (!outcome.miss) continue;
488    misses.push(outcome.miss);
489    if (outcome.unrepairable) {
490      refused = true;
491      break;
492    }
493  }
494
495  const first = actions.some((a) => a.type === "recheck") ? await readHealth(env) : null;
496  // Its own sync_ask can leave the Ask index listing short for about a minute; a cron run may wait
497  // up to 15 minutes of wall time, and waiting spends no CPU, so the re-check settles here.
498  const { reading: recheck, polls: settlePolls } = await settleAskRecheck({
499    recheck: first,
500    attempted,
501    reread: () => readHealth(env),
502    sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
503  });
504  const outcome = watchdogOutcome({ misses, recheck });
505
506  console.log(
507    JSON.stringify({
508      watchdog: "repaired",
509      attempted,
510      misses,
511      recheckStatus: recheck?.status ?? null,
512      settlePolls,
513      alerting: outcome.length > 0,
514    }),
515  );
516
517  if (outcome.length === 0) return QUIET;
518  return {
519    alerting: true,
520    failing: failingCheckNames((recheck ?? reading).body),
521    subject: refused
522      ? "dustinedwards.info: self-repair was refused, a deploy is the repair"
523      : "dustinedwards.info: self-repair did not settle it",
524    lines: [
525      ...outcome.map((a) => (a as { reason: string }).reason),
526      ...(refused
527        ? [
528            "",
529            "The operator API REFUSED the content itself (422), so repeating this " +
530              "cannot converge. That is what a repository ahead of the deployed " +
531              "build looks like: the content is valid and the renderer in " +
532              "production is older than it. Deploy (npm run ship), then let the " +
533              "next poll converge it.",
534          ]
535        : []),
536      "",
537      `Attempted: ${attempted.join(", ") || "(nothing)"}`,
538      "",
539      "Health BEFORE the repair:",
540      renderBody(reading),
541      "",
542      "Health AFTER the repair:",
543      renderBody(recheck),
544    ],
545  };
546}
547
548// A throwing Worker alerts even when every health check is green.
549function mergeErrorRate(
550  draft: AlertDraft,
551  errorRate: { ok: boolean; detail: string },
552): AlertDraft {
553  if (errorRate.ok) return draft;
554  return {
555    alerting: true,
556    subject: draft.alerting ? draft.subject : "dustinedwards.info is throwing",
557    lines: draft.alerting
558      ? [...draft.lines, "", `Error rate: ${errorRate.detail}`]
559      : [errorRate.detail],
560    failing: [...draft.failing, "error-rate"],
561  };
562}
563
564// Mails only on a change of state; throws when that mail did not send, so the next firing retries.
565async function mailTransition(
566  env: WatchdogEnv,
567  draft: AlertDraft,
568  reading: Reading,
569  errorRateNote: string[],
570): Promise<void> {
571  const { alerting, failing, subject, lines } = draft;
572  const { readable, stored } = await readState(env);
573  const now = new Date().toISOString();
574  const transition = alertTransition({ stored, storedReadable: readable, alerting, failing, now });
575
576  console.log(
577    JSON.stringify({
578      watchdog: "alert-decision",
579      alerting,
580      failing,
581      storedReadable: readable,
582      email: transition.email?.kind ?? "none",
583      reason: transition.reason,
584    }),
585  );
586
587  const email = transition.email;
588  const delivered = await deliverTransition(transition, {
589    send: () =>
590      email?.kind === "opened"
591        ? alert(env, subject, [
592            `Changed to unhealthy. Failing now: ${email.checks.join(", ") || "(the endpoint named none)"}.`,
593            "",
594            "You will not be mailed again about this until it recovers.",
595            "",
596            ...lines,
597            ...errorRateNote,
598          ])
599        : alert(env, "dustinedwards.info recovered", [
600            `Health is green again after ${formatDuration(email?.durationMs ?? null)}.`,
601            "",
602            `Was failing: ${email?.checks.join(", ") || "(the endpoint named none)"}.`,
603            "",
604            renderBody(reading),
605            ...errorRateNote,
606          ]),
607    write: () => writeState(env, transition.state),
608  });
609  // Thrown so the firing is recorded as failed; the unchanged state makes the next one retry.
610  if (!delivered) {
611    throw new Error(`the ${email?.kind} mail did not send, so the alert state was left unchanged`);
612  }
613}
614
615export default {
616  // Re-raised: a throwing Cron Trigger is recorded as a failed invocation, the last signal left.
617  async scheduled(controller: ScheduledController, env: WatchdogEnv): Promise<void> {
618    try {
619      const token = env.OPERATOR_TOKEN ?? "";
620      const reading = await readHealth(env);
621      const actions = watchdogActions(reading, { hasToken: token.length > 0 });
622
623      let draft = QUIET;
624      if (actions.length === 0) {
625        console.log(JSON.stringify({ watchdog: "healthy", status: reading.status }));
626      } else {
627        const notify = actions.filter((a) => a.type === "notify");
628        if (notify.length > 0) {
629          const reasons = notify.map((a) => (a as { reason: string }).reason);
630          console.error(JSON.stringify({ watchdog: "alerting", reasons }));
631          draft = {
632            alerting: true,
633            failing: failingCheckNames(reading.body),
634            subject: "dustinedwards.info is unhealthy",
635            lines: [...reasons, "", "Nothing was repaired.", "", renderBody(reading)],
636          };
637        } else {
638          draft = await runRepairs(env, token, reading, actions);
639        }
640      }
641
642      // Read every firing, whatever health said: checks can all be green while the Worker throws.
643      // Never repaired; a rebuild aimed at a throwing Worker treats an unclassified symptom.
644      const errorRate = await readErrorRate(env);
645      console.log(
646        JSON.stringify({
647          watchdog: "error-rate",
648          ok: errorRate.ok,
649          configured: errorRate.configured,
650          detail: errorRate.detail,
651        }),
652      );
653      draft = mergeErrorRate(draft, errorRate);
654      // An unconfigured rate reads ok, so every mail says it was not checked rather than implying it was.
655      const errorRateNote = errorRate.configured ? [] : ["", errorRate.detail];
656
657      await mailTransition(env, draft, reading, errorRateNote);
658
659      // Each runs independently, and both are raised after the health mail, so neither a bad week of
660      // counts nor a missed purge hides a bad site, and one firing's two unrelated jobs cannot mask
661      // each other (they fall in different quarter hours, so this is belt and suspenders).
662      const refreshProblem = await refreshCitationsWeekly(env, token, controller.scheduledTime);
663      const purgeProblem = await purgeZeroResultsDaily(env, token, controller.scheduledTime);
664      const backupProblem = await backupD1Daily(env, token, controller.scheduledTime);
665      const publicationProblem = await checkPublicationsDaily(env, token, controller.scheduledTime);
666      const scheduledProblems = [refreshProblem, purgeProblem, backupProblem, publicationProblem].filter((p): p is string => p !== null);
667      if (scheduledProblems.length > 0) throw new Error(scheduledProblems.join(" "));
668    } catch (error) {
669      const message = errorMessage(error);
670      console.error(JSON.stringify({ watchdog: "threw", error: message }));
671      await alert(env, "dustinedwards.info watchdog THREW", [
672        "The watchdog itself failed, so this firing proved nothing about the site.",
673        "",
674        message,
675      ]);
676      throw error;
677    }
678  },
679} satisfies ExportedHandler<WatchdogEnv>;