app/lib/health.server.ts

Functions

Imports

It imports app/db/schema.ts, app/lib/google/oauth.server.ts, app/lib/google/service-account.server.ts, app/lib/novels/repo.server.ts, app/lib/sites.server.ts, app/lib/social/queue.server.ts, app/lib/timed-fetch.server.ts. Imported by workers/app.ts.

Source

1// Carrel's own health check, run by the cron trigger. It emails Dustin only when a check changes
2// state, like the site's watchdog, so a standing failure sends one message, not one every 15 minutes.
3//
4// It checks the settings the gate needs, the database and its Owner, the Access signing keys, and
5// each site: its key, and whether the site still conforms to the site-api contract Carrel was built
6// against, the GitHub App for the writing repository, that every AI publish reached Dustin's inbox,
7// Google (the service account key's age and whether Google still takes it, and Dustin's drive.file
8// grant), and any template social post that went out, which Dustin should know about (design
9// decision 5, third step).
10
11import { backupHealth } from "@dustinedwards/d1-dump";
12import { runConformance } from "@dustinedwards/site-api/conformance";
13import { accessIssuer } from "@dustinedwards/site-runtime/access";
14import { eq, isNull } from "drizzle-orm";
15import { drizzle } from "drizzle-orm/d1";
16
17import { aiPublications, googleKeysSeen, googleTokens, healthState, people } from "~/db/schema";
18import { oauthConnection, userAccessToken } from "~/lib/google/oauth.server";
19import { saAccessToken, saConnection } from "~/lib/google/service-account.server";
20import { installationToken, NOVELS_REPO, novelsConnection } from "~/lib/novels/repo.server";
21import { SITE_IDS, siteConnection, siteEntry, type SiteId } from "~/lib/sites.server";
22import { unseenTemplates } from "~/lib/social/queue.server";
23import { timedFetch } from "~/lib/timed-fetch.server";
24
25export type CheckResult = { name: string; ok: boolean; detail: string };
26
27type Fetch = (input: string, init?: RequestInit) => Promise<Response>;
28
29const PLACEHOLDER = /example\.(com|cloudflareaccess\.com)/i;
30
31async function checkConfig(env: Env): Promise<CheckResult> {
32  const missing: string[] = [];
33  if (!env.ACCESS_AUD?.trim()) missing.push("ACCESS_AUD");
34  if (!env.ACCESS_TEAM_DOMAIN?.trim() || PLACEHOLDER.test(env.ACCESS_TEAM_DOMAIN)) missing.push("ACCESS_TEAM_DOMAIN");
35  if (!env.ALERT_EMAIL?.trim() || PLACEHOLDER.test(env.ALERT_EMAIL)) missing.push("ALERT_EMAIL");
36  return missing.length === 0
37    ? { name: "config", ok: true, detail: "Access and alert settings are present." }
38    : { name: "config", ok: false, detail: `Missing or placeholder: ${missing.join(", ")}.` };
39}
40
41async function checkOwner(env: Env): Promise<CheckResult> {
42  try {
43    const owners = await drizzle(env.DB).select({ id: people.id }).from(people).where(eq(people.isOwner, true)).all();
44    return owners.length === 1
45      ? { name: "owner", ok: true, detail: "The database answers and has its Owner." }
46      : { name: "owner", ok: false, detail: "The database has no Owner, so nobody can use Carrel. Run npm run seed:owner." };
47  } catch (error) {
48    return { name: "owner", ok: false, detail: `The database did not answer: ${message(error)}` };
49  }
50}
51
52async function checkAccessKeys(env: Env, fetcher: Fetch): Promise<CheckResult> {
53  const url = `${accessIssuer(env.ACCESS_TEAM_DOMAIN) ?? ""}/cdn-cgi/access/certs`;
54  try {
55    const response = await fetcher(url);
56    if (!response.ok) return { name: "access-keys", ok: false, detail: `${url} answered ${response.status}.` };
57    const body = (await response.json()) as { keys?: unknown[] };
58    return Array.isArray(body.keys) && body.keys.length > 0
59      ? { name: "access-keys", ok: true, detail: "Access signing keys are published." }
60      : { name: "access-keys", ok: false, detail: `${url} lists no signing keys.` };
61  } catch (error) {
62    return { name: "access-keys", ok: false, detail: `${url} could not be read: ${message(error)}` };
63  }
64}
65
66/**
67 * A site with no key is not connected yet, which is not a failure: its stage has not arrived. With a
68 * key, the conformance suite runs, including its writes, each of which a conforming site refuses. On
69 * a site that offers media (site-api v0.2.0) that includes the media checks: limits declared, list in
70 * shape, a delete of an unknown id and an upload of an undeclared type both refused. On a site that
71 * declares the mentions group (v0.5.0) it includes the mentions checks: the list in shape, writes without
72 * the key refused, a decision and a delete of an unknown id refused, a stale version refused on the first
73 * real mention, and a sweep with no change id refused; a site without the group must answer 501. The suite's real
74 * upload round trip stays off: the health check never puts a file into a site's storage.
75 */
76async function checkSite(env: Env, id: SiteId, fetcher: Fetch): Promise<CheckResult> {
77  const name = `site-${id}`;
78  const site = siteEntry(id).name;
79  const connection = siteConnection(env, id);
80  if (connection.state === "not-connected") return { name, ok: true, detail: `Not connected: ${connection.detail}` };
81  if (connection.state === "misconfigured") return { name, ok: false, detail: connection.detail };
82  try {
83    const report = await runConformance({
84      baseUrl: connection.origin,
85      key: connection.key,
86      probeMediaUpload: false,
87      fetch: ((input: RequestInfo | URL, init?: RequestInit) =>
88        fetcher(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, init)) as typeof fetch,
89    });
90    if (report.ok) return { name, ok: true, detail: `${site} conforms (${report.checks[0]?.detail ?? "meta ok"}).` };
91    const failed = report.checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`);
92    return { name, ok: false, detail: `${site} does not conform. ${failed.join(" | ")}` };
93  } catch (error) {
94    return { name, ok: false, detail: `${site} could not be checked: ${message(error)}` };
95  }
96}
97
98/** The newest complete D1 dump must be under 26 hours old (d1-dump's default). No complete dump at all fails too. */
99async function checkBackup(env: Env, now: Date): Promise<CheckResult> {
100  try {
101    const { last_ok, age_hours, warning } = await backupHealth(env.BACKUPS, { now });
102    if (last_ok === null || warning) return { name: "backup", ok: false, detail: warning ?? "No complete D1 dump exists yet." };
103    return { name: "backup", ok: true, detail: `The newest complete D1 dump finished ${last_ok}, ${age_hours} hours ago.` };
104  } catch (error) {
105    return { name: "backup", ok: false, detail: `The backup bucket could not be read: ${message(error)}` };
106  }
107}
108
109/** Not set up yet is not a failure, as with a site; set up, the App must still get a token for the repository. */
110async function checkNovels(env: Env, fetcher: Fetch): Promise<CheckResult> {
111  const connection = novelsConnection(env);
112  if (connection.state !== "connected") {
113    return connection.state === "not-connected"
114      ? { name: "writing", ok: true, detail: `Not connected: ${connection.detail}` }
115      : { name: "writing", ok: false, detail: connection.detail };
116  }
117  try {
118    await installationToken(connection.appId, connection.privateKey, ((input: RequestInfo | URL, init?: RequestInit) =>
119      fetcher(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, init)) as typeof fetch);
120    return { name: "writing", ok: true, detail: `The GitHub App can write to ${NOVELS_REPO}.` };
121  } catch (error) {
122    return { name: "writing", ok: false, detail: `The GitHub App could not reach ${NOVELS_REPO}: ${message(error)}` };
123  }
124}
125
126/** Decision 2c: every AI publish emails Dustin. One whose email failed stays a failure until it is sent. */
127async function checkAiPublishEmails(env: Env): Promise<CheckResult> {
128  try {
129    const unsent = await drizzle(env.DB)
130      .select({ changeId: aiPublications.changeId, itemId: aiPublications.itemId, client: aiPublications.client, error: aiPublications.emailError })
131      .from(aiPublications)
132      .where(isNull(aiPublications.emailedAt))
133      .all();
134    if (unsent.length === 0) return { name: "ai-publish-email", ok: true, detail: "Every AI publish was emailed." };
135    return {
136      name: "ai-publish-email",
137      ok: false,
138      detail: `${unsent.length} AI publish${unsent.length === 1 ? " was" : "es were"} not emailed: ${unsent
139        .map((u) => `${u.itemId} by ${u.client} (${u.error ?? "no error recorded"})`)
140        .join("; ")}.`,
141    };
142  } catch (error) {
143    return { name: "ai-publish-email", ok: false, detail: `The record could not be read: ${message(error)}` };
144  }
145}
146
147/** Google recommends rotating service account keys at least every 90 days. */
148const KEY_MAX_AGE_DAYS = 90;
149
150/**
151 * The service account key: absent is "not connected" (its stage's setup is Dustin's), a key Google
152 * refuses (deleted, disabled, expired) fails, and so does one older than KEY_MAX_AGE_DAYS. A key file
153 * carries no creation date, so age counts from the first time this check saw the key.
154 */
155async function checkServiceAccount(env: Env, fetcher: Fetch, now: Date): Promise<CheckResult> {
156  const connection = saConnection(env);
157  if (connection.state === "not-connected") return { name: "google-service-account", ok: true, detail: `Not connected: ${connection.detail}` };
158  if (connection.state === "misconfigured") return { name: "google-service-account", ok: false, detail: connection.detail };
159  const { key } = connection;
160  const d = drizzle(env.DB);
161  await d.insert(googleKeysSeen).values({ privateKeyId: key.privateKeyId, clientEmail: key.clientEmail, firstSeenAt: now.toISOString() }).onConflictDoNothing();
162  const seen = await d.select({ firstSeenAt: googleKeysSeen.firstSeenAt }).from(googleKeysSeen).where(eq(googleKeysSeen.privateKeyId, key.privateKeyId)).get();
163  const days = Math.floor((now.getTime() - Date.parse(seen!.firstSeenAt)) / 86_400_000);
164  const id = key.privateKeyId.slice(0, 8);
165  try {
166    await saAccessToken(key, ((input: RequestInfo | URL, init?: RequestInit) =>
167      fetcher(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, init)) as typeof fetch);
168  } catch (error) {
169    return { name: "google-service-account", ok: false, detail: `Key ${id} (${key.clientEmail}) no longer works: ${message(error)} Create a new key and set GOOGLE_SA_KEY.` };
170  }
171  if (days > KEY_MAX_AGE_DAYS) {
172    return { name: "google-service-account", ok: false, detail: `Key ${id} is ${days} days old, past ${KEY_MAX_AGE_DAYS}. Create a new key, set GOOGLE_SA_KEY, then delete the old one in Google Cloud.` };
173  }
174  return { name: "google-service-account", ok: true, detail: `Key ${id} (${key.clientEmail}) works; ${days} days old.` };
175}
176
177/** Dustin's drive.file grant, if he has connected it: Google must still refresh it. */
178async function checkDriveFile(env: Env, fetcher: Fetch): Promise<CheckResult> {
179  const connection = await oauthConnection(env);
180  if (connection.state === "not-configured") return { name: "google-drive-file", ok: true, detail: `Not connected: ${connection.detail}` };
181  if (connection.state === "misconfigured") return { name: "google-drive-file", ok: false, detail: connection.detail };
182  const grants = await drizzle(env.DB).select({ personId: googleTokens.personId }).from(googleTokens).all();
183  if (grants.length === 0) return { name: "google-drive-file", ok: true, detail: "Not connected: no one has connected Google for Send to Docs yet." };
184  try {
185    for (const g of grants) {
186      await userAccessToken(env, g.personId, ((input: RequestInfo | URL, init?: RequestInit) =>
187        fetcher(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, init)) as typeof fetch);
188    }
189    return { name: "google-drive-file", ok: true, detail: "The drive.file grant refreshes." };
190  } catch (error) {
191    return { name: "google-drive-file", ok: false, detail: message(error) };
192  }
193}
194
195/** A template post went out: the routine failed or hit its cap. Fails until Dustin marks it seen. */
196async function checkSocialTemplates(env: Env): Promise<CheckResult> {
197  try {
198    const unseen = await unseenTemplates(env.DB);
199    if (unseen.length === 0) return { name: "social-templates", ok: true, detail: "No template post is waiting to be seen." };
200    return {
201      name: "social-templates",
202      ok: false,
203      detail: `${unseen.length} template post${unseen.length === 1 ? "" : "s"} went out because the routine did not draft ${unseen.length === 1 ? "it" : "them"}: ${unseen
204        .map((u) => `${u.accountKey}: "${u.text}"`)
205        .join("; ")}. Mark ${unseen.length === 1 ? "it" : "them"} seen in Carrel's social page.`,
206    };
207  } catch (error) {
208    return { name: "social-templates", ok: false, detail: `The social record could not be read: ${message(error)}` };
209  }
210}
211
212export async function runChecks(
213  env: Env,
214  fetcher: Fetch = (url, init) => timedFetch(url, init),
215  now: Date = new Date(),
216): Promise<CheckResult[]> {
217  return Promise.all([
218    checkConfig(env),
219    checkOwner(env),
220    checkAccessKeys(env, fetcher),
221    ...SITE_IDS.map((id) => checkSite(env, id, fetcher)),
222    checkNovels(env, fetcher),
223    checkAiPublishEmails(env),
224    checkBackup(env, now),
225    checkServiceAccount(env, fetcher, now),
226    checkDriveFile(env, fetcher),
227    checkSocialTemplates(env),
228  ]);
229}
230
231/**
232 * Runs the checks, emails when any check changed state, and only then records the new states. If
233 * the email fails, nothing is recorded, so the next run tries again rather than losing the alert.
234 * A check seen for the first time counts as a change only when it fails.
235 */
236export async function runHealth(
237  env: Env,
238  fetcher?: Fetch,
239  now: () => Date = () => new Date(),
240): Promise<{ results: CheckResult[]; emailed: boolean }> {
241  const results = await runChecks(env, fetcher, now());
242  const db = drizzle(env.DB);
243
244  let previous: Map<string, boolean>;
245  try {
246    const rows = await db.select().from(healthState).all();
247    previous = new Map(rows.map((r) => [r.checkName, r.ok]));
248  } catch {
249    previous = new Map();
250  }
251
252  const changed = results.filter((r) => {
253    const before = previous.get(r.name);
254    return before === undefined ? !r.ok : before !== r.ok;
255  });
256  const unseen = results.filter((r) => !previous.has(r.name));
257  if (changed.length === 0 && unseen.length === 0) return { results, emailed: false };
258
259  let emailed = false;
260  if (changed.length > 0) {
261    const failing = results.filter((r) => !r.ok);
262    try {
263      await env.EMAIL.send({
264        from: env.ALERT_FROM,
265        to: env.ALERT_EMAIL,
266        subject: failing.length === 0 ? "Carrel: all checks pass" : `Carrel: ${failing.length} check${failing.length === 1 ? "" : "s"} failing`,
267        text: results.map((r) => `${r.ok ? "ok  " : "FAIL"} ${r.name}: ${r.detail}`).join("\n"),
268      });
269      emailed = true;
270    } catch (error) {
271      console.error(JSON.stringify({ health: "email-failed", error: message(error) }));
272      return { results, emailed: false };
273    }
274  }
275
276  const at = now().toISOString();
277  try {
278    for (const r of [...new Set([...changed, ...unseen])]) {
279      await db
280        .insert(healthState)
281        .values({ checkName: r.name, ok: r.ok, detail: r.detail, changedAt: at })
282        .onConflictDoUpdate({ target: healthState.checkName, set: { ok: r.ok, detail: r.detail, changedAt: at } });
283    }
284  } catch (error) {
285    console.error(JSON.stringify({ health: "record-failed", error: message(error) }));
286  }
287  return { results, emailed };
288}
289
290function message(error: unknown): string {
291  return error instanceof Error ? error.message : String(error);
292}