app/lib/health/alert-state.mjs

Functions

Imports

It imports no other module of this repo. Imported by workers/watchdog.ts.

Source

1// Mails only on a change of state. Absent state is a baseline and mails nothing; UNREADABLE state
2// mails if red, since a blind dedupe that stays silent has quietly stopped monitoring.
3
4/** @param {unknown} list @returns {string[]} sorted, unique, strings only */
5function names(list) {
6  if (!Array.isArray(list)) return [];
7  return [...new Set(list.filter((n) => typeof n === "string" && n.length > 0))].sort();
8}
9
10/** @param {string[]} a @param {string[]} b @returns {boolean} */
11function sameNames(a, b) {
12  return a.length === b.length && a.every((n, i) => n === b[i]);
13}
14
15/**
16 * Anything else is absent, not coerced: a half-parsed record would time a duration from nothing.
17 *
18 * @param {unknown} value
19 * @returns {{ red: boolean, since: string, checks: string[] } | null}
20 */
21function readStored(value) {
22  if (!value || typeof value !== "object") return null;
23  const record = /** @type {any} */ (value);
24  if (typeof record.red !== "boolean") return null;
25  if (typeof record.since !== "string" || record.since.length === 0) return null;
26  return { red: record.red, since: record.since, checks: names(record.checks) };
27}
28
29/**
30 * Null, not zero, when unknown: "recovered after 0 seconds" reads as a measurement and is not one.
31 *
32 * @param {string} since @param {string} now
33 */
34function elapsed(since, now) {
35  const a = Date.parse(since);
36  const b = Date.parse(now);
37  if (!Number.isFinite(a) || !Number.isFinite(b) || b < a) return null;
38  return b - a;
39}
40
41/**
42 * @param {number | null} ms
43 * @returns {string}
44 */
45export function formatDuration(ms) {
46  if (ms === null || !Number.isFinite(ms) || ms < 0) return "an unknown time";
47  const total = Math.floor(ms / 1000);
48  if (total < 60) return `${total}s`;
49  const minutes = Math.floor(total / 60);
50  if (minutes < 60) return `${minutes}m`;
51  const hours = Math.floor(minutes / 60);
52  const restMinutes = minutes % 60;
53  if (hours < 24) return restMinutes > 0 ? `${hours}h ${restMinutes}m` : `${hours}h`;
54  const days = Math.floor(hours / 24);
55  const restHours = hours % 24;
56  return restHours > 0 ? `${days}d ${restHours}h` : `${days}d`;
57}
58
59/**
60 * Mails first and stores second. Storing first made a failed "unhealthy" mail permanent: the next
61 * firing read red as already reported and stayed silent for the whole incident.
62 *
63 * @param {{ email: unknown, write: boolean }} transition  from alertTransition
64 * @param {{ send: () => Promise<boolean>, write: () => Promise<void> }} io
65 * @returns {Promise<boolean>} false when a mail was due and did not go; the state is then left as it was
66 */
67export async function deliverTransition(transition, { send, write }) {
68  if (transition.email && !(await send())) return false;
69  if (transition.write) await write();
70  return true;
71}
72
73/**
74 * @param {object} input
75 * @param {unknown} input.stored       what KV held, or null when the key is absent
76 * @param {boolean} input.storedReadable  false when the read itself FAILED
77 * @param {boolean} input.alerting     this firing ended in an alert condition
78 * @param {string[]} input.failing     check names failing now
79 * @param {string} input.now           ISO timestamp, supplied so tests can hold time
80 * @returns {{
81 *   email: null | { kind: "opened" | "recovered", checks: string[], durationMs: number | null },
82 *   state: { red: boolean, since: string, checks: string[] },
83 *   write: boolean,
84 *   reason: string,
85 * }}
86 */
87export function alertTransition({ stored, storedReadable = true, alerting, failing, now }) {
88  const current = names(failing);
89
90  if (!storedReadable) {
91    return {
92      email: alerting ? { kind: "opened", checks: current, durationMs: null } : null,
93      state: { red: alerting, since: now, checks: alerting ? current : [] },
94      write: true,
95      reason: alerting
96        ? "the stored state could not be read, so this mails rather than risk silence"
97        : "the stored state could not be read; nothing is wrong, so the baseline is rewritten",
98    };
99  }
100
101  const previous = readStored(stored);
102
103  if (previous === null) {
104    return {
105      email: null,
106      state: { red: alerting, since: now, checks: alerting ? current : [] },
107      write: true,
108      reason: "no usable stored state: this poll is the baseline and mails nothing",
109    };
110  }
111
112  if (!previous.red && alerting) {
113    return {
114      email: { kind: "opened", checks: current, durationMs: null },
115      state: { red: true, since: now, checks: current },
116      write: true,
117      reason: "healthy to unhealthy",
118    };
119  }
120
121  if (previous.red && !alerting) {
122    return {
123      email: {
124        kind: "recovered",
125        checks: previous.checks,
126        durationMs: elapsed(previous.since, now),
127      },
128      state: { red: false, since: now, checks: [] },
129      write: true,
130      reason: "unhealthy to healthy",
131    };
132  }
133
134  if (previous.red && alerting) {
135    const union = names([...previous.checks, ...current]);
136    const grew = !sameNames(union, previous.checks);
137    return {
138      email: null,
139      state: { red: true, since: previous.since, checks: union },
140      write: grew,
141      reason: grew
142        ? "still unhealthy, and the failing set grew; recorded, not mailed"
143        : "still unhealthy; suppressed",
144    };
145  }
146
147  return {
148    email: null,
149    state: previous,
150    write: false,
151    reason: "still healthy",
152  };
153}