app/lib/health/error-rate.mjs

Functions

Imports

It imports no other module of this repo. Imported by workers/watchdog.ts.

Source

1// Reads Cloudflare's workersInvocationsAdaptive: invocation logs are off and the site keeps no request counter of its own.
2// It reports the outcome, not the HTTP status, so a deliberate 503 is success.
3
4/** The window the rate is measured over, in minutes. Matches the cron cadence. */
5export const ERROR_WINDOW_MINUTES = 15;
6
7/**
8 * Measured over 30 days: the median 15-minute bucket held 3 invocations, so a bare ratio pages on one
9 * failed request. Below this the window is not judged; total >= 20 at ratio >= 0.25 fired on 2 of
10 * 1,296 buckets, both real outages, where the 0.50 variants never fired.
11 */
12export const ERROR_MIN_SAMPLE = 20;
13
14const ERROR_RATIO_THRESHOLD = 0.25;
15
16/** Not clientDisconnected: the dataset reports it with errors: 0, since it is the reader leaving. */
17const ERROR_STATUSES = /** @type {const} */ (["scriptThrewException", "loadShed"]);
18
19/**
20 * @param {{ accountId: string, scriptName: string, since: string, until: string }} args
21 * @returns {{ query: string, variables: Record<string, unknown> }}
22 */
23export function errorRateQuery({ accountId, scriptName, since, until }) {
24  return {
25    query:
26      "query WorkerErrors($account: String!, $script: String!, $since: Time!, $until: Time!) {" +
27      " viewer { accounts(filter: {accountTag: $account}) {" +
28      " workersInvocationsAdaptive(limit: 1000, filter: {" +
29      " scriptName: $script, datetime_geq: $since, datetime_leq: $until" +
30      " }) { sum { requests } dimensions { status } } } } }",
31    variables: { account: accountId, script: scriptName, since, until },
32  };
33}
34
35/**
36 * An unparseable body is ok: false: summing an unreadable answer to zero would report perfect health.
37 *
38 * @param {unknown} rows the `workersInvocationsAdaptive` array
39 * @param {{ minSample?: number, ratioThreshold?: number }} [options]
40 * @returns {{ ok: boolean, detail: string, total: number, errors: number, ratio: number }}
41 */
42export function errorRateVerdict(rows, options = {}) {
43  const minSample = options.minSample ?? ERROR_MIN_SAMPLE;
44  const ratioThreshold = options.ratioThreshold ?? ERROR_RATIO_THRESHOLD;
45
46  if (!Array.isArray(rows)) {
47    return {
48      ok: false,
49      detail:
50        "the invocations query returned no rows array, so the error rate is unknown. " +
51        "An unreadable answer is not evidence of health.",
52      total: 0,
53      errors: 0,
54      ratio: 0,
55    };
56  }
57
58  let total = 0;
59  let errors = 0;
60  for (const row of rows) {
61    const requests = Number(row?.sum?.requests);
62    // NaN is skipped, not coerced: an unparsed count must not become zero on either side of the ratio.
63    if (!Number.isFinite(requests)) continue;
64    total += requests;
65    if (ERROR_STATUSES.includes(row?.dimensions?.status)) errors += requests;
66  }
67
68  const ratio = total > 0 ? errors / total : 0;
69  const percent = (ratio * 100).toFixed(1);
70
71  if (total < minSample) {
72    return {
73      ok: true,
74      detail:
75        `${total} invocation(s) in the last ${ERROR_WINDOW_MINUTES} minutes, under the ` +
76        `${minSample} needed for a rate to mean anything (${errors} error(s)). Not judged.`,
77      total,
78      errors,
79      ratio,
80    };
81  }
82
83  if (ratio >= ratioThreshold) {
84    return {
85      ok: false,
86      detail:
87        `${errors} of ${total} invocations failed in the last ${ERROR_WINDOW_MINUTES} ` +
88        `minutes (${percent}%), at or above the ${(ratioThreshold * 100).toFixed(0)}% ` +
89        `threshold. Counted outcomes: ${ERROR_STATUSES.join(", ")}.`,
90      total,
91      errors,
92      ratio,
93    };
94  }
95
96  return {
97    ok: true,
98    detail:
99      `${errors} of ${total} invocations failed in the last ${ERROR_WINDOW_MINUTES} ` +
100      `minutes (${percent}%), under the ${(ratioThreshold * 100).toFixed(0)}% threshold.`,
101    total,
102    errors,
103    ratio,
104  };
105}