src/ops-cloudflare.ts

Functions

Imports

It imports src/env.ts, src/log.ts, src/ops-sites.ts, src/ops-types/sites.ts, src/ops-types/views.ts, src/time.ts. Imported by src/ops-cloudflare-config.ts, src/site-key.ts, src/watcher-gather.ts, src/workers-builds.ts.

Source

1import type { Env } from "./env";
2import type { OpsSite } from "./ops-sites";
3import type { CfDeploy, HourBucket, SiteCloudflare } from "./ops-types/sites";
4import type { WebAnalyticsSite } from "./ops-types/views";
5import { logEvent } from "./log";
6import { HOUR_MS } from "./time";
7
8// The watcher's read of Cloudflare, for the Watch Floor's deploy and error columns
9// (capsid/research/design-ops-console.md). It runs inside the watcher pass only, never
10// per dashboard request. Two modules import it (test/ops-cloudflare.test.ts):
11// src/watcher-gather.ts, and src/ops-cloudflare-config.ts, the admin-only capsid_admin_read_ops
12// tool's Access and Email Routing reads, which reuse this envelope and paging.
13//
14// The token, CF_OPS_TOKEN, is read-only and holds these permissions:
15//   Workers Scripts Read              GET /accounts/{id}/workers/domains and
16//                                     GET /accounts/{id}/workers/scripts/{script}/deployments
17//   Account Analytics Read            POST /graphql (workersInvocationsAdaptive)
18//   Access: Apps and Policies Read    GET /accounts/{id}/access/apps and /access/policies
19//   Email Routing Addresses Read      GET /accounts/{id}/email/routing/addresses
20//   Zone Read                         GET /zones
21//   Email Routing Rules Read          GET /zones/{zone}/email/routing/rules
22//   Account Settings Read             GET /accounts/{id}/rum/site_info/list (the live checks'
23//                                     Web Analytics setting; developers.cloudflare.com/api/resources/rum/subresources/site_info/methods/list/)
24// The Access, Email Routing and Zone permissions are capsid_admin_read_ops's; Access is also
25// the watcher's, for a blocked job's "check: access app" line (src/manual-steps.ts). To add them to the existing token:
26// dash.cloudflare.com/profile/api-tokens, the token capsid-portal-read, Edit; add
27// Account / Access: Apps and Policies / Read, Account / Email Routing Addresses / Read,
28// Zone / Email Routing Rules / Read and Zone / Zone / Read; Zone Resources: All zones
29// from the account; Continue to summary, Update token. The secret value does not change.
30//
31// What this read cannot see: a failed deploy. Cloudflare's deployments list records
32// only the deployments that happened, so a deploy that failed before it reached
33// Cloudflare leaves nothing here to find. No finding is invented for it; a failed
34// deploy workflow is already the watcher's ci-red finding on the default branch.
35
36export const CF_API = "https://api.cloudflare.com/client/v4";
37export const CF_GRAPHQL = `${CF_API}/graphql`;
38const CF_TIMEOUT_MS = 10_000;
39export const DEPLOYS_KEPT = 10;
40const ERROR_HOURS = 24;
41
42// Script names are inlined into the GraphQL query as a list literal, so only a name
43// that cannot break out of a string literal is sent.
44const SCRIPT_NAME = /^[A-Za-z0-9_-]{1,63}$/;
45
46export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
47
48export type CfCredentials = { ok: true; token: string; account: string } | { ok: false; reason: string };
49
50/** The token and the account it reads. The account is CF_ACCOUNT_ID, or R2_ACCOUNT_ID
51 *  where that is unset: R2 lives in the same Cloudflare account. The reason names
52 *  exactly what is unset. */
53export function cloudflareCredentials(env: Pick<Env, "CF_OPS_TOKEN" | "CF_ACCOUNT_ID" | "R2_ACCOUNT_ID">): CfCredentials {
54  const token = env.CF_OPS_TOKEN?.trim() ?? "";
55  const account = env.CF_ACCOUNT_ID?.trim() || env.R2_ACCOUNT_ID?.trim() || "";
56  const missing: string[] = [];
57  if (!token) missing.push("CF_OPS_TOKEN is not set");
58  if (!account) missing.push("neither CF_ACCOUNT_ID nor R2_ACCOUNT_ID is set");
59  if (missing.length) return { ok: false, reason: missing.join("; ") };
60  return { ok: true, token, account };
61}
62
63const hostOf = (origin: string): string => new URL(origin).hostname.toLowerCase();
64
65/** A failed Cloudflare read, carrying the HTTP status so a caller can say what a 403
66 *  means for its endpoint (the permission the token lacks). */
67export class CfReadError extends Error {
68  readonly status: number;
69  constructor(message: string, status: number) {
70    super(message);
71    this.name = "CfReadError";
72    this.status = status;
73  }
74}
75
76type CfEnvelope = { success?: unknown; errors?: unknown; result?: unknown; result_info?: unknown };
77
78// The Cloudflare REST envelope. `success` false or a non-2xx is a failed read, with
79// Cloudflare's own messages as the reason. The token is never part of a reason.
80export async function cfGet(fetchImpl: FetchLike, token: string, url: string, what: string): Promise<unknown> {
81  return (await cfEnvelope(fetchImpl, token, url, what)).result;
82}
83
84async function cfEnvelope(fetchImpl: FetchLike, token: string, url: string, what: string): Promise<CfEnvelope> {
85  const res = await fetchImpl(url, {
86    method: "GET",
87    headers: { Authorization: `Bearer ${token}`, "User-Agent": "capsid-watcher (ops read)" },
88    signal: AbortSignal.timeout(CF_TIMEOUT_MS),
89  });
90  const body = parseJson(await res.text(), `${what} answered ${res.status} with a body that is not JSON`) as CfEnvelope | null;
91  if (!res.ok || body?.success !== true) {
92    throw new CfReadError(`${what} answered ${res.status}: ${messagesOf(body?.errors) || "no error message"}`, res.status);
93  }
94  return body;
95}
96
97// A list read stops here rather than returning part of a list as if it were the whole.
98export const CF_MAX_PAGES = 20;
99
100/** Every item of a paged Cloudflare list, following result_info.total_pages. `url`
101 *  carries no page or per_page; both are set here. A list with more than CF_MAX_PAGES
102 *  pages fails with the count, never a truncated answer. */
103export async function cfGetAll(fetchImpl: FetchLike, token: string, url: string, what: string, perPage: number): Promise<unknown[]> {
104  const out: unknown[] = [];
105  for (let page = 1; ; page++) {
106    const u = new URL(url);
107    u.searchParams.set("page", String(page));
108    u.searchParams.set("per_page", String(perPage));
109    const body = await cfEnvelope(fetchImpl, token, u.toString(), what);
110    if (!Array.isArray(body.result)) throw new Error(`${what} returned no result array`);
111    out.push(...body.result);
112    const pages = (body.result_info as { total_pages?: unknown } | null | undefined)?.total_pages;
113    // No result_info, or no total_pages, is a list Cloudflare does not page: one page.
114    const total = typeof pages === "number" && Number.isFinite(pages) ? pages : 1;
115    if (page >= total) return out;
116    if (page >= CF_MAX_PAGES) throw new Error(`${what} has ${total} pages of ${perPage}, more than the ${CF_MAX_PAGES} this read follows`);
117  }
118}
119
120function parseJson(text: string, failure: string): unknown {
121  try {
122    return JSON.parse(text);
123  } catch {
124    throw new Error(failure);
125  }
126}
127
128function messagesOf(errors: unknown): string {
129  if (!Array.isArray(errors)) return "";
130  return errors
131    .map((e) => (e && typeof e === "object" && typeof (e as { message?: unknown }).message === "string" ? (e as { message: string }).message : ""))
132    .filter(Boolean)
133    .join("; ")
134    .slice(0, 300);
135}
136
137const reasonOf = (err: unknown): string => (err instanceof Error ? err.message : String(err)).slice(0, 300);
138
139/** GET /accounts/{id}/workers/domains: every Workers custom domain on the account, as
140 *  hostname to the script (`service`) it serves. */
141async function readCustomDomains(fetchImpl: FetchLike, token: string, account: string): Promise<Map<string, string>> {
142  const result = await cfGet(fetchImpl, token, `${CF_API}/accounts/${encodeURIComponent(account)}/workers/domains`, "the Workers custom domains list");
143  if (!Array.isArray(result)) throw new Error("the Workers custom domains list returned no result array");
144  const out = new Map<string, string>();
145  for (const d of result as Array<{ hostname?: unknown; service?: unknown }>) {
146    if (typeof d?.hostname === "string" && typeof d.service === "string" && d.service) out.set(d.hostname.toLowerCase(), d.service);
147  }
148  return out;
149}
150
151export const WEB_ANALYTICS_PERMISSION = "Account / Account Settings / Read";
152
153/** GET /accounts/{id}/rum/site_info/list: every Web Analytics site in the account. A 403
154 *  is turned into the permission the token lacks. */
155export async function readWebAnalyticsSites(fetchImpl: FetchLike, token: string, account: string): Promise<WebAnalyticsSite[]> {
156  const what = "the Web Analytics sites list";
157  let rows: unknown[];
158  try {
159    rows = await cfGetAll(fetchImpl, token, `${CF_API}/accounts/${encodeURIComponent(account)}/rum/site_info/list`, what, 50);
160  } catch (err) {
161    if (err instanceof CfReadError && err.status === 403) {
162      throw new Error(`${what} was refused with 403: CF_OPS_TOKEN lacks ${WEB_ANALYTICS_PERMISSION}. Add it at dash.cloudflare.com/profile/api-tokens, token capsid-portal-read, Edit (docs/portal.md, "The Cloudflare token"). Cloudflare said: ${err.message}`);
163    }
164    throw err;
165  }
166  const out: WebAnalyticsSite[] = [];
167  for (const row of rows as Array<{ host?: unknown; auto_install?: unknown; ruleset?: { enabled?: unknown } | null }>) {
168    if (typeof row?.host !== "string" || row.host === "") continue;
169    out.push({
170      host: row.host.toLowerCase(),
171      auto_install: typeof row.auto_install === "boolean" ? row.auto_install : null,
172      enabled: typeof row.ruleset?.enabled === "boolean" ? row.ruleset.enabled : null,
173    });
174  }
175  return out;
176}
177
178interface RawDeploy {
179  id?: unknown;
180  created_on?: unknown;
181  versions?: unknown;
182  annotations?: unknown;
183  author_email?: unknown;
184}
185
186const str = (v: unknown): string | null => (typeof v === "string" && v.length > 0 ? v : null);
187
188/** One deployment as the dashboard shows it. version_id is the version serving the
189 *  largest percentage of traffic. */
190function toCfDeploy(raw: RawDeploy): CfDeploy {
191  let version_id: string | null = null;
192  let best = -1;
193  for (const v of Array.isArray(raw.versions) ? (raw.versions as Array<{ percentage?: unknown; version_id?: unknown }>) : []) {
194    const pct = typeof v?.percentage === "number" ? v.percentage : -1;
195    if (pct > best && typeof v.version_id === "string") {
196      best = pct;
197      version_id = v.version_id;
198    }
199  }
200  const annotations = raw.annotations && typeof raw.annotations === "object" ? (raw.annotations as Record<string, unknown>) : {};
201  return {
202    id: str(raw.id) ?? "",
203    created_on: str(raw.created_on) ?? "",
204    version_id,
205    message: str(annotations["workers/message"]),
206    triggered_by: str(annotations["workers/triggered_by"]),
207    author_email: str(raw.author_email),
208  };
209}
210
211/** GET /accounts/{id}/workers/scripts/{script}/deployments. Cloudflare lists the
212 *  newest first ("the first deployment in the list is the latest deployment actively
213 *  serving traffic"); sorted again on created_on so the order does not rest on that. */
214async function readDeploys(fetchImpl: FetchLike, token: string, account: string, script: string): Promise<CfDeploy[]> {
215  const url = `${CF_API}/accounts/${encodeURIComponent(account)}/workers/scripts/${encodeURIComponent(script)}/deployments`;
216  const result = await cfGet(fetchImpl, token, url, `the deployments list for ${script}`);
217  const list = (result as { deployments?: unknown } | null)?.deployments;
218  if (!Array.isArray(list)) throw new Error(`the deployments list for ${script} returned no deployments array`);
219  return (list as RawDeploy[])
220    .map(toCfDeploy)
221    .sort((a, b) => b.created_on.localeCompare(a.created_on))
222    .slice(0, DEPLOYS_KEPT);
223}
224
225/** The 24 complete hours before `now`, oldest first. The last is the most recent
226 *  complete hour, the one the error-rate finding reads. */
227export function errorWindow(now: Date): { start: Date; end: Date; hours: string[] } {
228  const end = new Date(Math.floor(now.getTime() / HOUR_MS) * HOUR_MS);
229  const start = new Date(end.getTime() - ERROR_HOURS * HOUR_MS);
230  const hours = Array.from({ length: ERROR_HOURS }, (_, i) => new Date(start.getTime() + i * HOUR_MS).toISOString());
231  return { start, end, hours };
232}
233
234function errorsQuery(scripts: string[]): string {
235  for (const s of scripts) if (!SCRIPT_NAME.test(s)) throw new Error(`script name ${JSON.stringify(s)} is not one the query will carry`);
236  return `query WatchFloorErrors($accountTag: string, $datetimeStart: string, $datetimeEnd: string) {
237  viewer {
238    accounts(filter: { accountTag: $accountTag }) {
239      workersInvocationsAdaptive(
240        limit: 10000
241        filter: { scriptName_in: ${JSON.stringify(scripts)}, datetime_geq: $datetimeStart, datetime_lt: $datetimeEnd }
242      ) {
243        sum { requests errors }
244        dimensions { scriptName datetimeHour }
245      }
246    }
247  }
248}`;
249}
250
251/** ONE GraphQL Analytics query for every script, hourly, over errorWindow. A script
252 *  with no row in an hour had no invocations then, which is a real zero from a query
253 *  that succeeded. A query that failed throws, and the caller records no data with the
254 *  reason, never zeros. */
255async function readErrors(
256  fetchImpl: FetchLike,
257  token: string,
258  account: string,
259  scripts: string[],
260  now: Date
261): Promise<Map<string, HourBucket[]>> {
262  const { start, end, hours } = errorWindow(now);
263  const res = await fetchImpl(CF_GRAPHQL, {
264    method: "POST",
265    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": "capsid-watcher (ops read)" },
266    body: JSON.stringify({
267      query: errorsQuery(scripts),
268      variables: { accountTag: account, datetimeStart: start.toISOString(), datetimeEnd: end.toISOString() },
269    }),
270    signal: AbortSignal.timeout(CF_TIMEOUT_MS),
271  });
272  const body = parseJson(await res.text(), `the analytics query answered ${res.status} with a body that is not JSON`) as {
273    data?: unknown;
274    errors?: unknown;
275  } | null;
276  const graphErrors = messagesOf(body?.errors);
277  if (!res.ok || graphErrors || (Array.isArray(body?.errors) && body.errors.length > 0)) {
278    throw new Error(`the analytics query answered ${res.status}: ${graphErrors || "no error message"}`);
279  }
280  const accounts = (body?.data as { viewer?: { accounts?: unknown } } | null | undefined)?.viewer?.accounts;
281  if (!Array.isArray(accounts) || accounts.length !== 1) throw new Error("the analytics query returned no account");
282  const rows = (accounts[0] as { workersInvocationsAdaptive?: unknown }).workersInvocationsAdaptive;
283  if (!Array.isArray(rows)) throw new Error("the analytics query returned no workersInvocationsAdaptive rows");
284
285  const index = new Map(hours.map((h, i) => [h, i]));
286  const out = new Map<string, HourBucket[]>(scripts.map((s) => [s, hours.map((hour) => ({ hour, requests: 0, errors: 0 }))]));
287  for (const row of rows as Array<{ sum?: { requests?: unknown; errors?: unknown }; dimensions?: { scriptName?: unknown; datetimeHour?: unknown } }>) {
288    const script = row?.dimensions?.scriptName;
289    const hourRaw = row?.dimensions?.datetimeHour;
290    const requests = row?.sum?.requests;
291    const errors = row?.sum?.errors;
292    if (typeof script !== "string" || typeof hourRaw !== "string" || typeof requests !== "number" || typeof errors !== "number") {
293      throw new Error("the analytics query returned a row without scriptName, datetimeHour, requests and errors");
294    }
295    const at = Date.parse(hourRaw);
296    const i = Number.isNaN(at) ? undefined : index.get(new Date(at).toISOString());
297    const buckets = out.get(script);
298    // A row outside the window or for a script not asked for means the filter is not
299    // what this code believes, so the whole read fails rather than being half right.
300    if (i === undefined || !buckets) throw new Error(`the analytics query returned a row outside the asked window or scripts (${script} ${hourRaw})`);
301    buckets[i].requests += requests;
302    buckets[i].errors += errors;
303  }
304  return out;
305}
306
307export interface CloudflareRead {
308  // Every site's Cloudflare state, keyed by namespace.
309  bySite: Record<string, SiteCloudflare>;
310  // True only when the token is set and every read this pass needed succeeded, so a
311  // finding the "cloudflare" check owns clears only on a full read.
312  ran: boolean;
313}
314
315/** Every mapped site's Cloudflare state for this pass. Each failed read is logged and
316 *  recorded on the sites it affects with its reason; nothing is reported as zero. */
317export async function readCloudflare(
318  env: Pick<Env, "CF_OPS_TOKEN" | "CF_ACCOUNT_ID" | "R2_ACCOUNT_ID">,
319  sites: readonly OpsSite[],
320  fetchImpl: FetchLike,
321  now: Date
322): Promise<CloudflareRead> {
323  const bySite: Record<string, SiteCloudflare> = {};
324  const creds = cloudflareCredentials(env);
325  const onCloudflare = sites.filter((s) => s.platform === "cloudflare");
326  for (const s of sites) {
327    if (s.platform !== "cloudflare") bySite[s.namespace] = { state: "not-cloudflare", reason: `${s.name} is served by ${s.platform}, not Cloudflare` };
328  }
329  if (!creds.ok) {
330    for (const s of onCloudflare) bySite[s.namespace] = { state: "no-token", reason: creds.reason };
331    return { bySite, ran: false };
332  }
333  const failed = (what: string, err: unknown): string => {
334    const reason = reasonOf(err);
335    logEvent("error", "WATCHER_READ_FAILED", { message: `WATCHER_READ_FAILED cloudflare ${what}: ${reason}` });
336    return reason;
337  };
338
339  let ran = true;
340
341  // Script per site: named in the map where the host proves it (a workers.dev host),
342  // otherwise the Workers custom domain for the site's host. Never guessed.
343  let domains: Map<string, string> | null = null;
344  let domainsReason: string | null = null;
345  if (onCloudflare.some((s) => !s.script)) {
346    try {
347      domains = await readCustomDomains(fetchImpl, creds.token, creds.account);
348    } catch (err) {
349      domainsReason = failed("custom domains", err);
350      ran = false;
351    }
352  }
353  const scriptOf = new Map<string, string>();
354  for (const s of onCloudflare) {
355    if (s.script) {
356      scriptOf.set(s.namespace, s.script);
357      continue;
358    }
359    if (domainsReason !== null) {
360      bySite[s.namespace] = { state: "error", reason: `the script could not be resolved: ${domainsReason}` };
361      continue;
362    }
363    const host = hostOf(s.origin);
364    const service = domains?.get(host);
365    if (service) scriptOf.set(s.namespace, service);
366    else bySite[s.namespace] = { state: "unresolved", reason: `no Workers custom domain on the account names ${host}, so its script is not known` };
367  }
368
369  const deploys = new Map<string, CfDeploy[]>();
370  for (const [namespace, script] of scriptOf) {
371    try {
372      deploys.set(namespace, await readDeploys(fetchImpl, creds.token, creds.account, script));
373    } catch (err) {
374      bySite[namespace] = { state: "error", reason: failed(`deployments ${script}`, err) };
375      ran = false;
376    }
377  }
378
379  const scripts = [...new Set([...deploys.keys()].map((ns) => scriptOf.get(ns) as string))].sort();
380  let errors: Map<string, HourBucket[]> | null = null;
381  let errorsReason: string | null = null;
382  if (scripts.length > 0) {
383    try {
384      errors = await readErrors(fetchImpl, creds.token, creds.account, scripts, now);
385    } catch (err) {
386      errorsReason = failed("analytics", err);
387      ran = false;
388    }
389  }
390  for (const [namespace, list] of deploys) {
391    const script = scriptOf.get(namespace) as string;
392    const buckets = errors?.get(script) ?? null;
393    bySite[namespace] = {
394      state: "ok",
395      script,
396      deploys: list,
397      errors24: buckets,
398      errors_reason: buckets ? null : (errorsReason ?? "the analytics query returned nothing for this script"),
399    };
400  }
401  return { bySite, ran };
402}
403
404// THE SECRET-PRESENCE READ (job_09e5f6cbf782, shared with job_5ac1139641e0's credentials
405// inventory): which secrets a Worker has, by NAME. Cloudflare's secrets list returns each
406// secret's name and type and has no way to return a value, so none can reach Capsid. Read
407// on the watcher's pass only, like every Cloudflare read here; the Portal shows what the
408// last pass stored (src/secret-presence.ts).
409const SECRETS_PERMISSION = "Account / Workers Scripts / Read";
410
411/** GET /accounts/{id}/workers/scripts/{script}/secrets: the names. A 403 names the
412 *  permission the token lacks. */
413export async function readSecretNames(fetchImpl: FetchLike, token: string, account: string, script: string): Promise<string[]> {
414  if (!SCRIPT_NAME.test(script)) throw new Error(`'${script}' is not a Worker script name`);
415  const what = `the secrets list of ${script}`;
416  let rows: unknown[];
417  try {
418    rows = await cfGetAll(fetchImpl, token, `${CF_API}/accounts/${encodeURIComponent(account)}/workers/scripts/${encodeURIComponent(script)}/secrets`, what, 100);
419  } catch (err) {
420    if (err instanceof CfReadError && err.status === 403) {
421      throw new Error(`${what} was refused with 403: CF_OPS_TOKEN lacks ${SECRETS_PERMISSION}. Cloudflare said: ${err.message}`);
422    }
423    throw err;
424  }
425  // The name only: whatever else a row carries is dropped here, unread.
426  return rows.map((r) => (r && typeof r === "object" ? (r as { name?: unknown }).name : null)).filter((n): n is string => typeof n === "string" && n.length > 0);
427}
428
429// The Access applications' names, for a blocked job's "check: access app <name>" line
430// (src/manual-steps.ts), read on the watcher's pass. Names only: an Access for SaaS app
431// carries a client secret beside its settings (src/ops-cloudflare-config.ts), and nothing
432// but the name is copied out here.
433const ACCESS_PERMISSION = "Account / Access: Apps and Policies / Read";
434
435/** GET /accounts/{id}/access/apps: the names. A 403 names the permission the token lacks. */
436export async function readAccessAppNames(fetchImpl: FetchLike, token: string, account: string): Promise<string[]> {
437  const what = "the Access applications list";
438  let rows: unknown[];
439  try {
440    rows = await cfGetAll(fetchImpl, token, `${CF_API}/accounts/${encodeURIComponent(account)}/access/apps`, what, 50);
441  } catch (err) {
442    if (err instanceof CfReadError && err.status === 403) {
443      throw new Error(`${what} was refused with 403: CF_OPS_TOKEN lacks ${ACCESS_PERMISSION}. Cloudflare said: ${err.message}`);
444    }
445    throw err;
446  }
447  return rows.map((r) => (r && typeof r === "object" ? (r as { name?: unknown }).name : null)).filter((n): n is string => typeof n === "string" && n.length > 0);
448}