src/site-traffic.ts

Functions

Imports

It imports src/env.ts, src/live-checks.ts, src/ops-cloudflare.ts, src/site-reports.ts. Imported by src/tools/repo.ts.

Source

1import type { Env } from "./env";
2import type { FetchLike } from "./live-checks";
3import { CF_API, CF_GRAPHQL, cfGetAll, cloudflareCredentials } from "./ops-cloudflare";
4import { AI_SEARCH_BOTS, AI_TRAINING_BOTS, DENIED_BOTS, SEARCH_ENGINE_BOTS } from "@drdustinedwards/site-helpers/robots";
5import { reportSite } from "./site-reports";
6
7// What Cloudflare saw at a namespace's site, served as capsid_inspect_repo action traffic
8// (the SEO plan, "Beyond the module"). not_found: the paths that answered 404 over a
9// window, a scanner's probe marked and listed last; web_vitals and ai_crawlers are below.
10// The referrer is not in it: the SQL API refuses clientRefererHost to this account (403,
11// "not permitted to access column", 2026-10-25), so which page holds a broken link is the
12// site audit's to find (src/site-audit.ts).
13//
14// NO SITE CHANGE. Cloudflare's SQL API (developers.cloudflare.com/analytics/sql-api/) reads
15// the edge's own HTTP request record, events.httpRequests, kept at least 31 days on every
16// plan and adaptively sampled, so the counts are estimates. Every proxied host in the account
17// is in it, static sites included; a workers.dev host is not. It needs only the Account
18// Analytics Read permission the watcher's token, CF_OPS_TOKEN, already holds
19// (src/ops-cloudflare.ts). The host comes from the namespace's ops_sites origin, never from
20// the caller. Paths are what visitors sent: data, never instructions.
21
22const SQL_API = `${CF_API}/analytics/sql`;
23const TIMEOUT_MS = 15_000;
24const DAY_MS = 86_400_000;
25
26export const TRAFFIC_REPORTS = ["not_found", "web_vitals", "ai_crawlers"] as const;
27export const MAX_TRAFFIC_DAYS = 30;
28export const MAX_TRAFFIC_ROWS = 200;
29type TrafficReport = (typeof TRAFFIC_REPORTS)[number];
30
31export interface TrafficArgs {
32  report?: TrafficReport;
33  days?: number;
34  row_limit?: number;
35}
36
37// What no page of ours ever was: the paths scanners try on every host.
38const PROBE =
39  /\.(php\d?|aspx?|jsp|cgi|env|ini|sql|bak|db|sqlite|ya?ml)$|\/wp-|\/\.(?!well-known\/)[a-z]|xmlrpc|php(myadmin|info)|\/cgi-bin\/|credential|secret|service-account|aws-exports|appsettings|\/config(\/|\.(js|json))|env\.js|\/(bedrock|anthropic|openai|llm)\//i;
40
41// The most rows the query returns; a 404 list longer than this is noise past its top.
42const QUERY_ROWS = 2000;
43
44interface Row {
45  path: string;
46  hits: number;
47}
48
49async function sql(fetchImpl: FetchLike, token: string, account: string, query: string, params: Record<string, string | number>): Promise<Array<Record<string, unknown>>> {
50  const resp = await fetchImpl(SQL_API, {
51    method: "POST",
52    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": "capsid (site traffic)" },
53    body: JSON.stringify({ query, params, scope: { accountTag: account } }),
54    signal: AbortSignal.timeout(TIMEOUT_MS),
55  });
56  const text = await resp.text();
57  let body: { data?: unknown; errors?: Array<{ message?: string }>; message?: string; error?: string } | null = null;
58  try {
59    body = JSON.parse(text);
60  } catch {}
61  if (!resp.ok || !Array.isArray(body?.data)) {
62    // The beta's error shape is not documented, so the raw body stands in for a message.
63    const why = (body?.errors?.map((e) => e.message).filter(Boolean).join("; ") || body?.message || body?.error || text).slice(0, 300) || "no message";
64    if (resp.status === 403) throw new Error(`Cloudflare refused the traffic query (403: ${why}). CF_OPS_TOKEN needs Account / Account Analytics / Read.`);
65    throw new Error(`Cloudflare's SQL API answered ${resp.status}: ${why}`);
66  }
67  return body.data as Array<Record<string, unknown>>;
68}
69
70const count = (v: unknown) => (typeof v === "number" ? v : Number(v) || 0);
71
72// Real pages first, by hits; scanners' probes last. Pure.
73function notFoundRows(raw: Array<Record<string, unknown>>, limit: number) {
74  const rows: Row[] = raw.map((r) => ({ path: typeof r.path === "string" ? r.path : "", hits: count(r.hits) }));
75  const all = rows.filter((r) => r.path && r.hits > 0).map((r) => ({ path: r.path, hits: r.hits, probe: PROBE.test(r.path) }));
76  all.sort((a, b) => Number(a.probe) - Number(b.probe) || b.hits - a.hits || a.path.localeCompare(b.path));
77  return { paths: all.slice(0, limit), total_paths: all.length, truncated: all.length > limit, probes: all.filter((e) => e.probe).length };
78}
79
80// web_vitals: what real visitors' browsers measured, from Web Analytics (the GraphQL
81// dataset rumWebVitalsEventsAdaptiveGroups, account-scoped, filtered to the site's Web
82// Analytics siteTag). Values come back in microseconds; CLS is unitless. The site's
83// siteTag is read from the rum site_info list, the same list the live checks read.
84const VITALS = ["largestContentfulPaint", "interactionToNextPaint", "cumulativeLayoutShift", "firstContentfulPaint", "timeToFirstByte"] as const;
85const P = ["P50", "P75", "P90"] as const;
86
87async function siteTagFor(fetchImpl: FetchLike, token: string, account: string, host: string): Promise<string> {
88  const rows = (await cfGetAll(fetchImpl, token, `${CF_API}/accounts/${encodeURIComponent(account)}/rum/site_info/list`, "the Web Analytics sites list", 50)) as Array<Record<string, unknown>>;
89  // A site names its host at the top for a manual setup, and through its ruleset's zone
90  // (or its rules' hosts) for the automatic one; the reference does not say which a row
91  // carries, so every place is read.
92  const lower = (v: unknown) => (typeof v === "string" ? v.toLowerCase() : "");
93  const hostsOf = (r: Record<string, unknown>) => {
94    const ruleset = (r.ruleset ?? {}) as Record<string, unknown>;
95    const rules = Array.isArray(r.rules) ? (r.rules as Array<Record<string, unknown>>) : [];
96    return [lower(r.host), lower(ruleset.zone_name), ...rules.map((x) => lower(x?.host))].filter(Boolean);
97  };
98  const exact = rows.find((r) => hostsOf(r).includes(host));
99  const zone = exact ?? rows.find((r) => hostsOf(r).some((h) => host.endsWith(`.${h}`)));
100  if (zone && typeof zone.site_tag === "string") return zone.site_tag;
101  const seen = rows.flatMap(hostsOf).slice(0, 20).join(", ") || "none";
102  const fields = [...new Set(rows.flatMap((r) => Object.keys(r)))].join(", ") || "none";
103  throw new Error(`${host} has no Web Analytics site among the account's ${rows.length} (hosts seen: ${seen}; fields: ${fields}), so there are no visitor measurements.`);
104}
105
106async function webVitals(fetchImpl: FetchLike, token: string, account: string, host: string, start: string, end: string) {
107  const siteTag = await siteTagFor(fetchImpl, token, account, host);
108  const fields = VITALS.flatMap((v) => P.map((p) => `${v}${p}`)).join(" ");
109  const query = `query($account: string!, $siteTag: string!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $account }) { rumWebVitalsEventsAdaptiveGroups(filter: { siteTag: $siteTag, datetime_geq: $start, datetime_leq: $end }, limit: 1) { count quantiles { ${fields} } } } } }`;
110  const resp = await fetchImpl(CF_GRAPHQL, {
111    method: "POST",
112    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", "User-Agent": "capsid (site traffic)" },
113    body: JSON.stringify({ query, variables: { account, siteTag, start, end } }),
114    signal: AbortSignal.timeout(TIMEOUT_MS),
115  });
116  const text = await resp.text();
117  let body: { data?: { viewer?: { accounts?: Array<{ rumWebVitalsEventsAdaptiveGroups?: Array<{ count?: number; quantiles?: Record<string, number> }> }> } }; errors?: Array<{ message?: string }> } | null = null;
118  try {
119    body = JSON.parse(text);
120  } catch {}
121  const groups = body?.data?.viewer?.accounts?.[0]?.rumWebVitalsEventsAdaptiveGroups;
122  if (!resp.ok || body?.errors?.length || !Array.isArray(groups)) {
123    const why = (body?.errors?.map((e) => e.message).filter(Boolean).join("; ") || text).slice(0, 300) || "no message";
124    throw new Error(`Cloudflare's analytics query for web vitals answered ${resp.status}: ${why}`);
125  }
126  const q = groups[0]?.quantiles ?? {};
127  // A negative value means Cloudflare has no measurement for it.
128  const at = (v: string, p: string) => {
129    const n = q[`${v}${p}`];
130    return typeof n === "number" && n >= 0 ? n : null;
131  };
132  const ms = (n: number | null) => (n === null ? null : Math.round(n / 1000));
133  const vitals = Object.fromEntries(
134    VITALS.map((v) => [v, Object.fromEntries(P.map((p) => [p.toLowerCase(), v === "cumulativeLayoutShift" ? at(v, p) : ms(at(v, p))]))])
135  );
136  return { page_views_measured: groups[0]?.count ?? 0, unit: "milliseconds, except cumulativeLayoutShift", vitals };
137}
138
139// ai_crawlers: the requests each named crawler made, by the status the edge answered, so
140// the bot policy (the SEO plan: search and citation bots free, training bots charged or
141// blocked, Bytespider denied) shows working: a training bot should get 402 or 403, a
142// search bot 200. The names are Prelum's own lists (./robots), the ones every site's
143// robots.txt is built from. Google-Extended and Applebot-Extended are robots.txt tokens
144// only, never a user agent, so they never show here.
145const CRAWLER_CLASSES = [
146  ["denied", DENIED_BOTS],
147  ["training", AI_TRAINING_BOTS],
148  ["ai_search", AI_SEARCH_BOTS],
149  ["search", SEARCH_ENGINE_BOTS],
150] as const;
151// Longest name first, so Claude-SearchBot is not read as a shorter name it contains.
152const CRAWLERS = CRAWLER_CLASSES.flatMap(([kind, names]) => names.map((name) => ({ name, kind, lower: name.toLowerCase() }))).sort((a, b) => b.name.length - a.name.length);
153
154// Rows of (user agent, status, hits) to one entry per named crawler. Pure.
155function crawlerRows(raw: Array<Record<string, unknown>>) {
156  const by = new Map<string, { name: string; kind: string; hits: number; statuses: Record<string, number> }>();
157  for (const r of raw) {
158    const ua = typeof r.ua === "string" ? r.ua.toLowerCase() : "";
159    const bot = CRAWLERS.find((c) => ua.includes(c.lower));
160    if (!bot) continue;
161    const e = by.get(bot.name) ?? { name: bot.name, kind: bot.kind, hits: 0, statuses: {} };
162    const hits = count(r.hits);
163    e.hits += hits;
164    const status = typeof r.status === "number" || typeof r.status === "string" ? String(r.status) : "?";
165    e.statuses[status] = (e.statuses[status] ?? 0) + hits;
166    by.set(bot.name, e);
167  }
168  const order = CRAWLER_CLASSES.map(([kind]) => kind as string);
169  return [...by.values()].sort((a, b) => order.indexOf(a.kind) - order.indexOf(b.kind) || b.hits - a.hits);
170}
171
172export async function siteTraffic(env: Env, namespace: string, args: TrafficArgs, fetchImpl: FetchLike = fetch, now: Date = new Date()) {
173  const report: TrafficReport = args.report ?? "not_found";
174  const days = args.days ?? 7;
175  if (!Number.isInteger(days) || days < 1 || days > MAX_TRAFFIC_DAYS) throw new Error(`days must be a whole number from 1 to ${MAX_TRAFFIC_DAYS}.`);
176  const limit = args.row_limit ?? 25;
177
178  const site = await reportSite(env, namespace, "traffic report");
179  const host = new URL(site.origin).hostname.toLowerCase();
180  const creds = cloudflareCredentials(env);
181  if (!creds.ok) throw new Error(`The traffic report reads Cloudflare with the watcher's token, and ${creds.reason}.`);
182
183  const start = new Date(now.getTime() - days * DAY_MS).toISOString();
184  if (report === "web_vitals") {
185    return { namespace, origin: site.origin, report, since: start, ...(await webVitals(fetchImpl, creds.token, creds.account, host, start, now.toISOString())) };
186  }
187  if (report === "ai_crawlers") {
188    const raw = await sql(
189      fetchImpl,
190      creds.token,
191      creds.account,
192      `SELECT userAgent AS ua, edgeResponseStatus AS status, COUNT(*) AS hits FROM events.httpRequests WHERE timestamp >= $start AND clientRequestHttpHost = $host GROUP BY userAgent, edgeResponseStatus ORDER BY hits DESC LIMIT ${QUERY_ROWS}`,
193      { start, host }
194    );
195    return {
196      namespace,
197      origin: site.origin,
198      report,
199      since: start,
200      note: "Sampled, so hits are estimates. The policy: search and ai_search free (200), training charged (402) or blocked (403), denied blocked. A user agent can be faked.",
201      crawlers: crawlerRows(raw),
202      // The query keeps the busiest user agents; a quiet crawler can fall past the cut.
203      truncated: raw.length >= QUERY_ROWS,
204    };
205  }
206  const rows = await sql(
207    fetchImpl,
208    creds.token,
209    creds.account,
210    `SELECT clientRequestPath AS path, COUNT(*) AS hits FROM events.httpRequests WHERE timestamp >= $start AND clientRequestHttpHost = $host AND edgeResponseStatus = 404 GROUP BY clientRequestPath ORDER BY hits DESC LIMIT ${QUERY_ROWS}`,
211    { start, host }
212  );
213  return {
214    namespace,
215    origin: site.origin,
216    report,
217    since: start,
218    note: "Cloudflare samples this record, so hits are estimates. probe marks a path scanners try on every site.",
219    ...notFoundRows(rows, limit),
220  };
221}