src/public-data.ts

Functions

Imports

It imports src/env.ts, src/github/client.ts, src/public-code.ts, src/public-stats.ts, src/rate-limit.ts, src/system-model.ts, src/time.ts. Imported by src/maintenance.ts, src/routes.ts, src/tick.ts.

Source

1import type { Env } from "./env";
2import rawModel from "../system/model.json" with { type: "json" };
3import { listInstalledRepos, type InstalledRepo } from "./github/client";
4import { callerIp, checkRate, rateLimitedResponse, type RateLimitPolicy } from "./rate-limit";
5import { parseSystemModel, toPublic, type SystemModel } from "./system-model";
6import { readStatsRows, summarize, type StatsRows } from "./public-stats";
7import { buildPublicCode, readCodeMap } from "./public-code";
8import { HOUR_MS } from "./time";
9
10// The public data build (design capsid/research/design-system-explorer.md, section 2;
11// docs/public-data.md). Once a UTC day the tick reads every repo's visibility from
12// GitHub, runs the model through toPublic, computes the aggregate numbers, and stores
13// both documents in KV. The routes below serve those stored documents and nothing else:
14// they never read D1 or GitHub, so a request cannot make the Worker compute anything.
15//
16// Their readers are the explorer's and dustinedwards.info's builds, not browsers
17// (answer 4), so the responses carry no CORS header.
18
19export const PUBLIC_MODEL_PATH = "/public/model/v1.json";
20export const PUBLIC_MODEL_KEY = "public:model:v1";
21export const PUBLIC_STATS_PATH = "/public/stats/v1.json";
22export const PUBLIC_STATS_KEY = "public:stats:v1";
23export const PUBLIC_CODE_PATH = "/public/code/v1.json";
24export const PUBLIC_CODE_KEY = "public:code:v1";
25export const PUBLIC_BUILD_LAST_KEY = "public:model:last";
26const PUBLIC_CODE_LAST_KEY = "public:code:last";
27// A repo publishes a new code map after each push, so the code document is rebuilt hourly, not daily.
28const CODE_REFRESH_MS = HOUR_MS;
29const CACHE_SECONDS = 3600;
30
31// Fail closed: an unreadable counter refuses rather than serve without a limit. The
32// edge cache answers most requests before the counter is read at all. One budget for
33// both routes.
34export const PUBLIC_DATA_LIMIT: RateLimitPolicy = {
35  prefix: "public:rate:",
36  perHour: 120,
37  perDay: 1000,
38  label: "PUBLIC_DATA",
39  onUnavailable: "refuse",
40};
41
42/** The model the Worker was built with. A model that fails its schema fails the
43 *  build step loudly, every day, rather than publish a partial map. */
44export function bundledModel(): SystemModel {
45  const parsed = parseSystemModel(rawModel);
46  if (!parsed.ok) throw new Error(`system/model.json is invalid: ${parsed.problems.join("; ")}`);
47  return parsed.model;
48}
49
50export type PublicBuildReport = { ran: false; note: string } | { ran: true; note: string; codeFailed?: true };
51
52export interface PublicBuildReaders {
53  listRepos: (env: Env) => Promise<{ repos: InstalledRepo[] }>;
54  statsRows: (env: Env, now: Date, systems: StatsRows["systems"]) => Promise<StatsRows>;
55  codeMap: (env: Env, repo: string) => Promise<unknown>;
56}
57
58const LIVE: PublicBuildReaders = { listRepos: listInstalledRepos, statsRows: readStatsRows, codeMap: readCodeMap };
59
60/**
61 * Builds and stores the public documents once per UTC day. The model and the numbers
62 * are read and computed before anything is written, so a failed read leaves the last
63 * good documents served and the day unstamped; a partial visibility read cannot happen
64 * (listInstalledRepos throws on one). The code maps come after, on their own: a repo's
65 * broken map keeps the last good code document and is reported, and does not hold back
66 * the model or retry every tick. Later in the day the code document alone is rebuilt
67 * each hour, so a repo's new map shows the same day.
68 */
69export async function publicBuildTick(env: Env, now: Date, readers: PublicBuildReaders = LIVE, model: SystemModel = bundledModel()): Promise<PublicBuildReport> {
70  const today = now.toISOString().slice(0, 10);
71  const last = await env.APP_KV.get(PUBLIC_BUILD_LAST_KEY);
72  if (last && last.slice(0, 10) === today) {
73    const codeLast = await env.APP_KV.get(PUBLIC_CODE_LAST_KEY);
74    if (codeLast && now.getTime() - Date.parse(codeLast) < CODE_REFRESH_MS) return { ran: false, note: `already built ${last}` };
75    const { repos } = await readers.listRepos(env);
76    const code = await storeCode(env, model, privacyOf(repos), today, now, readers.codeMap);
77    const note = `code maps only, the model was built ${last}: ${code.note}`;
78    return code.failed ? { ran: true, note, codeFailed: true } : { ran: true, note };
79  }
80
81  const { repos } = await readers.listRepos(env);
82  const privacy = privacyOf(repos);
83  const doc = toPublic(model, privacy, today);
84  const stats = summarize(await readers.statsRows(env, now, { public: doc.counts.public, private: doc.counts.private }), now);
85
86  await env.APP_KV.put(PUBLIC_MODEL_KEY, JSON.stringify(doc));
87  await env.APP_KV.put(PUBLIC_STATS_KEY, JSON.stringify(stats));
88  await env.APP_KV.put(PUBLIC_BUILD_LAST_KEY, now.toISOString());
89
90  const code = await storeCode(env, model, privacy, today, now, readers.codeMap);
91
92  const c = doc.counts;
93  const note = `${c.public} public and ${c.private} private system(s), ${c.containers} container(s), ${c.relationships} relationship(s), ${doc.flows?.length ?? 0} of ${model.flows?.length ?? 0} flow(s) whole; visibility read for ${repos.length} repo(s); stats as of ${stats.as_of}, ${stats.jobs.completed_30d} job(s) finished in 30 days; ${code.note}`;
94  return code.failed ? { ran: true, note, codeFailed: true } : { ran: true, note };
95}
96
97function privacyOf(repos: InstalledRepo[]): Map<string, boolean> {
98  return new Map(repos.map((r) => [r.full_name.toLowerCase(), r.private] as const));
99}
100
101/** Builds and stores the code document. A map that fails its parse keeps the last good
102 *  document. Either way the attempt is stamped, so a broken map is retried the next hour, not every tick. */
103async function storeCode(env: Env, model: SystemModel, privacy: Map<string, boolean>, today: string, now: Date, read: PublicBuildReaders["codeMap"]): Promise<{ note: string; failed: boolean }> {
104  let result: { note: string; failed: boolean };
105  try {
106    const built = await buildPublicCode(env, model, privacy, today, read);
107    await env.APP_KV.put(PUBLIC_CODE_KEY, JSON.stringify(built));
108    result = { note: `${built.maps.length} code map(s)${built.maps.length ? ` (${built.maps.map((m) => m.repo.split("/")[1]).join(", ")})` : ""}`, failed: false };
109  } catch (e) {
110    result = { note: `code maps kept from the last build: ${e instanceof Error ? e.message : String(e)}`, failed: true };
111  }
112  await env.APP_KV.put(PUBLIC_CODE_LAST_KEY, now.toISOString());
113  return result;
114}
115
116function json(body: string, status: number, cache: string): Response {
117  return new Response(body, { status, headers: { "Content-Type": "application/json;charset=utf-8", "Cache-Control": cache } });
118}
119
120/** Which stored document a route serves. */
121export const PUBLIC_DOCS = {
122  model: { path: PUBLIC_MODEL_PATH, key: PUBLIC_MODEL_KEY },
123  stats: { path: PUBLIC_STATS_PATH, key: PUBLIC_STATS_KEY },
124  code: { path: PUBLIC_CODE_PATH, key: PUBLIC_CODE_KEY },
125} as const;
126
127/**
128 * GET of one public document. The colo's cache first, so a cached answer costs no KV
129 * read and no counter; then the per-IP limit; then the stored document. 503 until the
130 * first build, and never a computed fallback.
131 */
132export async function handlePublicDocument(
133  request: Request,
134  env: Env,
135  ctx: Pick<ExecutionContext, "waitUntil">,
136  doc: (typeof PUBLIC_DOCS)[keyof typeof PUBLIC_DOCS],
137  now: Date = new Date(),
138  cache: Cache | undefined = (globalThis as { caches?: CacheStorage & { default?: Cache } }).caches?.default
139): Promise<Response> {
140  // The query string is not part of the key, so it cannot be used to miss the cache.
141  const key = new Request(new URL(doc.path, request.url).toString());
142  const hit = await cache?.match(key);
143  if (hit) return hit;
144
145  const verdict = await checkRate(env.APP_KV, callerIp(request), now, PUBLIC_DATA_LIMIT);
146  if (!verdict.allowed) return rateLimitedResponse(verdict, "requests");
147
148  const stored = await env.APP_KV.get(doc.key);
149  if (!stored) return json(JSON.stringify({ error: "not computed yet" }), 503, "no-store");
150  const response = json(stored, 200, `public, max-age=${CACHE_SECONDS}, s-maxage=${CACHE_SECONDS}`);
151  if (cache) ctx.waitUntil(cache.put(key, response.clone()));
152  return response;
153}