src/shared-code.ts

Functions

Imports

It imports src/encoding.ts, src/env.ts, src/github/client.ts, src/github/contents.ts, src/ops-packages.ts, src/ops-sites.ts, src/ops-types/feed.ts, src/ops-types/sites.ts. Imported by src/maintenance-releases.ts, src/portal-packages.ts.

Source

1import type { Env } from "./env";
2import { base64Decode } from "./encoding";
3import { ghFetch } from "./github/client";
4import { readRepoFile, repoBlobPaths } from "./github/contents";
5import { readNpm, type FetchLike } from "./ops-packages";
6import { readSiteConfig } from "./ops-sites";
7import type { PackageSnapshot } from "./ops-types/sites";
8import type { SharedCodeApp, SharedCodePackage, SharedCodeView } from "./ops-types/feed";
9
10// SHARED CODE: how far the centralization has got (job_584b4e7f2824; ruling
11// capsid/rulings/shared-homes-2026-10-06.md). For each shared package in configuration
12// (shared_packages, migrations/0036): its newest tag, the tag each app pins, how many
13// releases behind each app is, and how many known local copies are still in the apps.
14//
15// The apps are the configured sites' namespaces (ops_sites, the same list the inbox and
16// the watcher use), each read on its default branch through the GitHub App the watcher
17// already uses: one recursive tree, then every package.json outside node_modules (up to
18// MANIFESTS_PER_APP) and the Renovate config. Every shared package is a git dependency
19// ("github:DrDustinEdwards/<repo>#v1.2.3"), so the manifest names the tag itself; the
20// lockfile is read only for a spec with no tag, for the version it resolved. devkit is
21// a Renovate preset ("github>DrDustinEdwards/devkit"), so it is found in the Renovate
22// config, not in a manifest.
23//
24// Release tracking (Capsid improvement list, item 13): each package's own repo is read
25// too, for the version in its root package.json on the default branch, whether that
26// version has a release tag yet (untagged), and, when the package.json is not private,
27// npm's latest dist-tag from the registry's abbreviated document (src/ops-packages.ts,
28// readNpm). So each panel says tagged, on npm, and which apps are behind. The daily
29// maintenance pass tags an untagged version (src/maintenance-releases.ts), using
30// releaseDecision below, so the view and the pass agree on what "untagged" means.
31//
32// Read on demand by the Portal (GET /portal/api/shared-code) and kept for an hour in KV:
33// a few dozen GitHub reads per refresh, not per watcher pass. Read-only: it writes
34// nothing but its own cache. The key's version moves when the view's shape does, so a
35// cached answer without the newer fields is never served.
36
37export const SHARED_CODE_CACHE_KEY = "portal:shared-code:v2";
38const CACHE_TTL_SECONDS = 3600;
39const MANIFESTS_PER_APP = 8;
40const TAGS_PER_PAGE = 100;
41const RENOVATE_FILES = ["renovate.json", ".github/renovate.json", "renovate.json5", ".github/renovate.json5", ".renovaterc", ".renovaterc.json"];
42
43interface SharedRow {
44  name: string;
45  repo: string;
46  formerly: string;
47  local_paths: string;
48}
49
50export interface SharedConfig {
51  name: string;
52  repo: string;
53  // Earlier names of the repo (security-headers for site-runtime): an app still pinned
54  // through GitHub's redirect counts as a consumer.
55  formerly: string[];
56  // Known local copies the package replaces, "namespace:path".
57  local_paths: string[];
58}
59
60function jsonList(raw: string | null | undefined): string[] {
61  try {
62    const v: unknown = JSON.parse(raw ?? "[]");
63    return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
64  } catch {
65    return [];
66  }
67}
68
69export async function readSharedConfig(db: D1Database): Promise<SharedConfig[]> {
70  const { results } = await db.prepare("SELECT name, repo, formerly, local_paths FROM shared_packages ORDER BY name LIMIT 50").all<SharedRow>();
71  return (results ?? []).map((r) => ({ name: r.name, repo: r.repo, formerly: jsonList(r.formerly), local_paths: jsonList(r.local_paths) }));
72}
73
74// ---- pure parts ----------------------------------------------------------------------
75
76/** "owner/repo" lower-cased, from any GitHub spelling npm accepts, with its ref; or null. */
77export function gitSpec(spec: string): { repo: string; ref: string | null } | null {
78  const s = spec.trim();
79  const m =
80    /^(?:github:)?([A-Za-z0-9-]+\/[A-Za-z0-9._-]+?)(?:\.git)?(?:#(.+))?$/.exec(s) ??
81    /^git\+(?:https|ssh):\/\/(?:git@)?github\.com[/:]([A-Za-z0-9-]+\/[A-Za-z0-9._-]+?)(?:\.git)?(?:#(.+))?$/.exec(s) ??
82    /^https:\/\/github\.com\/([A-Za-z0-9-]+\/[A-Za-z0-9._-]+?)(?:\.git)?(?:#(.+))?$/.exec(s);
83  if (!m) return null;
84  // A bare "a/b" without github: and without a ref is a path or a scope, not a repo.
85  if (!s.startsWith("github:") && !s.startsWith("git+") && !s.startsWith("https://") && m[2] === undefined) return null;
86  return { repo: m[1].toLowerCase(), ref: m[2] ?? null };
87}
88
89/** A Renovate extends entry naming a GitHub preset repo, with its ref. */
90export function presetSpec(entry: string): { repo: string; ref: string | null } | null {
91  const m = /^github>([A-Za-z0-9-]+\/[A-Za-z0-9._-]+?)(?::[^#]+)?(?:#(.+))?$/.exec(entry.trim());
92  return m ? { repo: m[1].toLowerCase(), ref: m[2] ?? null } : null;
93}
94
95const SEMVER = /^v?(\d+)\.(\d+)\.(\d+)$/;
96
97function semver(tag: string): [number, number, number] | null {
98  const m = SEMVER.exec(tag);
99  return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;
100}
101
102function cmp(a: [number, number, number], b: [number, number, number]): number {
103  return a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
104}
105
106/** The newest release tag, by version, ignoring tags that are not x.y.z. */
107export function latestTag(tags: readonly string[]): string | null {
108  let best: { tag: string; v: [number, number, number] } | null = null;
109  for (const tag of tags) {
110    const v = semver(tag);
111    if (v && (!best || cmp(v, best.v) > 0)) best = { tag, v };
112  }
113  return best?.tag ?? null;
114}
115
116/** How many release tags are newer than `pinned`, or null when the pin is not a release. */
117export function behindBy(pinned: string | null, tags: readonly string[]): number | null {
118  const p = pinned ? semver(pinned) : null;
119  if (!p) return null;
120  return tags.filter((t) => {
121    const v = semver(t);
122    return v !== null && cmp(v, p) > 0;
123  }).length;
124}
125
126/** What a repo's root package.json says about a release: the name npm knows it by, its
127 *  version, and whether it is private (never published). */
128export interface RootManifest {
129  name: string | null;
130  version: string | null;
131  private: boolean;
132}
133
134/** The release fields of a package.json's text. Throws when it is not a JSON object. */
135export function rootManifest(raw: string): RootManifest {
136  const j: unknown = JSON.parse(raw);
137  if (!j || typeof j !== "object" || Array.isArray(j)) throw new Error("package.json is not a JSON object");
138  const o = j as Record<string, unknown>;
139  return { name: typeof o.name === "string" ? o.name : null, version: typeof o.version === "string" ? o.version : null, private: o.private === true };
140}
141
142/**
143 * What a package's version asks for, given the repo's tags:
144 * - tagged: a tag names this version already (v1.2.3 or 1.2.3);
145 * - tag: no tag names it and it is newer than every release tag, so it is the next release;
146 * - backwards: no tag names it but a newer release tag exists. Never tagged: the tag would
147 *   start bump-consumers on an older version and move the apps backwards. Listed instead;
148 * - not-a-release: no version, or not a plain x.y.z (a pre-release is tagged by hand).
149 */
150export type ReleaseDecision =
151  | { kind: "tagged"; tag: string }
152  | { kind: "tag"; tag: string; newest: string | null }
153  | { kind: "backwards"; tag: string; newest: string }
154  | { kind: "not-a-release"; version: string | null };
155
156export function releaseDecision(version: string | null, tags: readonly string[]): ReleaseDecision {
157  const v = version !== null && /^\d+\.\d+\.\d+$/.test(version) ? semver(version) : null;
158  if (!v) return { kind: "not-a-release", version };
159  const existing = tags.find((t) => {
160    const s = semver(t);
161    return s !== null && cmp(s, v) === 0;
162  });
163  if (existing) return { kind: "tagged", tag: existing };
164  const tag = `v${version}`;
165  const newest = latestTag(tags);
166  if (newest && cmp(v, semver(newest)!) < 0) return { kind: "backwards", tag, newest };
167  return { kind: "tag", tag, newest };
168}
169
170/** What was read of a package's own repo and of npm, or why the repo was not read. */
171export interface ReleaseRead {
172  manifest: RootManifest | string;
173  npm: PackageSnapshot["npm"] | null;
174}
175
176interface AppRead {
177  namespace: string;
178  name: string;
179  blobs: Set<string> | null;
180  // Each manifest's dependencies of every kind, by path.
181  manifests: Array<{ path: string; deps: Record<string, string>; lockVersions: Record<string, string> }>;
182  presets: string[];
183  error: string | null;
184}
185
186/** The view, from what was read. Pure, so every rule is tested without GitHub. */
187export function sharedCodeView(
188  config: readonly SharedConfig[],
189  tags: ReadonlyMap<string, string[] | string>,
190  apps: readonly AppRead[],
191  now: Date,
192  releases: ReadonlyMap<string, ReleaseRead> = new Map()
193): SharedCodeView {
194  const packages: SharedCodePackage[] = config.map((pkg) => {
195    const names = new Set([pkg.repo, ...pkg.formerly].map((r) => r.toLowerCase()));
196    const t = tags.get(pkg.name);
197    const list = Array.isArray(t) ? t : [];
198    const latest = latestTag(list);
199    // Without the tags, how far behind is unknown, never zero.
200    const behind = (pinned: string | null) => (Array.isArray(t) ? behindBy(pinned, list) : null);
201    const users: SharedCodeApp[] = [];
202    for (const app of apps) {
203      for (const m of app.manifests) {
204        for (const [dep, spec] of Object.entries(m.deps)) {
205          const g = gitSpec(spec);
206          if (!g || !names.has(g.repo)) continue;
207          const pinned = g.ref ?? m.lockVersions[dep] ?? null;
208          users.push({ namespace: app.namespace, name: app.name, where: m.path, pinned, behind: behind(pinned), via_old_name: g.repo !== pkg.repo.toLowerCase() });
209        }
210      }
211      for (const entry of app.presets) {
212        const p = presetSpec(entry);
213        if (!p || !names.has(p.repo)) continue;
214        users.push({ namespace: app.namespace, name: app.name, where: "renovate", pinned: p.ref, behind: behind(p.ref), via_old_name: p.repo !== pkg.repo.toLowerCase() });
215      }
216    }
217    const local = pkg.local_paths.map((entry) => {
218      const at = entry.indexOf(":");
219      const namespace = at > 0 ? entry.slice(0, at) : "";
220      const path = (at > 0 ? entry.slice(at + 1) : entry).replace(/\/+$/, "");
221      const blobs = apps.find((a) => a.namespace === namespace)?.blobs ?? null;
222      const present = blobs ? [...blobs].some((b) => b === path || b.startsWith(`${path}/`)) : null;
223      return { namespace, path, present };
224    });
225    const release = releaseFields(releases.get(pkg.name), Array.isArray(t) ? list : null);
226    return {
227      name: pkg.name,
228      repo: pkg.repo,
229      latest,
230      tags_error: typeof t === "string" ? t : null,
231      ...release,
232      users,
233      behind: users.filter((u) => (u.behind ?? 0) > 0).length,
234      local_copies: local,
235      local_left: local.filter((l) => l.present === true).length,
236    };
237  });
238  return {
239    generated: now.toISOString(),
240    configured: true,
241    error: null,
242    apps_read: apps.filter((a) => a.error === null).length,
243    apps_failed: apps.filter((a) => a.error !== null).map((a) => ({ namespace: a.namespace, error: a.error! })),
244    packages,
245  };
246}
247
248type ReleaseFields = Pick<SharedCodePackage, "version" | "untagged" | "private" | "npm_latest" | "npm_note" | "manifest_error">;
249
250/** A package's release fields. Untagged only when the tags were read and none names the
251 *  version: without the tags it is unknown, and unknown is not untagged. */
252function releaseFields(read: ReleaseRead | undefined, tags: readonly string[] | null): ReleaseFields {
253  const unread = { version: null, untagged: false, private: false, npm_latest: null, npm_note: null };
254  if (!read) return { ...unread, manifest_error: null };
255  if (typeof read.manifest === "string") return { ...unread, manifest_error: read.manifest };
256  const m = read.manifest;
257  const decision = tags ? releaseDecision(m.version, tags) : null;
258  const untagged = decision?.kind === "tag" || decision?.kind === "backwards";
259  let npm_latest: string | null = null;
260  let npm_note: string | null = null;
261  if (m.private) npm_note = null;
262  else if (!m.name) npm_note = "package.json has no name";
263  else if (!read.npm) npm_note = "npm was not read";
264  else if (read.npm.state === "ok") {
265    npm_latest = read.npm.latest;
266    if (!npm_latest) npm_note = "npm has no latest dist-tag";
267  } else npm_note = read.npm.reason;
268  return { version: m.version, untagged, private: m.private, npm_latest, npm_note, manifest_error: null };
269}
270
271// ---- reading -------------------------------------------------------------------------
272
273/** A repo's root package.json, on its default branch or at `ref`, through the App.
274 *  Uncached: the release-tag rule reads it to decide a write. */
275export async function readRootManifest(env: Env, full: string, ref?: string): Promise<RootManifest> {
276  const [owner, repo] = full.split("/");
277  const query = ref ? `?ref=${encodeURIComponent(ref)}` : "";
278  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/contents/package.json${query}`);
279  if (!resp.ok) {
280    await resp.body?.cancel();
281    throw new Error(`GitHub answered ${resp.status} for ${full}'s package.json${ref ? ` at ${ref.slice(0, 12)}` : ""}`);
282  }
283  const data = (await resp.json()) as { content?: unknown; encoding?: unknown };
284  if (data.encoding !== "base64" || typeof data.content !== "string") throw new Error(`${full}'s package.json came back without its content`);
285  return rootManifest(base64Decode(data.content));
286}
287
288async function readRelease(env: Env, full: string, fetchImpl: FetchLike): Promise<ReleaseRead> {
289  let manifest: RootManifest;
290  try {
291    manifest = await readRootManifest(env, full);
292  } catch (err) {
293    return { manifest: err instanceof Error ? err.message : String(err), npm: null };
294  }
295  const npm = manifest.private || !manifest.name ? null : await readNpm(fetchImpl, manifest.name);
296  return { manifest, npm };
297}
298
299function depsOf(raw: string): Record<string, string> {
300  const j = JSON.parse(raw) as Record<string, unknown>;
301  const out: Record<string, string> = {};
302  for (const key of ["dependencies", "devDependencies", "optionalDependencies", "peerDependencies"]) {
303    const block = j[key];
304    if (block && typeof block === "object") for (const [k, v] of Object.entries(block)) if (typeof v === "string") out[k] = v;
305  }
306  return out;
307}
308
309async function readApp(env: Env, namespace: string, name: string): Promise<AppRead> {
310  const app: AppRead = { namespace, name, blobs: null, manifests: [], presets: [], error: null };
311  try {
312    app.blobs = await repoBlobPaths(env, namespace);
313    if (!app.blobs) throw new Error("its file tree could not be read (missing, or too large)");
314    const manifests = [...app.blobs].filter((p) => (p === "package.json" || p.endsWith("/package.json")) && !p.includes("node_modules/")).sort((a, b) => a.split("/").length - b.split("/").length).slice(0, MANIFESTS_PER_APP);
315    for (const path of manifests) {
316      const deps = depsOf((await readRepoFile(env, namespace, path)).content);
317      const lockVersions: Record<string, string> = {};
318      const untagged = Object.entries(deps).filter(([, spec]) => gitSpec(spec)?.ref === null);
319      const lockPath = path.replace(/package\.json$/, "package-lock.json");
320      if (untagged.length && app.blobs.has(lockPath)) {
321        const lock = JSON.parse((await readRepoFile(env, namespace, lockPath)).content) as { packages?: Record<string, { version?: string }> };
322        for (const [dep] of untagged) {
323          const v = lock.packages?.[`node_modules/${dep}`]?.version;
324          if (v) lockVersions[dep] = `v${v}`;
325        }
326      }
327      app.manifests.push({ path, deps, lockVersions });
328    }
329    for (const path of RENOVATE_FILES) {
330      if (!app.blobs.has(path)) continue;
331      // renovate.json5 allows comments; the extends line is read either way.
332      const text = (await readRepoFile(env, namespace, path)).content;
333      app.presets.push(...[...text.matchAll(/"(github>[^"]+)"/g)].map((m) => m[1]));
334    }
335  } catch (err) {
336    app.error = err instanceof Error ? err.message : String(err);
337  }
338  return app;
339}
340
341async function readTags(env: Env, full: string): Promise<string[] | string> {
342  const [owner, repo] = full.split("/");
343  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/tags?per_page=${TAGS_PER_PAGE}`);
344  if (!resp.ok) return `GitHub answered ${resp.status} for ${full}'s tags`;
345  return ((await resp.json()) as Array<{ name: string }>).map((t) => t.name);
346}
347
348/** The view, from the hour's cache unless `fresh`. A missing table says so. */
349export async function readSharedCode(env: Env, now: Date, fresh = false, fetchImpl: FetchLike = (url, init) => fetch(url, init)): Promise<SharedCodeView> {
350  if (!fresh) {
351    const cached = await env.APP_KV.get(SHARED_CODE_CACHE_KEY);
352    if (cached) return JSON.parse(cached) as SharedCodeView;
353  }
354  let config: SharedConfig[];
355  try {
356    config = await readSharedConfig(env.DB);
357  } catch (err) {
358    const why = err instanceof Error ? err.message : String(err);
359    return { generated: now.toISOString(), configured: false, error: /no such table/i.test(why) ? "shared_packages does not exist yet: migrations/0036_shared_packages.sql is the seat's to apply" : why, apps_read: 0, apps_failed: [], packages: [] };
360  }
361  if (config.length === 0) return { generated: now.toISOString(), configured: false, error: null, apps_read: 0, apps_failed: [], packages: [] };
362  const sites = await readSiteConfig(env.DB);
363  const apps: AppRead[] = [];
364  for (const s of sites) apps.push(await readApp(env, s.namespace, s.name));
365  const tags = new Map<string, string[] | string>();
366  for (const pkg of config) tags.set(pkg.name, await readTags(env, pkg.repo).catch((err: unknown) => (err instanceof Error ? err.message : String(err))));
367  const releases = new Map<string, ReleaseRead>();
368  for (const pkg of config) releases.set(pkg.name, await readRelease(env, pkg.repo, fetchImpl));
369  const view = sharedCodeView(config, tags, apps, now, releases);
370  await env.APP_KV.put(SHARED_CODE_CACHE_KEY, JSON.stringify(view), { expirationTtl: CACHE_TTL_SECONDS });
371  return view;
372}