src/github/client.ts

Functions

Imports

It imports src/encoding.ts, src/env.ts, src/limits.ts, src/log.ts. Imported by src/auto-merge-tick.ts, src/complete-ci.ts, src/deploy-outcomes.ts, src/fail-exemption.ts, src/github.ts, src/github/actions.ts, src/github/contents.ts, src/github/dispatch.ts, src/github/pr-files.ts, src/github/prune.ts, src/github/refs.ts, src/github/releases.ts, src/github/search.ts, src/job-fact-check.ts, src/job-outcomes.ts, src/job-overlaps.ts, src/jobs-holder.ts, src/jobs-seat.ts, src/live-checks.ts, src/maintenance-branches.ts, src/maintenance-deploys.ts, src/maintenance-prs.ts, src/maintenance-releases.ts, src/manual-steps.ts, src/merge-resume.ts, src/merge-safety.ts, src/model-drift.ts, src/model-routing-pr.ts, src/ops-packages.ts, src/outcome-prs.ts, src/overnight-plan.ts, src/pr-approvals.ts, src/public-code.ts, src/public-data.ts, src/review.ts, src/shadow.ts, src/shared-code.ts, src/stale-canon.ts, src/tools/repo-guards.ts, src/trends.ts, src/unmapped-repos.ts, src/whats-next.ts.

Source

1// GitHub App repo access for Capsid's repo fallthrough.
2//
3// Mints short-lived installation tokens from the App private key (RS256 JWT via Web
4// Crypto) and caches them in APP_KV for ~55 minutes. Resolves the target repo per
5// namespace from the D1 namespaces table, then reads and writes over the GitHub
6// REST API. No PAT, no clone.
7
8import { b64urlFromBytes, b64urlEncode } from "../encoding";
9import { repoPathProblem } from "../limits";
10// RepoEnv, not Env: this module must not be able to name the credentials it omits.
11import type { RepoEnv as Env } from "../env";
12import { logEvent } from "../log";
13
14const GH = "https://api.github.com";
15const GH_HEADERS: Record<string, string> = {
16  Accept: "application/vnd.github+json",
17  "X-GitHub-Api-Version": "2022-11-28",
18  "User-Agent": "capsid",
19};
20
21const TOKEN_TTL_SECONDS = 3300; // installation tokens live 60 min; refresh a little early
22const INSTALL_TTL_SECONDS = 86400; // installation id is stable
23const READ_CACHE_TTL_SECONDS = 60; // brief cache for read tools
24// Every GitHub call is bounded, so a hung connection fails the call instead of holding
25// the request (or a cron pass) until the runtime kills it. Long enough for a large
26// contents PUT; a caller's own signal wins.
27const GH_TIMEOUT_MS = 30_000;
28
29const installKey = (owner: string) => `gh:install:v2:${owner}`;
30const tokenKey = (owner: string, repo: string) => `gh:token:v3:${owner}/${repo}`;
31const readKey = (path: string) => `gh:get:${path}`;
32// The trailing slash matters: without it, owner/r would also match owner/repo2.
33const readPrefix = (owner: string, repo: string) => `gh:get:/repos/${owner}/${repo}/`;
34
35export interface RepoRef {
36  owner: string;
37  repo: string;
38  full: string;
39}
40
41let cachedKey: { pem: string; key: CryptoKey } | null = null;
42
43async function importPrivateKey(pem: string): Promise<CryptoKey> {
44  if (cachedKey && cachedKey.pem === pem) return cachedKey.key;
45  const body = pem
46    .replace(/-----BEGIN [^-]+-----/, "")
47    .replace(/-----END [^-]+-----/, "")
48    .replace(/\s+/g, "");
49  const der = Uint8Array.from(atob(body), (c) => c.charCodeAt(0));
50  const key = await crypto.subtle.importKey(
51    "pkcs8",
52    der,
53    { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
54    false,
55    ["sign"]
56  );
57  cachedKey = { pem, key };
58  return key;
59}
60
61async function createAppJwt(env: Env): Promise<string> {
62  if (!env.GITHUB_APP_CLIENT_ID || !env.GITHUB_APP_PRIVATE_KEY) {
63    throw new Error("GitHub App not configured: set GITHUB_APP_CLIENT_ID and GITHUB_APP_PRIVATE_KEY");
64  }
65  const now = Math.floor(Date.now() / 1000);
66  const header = b64urlEncode(JSON.stringify({ alg: "RS256", typ: "JWT" }));
67  const payload = b64urlEncode(JSON.stringify({ iss: env.GITHUB_APP_CLIENT_ID, iat: now - 60, exp: now + 540 }));
68  const signingInput = `${header}.${payload}`;
69  const key = await importPrivateKey(env.GITHUB_APP_PRIVATE_KEY);
70  const signature = await crypto.subtle.sign("RSASSA-PKCS1-v1_5", key, new TextEncoder().encode(signingInput));
71  return `${signingInput}.${b64urlFromBytes(new Uint8Array(signature))}`;
72}
73
74async function appFetch(env: Env, path: string, init?: RequestInit): Promise<Response> {
75  const jwt = await createAppJwt(env);
76  return fetch(`${GH}${path}`, {
77    ...init,
78    headers: { ...GH_HEADERS, ...(init?.headers as Record<string, string>), Authorization: `Bearer ${jwt}` },
79    signal: init?.signal ?? AbortSignal.timeout(GH_TIMEOUT_MS),
80  });
81}
82
83
84async function getInstallationId(env: Env, owner: string, repo: string): Promise<string> {
85  const cached = await env.APP_KV.get(installKey(owner));
86  if (cached) return cached;
87  const resp = await appFetch(env, `/repos/${owner}/${repo}/installation`);
88  if (!resp.ok) {
89    // A bad JWT answers 401; a valid one with no installation covering the repo
90    // answers 404. So a 404 means the credentials are fine and the App is not
91    // installed on that repo.
92    throw new Error(
93      `could not resolve GitHub App installation for ${owner}/${repo} (${resp.status}): ${await resp.text()}` +
94        (resp.status === 404 ? " (404 means the App is not installed on this repo; the credentials are fine)" : "")
95    );
96  }
97  const data = (await resp.json()) as { id: number };
98  const id = String(data.id);
99  await env.APP_KV.put(installKey(owner), id, { expirationTtl: INSTALL_TTL_SECONDS });
100  return id;
101}
102
103async function getInstallationToken(env: Env, owner: string, repo: string): Promise<string> {
104  const cacheKey = tokenKey(owner, repo);
105  const cached = await env.APP_KV.get(cacheKey);
106  if (cached) return cached;
107  const installationId = await getInstallationId(env, owner, repo);
108  // Scoped to the one repo asked about. An access_tokens POST with no body mints a
109  // token for every repo the installation covers, and that token then sits in APP_KV.
110  // With `repositories` it holds exactly the repo the caller resolved.
111  const resp = await appFetch(env, `/app/installations/${installationId}/access_tokens`, {
112    method: "POST",
113    headers: { "Content-Type": "application/json" },
114    body: JSON.stringify({ repositories: [repo] }),
115  });
116  if (!resp.ok) throw new Error(`installation token request failed (${resp.status}): ${await resp.text()}`);
117  const data = (await resp.json()) as { token: string };
118  await env.APP_KV.put(cacheKey, data.token, { expirationTtl: TOKEN_TTL_SECONDS });
119  return data.token;
120}
121
122// What the App can see, for the watcher's unmapped-repo check (src/unmapped-repos.ts).
123// Listing /installation/repositories needs an installation token covering every repo,
124// which getInstallationToken deliberately never mints. This one is minted for the call,
125// limited to the metadata:read permission (names, visibility, archived and push date, and
126// nothing else: no contents, no writes), held in a local variable and never stored in
127// KV or logged. It is the only unscoped token in the Worker.
128export interface InstalledRepo {
129  full_name: string;
130  private: boolean;
131  archived: boolean;
132  pushed_at: string | null;
133}
134
135const LIST_PER_PAGE = 100;
136const LIST_MAX_PAGES = 10;
137
138/** Every repo every installation of the App can see. Throws on any failed or partial
139 *  read: a list read in part would read as "everything else is unmapped" or, worse,
140 *  "nothing is". */
141export async function listInstalledRepos(env: Env): Promise<{ repos: InstalledRepo[]; installations: number }> {
142  const installations: number[] = [];
143  for (let page = 1; ; page++) {
144    if (page > LIST_MAX_PAGES) throw new Error(`the App has more than ${LIST_MAX_PAGES * LIST_PER_PAGE} installations`);
145    const resp = await appFetch(env, `/app/installations?per_page=${LIST_PER_PAGE}&page=${page}`);
146    if (!resp.ok) throw new Error(`could not list the App's installations (${resp.status})`);
147    const rows = (await resp.json()) as Array<{ id: number }>;
148    installations.push(...rows.map((r) => r.id));
149    if (rows.length < LIST_PER_PAGE) break;
150  }
151
152  const repos: InstalledRepo[] = [];
153  for (const id of installations) {
154    const minted = await appFetch(env, `/app/installations/${id}/access_tokens`, {
155      method: "POST",
156      headers: { "Content-Type": "application/json" },
157      body: JSON.stringify({ permissions: { metadata: "read" } }),
158    });
159    if (!minted.ok) throw new Error(`could not mint a metadata-only token for installation ${id} (${minted.status})`);
160    const { token } = (await minted.json()) as { token: string };
161    let total = Infinity;
162    const seen: InstalledRepo[] = [];
163    for (let page = 1; seen.length < total; page++) {
164      if (page > LIST_MAX_PAGES) throw new Error(`installation ${id} lists more than ${LIST_MAX_PAGES * LIST_PER_PAGE} repos`);
165      const resp = await fetch(`${GH}/installation/repositories?per_page=${LIST_PER_PAGE}&page=${page}`, {
166        headers: { ...GH_HEADERS, Authorization: `Bearer ${token}` },
167        signal: AbortSignal.timeout(GH_TIMEOUT_MS),
168      });
169      if (!resp.ok) throw new Error(`could not list the repos of installation ${id} (${resp.status})`);
170      const body = (await resp.json()) as {
171        total_count: number;
172        repositories: Array<{ full_name: string; private: boolean; archived?: boolean; pushed_at?: string | null }>;
173      };
174      total = body.total_count;
175      if (body.repositories.length === 0) break;
176      for (const r of body.repositories) {
177        seen.push({ full_name: r.full_name, private: r.private, archived: r.archived === true, pushed_at: r.pushed_at ?? null });
178      }
179    }
180    if (seen.length < total) throw new Error(`installation ${id} listed ${seen.length} of ${total} repos`);
181    repos.push(...seen);
182  }
183  return { repos, installations: installations.length };
184}
185
186// One installation covers every repo under a single owner, but each minted token is
187// scoped to one repo, so the token cache is per owner+repo to match.
188
189// The URL built for a repo call cannot escape /repos/<owner>/<repo>/. A "../.."
190// smuggled through a file path or branch would normalize out of it once fetch()
191// parsed the string as a URL. This check runs after assembly and after WHATWG
192// normalization, so it sees the escape the input-level repoPathProblem guard is also
193// there to stop. owner and repo come from the namespace mapping (REPO_SHAPE excludes
194// "." and ".."), so the expected base is itself traversal-free.
195//
196// It builds the same URL new URL() will, which is why a fake fetch keyed on the raw
197// string cannot substitute for this guard: the raw string still contains the "..",
198// the parsed pathname does not.
199function repoApiUrl(owner: string, repo: string, path: string): string {
200  const url = `${GH}${path}`;
201  const base = `/repos/${owner}/${repo}`;
202  const { pathname } = new URL(url);
203  if (pathname !== base && !pathname.startsWith(`${base}/`)) {
204    throw new Error(`refusing GitHub request that escapes ${base}: it resolves to ${pathname}`);
205  }
206  return url;
207}
208
209export async function ghFetch(env: Env, owner: string, repo: string, path: string, init?: RequestInit): Promise<Response> {
210  const url = repoApiUrl(owner, repo, path);
211  const call = (token: string) =>
212    fetch(url, {
213      ...init,
214      headers: { ...GH_HEADERS, ...(init?.headers as Record<string, string>), Authorization: `Bearer ${token}` },
215      signal: init?.signal ?? AbortSignal.timeout(GH_TIMEOUT_MS),
216    });
217  let resp = await call(await getInstallationToken(env, owner, repo));
218  if (resp.status === 401) {
219    await env.APP_KV.delete(tokenKey(owner, repo));
220    resp = await call(await getInstallationToken(env, owner, repo));
221  }
222  return resp;
223}
224
225// The cache key is the full API path, including ?ref=, so different refs are
226// different keys.
227export async function cachedGet(env: Env, owner: string, repo: string, path: string): Promise<Response> {
228  // Assert the URL is in-bounds before it is ever used as a cache key, so a
229  // traversal path cannot be stored under a key the invalidator will not find.
230  repoApiUrl(owner, repo, path);
231  const cacheKey = readKey(path);
232  const cached = (await env.APP_KV.get(cacheKey, "json")) as { status: number; body: string } | null;
233  if (cached) return new Response(cached.body, { status: cached.status });
234  const resp = await ghFetch(env, owner, repo, path);
235  const body = await resp.text();
236  if (resp.ok) {
237    await env.APP_KV.put(cacheKey, JSON.stringify({ status: resp.status, body }), { expirationTtl: READ_CACHE_TTL_SECONDS });
238  }
239  return new Response(body, { status: resp.status });
240}
241
242// Invalidation after a write. Without it, for up to READ_CACHE_TTL_SECONDS after a
243// commit a read returns the body the write replaced, and Capsid writes and then
244// reads back as its ordinary path.
245//
246// Swept by repo prefix rather than by computed key. A key-precise invalidation has to
247// reproduce the exact spelling of every affected entry (the encoded path, the parent
248// listing, the root listing's trailing slash, each with and without ?ref=), and one
249// missed spelling is a stale read. The prefix sweep also covers a merge, whose
250// affected paths are not known here. It over-invalidates by one GitHub GET.
251export async function invalidateRepoReads(env: Env, owner: string, repo: string): Promise<number> {
252  const prefix = readPrefix(owner, repo);
253  let cursor: string | undefined;
254  let deleted = 0;
255  try {
256    do {
257      const page = await env.APP_KV.list({ prefix, cursor });
258      // Deletes within a page run in parallel, since they are independent and this
259      // runs on the write path where a caller is waiting. Pages stay sequential
260      // because the next cursor is only known once the current page returns.
261      await Promise.all(page.keys.map((key) => env.APP_KV.delete(key.name)));
262      deleted += page.keys.length;
263      cursor = page.list_complete ? undefined : page.cursor;
264    } while (cursor);
265  } catch (err) {
266    // The commit already landed. Failing the tool call because a cache sweep failed
267    // would misreport the write, so this is logged by name and the stale window
268    // stays bounded by the 60 second TTL.
269    logEvent("error", "GH_CACHE_INVALIDATION_FAILED", {
270      message: `GH_CACHE_INVALIDATION_FAILED ${owner}/${repo}: ${err instanceof Error ? err.message : String(err)}`,
271    });
272  }
273  return deleted;
274}
275
276// The shape of a single repos entry: "owner/name". A looser shape would admit
277// "../other" as a legal mapping and make the authorization boundary traversable.
278// GitHub owner and repo names are drawn from [A-Za-z0-9._-]; the charset still
279// matches "..", so repoTokenOk below also rejects "." and "..".
280export const REPO_SHAPE = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
281
282// Neither side of an owner/name may be "." or "..": those walk a URL out of
283// /repos/<owner>/<repo>/ even though they satisfy the charset.
284export function repoTokenOk(repoFull: string): boolean {
285  if (!REPO_SHAPE.test(repoFull)) return false;
286  const [owner, name] = repoFull.split("/");
287  return owner !== "." && owner !== ".." && name !== "." && name !== "..";
288}
289
290export interface RepoEntry {
291  repo: string;
292  label: string;
293}
294
295// Parse and validate a repos JSON array, shared by register_namespace and
296// update_namespace: a non-empty array of { repo: "owner/name", label? } with label
297// defaulting to "primary". Returns the normalized list or a caller-facing error.
298// It does NOT enforce a single primary; update_namespace layers that on top.
299export function parseReposList(reposJson: string): { list: RepoEntry[] } | { error: string } {
300  let parsed: unknown;
301  try {
302    parsed = JSON.parse(reposJson);
303  } catch (err) {
304    return { error: `invalid repos JSON: ${err instanceof Error ? err.message : String(err)}` };
305  }
306  if (!Array.isArray(parsed) || parsed.length === 0) {
307    return { error: "repos must be a non-empty JSON array of { repo, label } entries" };
308  }
309  const list: RepoEntry[] = [];
310  for (const r of parsed) {
311    if (!r || typeof r.repo !== "string" || !repoTokenOk(r.repo)) {
312      return { error: `each repos entry needs a "repo" of the form owner/name (got ${JSON.stringify(r)})` };
313    }
314    list.push({ repo: r.repo, label: typeof r.label === "string" && r.label.trim() ? r.label.trim() : "primary" });
315  }
316  return { list };
317}
318
319// update_namespace requires exactly one primary (register_namespace does not, so
320// this is not folded into parseReposList). Returns an error string or null.
321export function requireSinglePrimary(list: RepoEntry[]): string | null {
322  const primaries = list.filter((r) => r.label === "primary").length;
323  return primaries === 1 ? null : `repos must have exactly one entry labeled "primary" (found ${primaries})`;
324}
325
326// https://github.com/<owner>/<repo>/pull/<number>, as capsid_open_pr action open returns it and a
327// driver pastes it. The one spelling for every site that reads one (the review gate,
328// the evidence verifier and the prose scan in outcome-prs.ts): separate spellings
329// with different character classes let the gate accept owner/repo strings the
330// verifier refused. The owner and repo are what the URL says, not what the namespace
331// maps: a caller resolves them through resolveRepo before reaching GitHub.
332export const PR_URL_SOURCE = String.raw`https:\/\/github\.com\/([A-Za-z0-9._-]+)\/([A-Za-z0-9._-]+)\/pull\/(\d+)`;
333const PR_URL_EXACT = new RegExp(`^${PR_URL_SOURCE}(?:[/?#].*)?$`);
334
335export interface PrUrl {
336  owner: string;
337  repo: string;
338  number: number;
339}
340
341// Anchored at both ends: a reference that merely mentions a pull request is not one.
342export function parsePrUrl(ref: string | null | undefined): PrUrl | null {
343  if (!ref) return null;
344  const match = PR_URL_EXACT.exec(ref.trim());
345  if (!match) return null;
346  return { owner: match[1], repo: match[2], number: Number(match[3]) };
347}
348
349// Resolve a namespace to one of its mapped repos. `selector` is the optional `repo`
350// tool argument: a label from the namespace's repos array ("primary", "legacy") or a
351// full "owner/name" that MUST appear in that array. The namespace mapping is the
352// authorization boundary, so an unknown selector is rejected with the valid values
353// rather than falling through to an arbitrary repo. With no selector the default is
354// the entry labeled "primary", or the first entry.
355export async function resolveRepo(env: Env, namespace: string, selector?: string): Promise<RepoRef> {
356  const row = await env.DB.prepare("SELECT repos FROM namespaces WHERE namespace = ?1")
357    .bind(namespace)
358    .first<{ repos: string }>();
359  if (!row) throw new Error(`unknown namespace: ${namespace}`);
360  // Fails closed: a corrupt repos column is reported as corrupt, not as "has no repo
361  // mapping". That is a different fact with a different fix, and reading the damage
362  // as an unconfigured namespace invites "fixing" it by overwriting the mapping.
363  let list: Array<{ repo: string; label?: string }>;
364  try {
365    list = JSON.parse(row.repos || "[]");
366  } catch (err) {
367    throw new Error(
368      `namespace ${namespace} has a CORRUPT repos mapping: the stored value is not valid JSON (${err instanceof Error ? err.message : String(err)}). Nothing was resolved. Repair it with capsid_admin_manage_namespaces action update.`
369    );
370  }
371  if (!Array.isArray(list)) {
372    throw new Error(`namespace ${namespace} has a CORRUPT repos mapping: expected a JSON array, got ${typeof list}.`);
373  }
374  if (list.length === 0) throw new Error(`namespace ${namespace} has no repo mapping`);
375  let chosen: { repo: string; label?: string } | undefined;
376  if (selector) {
377    chosen = list.find((r) => r.label === selector) ?? list.find((r) => r.repo === selector);
378    if (!chosen) {
379      const labels = list.map((r) => r.label).filter(Boolean).join(", ") || "(none)";
380      const repos = list.map((r) => r.repo).join(", ");
381      throw new Error(
382        `repo '${selector}' is not mapped to namespace ${namespace}. Valid labels: ${labels}. Valid repos: ${repos}.`
383      );
384    }
385  } else {
386    chosen = list.find((r) => r.label === "primary") ?? list[0];
387  }
388  const [owner, repo] = chosen.repo.split("/");
389  if (!owner || !repo) throw new Error(`invalid repo entry for ${namespace}: ${chosen.repo}`);
390  return { owner, repo, full: chosen.repo };
391}
392
393// encodePath keeps the "/" between segments, so it cannot stop traversal on its own.
394// It refuses a "." or ".." segment as an inner guard, and repoApiUrl asserts the
395// assembled URL as the outer one. Both are kept: this gives a clear caller-facing
396// error, that catches anything this misses. Exported for the traversal test.
397export function encodePath(path: string): string {
398  return path
399    .split("/")
400    .filter(Boolean)
401    .map((segment) => {
402      if (segment === "." || segment === "..") {
403        throw new Error(`refusing repo path with a '${segment}' segment: ${path}`);
404      }
405      return encodeURIComponent(segment);
406    })
407    .join("/");
408}
409
410// Refuse a caller-supplied repo argument (path, branch, ref or workflow) at the
411// door, naming the argument. The URL assertion in repoApiUrl is the backstop; this
412// is the friendly error, and the guard for arguments that reach GitHub through
413// encodeURIComponent (refs, workflow names) rather than encodePath.
414export function assertRepoArg(kind: string, value: string): void {
415  const problem = repoPathProblem(value);
416  if (problem) throw new Error(`${kind} ${problem}`);
417}
418
419export async function getDefaultBranch(env: Env, owner: string, repo: string): Promise<string> {
420  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}`);
421  if (!resp.ok) throw new Error(`repo lookup failed (${resp.status}): ${await resp.text()}`);
422  return ((await resp.json()) as { default_branch: string }).default_branch;
423}
424
425export async function getRefSha(env: Env, owner: string, repo: string, branch: string): Promise<string> {
426  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/git/ref/heads/${encodeURIComponent(branch)}`);
427  if (!resp.ok) throw new Error(`ref lookup failed for ${branch} (${resp.status}): ${await resp.text()}`);
428  return ((await resp.json()) as { object: { sha: string } }).object.sha;
429}
430
431export async function getFileSha(env: Env, owner: string, repo: string, path: string, ref: string): Promise<string | undefined> {
432  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/contents/${encodePath(path)}?ref=${encodeURIComponent(ref)}`);
433  if (resp.status === 404) return undefined;
434  if (!resp.ok) throw new Error(`file sha lookup failed (${resp.status}): ${await resp.text()}`);
435  const data = (await resp.json()) as { sha: string } | unknown[];
436  if (Array.isArray(data)) throw new Error(`${path} is a directory`);
437  return data.sha;
438}
439
440// The default branch's head commit sha. listRepoTree has no top-level sha (its shas
441// are per-entry blob shas), so it cannot answer this.
442export async function defaultBranchSha(env: Env, namespace: string, repoSelector?: string): Promise<string> {
443  const { owner, repo } = await resolveRepo(env, namespace, repoSelector);
444  const branch = await getDefaultBranch(env, owner, repo);
445  return getRefSha(env, owner, repo, branch);
446}