src/github/contents.ts

Functions

Imports

It imports src/encoding.ts, src/env.ts, src/github/client.ts, src/github/refs.ts, src/github/search.ts, src/limits.ts. Imported by src/github.ts, src/github/releases.ts, src/jobs-seat.ts, src/policy-sign.ts, src/shared-code.ts.

Source

1import { base64Decode, base64Encode } from "../encoding";
2import { pathProblem } from "../limits";
3import type { RepoEnv as Env } from "../env";
4import {
5  assertRepoArg,
6  cachedGet,
7  encodePath,
8  getDefaultBranch,
9  getFileSha,
10  getRefSha,
11  ghFetch,
12  invalidateRepoReads,
13  resolveRepo,
14} from "./client";
15import { ensureBranch, openPr } from "./refs";
16import { SEARCH_TREE_LIMIT } from "./search";
17
18export async function listRepoTree(env: Env, namespace: string, path = "", ref?: string, repoSelector?: string) {
19  if (path) assertRepoArg("path", path);
20  if (ref) assertRepoArg("ref", ref);
21  const { owner, repo } = await resolveRepo(env, namespace, repoSelector);
22  const query = ref ? `?ref=${encodeURIComponent(ref)}` : "";
23  const resp = await cachedGet(env, owner, repo, `/repos/${owner}/${repo}/contents/${encodePath(path)}${query}`);
24  if (!resp.ok) throw new Error(`list_repo_tree failed (${resp.status}): ${await resp.text()}`);
25  const data = (await resp.json()) as unknown;
26  const entries = Array.isArray(data) ? data : [data];
27  return {
28    repo: `${owner}/${repo}`,
29    path: path || "/",
30    entries: (entries as Array<{ path: string; type: string; size: number; sha: string }>).map((e) => ({
31      path: e.path,
32      type: e.type,
33      size: e.size,
34      sha: e.sha,
35    })),
36  };
37}
38
39// fresh: skip the KV read cache, for a caller that compares two reads (the second must be
40// GitHub's answer now, not the first one served back).
41export async function readRepoFile(env: Env, namespace: string, path: string, ref?: string, repoSelector?: string, fresh = false) {
42  assertRepoArg("path", path);
43  if (ref) assertRepoArg("ref", ref);
44  const { owner, repo } = await resolveRepo(env, namespace, repoSelector);
45  const query = ref ? `?ref=${encodeURIComponent(ref)}` : "";
46  const api = `/repos/${owner}/${repo}/contents/${encodePath(path)}${query}`;
47  const resp = fresh ? await ghFetch(env, owner, repo, api) : await cachedGet(env, owner, repo, api);
48  if (!resp.ok) throw new Error(`read_repo_file failed (${resp.status}): ${await resp.text()}`);
49  const data = (await resp.json()) as
50    | { type: string; content?: string; encoding?: string; size: number; sha: string }
51    | unknown[];
52  if (Array.isArray(data)) throw new Error(`${path} is a directory; use capsid_read_repo action tree`);
53  if (data.type !== "file") throw new Error(`${path} is not a file (type: ${data.type})`);
54  let content: string;
55  if (data.encoding === "base64" && data.content) {
56    content = base64Decode(data.content);
57  } else {
58    // Files over 1 MB come back without inline content; fetch the blob by sha.
59    const blob = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/git/blobs/${data.sha}`);
60    if (!blob.ok) throw new Error(`read_repo_file blob fetch failed (${blob.status})`);
61    const blobData = (await blob.json()) as { content: string; encoding: string };
62    content = base64Decode(blobData.content);
63  }
64  return { repo: `${owner}/${repo}`, path, size: data.size, sha: data.sha, content };
65}
66
67// Commits the repo tool makes are authored by the owner, not by the App's bot
68// (improvement list item 22). The App still commits them, so GitHub shows the owner as
69// author and capsid-repo-access[bot] as committer. Without ADMIN_EMAIL, GitHub's
70// default (the bot as both) stands.
71const COMMIT_AUTHOR_NAME = "Dustin Edwards";
72
73function commitAuthor(env: Pick<Env, "ADMIN_EMAIL">): { author?: { name: string; email: string } } {
74  const email = env.ADMIN_EMAIL?.trim();
75  return email ? { author: { name: COMMIT_AUTHOR_NAME, email } } : {};
76}
77
78async function putFile(
79  env: Env,
80  owner: string,
81  repo: string,
82  path: string,
83  content: string,
84  message: string,
85  branch: string,
86  allowWorkflowWrite?: boolean
87): Promise<{ commitSha: string; fileSha: string }> {
88  // The write primitive's own copy of the workflow refusal, so a caller added later
89  // inherits it. See workflowWriteRefusal.
90  const workflowRefusal = workflowWriteRefusal(path, allowWorkflowWrite);
91  if (workflowRefusal) throw new Error(workflowRefusal);
92  const sha = await getFileSha(env, owner, repo, path, branch);
93  const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/contents/${encodePath(path)}`, {
94    method: "PUT",
95    headers: { "Content-Type": "application/json" },
96    body: JSON.stringify({
97      message,
98      content: base64Encode(content),
99      branch,
100      ...(sha ? { sha } : {}),
101      ...commitAuthor(env),
102    }),
103  });
104  if (!resp.ok) throw new Error(`commit failed (${resp.status}): ${await resp.text()}`);
105  const data = (await resp.json()) as { commit: { sha: string }; content: { sha: string } };
106  // Cache invalidation is in commitOnBranch, the one site for every verb.
107  return { commitSha: data.commit.sha, fileSha: data.content.sha };
108}
109
110// The repo this Worker deploys from. A write landing on its default branch is a
111// production deploy of the server itself, which is why commitOnBranch refuses direct
112// mode, and pr mode aimed at the default branch, against it.
113export const SELF_REPO = "DrDustinEdwards/capsid";
114
115// The workflow directory is not ordinary repo content. The App holds Workflows:
116// write, and a workflow is code CI executes with that repo's secrets in scope.
117//
118// Refused unless the caller passes allow_workflow_write, which is audit-logged.
119// Checked in two places: commitOnBranch covers both verbs including the delete path
120// (which never reaches putFile), and putFile covers the write primitive, so a third
121// caller added later inherits the refusal.
122const WORKFLOW_DIR = ".github/workflows/";
123
124export function workflowWriteRefusal(path: string, allow: boolean | undefined): string | null {
125  if (allow === true) return null;
126  // Normalised the way encodePath will see it, so "./.github/workflows/x.yml" and a
127  // leading slash cannot slip past a bare startsWith.
128  const segments = path.split("/").filter((seg) => seg.length > 0 && seg !== ".");
129  const joined = `${segments.join("/")}/`;
130  if (!joined.startsWith(WORKFLOW_DIR)) return null;
131  return (
132    `refuses: ${path} is under ${WORKFLOW_DIR}, and a workflow is code CI executes with this repo's secrets in scope, not ordinary file content. ` +
133    `Pass allow_workflow_write: true to write it anyway; the flag is audit-logged.`
134  );
135}
136
137function branchSlug(path: string): string {
138  return path.replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "file";
139}
140
141// The steps write_repo_file and delete_repo_file share: resolve the repo, find the
142// default branch, cut a work branch in pr mode, mutate, invalidate the read cache,
143// and open a PR in pr mode. The invalidation is the hazard: a stale read cache serves
144// the pre-write body for up to 60 seconds, and it is only enforced by being written
145// once here.
146//
147// Response shaping stays with each caller. The two responses differ (write carries a
148// fileSha, delete does not, and write's pr mode omits it), and folding that in here
149// would either change what a caller sees or push per-verb conditionals into the
150// shared step.
151async function commitOnBranch<R>(
152  env: Env,
153  namespace: string,
154  path: string,
155  message: string,
156  mode: "pr" | "direct",
157  branch: string | undefined,
158  repoSelector: string | undefined,
159  op: {
160    branchPrefix: string;
161    fallbackTitle: string;
162    prBody: string;
163    allowWorkflowWrite?: boolean;
164    existingPr?: number;
165    mutate: (owner: string, repo: string, target: string) => Promise<R>;
166  }
167): Promise<{
168  base: { repo: string; mode: "pr" | "direct"; branch: string; path: string };
169  result: R;
170  pr: { number: number; url: string; existing?: true } | null;
171  pr_error?: string;
172}> {
173  assertRepoArg("path", path);
174  if (branch) assertRepoArg("branch", branch);
175  // `pr` names the open pull request whose head is `branch`, so it means nothing in
176  // direct mode or without a branch (a generated branch has no PR yet).
177  if (op.existingPr !== undefined && (mode !== "pr" || !branch)) {
178    throw new Error(
179      `refuses: pr names an existing pull request to commit to, which needs mode "pr" and branch set to that pull request's head branch.`
180    );
181  }
182  // Before any network call, so the refusal cannot be probed by timing. This also
183  // covers delete_repo_file, whose mutate never reaches putFile.
184  const workflowRefusal = workflowWriteRefusal(path, op.allowWorkflowWrite);
185  if (workflowRefusal) throw new Error(workflowRefusal);
186  const { owner, repo } = await resolveRepo(env, namespace, repoSelector);
187  // The server's own default branch cannot be written: CI deploys this Worker on
188  // every push to it, so a commit landing there is a production deploy behind a
189  // write-grant key.
190  //
191  // Only the default branch is refused: a direct commit to a work branch is ordinary.
192  // Direct mode with no branch always means the default branch, so it is refused without a
193  // round trip.
194  if (`${owner}/${repo}` === SELF_REPO && mode === "direct" && !branch) {
195    throw new Error(
196      `refuses: a direct commit with no branch lands on the default branch of this server's own repo (${SELF_REPO}) and redeploys the Worker. Name a work branch, or use mode "pr" and merge through capsid_close_pr action merge.`
197    );
198  }
199  const defaultBranch = await getDefaultBranch(env, owner, repo);
200  if (`${owner}/${repo}` === SELF_REPO && branch === defaultBranch) {
201    throw new Error(
202      `refuses: ${defaultBranch} is the default branch of this server's own repo (${SELF_REPO}), and a commit landing there redeploys the Worker. Use mode "pr" with a work branch and merge through capsid_close_pr action merge, or name a non-default branch.`
203    );
204  }
205  // PR mode never commits to the default branch, on any repo. can_direct_write is
206  // required only for mode "direct" (repoWriteFlags), so a pr-mode call naming the
207  // default branch as its work branch would commit straight onto it without that
208  // flag, and several mapped repos deploy on push. Refused before the branch step and
209  // the commit, so nothing lands.
210  if (mode === "pr" && branch === defaultBranch) {
211    throw new Error(
212      `refuses: mode "pr" with branch ${defaultBranch}, which is the default branch of ${owner}/${repo}; the commit would land on it without a pull request. Omit branch, name a work branch, or use mode "direct" (which needs can_direct_write).`
213    );
214  }
215  // A branch an open PR already holds is refused unless the call names that PR, since
216  // committing to it changes a PR that may not be the caller's. Uncached, so a PR
217  // opened seconds ago is seen, and fail closed: a lookup that errors is not "no open
218  // PR".
219  let existing: { number: number; url: string } | null = null;
220  if (mode === "pr" && branch) {
221    const prResp = await ghFetch(
222      env,
223      owner,
224      repo,
225      `/repos/${owner}/${repo}/pulls?state=open&head=${encodeURIComponent(`${owner}:${branch}`)}`
226    );
227    if (!prResp.ok) {
228      throw new Error(
229        `refuses: could not check open pull requests for ${branch} on ${owner}/${repo} (${prResp.status}), so the write cannot tell whether another pull request holds that branch. Retry.`
230      );
231    }
232    const open = (await prResp.json()) as Array<{ number: number; html_url: string }>;
233    const held = open[0];
234    if (op.existingPr !== undefined && held?.number !== op.existingPr) {
235      throw new Error(
236        held
237          ? `refuses: pr ${op.existingPr} is not the open pull request on ${branch}; ${branch} on ${owner}/${repo} has open pull request #${held.number} (${held.html_url}).`
238          : `refuses: pr ${op.existingPr} is not the open pull request on ${branch}; ${branch} has no open pull request on ${owner}/${repo}. Omit pr to commit and open one.`
239      );
240    }
241    if (held && op.existingPr === undefined) {
242      throw new Error(
243        `refuses: branch ${branch} on ${owner}/${repo} already has open pull request #${held.number} (${held.html_url}), and this commit would change it. Pass pr: ${held.number} to commit to that pull request, or name a different branch.`
244      );
245    }
246    if (held) existing = { number: held.number, url: held.html_url };
247  }
248  const target =
249    mode === "direct"
250      ? branch || defaultBranch
251      : branch || `${op.branchPrefix}${branchSlug(path)}-${Date.now().toString(36)}`;
252
253  if (mode === "pr") {
254    const headSha = await getRefSha(env, owner, repo, defaultBranch);
255    await ensureBranch(env, owner, repo, target, headSha);
256  }
257
258  const result = await op.mutate(owner, repo, target);
259
260  // The commit has landed, so every cached read of this repo is potentially stale.
261  // Swept here, for both verbs, before the PR is opened.
262  await invalidateRepoReads(env, owner, repo);
263
264  const base = { repo: `${owner}/${repo}`, mode, branch: target, path };
265  if (mode === "direct") return { base, result, pr: null };
266  // The commit landed on a named PR's head, so that PR carries it; a second PR from
267  // the same head would be refused by GitHub anyway.
268  if (existing) return { base, result, pr: { ...existing, existing: true } };
269
270  const title = message.split("\n")[0] || op.fallbackTitle;
271  // The commit has landed, so a failed PR open is reported, not thrown. A throw would
272  // reach guardedWrite as a failed call with no audit row, and a caller told "failed"
273  // retries into a second commit. The result carries pr: null and pr_error, and
274  // guardedWrite audits it like any other landed write.
275  //
276  // The resolved repo is passed so the PR lands on the repo the file was committed
277  // to, not the namespace default.
278  try {
279    const pr = await openPr(env, namespace, title, target, defaultBranch, op.prBody, `${owner}/${repo}`);
280    return { base, result, pr: { number: pr.number, url: pr.url } };
281  } catch (err) {
282    const reason = err instanceof Error ? err.message : String(err);
283    return {
284      base,
285      result,
286      pr: null,
287      pr_error:
288        `THE COMMIT LANDED on ${target} of ${owner}/${repo}, but the pull request could not be opened: ${reason}. ` +
289        `Do not retry this write; open the pull request with capsid_open_pr action open (head ${target}).`,
290    };
291  }
292}
293
294export async function writeRepoFile(
295  env: Env,
296  namespace: string,
297  path: string,
298  content: string,
299  message: string,
300  mode: "pr" | "direct" = "pr",
301  branch?: string,
302  repoSelector?: string,
303  allowWorkflowWrite?: boolean,
304  existingPr?: number
305) {
306  const { base, result, pr, pr_error } = await commitOnBranch(env, namespace, path, message, mode, branch, repoSelector, {
307    branchPrefix: "capsid/",
308    fallbackTitle: `Update ${path}`,
309    prBody: `Automated change to \`${path}\` via Capsid.`,
310    allowWorkflowWrite,
311    existingPr,
312    mutate: (owner, repo, target) => putFile(env, owner, repo, path, content, message, target, allowWorkflowWrite),
313  });
314  // Returned so it lands in audit_log via guardedWrite, which files the whole result,
315  // and a workflow authored through this tool is greppable afterwards.
316  const flag = allowWorkflowWrite === true ? { allow_workflow_write: true } : {};
317  // direct carries the file sha as well as the commit sha; pr mode does not. Kept,
318  // because tidying it would change what a caller receives.
319  if (pr_error) return { ...base, commitSha: result.commitSha, pr: null, pr_error, ...flag };
320  if (!pr) return { ...base, ...result, ...flag };
321  return { ...base, commitSha: result.commitSha, pr, ...flag };
322}
323
324// Delete a file from a namespace's repo. PR mode (default) commits the deletion
325// to a work branch and opens a PR; direct mode deletes on the default branch.
326export async function deleteRepoFile(
327  env: Env,
328  namespace: string,
329  path: string,
330  message: string,
331  mode: "pr" | "direct" = "pr",
332  branch?: string,
333  repoSelector?: string,
334  allowWorkflowWrite?: boolean,
335  existingPr?: number
336) {
337  const { base, result, pr, pr_error } = await commitOnBranch(env, namespace, path, message, mode, branch, repoSelector, {
338    branchPrefix: "capsid/rm-",
339    fallbackTitle: `Delete ${path}`,
340    prBody: `Delete \`${path}\` via Capsid.`,
341    allowWorkflowWrite,
342    existingPr,
343    mutate: async (owner, repo, target) => {
344      // GitHub's contents DELETE needs the CURRENT file sha, so a missing file is an
345      // error rather than a no-op. Read on the target branch, which in pr mode is
346      // the work branch just cut from the default.
347      const sha = await getFileSha(env, owner, repo, path, target);
348      if (!sha) throw new Error(`delete_repo_file: ${path} does not exist on ${owner}/${repo}@${target}`);
349      const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/contents/${encodePath(path)}`, {
350        method: "DELETE",
351        headers: { "Content-Type": "application/json" },
352        body: JSON.stringify({ message, sha, branch: target, ...commitAuthor(env) }),
353      });
354      if (!resp.ok) throw new Error(`delete failed (${resp.status}): ${await resp.text()}`);
355      const data = (await resp.json()) as { commit: { sha: string } };
356      return { commitSha: data.commit.sha };
357    },
358  });
359  const flag = allowWorkflowWrite === true ? { allow_workflow_write: true } : {};
360  if (pr_error) return { ...base, commitSha: result.commitSha, pr: null, pr_error, ...flag };
361  if (!pr) return { ...base, commitSha: result.commitSha, ...flag };
362  return { ...base, commitSha: result.commitSha, pr, ...flag };
363}
364
365export const REPO_BATCH_MAX_FILES = 20;
366export const REPO_FILE_BUDGET = 200 * 1024;
367
368/** Read up to REPO_BATCH_MAX_FILES files, each independently. One missing path does
369 *  not fail the batch: a batch is triage, and a triage call that throws because one
370 *  of twenty paths moved has to be retried by bisection.
371 *
372 *  The per-file budget applies here and not to the single-path read, so an existing
373 *  readRepoFile call never starts truncating silently. */
374export async function readRepoFiles(env: Env, namespace: string, paths: string[], ref?: string, repoSelector?: string) {
375  if (paths.length === 0) throw new Error("read_repo_file: paths was empty");
376  if (paths.length > REPO_BATCH_MAX_FILES) {
377    throw new Error(`read_repo_file: ${paths.length} paths exceeds the batch maximum of ${REPO_BATCH_MAX_FILES}`);
378  }
379  const { full } = await resolveRepo(env, namespace, repoSelector);
380  const files = await Promise.all(
381    paths.map(async (path) => {
382      const problem = pathProblem(path);
383      if (problem) return { path, error: problem };
384      try {
385        const one = await readRepoFile(env, namespace, path, ref, repoSelector);
386        const truncated = one.content.length > REPO_FILE_BUDGET;
387        return {
388          path,
389          size: one.size,
390          sha: one.sha,
391          truncated,
392          content: truncated ? one.content.slice(0, REPO_FILE_BUDGET) : one.content,
393        };
394      } catch (err) {
395        return { path, error: err instanceof Error ? err.message : String(err) };
396      }
397    })
398  );
399  const failed = files.filter((f) => "error" in f).length;
400  return {
401    repo: full,
402    requested: paths.length,
403    ok: files.length - failed,
404    failed,
405    bytes: files.reduce((n, f) => n + ("content" in f && f.content ? f.content.length : 0), 0),
406    files,
407  };
408}
409
410// Every blob path on the default branch, in one call, for the truth report's
411// doc-vs-code drift check, which asks a set-membership question about a few hundred
412// paths. It reuses searchCode's tree fetch and size refusal: a recursive tree on a
413// large repo is one response, and GitHub truncates it silently past a limit.
414//
415// Returns null rather than throwing when the tree cannot be read or is too large, so
416// the caller reports the check as unrun; an empty set would report every cited path
417// as drift.
418export async function repoBlobPaths(env: Env, namespace: string, repoSelector?: string): Promise<Set<string> | null> {
419  try {
420    const { owner, repo } = await resolveRepo(env, namespace, repoSelector);
421    const ref = await getDefaultBranch(env, owner, repo);
422    const resp = await ghFetch(env, owner, repo, `/repos/${owner}/${repo}/git/trees/${encodeURIComponent(ref)}?recursive=1`);
423    if (!resp.ok) return null;
424    const tree = (await resp.json()) as { tree?: Array<{ path: string; type: string }>; truncated?: boolean };
425    if (!Array.isArray(tree.tree) || tree.truncated || tree.tree.length > SEARCH_TREE_LIMIT) return null;
426    return new Set(tree.tree.filter((e) => e.type === "blob").map((e) => e.path));
427  } catch {
428    return null;
429  }
430}