app/lib/novels/repo.server.ts

Functions

Imports

It imports app/lib/timed-fetch.server.ts. Imported by app/lib/binder.server.ts, app/lib/book-history.server.ts, app/lib/books.server.ts, app/lib/health.server.ts.

Source

1// The writing repository, reached through the GitHub App `carrel-writer` (design setup step 8),
2// installed on that one repository with contents read and write and nothing else. Carrel reads the
3// tree and files and commits one file at a time, each with the version it expects to replace. A move
4// that renames several files is one commit made through Git's data API (commitMany), and a file's
5// history is the commits that touched its path.
6
7import { SignJWT } from "jose";
8import { timedFetch } from "~/lib/timed-fetch.server";
9
10export const NOVELS_REPO = "DrDustinEdwards/writing";
11const NOVELS_BRANCH = "main";
12const API = "https://api.github.com";
13
14type RepoFile = { source: string; sha: string };
15
16/** One file renamed in a multi-file commit: its blob is kept, only its path changes. */
17type Rename = { from: string; to: string; sha: string };
18
19/** A commit that touched a path, newest first, as GitHub lists it. */
20type PathCommit = { sha: string; message: string; author: string; email: string; at: string };
21
22/** A write refused because the file is not at the version the writer expected. */
23export class GitConflict extends Error {
24  constructor(readonly currentSha: string | null) {
25    super("version-conflict");
26  }
27}
28
29export class NovelsNotConnected extends Error {
30  constructor(readonly detail: string) {
31    super(detail);
32  }
33}
34
35export interface NovelsRepo {
36  /** Every file under `prefix` (a folder, no trailing slash), with its blob sha. */
37  tree(prefix: string): Promise<{ path: string; sha: string }[]>;
38  read(path: string): Promise<RepoFile | null>;
39  /**
40   * Commits one file. `expectedSha` null means the file must not exist yet. Throws GitConflict when
41   * the file is not at that version.
42   */
43  write(path: string, input: { source: string; expectedSha: string | null; message: string; author: { name: string; email: string } }): Promise<{ sha: string; commit: string }>;
44  /**
45   * Renames several files in ONE commit on the branch. Each file must be at `sha` in the branch head
46   * and no file may already be at a `to` path that is not also moving away; otherwise GitConflict.
47   * The branch moves only forward: a head that moved while the commit was built is a GitConflict too.
48   */
49  commitMany(input: { renames: Rename[]; message: string; author: { name: string; email: string } }): Promise<{ commit: string }>;
50  /** The commits on the branch that touched `path`, newest first, at most `limit`. */
51  history(path: string, limit?: number): Promise<PathCommit[]>;
52  /** The file as it was at a commit, or null when it was not there. */
53  readAt(path: string, commit: string): Promise<RepoFile | null>;
54}
55
56// ---------- the connection
57
58export type NovelsConnection =
59  | { state: "connected"; appId: string; privateKey: string }
60  | { state: "not-connected" | "misconfigured"; detail: string };
61
62export function novelsConnection(env: Env): NovelsConnection {
63  const appId = env.NOVELS_APP_ID?.trim();
64  const key = env.NOVELS_APP_PRIVATE_KEY?.trim();
65  if (!appId && !key) {
66    return { state: "not-connected", detail: "The GitHub App for the writing repository is not set up yet (NOVELS_APP_ID, NOVELS_APP_PRIVATE_KEY)." };
67  }
68  if (!appId || !/^\d+$/.test(appId)) return { state: "misconfigured", detail: "NOVELS_APP_ID is missing or is not the App's numeric id." };
69  if (!key || !/-----BEGIN (RSA )?PRIVATE KEY-----/.test(key)) {
70    return { state: "misconfigured", detail: "NOVELS_APP_PRIVATE_KEY is missing or is not a PEM private key." };
71  }
72  return { state: "connected", appId, privateKey: key };
73}
74
75// ---------- GitHub App authentication
76
77function base64ToBytes(b64: string): Uint8Array<ArrayBuffer> {
78  const binary = atob(b64.replace(/\s+/g, ""));
79  const out = new Uint8Array(binary.length);
80  for (let i = 0; i < binary.length; i++) out[i] = binary.charCodeAt(i);
81  return out;
82}
83
84function bytesToBase64(bytes: Uint8Array): string {
85  let binary = "";
86  for (let i = 0; i < bytes.length; i += 0x8000) binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
87  return btoa(binary);
88}
89
90function derLength(n: number): number[] {
91  if (n < 0x80) return [n];
92  const bytes: number[] = [];
93  for (let v = n; v > 0; v >>= 8) bytes.unshift(v & 0xff);
94  return [0x80 | bytes.length, ...bytes];
95}
96
97function der(tag: number, content: Uint8Array | number[]): number[] {
98  const body = Array.from(content);
99  return [tag, ...derLength(body.length), ...body];
100}
101
102/**
103 * GitHub hands out App keys as PKCS#1 ("BEGIN RSA PRIVATE KEY"); WebCrypto imports PKCS#8 only. The
104 * PKCS#8 form is the PKCS#1 key wrapped with the rsaEncryption algorithm id, so Dustin can paste the
105 * downloaded file as it is.
106 */
107export function pkcs8FromPem(pem: string): Uint8Array<ArrayBuffer> {
108  const body = pem.replace(/-----(BEGIN|END) [A-Z ]+-----/g, "");
109  const bytes = base64ToBytes(body);
110  if (!/BEGIN RSA PRIVATE KEY/.test(pem)) return bytes;
111  const version = der(0x02, [0x00]);
112  // SEQUENCE { OID 1.2.840.113549.1.1.1, NULL }
113  const algorithm = der(0x30, [0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00]);
114  return new Uint8Array(der(0x30, [...version, ...algorithm, ...der(0x04, bytes)]));
115}
116
117export async function appJwt(appId: string, pem: string, now = Date.now()): Promise<string> {
118  const key = await crypto.subtle.importKey("pkcs8", pkcs8FromPem(pem), { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["sign"]);
119  const iat = Math.floor(now / 1000) - 60; // GitHub allows for clock drift this way.
120  return new SignJWT({}).setProtectedHeader({ alg: "RS256" }).setIssuer(appId).setIssuedAt(iat).setExpirationTime(iat + 540).sign(key);
121}
122
123type Fetch = typeof fetch;
124
125function headers(token: string): HeadersInit {
126  return {
127    Authorization: `Bearer ${token}`,
128    Accept: "application/vnd.github+json",
129    "X-GitHub-Api-Version": "2022-11-28",
130    "User-Agent": "carrel",
131  };
132}
133
134/** Installation tokens last an hour; one per isolate, renewed five minutes early. */
135const tokenCache = new Map<string, { token: string; expires: number }>();
136
137export async function installationToken(appId: string, pem: string, fetcher: Fetch = timedFetch, now = Date.now()): Promise<string> {
138  const cached = tokenCache.get(appId);
139  if (cached && cached.expires - 5 * 60_000 > now) return cached.token;
140  const jwt = await appJwt(appId, pem, now);
141  const inst = await fetcher(`${API}/repos/${NOVELS_REPO}/installation`, { headers: headers(jwt) });
142  if (!inst.ok) throw new Error(`GitHub did not find the App's installation on ${NOVELS_REPO} (${inst.status}).`);
143  const { id } = (await inst.json()) as { id: number };
144  // Narrowed to the one repository and to contents, whatever the installation allows.
145  const res = await fetcher(`${API}/app/installations/${id}/access_tokens`, {
146    method: "POST",
147    headers: headers(jwt),
148    body: JSON.stringify({ repositories: [NOVELS_REPO.split("/")[1]], permissions: { contents: "write" } }),
149  });
150  if (!res.ok) throw new Error(`GitHub refused an installation token (${res.status}).`);
151  const body = (await res.json()) as { token: string; expires_at: string };
152  tokenCache.set(appId, { token: body.token, expires: Date.parse(body.expires_at) });
153  return body.token;
154}
155
156export function clearTokenCache(): void {
157  tokenCache.clear();
158}
159
160// ---------- the GitHub contents client
161
162function encodePath(path: string): string {
163  return path.split("/").map(encodeURIComponent).join("/");
164}
165
166function utf8ToBase64(text: string): string {
167  return bytesToBase64(new TextEncoder().encode(text));
168}
169
170function base64ToUtf8(b64: string): string {
171  return new TextDecoder().decode(base64ToBytes(b64));
172}
173
174export function githubRepo(token: () => Promise<string>, fetcher: Fetch = timedFetch): NovelsRepo {
175  const repoUrl = `${API}/repos/${NOVELS_REPO}`;
176  const call = async (url: string, init: RequestInit = {}) => fetcher(url, { ...init, headers: { ...headers(await token()), ...init.headers } });
177
178  const repo: NovelsRepo = {
179    async tree(prefix) {
180      const res = await call(`${repoUrl}/git/trees/${NOVELS_BRANCH}?recursive=1`);
181      if (!res.ok) throw new Error(`GitHub did not list the writing repository (${res.status}).`);
182      const body = (await res.json()) as { truncated: boolean; tree: { path: string; type: string; sha: string }[] };
183      // A truncated tree would make files look deleted; refuse it rather than index half a book.
184      if (body.truncated) throw new Error("GitHub truncated the writing repository's tree; it is too large to list in one call.");
185      const start = `${prefix}/`;
186      return body.tree.filter((e) => e.type === "blob" && e.path.startsWith(start)).map((e) => ({ path: e.path.slice(start.length), sha: e.sha }));
187    },
188
189    read(path) {
190      return repo.readAt(path, NOVELS_BRANCH);
191    },
192
193    async readAt(path, ref) {
194      const res = await call(`${repoUrl}/contents/${encodePath(path)}?ref=${encodeURIComponent(ref)}`);
195      if (res.status === 404) return null;
196      if (!res.ok) throw new Error(`GitHub did not return ${path} (${res.status}).`);
197      const body = (await res.json()) as { type: string; content?: string; encoding?: string; sha: string };
198      if (body.type !== "file" || body.encoding !== "base64" || body.content === undefined) throw new Error(`${path} is not a file GitHub returns inline.`);
199      return { source: base64ToUtf8(body.content), sha: body.sha };
200    },
201
202    async write(path, input) {
203      const res = await call(`${repoUrl}/contents/${encodePath(path)}`, {
204        method: "PUT",
205        headers: { "Content-Type": "application/json" },
206        body: JSON.stringify({
207          message: input.message,
208          content: utf8ToBase64(input.source),
209          branch: NOVELS_BRANCH,
210          author: input.author,
211          ...(input.expectedSha ? { sha: input.expectedSha } : {}),
212        }),
213      });
214      // 409: the sha is not the file's; 422: a create for a file that exists (no sha supplied).
215      if (res.status === 409 || res.status === 422) {
216        const current = await repo.read(path);
217        // A 422 for a file that is where the writer expected is some other refusal, not a conflict.
218        if ((current?.sha ?? null) === input.expectedSha) throw new Error(`GitHub refused the commit to ${path} (${res.status}).`);
219        throw new GitConflict(current?.sha ?? null);
220      }
221      if (!res.ok) throw new Error(`GitHub did not accept the commit to ${path} (${res.status}).`);
222      const body = (await res.json()) as { content: { sha: string }; commit: { sha: string } };
223      return { sha: body.content.sha, commit: body.commit.sha };
224    },
225
226    async commitMany(input) {
227      if (input.renames.length === 0) throw new Error("A multi-file commit needs at least one change.");
228      const json = async <T>(res: Response, what: string): Promise<T> => {
229        if (!res.ok) throw new Error(`GitHub did not ${what} (${res.status}).`);
230        return (await res.json()) as T;
231      };
232      const ref = await json<{ object: { sha: string } }>(await call(`${repoUrl}/git/ref/heads/${NOVELS_BRANCH}`), "return the branch head");
233      const head = ref.object.sha;
234      const commit = await json<{ tree: { sha: string } }>(await call(`${repoUrl}/git/commits/${head}`), "return the head commit");
235      const tree = await json<{ truncated: boolean; tree: { path: string; type: string; sha: string; mode: string }[] }>(
236        await call(`${repoUrl}/git/trees/${commit.tree.sha}?recursive=1`),
237        "list the head's tree",
238      );
239      if (tree.truncated) throw new Error("GitHub truncated the writing repository's tree; it is too large to move files in one commit.");
240      const at = new Map(tree.tree.filter((e) => e.type === "blob").map((e) => [e.path, e]));
241      // Every file must be where the mover saw it, and no move may land on a file that stays.
242      const leaving = new Set(input.renames.map((r) => r.from));
243      for (const r of input.renames) {
244        const entry = at.get(r.from);
245        if (!entry || entry.sha !== r.sha) throw new GitConflict(entry?.sha ?? null);
246        if (at.has(r.to) && !leaving.has(r.to)) throw new GitConflict(at.get(r.to)!.sha);
247      }
248      const arriving = new Set(input.renames.map((r) => r.to));
249      const entries = [
250        ...input.renames.filter((r) => !arriving.has(r.from)).map((r) => ({ path: r.from, mode: at.get(r.from)!.mode, type: "blob", sha: null })),
251        ...input.renames.map((r) => ({ path: r.to, mode: at.get(r.from)!.mode, type: "blob", sha: r.sha })),
252      ];
253      const post = (url: string, body: unknown) => call(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) });
254      const made = await json<{ sha: string }>(await post(`${repoUrl}/git/trees`, { base_tree: commit.tree.sha, tree: entries }), "build the new tree");
255      const next = await json<{ sha: string }>(
256        await post(`${repoUrl}/git/commits`, { message: input.message, tree: made.sha, parents: [head], author: input.author }),
257        "make the commit",
258      );
259      // Never forced: if anyone committed since the head was read, the branch refuses to move.
260      const moved = await call(`${repoUrl}/git/refs/heads/${NOVELS_BRANCH}`, {
261        method: "PATCH",
262        headers: { "Content-Type": "application/json" },
263        body: JSON.stringify({ sha: next.sha, force: false }),
264      });
265      if (moved.status === 422 || moved.status === 409) throw new GitConflict(null);
266      if (!moved.ok) throw new Error(`GitHub did not move the branch to the new commit (${moved.status}).`);
267      return { commit: next.sha };
268    },
269
270    async history(path, limit = 30) {
271      const res = await call(`${repoUrl}/commits?sha=${NOVELS_BRANCH}&path=${encodeURIComponent(path)}&per_page=${Math.min(100, Math.max(1, limit))}`);
272      if (!res.ok) throw new Error(`GitHub did not list the history of ${path} (${res.status}).`);
273      const body = (await res.json()) as { sha: string; commit: { message: string; author: { name: string; email: string; date: string } } }[];
274      return body.map((c) => ({ sha: c.sha, message: c.commit.message, author: c.commit.author.name, email: c.commit.author.email, at: c.commit.author.date }));
275    },
276  };
277  return repo;
278}
279
280/** The repository for this Worker's credentials; throws NovelsNotConnected until the App is set up. */
281export function novelsRepo(env: Env, fetcher: Fetch = timedFetch): NovelsRepo {
282  const connection = novelsConnection(env);
283  if (connection.state !== "connected") throw new NovelsNotConnected(connection.detail);
284  return githubRepo(() => installationToken(connection.appId, connection.privateKey, fetcher), fetcher);
285}