app/lib/phages/save.server.ts

Functions

Imports

It imports app/db/phages.ts, app/lib/cache-purge.server.ts, app/lib/carrel/errors.server.ts, app/lib/content/load-pipeline.server.ts, app/lib/editor/converge.mjs, app/lib/editor/github.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/pages/compile.mjs, app/lib/pages/save.server.ts, app/lib/phages/compile.mjs, app/lib/publications/parse.mjs. Imported by app/lib/carrel/phage-handler.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1// The one phage save, called by the site-api adapter for Carrel (app/lib/carrel/phage-handler.server.ts), and
2// the read the operator's get_phage tool answers. The phage table is edited through Carrel and nothing else:
3// there is no save tool and no admin editor. A save compiles the file the way CI does (compile.mjs), refuses
4// it with every message if it fails, commits it to the repository (unless it is byte-identical,
5// commitUnlessUnchanged), then writes its D1 row and re-derives the page that is drawn from the rows, so the
6// table changes on the next request with no build or deploy. The commit is never reverted when a derived write
7// fails: the error says the commit landed and how to repair it.
8//
9// /research/phages is compiled from these rows (its table, its sections, its twin, its search records), and
10// those live in the page's own row. So a write refreshes that page row FIRST, through the page compile door,
11// and the phage's own row LAST: the phage row's blob sha is what the drift check reads as "current", so a
12// failure between the two leaves the phage reading stale and the repair re-runs both. The purge is the pages
13// tag, because the page's HTML and its twin both embed the table.
14
15import { listPhages } from "~/db/phages";
16import { ContentInvalid } from "~/lib/carrel/errors.server";
17import { purgePhages, type PurgeOutcome } from "~/lib/cache-purge.server";
18import { loadPipeline } from "~/lib/content/load-pipeline.server";
19import { convergeWithRetry } from "~/lib/editor/converge.mjs";
20import { blobGuard, commitFiles, readFile } from "~/lib/editor/github.server";
21import { decideFileWrite, type Actor } from "~/lib/editor/publish-policy.mjs";
22import { commitUnlessUnchanged, UNCHANGED_NOTE } from "~/lib/editor/write-path.server";
23import { pageSlug, pageSourcePath } from "~/lib/pages/compile.mjs";
24import { PUBLICATIONS_DIR } from "~/lib/publications/parse.mjs";
25import { compile as compilePageFile, writeRow as writePageRow } from "~/lib/pages/save.server";
26
27import { PHAGES_PAGE_PATH, PHAGE_FACT_PAGES, compilePhage, phagePath, phageSetErrors, phageSlug, sortPhages, type Phage, type PhageHost } from "./compile.mjs";
28
29type PhageEnv = Env & { GITHUB_TOKEN?: string };
30
31/** A phage's file key is its name in lower case, which is also what keeps an id from naming any other path. */
32export const PHAGE_SLUG = /^[a-z][a-z0-9]*$/;
33
34/** A file the validator refused: carries every message, so the caller can fix its own edit. */
35class PhageInvalid extends ContentInvalid {
36  constructor(slug: string, errors: string[]) {
37    super(`The phage "${slug}" was not saved: it fails ${errors.length} check(s). Nothing was committed.`, errors);
38    this.name = "PhageInvalid";
39  }
40}
41
42/**
43 * The papers the site holds, asked of the repository, which is what CI asks too: a genome announcement is a paper
44 * this site has a file, and so a page, for. Asking the repository rather than D1 keeps a phage save and a
45 * sync_phages independent of whether the publications have been synced yet.
46 */
47function githubHost(env: PhageEnv): PhageHost {
48  return { paper: async (slug) => (await readFile(env, `${PUBLICATIONS_DIR}/${slug}.md`)) !== null };
49}
50
51/** The one compile door: the save, get_phage and sync_phages all read a file through it. */
52export async function compile(env: PhageEnv, slug: string, raw: string) {
53  const { findWideDashes } = await loadPipeline();
54  return compilePhage({ slug, raw, host: githubHost(env), pipeline: { findWideDashes } });
55}
56
57type Compiled = Extract<Awaited<ReturnType<typeof compile>>, { ok: true }>;
58
59/** The slug of an id, or a refusal: nothing but a lower-case name names a phage file. */
60function registered(slug: string) {
61  if (!PHAGE_SLUG.test(slug)) {
62    throw new PhageInvalid(slug, [`"${slug}" is not a phage file key; a phage's file is named for its name in lower case, such as acorn15`]);
63  }
64  return { slug, file: phagePath(slug) };
65}
66
67/** The phage the committed file holds, judged by the same validator, for get_phage. */
68export async function readPhage(env: PhageEnv, slug: string) {
69  const { file } = registered(slug);
70  const existing = await readFile(env, file);
71  if (!existing) return null;
72  const compiled = await compile(env, slug, existing.content);
73  return { slug, path: file, raw: existing.content, phage: compiled.ok ? compiled.phage : null, errors: compiled.ok ? [] : compiled.errors };
74}
75
76/** Every other phage's row, from D1: CI re-checks the whole set on the commit a save makes. */
77async function otherPhages(env: PhageEnv, slug: string) {
78  return (await listPhages(env)).filter((phage) => phageSlug(phage.name) !== slug);
79}
80
81/**
82 * Re-derives the page the table is drawn into: its HTML, twin and search records, compiled from the page's file
83 * with `phages` in place of whatever D1 holds. A page with no file has nothing derived to refresh, and that is an
84 * error here: the table has nowhere to be drawn. A page whose file no longer compiles is an error too, never
85 * skipped: the phage would otherwise read as converged while the page is stale.
86 */
87export async function refreshPage(env: PhageEnv, phages: Phage[]) {
88  // The table's page, then every page that states a phage fact by token ({{phages.count}}): each is derived from
89  // the same rows, so a phage write leaves none of them stale.
90  for (const path of [PHAGES_PAGE_PATH, ...PHAGE_FACT_PAGES]) {
91    const slug = pageSlug(path);
92    const file = await readFile(env, pageSourcePath(slug));
93    if (!file) throw new Error(`the page ${path} has no file (${pageSourcePath(slug)}), so what it draws from the phages has nowhere to be drawn`);
94    const compiled = await compilePageFile(env, slug, file.content, undefined, sortPhages(phages));
95    if (!compiled.ok) {
96      throw new Error(`the page ${path} does not compile with these phages, so its table, twin and search record were not refreshed: ${compiled.errors.join("; ")}`);
97    }
98    await writePageRow(env, compiled);
99  }
100}
101
102/** The phage's D1 row alone: the derived store the drift check reads. One row, in one statement. */
103export async function writePhageRow(env: PhageEnv, compiled: Compiled) {
104  const phage: Phage = compiled.phage;
105  await env.DB.prepare(
106    `INSERT INTO phages (slug, name, year, record, source_path, source_blob_sha, synced_at)
107     VALUES (?1, ?2, ?3, ?4, ?5, ?6, unixepoch())
108     ON CONFLICT(slug) DO UPDATE SET name = excluded.name, year = excluded.year, record = excluded.record,
109       source_path = excluded.source_path, source_blob_sha = excluded.source_blob_sha,
110       synced_at = excluded.synced_at`,
111  )
112    .bind(phageSlug(phage.name), phage.name, phage.year, compiled.record, compiled.sourcePath, compiled.sourceBlobSha)
113    .run();
114}
115
116/**
117 * Every derived store a phage feeds: the page's row first, then its own (see the header). Shared by the save,
118 * so a repair that re-runs it writes what a save writes.
119 */
120async function writeRow(env: PhageEnv, compiled: Compiled) {
121  const slug = phageSlug(compiled.phage.name);
122  await refreshPage(env, [...(await otherPhages(env, slug)), compiled.phage]);
123  await writePhageRow(env, compiled);
124}
125
126/** A phage whose file is gone: the page loses it first, then its row goes (the row is the drift marker). */
127export async function deletePhageRow(env: PhageEnv, row: { slug: string }) {
128  await refreshPage(env, await otherPhages(env, row.slug));
129  await env.DB.prepare(`DELETE FROM phages WHERE slug = ?1`).bind(row.slug).run();
130}
131
132/** True when D1's row for the phage was compiled from exactly this file (the git blob sha matches). */
133async function rowIsCurrent(env: PhageEnv, slug: string, blobSha: string) {
134  const row = await env.DB.prepare("SELECT source_blob_sha FROM phages WHERE slug = ?1")
135    .bind(slug)
136    .first<{ source_blob_sha: string | null }>();
137  return row?.source_blob_sha === blobSha;
138}
139
140export type SavedPhage = {
141  slug: string;
142  commitSha: string;
143  /** The git blob sha of the committed file: Carrel's version of the item. */
144  sourceBlobSha: string;
145  unchanged: boolean;
146  created: boolean;
147  purged: PurgeOutcome;
148  note?: string;
149};
150
151/** validate -> policy -> commitUnlessUnchanged -> blob verify -> derived writes (convergeWithRetry) -> purge by tag. */
152export async function savePhage(
153  env: PhageEnv,
154  options: { slug: string; raw: string; expectedHeadSha?: string | null;
155    /** The blob sha of this file as Carrel loaded it; undefined means no check. */
156    expectedBlobSha?: string | null; isNew: boolean; actor: Actor },
157): Promise<SavedPhage> {
158  const { raw, actor } = options;
159  const { slug, file } = registered(options.slug);
160  const existing = await readFile(env, file);
161  if (options.isNew && existing) throw new PhageInvalid(slug, [`the phage ${slug} already has a file (${file}); pass isNew false to edit it`]);
162  if (!options.isNew && !existing) throw new PhageInvalid(slug, [`the phage ${slug} has no file yet; pass isNew to create it`]);
163
164  const compiled = await compile(env, slug, raw);
165  if (!compiled.ok) throw new PhageInvalid(slug, compiled.errors);
166  // The set as it will stand: this phage against every other, so a record two phages claim is refused here.
167  const setErrors = phageSetErrors([...(await otherPhages(env, slug)), compiled.phage]);
168  if (setErrors.length > 0) throw new PhageInvalid(slug, setErrors);
169  // A phage has no draft state: it is public when it is saved, so the policy reads it as published.
170  decideFileWrite({ actor, noun: "phage", incomingDraft: false, priorRaw: existing?.content ?? null, isDraft: () => false });
171
172  const tag = actor.kind === "operator" ? ` [operator:${actor.id}]` : actor.kind === "carrel" ? ` [carrel:${actor.changeId}]` : "";
173  // After the gate, so a file that fails validation is still refused even when it is unchanged.
174  const written = await commitUnlessUnchanged(env, {
175    existing,
176    raw,
177    rowIsCurrent: () => rowIsCurrent(env, slug, compiled.sourceBlobSha),
178    commit: () =>
179      commitFiles(env, {
180        expectedHeadSha: options.expectedHeadSha,
181        expectedBlobs: blobGuard(file, options.expectedBlobSha),
182        message: `${options.isNew ? "Add" : "Update"} phage: ${compiled.phage.name}${tag}`,
183        changes: [{ path: file, content: raw }],
184      }),
185  });
186  if (written.action === "noop") {
187    return { slug, commitSha: written.commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: true, created: false, purged: null, note: UNCHANGED_NOTE };
188  }
189  const { commitSha } = written;
190  const blobSha = written.blobShas[file];
191  if (blobSha && blobSha !== compiled.sourceBlobSha) {
192    throw new Error(
193      `The phage "${slug}" WAS committed as ${commitSha}, but the committed bytes (${blobSha}) are not ` +
194        `the bytes compiled (${compiled.sourceBlobSha}), so its page was not updated. Run the content sync to ` +
195        `rebuild the row from the repository.`,
196    );
197  }
198
199  let purged: PurgeOutcome = false;
200  await convergeWithRetry({
201    write: async () => {
202      await writeRow(env, compiled);
203      // The pages tag: the page's HTML embeds the table and its twin embeds the rows, and both carry it.
204      purged = await purgePhages(`save phage ${slug}`);
205    },
206    // Nothing kept in KV for a phage: the thrown error names the commit and the repair (the content sync),
207    // and the next ship's sync converges the rows from the repository anyway.
208    recordDivergence: async () => undefined,
209    slug,
210    commitSha,
211  });
212
213  return { slug, commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: written.action === "repair", created: options.isNew, purged };
214}