app/lib/pages/save.server.ts

Functions

Imports

It imports app/db/dictionary.ts, app/db/index.ts, app/db/pages.ts, app/db/phages.ts, app/db/procedures.ts, app/lib/cache-purge.server.ts, app/lib/carrel/errors.server.ts, app/lib/content-pages.mjs, app/lib/content/load-pipeline.server.ts, app/lib/dictionary-entries.mjs, app/lib/editor/converge.mjs, app/lib/editor/github.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/pages/compile.mjs, app/lib/pages/links.mjs, app/lib/pages/protected.mjs, app/lib/phages/compile.mjs, app/lib/search/records.mjs, app/lib/search/replace.server.ts. Imported by app/lib/carrel/page-handler.server.ts, app/lib/dictionary/save.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts, app/lib/phages/save.server.ts.

Source

1// The one page save, called by the site-api adapter for Carrel (app/lib/pages/adapter.server.ts), and the
2// read the operator's get_page tool answers. Content is edited through Carrel and nothing else: there is no
3// save tool and no admin editor. A save compiles the file the way CI does (compile.mjs), judges its internal
4// links (links.mjs), refuses it with every message if it fails, commits it to the repository (unless it is
5// byte-identical, commitUnlessUnchanged), then writes its D1 row and search records, so the page changes on
6// the next request with no build or deploy. The commit is never reverted when the D1 write fails;
7// the error says the commit landed and how to repair the row. A save cannot create a path: a page's
8// address is structure and lives in CONTENT_PAGE_PATHS, in code.
9
10import matter from "gray-matter";
11
12import { listBlogPosts } from "~/db";
13import { getPublishedEntryByPath } from "~/db/dictionary";
14import { listPublishedPageHtml } from "~/db/pages";
15import { listPhages } from "~/db/phages";
16import { listPublishedProcedures } from "~/db/procedures";
17import { ContentInvalid } from "~/lib/carrel/errors.server";
18import { purgePages, type PurgeOutcome } from "~/lib/cache-purge.server";
19import { loadPipeline } from "~/lib/content/load-pipeline.server";
20import {
21  CONTENT_PAGES_FROM_DATA,
22  CONTENT_PAGES_OWN_ROUTE,
23  PAGE_FILE_PATHS,
24  contentPageSearchUid,
25} from "~/lib/content-pages.mjs";
26import { convergeWithRetry } from "~/lib/editor/converge.mjs";
27import { blobGuard, commitFiles, readFile } from "~/lib/editor/github.server";
28import { commitUnlessUnchanged, UNCHANGED_NOTE } from "~/lib/editor/write-path.server";
29import { PolicyError, WRITE_CAPABILITIES, type Actor } from "~/lib/editor/publish-policy.mjs";
30import type { DictionaryEntry } from "~/lib/dictionary-entries.mjs";
31import { PHAGES_PAGE_PATH, type Phage } from "~/lib/phages/compile.mjs";
32import { recordsForPages } from "~/lib/search/records.mjs";
33import { replaceSearchRecords } from "~/lib/search/replace.server";
34
35import { compilePage, pagePathForSlug, pageSlug, pageSourcePath } from "./compile.mjs";
36import { idsIn, pageLinkErrors, type AddressBook } from "./links.mjs";
37import { protectedPageReason } from "./protected.mjs";
38
39type PageEnv = Env & { GITHUB_TOKEN?: string };
40
41/** A file the validator refused: 422, with every message, so the caller can fix its own edit. */
42export class PageInvalid extends ContentInvalid {
43  constructor(path: string, errors: string[]) {
44    super(`The page "${path}" was not saved: it fails ${errors.length} check(s). Nothing was committed.`, errors);
45    this.name = "PageInvalid";
46  }
47}
48
49/**
50 * The page a path names. A path outside CONTENT_PAGE_PATHS is refused here, before any read or write: a
51 * save cannot create a path, and the error says so. /cv is generated from data and has no file.
52 */
53function registeredPage(path: string) {
54  const clean = path.trim();
55  if (!PAGE_FILE_PATHS.includes(clean)) {
56    throw new PageInvalid(clean, [
57      `"${clean}" is not a registered page path. A save cannot create a new path: a page's address is structure, ` +
58        "so a new page's path is added to CONTENT_PAGE_PATHS (app/lib/content-pages.mjs) in code first. " +
59        `Registered paths: ${PAGE_FILE_PATHS.filter((p) => !CONTENT_PAGES_FROM_DATA.includes(p)).join(", ")}`,
60    ]);
61  }
62  if (CONTENT_PAGES_FROM_DATA.includes(clean)) {
63    throw new PageInvalid(clean, [`${clean} is generated from structured data (CONTENT_PAGES_FROM_DATA); edit the data, not a markdown copy`]);
64  }
65  const slug = pageSlug(clean);
66  return { path: clean, slug, file: pageSourcePath(slug) };
67}
68
69/**
70 * The one compile door: the save and sync_pages both read a file through it (no link check; see judge). A page
71 * that opens with a dictionary entry carries it in its twin and its search record, so the compile reads the
72 * published entry from D1 (docs/DICTIONARY.md); a dictionary save passes the entry it is about to write
73 * instead (null for none), because D1 does not hold it yet. The phage page draws its table and sections from the
74 * phage rows (docs/PHAGES.md), so the compile reads them from D1; a phage save passes the set it is about to
75 * write instead, for the same reason.
76 */
77export async function compile(env: PageEnv, slug: string, raw: string, entry?: DictionaryEntry | null, phages?: Phage[]) {
78  const { renderBody, findWideDashes } = await loadPipeline();
79  const path = pagePathForSlug(slug);
80  const lead = entry !== undefined ? entry : path ? await getPublishedEntryByPath(env, path) : null;
81  // A page that states a phage fact by token ({{phages.count}}) reads the rows too, whichever page it is.
82  const needsPhages = path === PHAGES_PAGE_PATH || raw.includes("{{phage");
83  const rows = phages ?? (needsPhages ? await listPhages(env) : undefined);
84  return compilePage({ slug, raw, pipeline: { renderBody, findWideDashes }, entry: lead ?? undefined, phages: rows });
85}
86
87/** The address book the links are judged against: what D1 holds now, with this page's own HTML in place. */
88async function addressBook(env: PageEnv, own: { path: string; html: string }): Promise<AddressBook> {
89  const [procedures, posts, pageHtml] = await Promise.all([
90    listPublishedProcedures(env),
91    listBlogPosts(env, { perPage: 1000 }),
92    listPublishedPageHtml(env),
93  ]);
94  const htmlOf = new Map(pageHtml);
95  htmlOf.set(own.path, own.html);
96  const ids = new Map<string, ReadonlySet<string>>();
97  return {
98    procedurePaths: new Set(procedures.map((p) => p.path)),
99    posts: new Map(posts.posts.map((p) => [`/writing/${p.slug}`, true])),
100    idsOf: (path) => {
101      const html = htmlOf.get(path);
102      if (html === undefined) return null;
103      if (!ids.has(path)) ids.set(path, idsIn(html));
104      return ids.get(path) ?? null;
105    },
106  };
107}
108
109/** Compile and link check together: every message the file has now, which is what CI and a save both refuse. */
110async function judge(env: PageEnv, slug: string, raw: string) {
111  const compiled = await compile(env, slug, raw);
112  if (!compiled.ok) return compiled;
113  const errors = pageLinkErrors(compiled.page, await addressBook(env, compiled.page));
114  return errors.length > 0 ? { ok: false as const, errors } : compiled;
115}
116
117/** The file, and its front matter and body as parsed. */
118export async function readPage(env: PageEnv, path: string) {
119  const { slug, file } = registeredPage(path);
120  const existing = await readFile(env, file);
121  if (!existing) return null;
122  const parsed = matter(existing.content);
123  const judged = await judge(env, slug, existing.content);
124  return {
125    slug,
126    path,
127    raw: existing.content,
128    draft: parsed.data.draft === true,
129    record: { data: parsed.data, body: parsed.content },
130    errors: judged.ok ? [] : judged.errors,
131  };
132}
133
134/**
135 * An operator may edit, unpublish and republish a page, never publish one for the first time: the same
136 * rule as posts (the first-publish rule, `WRITE_CAPABILITIES` in `publish-policy.mjs`), read from the prior FILE, the only authority.
137 */
138function decidePage(actor: Actor, incomingDraft: boolean, prior: string | null) {
139  const may = WRITE_CAPABILITIES[actor.kind];
140  if (!may.write) {
141    throw new PolicyError("Refused: this credential is READ ONLY and may not save a page.", "smoke-is-read-only");
142  }
143  const everPublished = prior !== null && matter(prior).data.draft !== true;
144  if (!may.firstPublish && !incomingDraft && !everPublished) {
145    throw new PolicyError(
146      "Refused: publishing a page for the first time is reserved to the human admin. Save it with " +
147        "draft: true, and ask Dustin to publish it.",
148      "first-publish-requires-admin",
149    );
150  }
151}
152
153export async function writeRow(env: PageEnv, compiled: Extract<Awaited<ReturnType<typeof compile>>, { ok: true }>) {
154  const db = env.DB;
155  const r = compiled.record;
156  const uid = compiled.searchInput.uid;
157  // A page with a route of its own (About) has never been in the search index, so it gets no record.
158  const records = compiled.draft || CONTENT_PAGES_OWN_ROUTE.includes(r.path) ? [] : recordsForPages([compiled.searchInput]);
159  await db.batch([
160    db
161      .prepare(
162        `INSERT INTO pages (slug, path, title, description, status, record, markdown, source_path,
163           source_blob_sha, synced_at)
164         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, unixepoch())
165         ON CONFLICT(slug) DO UPDATE SET path = excluded.path, title = excluded.title,
166           description = excluded.description, status = excluded.status, record = excluded.record,
167           markdown = excluded.markdown, source_path = excluded.source_path,
168           source_blob_sha = excluded.source_blob_sha, synced_at = excluded.synced_at`,
169      )
170      .bind(
171        r.slug,
172        r.path,
173        r.title,
174        r.description,
175        compiled.draft ? "draft" : "published",
176        JSON.stringify(r),
177        compiled.markdown,
178        compiled.sourcePath,
179        compiled.sourceBlobSha,
180      ),
181    ...replaceSearchRecords(db, uid, records),
182  ]);
183}
184
185/** The row of a page whose file is gone and its search records: the one removal sync_pages makes. */
186export async function deletePageRow(env: PageEnv, row: { slug: string; path: string }) {
187  // /terms is the dataset license: a sync that finds its file gone repairs nothing here, it refuses (job_044f96426efa).
188  const keep = protectedPageReason(row.path);
189  if (keep) throw new PolicyError(`Refused: ${keep}.`, "terms-cannot-be-removed");
190  await env.DB.batch([
191    env.DB.prepare(`DELETE FROM pages WHERE slug = ?1`).bind(row.slug),
192    env.DB.prepare(`DELETE FROM search_docs WHERE doc_uid = ?1`).bind(contentPageSearchUid(row.path)),
193    env.DB.prepare(`INSERT INTO search_identity (search_identity) VALUES ('rebuild')`),
194    env.DB.prepare(`INSERT INTO search_prose (search_prose) VALUES ('rebuild')`),
195  ]);
196}
197
198/** True when D1's row for the page was compiled from exactly this file (the git blob sha matches). */
199async function rowIsCurrent(env: PageEnv, slug: string, blobSha: string) {
200  const row = await env.DB.prepare("SELECT source_blob_sha FROM pages WHERE slug = ?1")
201    .bind(slug)
202    .first<{ source_blob_sha: string | null }>();
203  return row?.source_blob_sha === blobSha;
204}
205
206/** The page save. */
207export async function savePage(
208  env: PageEnv,
209  options: { path: string; raw: string; expectedHeadSha?: string | null;
210    /** The blob sha of this file as Carrel loaded it; undefined means no check. */
211    expectedBlobSha?: string | null; isNew: boolean; actor: Actor },
212) {
213  const { raw, actor } = options;
214  const { path, slug, file } = registeredPage(options.path);
215  const existing = await readFile(env, file);
216  if (options.isNew && existing) throw new PageInvalid(path, [`the page ${path} already has a file (${file}); pass isNew false to edit it`]);
217  if (!options.isNew && !existing) throw new PageInvalid(path, [`the page ${path} has no file yet; pass isNew to create it`]);
218
219  const compiled = await judge(env, slug, raw);
220  if (!compiled.ok) throw new PageInvalid(path, compiled.errors);
221  decidePage(actor, compiled.draft, existing?.content ?? null);
222
223  const tag = actor.kind === "operator" ? ` [operator:${actor.id}]` : actor.kind === "carrel" ? ` [carrel:${actor.changeId}]` : "";
224  // After the gate, so a file that fails validation is still refused even when it is unchanged.
225  const written = await commitUnlessUnchanged(env, {
226    existing,
227    raw,
228    rowIsCurrent: () => rowIsCurrent(env, slug, compiled.sourceBlobSha),
229    commit: () =>
230      commitFiles(env, {
231        expectedHeadSha: options.expectedHeadSha,
232        expectedBlobs: blobGuard(file, options.expectedBlobSha),
233        message: `${options.isNew ? "Add" : "Update"} page: ${compiled.record.title}${tag}`,
234        changes: [{ path: file, content: raw }],
235      }),
236  });
237  const unchanged = written.action !== "commit";
238  if (written.action === "noop") {
239    return {
240      slug,
241      path,
242      commitSha: written.commitSha,
243      sourceBlobSha: compiled.sourceBlobSha,
244      unchanged: true,
245      note: UNCHANGED_NOTE,
246      created: false,
247      draft: compiled.draft,
248      purged: null as PurgeOutcome,
249    };
250  }
251  const { commitSha, blobShas } = written;
252  if (blobShas[file] && blobShas[file] !== compiled.sourceBlobSha) {
253    throw new Error(
254      `The page "${path}" WAS committed as ${commitSha}, but the committed bytes (${blobShas[file]}) are not ` +
255        `the bytes compiled (${compiled.sourceBlobSha}), so its page was not updated. Run the content sync to ` +
256        `rebuild the row from the repository.`,
257    );
258  }
259
260  let purged: PurgeOutcome = false;
261  await convergeWithRetry({
262    write: async () => {
263      await writeRow(env, compiled);
264      purged = await purgePages(`save page ${path}`);
265    },
266    // Nothing kept in KV for a page: the thrown error names the commit and the repair (the content sync),
267    // and the next ship's sync converges the row from the repository anyway.
268    recordDivergence: async () => undefined,
269    slug,
270    commitSha,
271  });
272
273  return {
274    slug,
275    path,
276    commitSha,
277    sourceBlobSha: compiled.sourceBlobSha,
278    unchanged,
279    created: options.isNew,
280    draft: compiled.draft,
281    purged,
282  };
283}