app/lib/dictionary/save.server.ts

Functions

Imports

It imports app/db/dictionary.ts, app/lib/cache-purge.server.ts, app/lib/carrel/errors.server.ts, app/lib/content/load-pipeline.server.ts, app/lib/dictionary-entries.mjs, app/lib/dictionary/compile.mjs, app/lib/dictionary/parse.mjs, app/lib/editor/converge.mjs, app/lib/editor/github.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/pages/compile.mjs, app/lib/pages/save.server.ts. Imported by app/lib/carrel/dictionary-handler.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1// A dictionary entry's write path, run for the Carrel adapter (app/lib/carrel/dictionary-handler.server.ts):
2// compile the file the way CI does (compile.mjs), refuse it with the validator's own messages if it fails,
3// commit it unless it is unchanged (commitUnlessUnchanged), then write the derived stores, so the page changes
4// on the next request with no build or deploy. The commit is never reverted when a derived write fails: the
5// error says the commit landed and how to repair it.
6//
7// A page's lead is drawn from this entry at request time, but its markdown twin and its search record carry
8// the entry too, and those live in the page's own row. So a write refreshes that page row FIRST, through the
9// page compile door, and the entry's own row LAST: the entry row's blob sha is what the drift check reads as
10// "current", so a failure between the two leaves the entry reading stale and the repair re-runs both. The
11// purge is the pages tag, because the page's HTML and its twin both embed the entry.
12
13import { listDictionaryRows } from "~/db/dictionary";
14import { ContentInvalid } from "~/lib/carrel/errors.server";
15import { purgePages, type PurgeOutcome } from "~/lib/cache-purge.server";
16import { loadPipeline } from "~/lib/content/load-pipeline.server";
17import type { DictionaryEntry } from "~/lib/dictionary-entries.mjs";
18import { convergeWithRetry } from "~/lib/editor/converge.mjs";
19import { blobGuard, commitFiles, GitHubError, listDirectory, readFile } from "~/lib/editor/github.server";
20import { decideFileWrite, type Actor } from "~/lib/editor/publish-policy.mjs";
21import { commitUnlessUnchanged, UNCHANGED_NOTE } from "~/lib/editor/write-path.server";
22import { pageSlug, pageSourcePath } from "~/lib/pages/compile.mjs";
23import { compile as compilePageFile, writeRow as writePageRow } from "~/lib/pages/save.server";
24
25import { compileDictionaryEntry, type DictionaryHost } from "./compile.mjs";
26import { dictionaryPath, parseDictionaryEntry } from "./parse.mjs";
27
28type DictionaryEnv = Env & { GITHUB_TOKEN?: string };
29
30/** A file the validator refused: carries every message, so the caller can fix its own edit. */
31class DictionaryInvalid extends ContentInvalid {
32  constructor(key: string, errors: string[]) {
33    super(`The dictionary entry "${key}" was not saved: it fails ${errors.length} check(s). Nothing was committed.`, errors);
34    this.name = "DictionaryInvalid";
35  }
36}
37
38/** The repository's clips, asked of GitHub: existence and size, never the bytes, which are binary. */
39function githubHost(env: DictionaryEnv): DictionaryHost {
40  return {
41    async audio(sitePath) {
42      const at = sitePath.lastIndexOf("/");
43      const dir = `public${sitePath.slice(0, at)}`;
44      const name = sitePath.slice(at + 1);
45      try {
46        const entry = (await listDirectory(env, dir)).find((e) => e.type === "file" && e.name === name);
47        return entry ? { size: entry.size } : null;
48      } catch (error) {
49        if (error instanceof GitHubError && error.status === 404) return null;
50        throw error;
51      }
52    },
53  };
54}
55
56/** Every other entry's key and path, from D1: CI re-checks the whole set on the commit this makes. */
57async function otherEntries(env: DictionaryEnv, key: string) {
58  return (await listDictionaryRows(env)).filter((row) => row.key !== key).map(({ key: other, path }) => ({ key: other, path }));
59}
60
61/** The one compile door: the save, sync_dictionary and the reads all go through it. */
62export async function compile(env: DictionaryEnv, key: string, raw: string) {
63  const { findWideDashes } = await loadPipeline();
64  return compileDictionaryEntry({ key, raw, pipeline: { findWideDashes }, host: githubHost(env), others: await otherEntries(env, key) });
65}
66
67type Compiled = Extract<Awaited<ReturnType<typeof compile>>, { ok: true }>;
68
69/** Whether a stored file is a draft, read from the file itself, the only authority. */
70export function fileIsDraft(raw: string) {
71  return parseDictionaryEntry({ file: "", raw }).data.draft === true;
72}
73
74/** The file, as the operator's get_dictionary reads it: raw, the entry when it compiles, every error when not. */
75export async function readDictionary(env: DictionaryEnv, key: string) {
76  const file = await readFile(env, dictionaryPath(key));
77  if (!file) return null;
78  const compiled = await compile(env, key, file.content);
79  return {
80    key,
81    raw: file.content,
82    draft: compiled.ok ? compiled.draft : fileIsDraft(file.content),
83    entry: compiled.ok ? compiled.entry : null,
84    errors: compiled.ok ? [] : compiled.errors,
85  };
86}
87
88/**
89 * Re-derives the page an entry opens: its twin and its search records, compiled from the page's file with
90 * `entry` (null for none) in place of whatever D1 holds. A page with no file has nothing derived to refresh. A
91 * page whose file no longer compiles is an error, never skipped: the entry would otherwise read as converged
92 * while the page's twin is stale.
93 */
94async function refreshPage(env: DictionaryEnv, path: string, entry: DictionaryEntry | null) {
95  const slug = pageSlug(path);
96  const file = await readFile(env, pageSourcePath(slug));
97  if (!file) return;
98  const compiled = await compilePageFile(env, slug, file.content, entry);
99  if (!compiled.ok) {
100    throw new Error(`the page ${path} does not compile, so its twin and search record were not refreshed: ${compiled.errors.join("; ")}`);
101  }
102  await writePageRow(env, compiled);
103}
104
105/** The entry's D1 row alone: the derived store the page reads its lead and JSON-LD from. */
106async function writeEntryRow(env: DictionaryEnv, compiled: Compiled) {
107  await env.DB.prepare(
108    `INSERT INTO dictionary_entries (key, path, term, status, record, source_path, source_blob_sha, synced_at)
109     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, unixepoch())
110     ON CONFLICT(key) DO UPDATE SET path = excluded.path, term = excluded.term, status = excluded.status,
111       record = excluded.record, source_path = excluded.source_path, source_blob_sha = excluded.source_blob_sha,
112       synced_at = excluded.synced_at`,
113  )
114    .bind(
115      compiled.key,
116      compiled.entry.path,
117      compiled.entry.term,
118      compiled.draft ? "draft" : "published",
119      JSON.stringify(compiled.entry),
120      compiled.sourcePath,
121      compiled.sourceBlobSha,
122    )
123    .run();
124}
125
126/**
127 * Every derived store an entry feeds: its page's row first, then its own (see the header). Shared by the save
128 * and sync_dictionary, so a repair writes what a save writes.
129 */
130export async function writeRow(env: DictionaryEnv, compiled: Compiled) {
131  await refreshPage(env, compiled.entry.path, compiled.draft ? null : compiled.entry);
132  await writeEntryRow(env, compiled);
133}
134
135/** An entry whose file is gone: its page loses the lead first, then its row goes (the row is the drift marker). */
136export async function deleteRow(env: DictionaryEnv, row: { key: string; path: string }) {
137  await refreshPage(env, row.path, null);
138  await env.DB.prepare(`DELETE FROM dictionary_entries WHERE key = ?1`).bind(row.key).run();
139}
140
141/** True when D1's row for the entry was compiled from exactly this file (the git blob sha matches). */
142async function rowIsCurrent(env: DictionaryEnv, key: string, blobSha: string) {
143  const row = await env.DB.prepare("SELECT source_blob_sha FROM dictionary_entries WHERE key = ?1")
144    .bind(key)
145    .first<{ source_blob_sha: string | null }>();
146  return row?.source_blob_sha === blobSha;
147}
148
149export type SavedDictionaryEntry = {
150  key: string;
151  commitSha: string;
152  /** The git blob sha of the committed file: Carrel's version of the item. */
153  sourceBlobSha: string;
154  unchanged: boolean;
155  created: boolean;
156  draft: boolean;
157  purged: PurgeOutcome;
158  note?: string;
159};
160
161/** validate -> policy -> commitUnlessUnchanged -> blob verify -> derived writes (convergeWithRetry) -> purge by tag. */
162export async function saveDictionaryEntry(
163  env: DictionaryEnv,
164  options: { key: string; raw: string; expectedHeadSha?: string | null;
165    /** The blob sha of this file as Carrel loaded it; undefined means no check. */
166    expectedBlobSha?: string | null; isNew: boolean; actor: Actor },
167): Promise<SavedDictionaryEntry> {
168  const { key, raw, actor } = options;
169  const path = dictionaryPath(key);
170  const existing = await readFile(env, path);
171  if (options.isNew && existing) throw new DictionaryInvalid(key, [`a dictionary entry named ${key} already exists`]);
172  if (!options.isNew && !existing) throw new DictionaryInvalid(key, [`no dictionary entry named ${key} exists; pass isNew to create it`]);
173
174  const compiled = await compile(env, key, raw);
175  if (!compiled.ok) throw new DictionaryInvalid(key, compiled.errors);
176  decideFileWrite({
177    actor,
178    noun: "dictionary entry",
179    incomingDraft: compiled.draft,
180    priorRaw: existing?.content ?? null,
181    isDraft: fileIsDraft,
182  });
183
184  const tag = actor.kind === "operator" ? ` [operator:${actor.id}]` : actor.kind === "carrel" ? ` [carrel:${actor.changeId}]` : "";
185  // After the gate, so a file that fails validation is still refused even when it is unchanged.
186  const written = await commitUnlessUnchanged(env, {
187    existing,
188    raw,
189    rowIsCurrent: () => rowIsCurrent(env, key, compiled.sourceBlobSha),
190    commit: () =>
191      commitFiles(env, {
192        expectedHeadSha: options.expectedHeadSha,
193        expectedBlobs: blobGuard(path, options.expectedBlobSha),
194        message: `${options.isNew ? "Add" : "Update"} dictionary entry: ${compiled.entry.term}${tag}`,
195        changes: [{ path, content: raw }],
196      }),
197  });
198  if (written.action === "noop") {
199    return { key, commitSha: written.commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: true, created: false, draft: compiled.draft, purged: null, note: UNCHANGED_NOTE };
200  }
201  const { commitSha } = written;
202  const blobSha = written.blobShas[path];
203  if (blobSha && blobSha !== compiled.sourceBlobSha) {
204    throw new Error(
205      `The dictionary entry "${key}" WAS committed as ${commitSha}, but the committed bytes (${blobSha}) are not ` +
206        `the bytes compiled (${compiled.sourceBlobSha}), so its page was not updated. Run the content sync to ` +
207        `rebuild the row from the repository.`,
208    );
209  }
210
211  let purged: PurgeOutcome = false;
212  await convergeWithRetry({
213    write: async () => {
214      await writeRow(env, compiled);
215      // The pages tag: the page's HTML embeds the lead and its twin embeds the entry, and both carry it.
216      purged = await purgePages(`save dictionary entry ${key}`);
217    },
218    // Nothing kept in KV for an entry: the thrown error names the commit and the repair (the content sync),
219    // and the next ship's sync converges the rows from the repository anyway.
220    recordDivergence: async () => undefined,
221    slug: key,
222    commitSha,
223  });
224
225  return {
226    key,
227    commitSha,
228    sourceBlobSha: compiled.sourceBlobSha,
229    unchanged: written.action === "repair",
230    created: options.isNew,
231    draft: compiled.draft,
232    purged,
233  };
234}