app/kb/registry/save.server.ts

Functions

Imports

It imports app/db/registry.ts, app/kb/procedures/primer-dependents.server.ts, app/kb/registry/compile.mjs, app/kb/registry/host.server.ts, app/kb/registry/kinds.mjs, app/kb/registry/search-inputs.mjs, app/lib/cache-purge.server.ts, app/lib/carrel/errors.server.ts, app/lib/editor/converge.mjs, app/lib/editor/github.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/search/records.mjs. Imported by app/kb/editor.server.ts, app/lib/carrel/registry-handler.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1// The one registry save, called by the site-api adapter for Carrel (app/lib/carrel/registry-handler.server.ts), and
2// the read the operator's get_registry tool answers. A save compiles the file the way CI does (compile.mjs), refuses
3// it with every message if it fails, commits it to the repository (unless it is byte-identical,
4// commitUnlessUnchanged), then writes its D1 row, so the registry changes on the next request with no build or
5// deploy. The commit is never reverted when a derived write fails: the error says the commit landed
6// and how to repair it (sync_registry).
7//
8// Nothing else is derived from a registry row yet, so a write is one statement: the item's own row, whose blob sha
9// is what the registry-drift check reads as current. The purge is the registry's tag.
10
11import matter from "gray-matter";
12
13import { listRegistry } from "~/db/registry";
14import { ContentInvalid } from "~/lib/carrel/errors.server";
15import { purgeRegistry, type PurgeOutcome } from "~/lib/cache-purge.server";
16import { convergeWithRetry } from "~/lib/editor/converge.mjs";
17import { blobGuard, commitFiles, readFile } from "~/lib/editor/github.server";
18import { decideFileWrite, type Actor } from "~/lib/editor/publish-policy.mjs";
19import { commitUnlessUnchanged, UNCHANGED_NOTE } from "~/lib/editor/write-path.server";
20import { refreshProceduresNaming } from "~/kb/procedures/primer-dependents.server";
21import { recordsForPapers } from "~/lib/search/records.mjs";
22
23import {
24  parseRegistrySlug,
25  registryPath,
26  registrySetErrors,
27  registrySlug,
28  type RegistryItem,
29} from "./compile.mjs";
30import { compile } from "./host.server";
31import { KINDS, type KindSpec } from "./kinds.mjs";
32import { registrySearchInput, registrySearchUid } from "./search-inputs.mjs";
33
34type RegistryEnv = Env & { GITHUB_TOKEN?: string };
35type Kinds = Readonly<Record<string, KindSpec>>;
36
37/** A file the validator refused: carries every message, so the caller can fix its own edit. */
38export class RegistryInvalid extends ContentInvalid {
39  constructor(slug: string, errors: string[]) {
40    super(`The registry item "${slug}" was not saved: it fails ${errors.length} check(s). Nothing was committed.`, errors);
41    this.name = "RegistryInvalid";
42  }
43}
44
45/** The one compile door (host.server.ts): the save, get_registry and sync_registry all read a file through it. */
46export { compile };
47
48type Compiled = Extract<Awaited<ReturnType<typeof compile>>, { ok: true }>;
49
50/** The address of a slug, or a refusal: nothing but `<kind>/<id>` in the allowed shapes names a registry file. */
51function registered(slug: string, kinds: Kinds) {
52  const address = parseRegistrySlug(slug);
53  if (!address) {
54    throw new RegistryInvalid(slug, [`"${slug}" is not a registry address; it is <kind>/<id>, in lower-case letters, digits and hyphens`]);
55  }
56  if (!kinds[address.kind]) {
57    throw new RegistryInvalid(slug, [`"${address.kind}" is not a registry kind; the kinds are ${Object.keys(kinds).join(", ") || "none yet"}`]);
58  }
59  return { ...address, slug, file: registryPath(slug) };
60}
61
62/** The item the committed file holds, judged by the same validator, for get_registry. */
63export async function readRegistryItem(env: RegistryEnv, slug: string, kinds: Kinds = KINDS) {
64  const { file } = registered(slug, kinds);
65  const existing = await readFile(env, file);
66  if (!existing) return null;
67  const compiled = await compile(env, slug, existing.content, kinds);
68  return {
69    slug,
70    path: file,
71    raw: existing.content,
72    item: compiled.ok ? compiled.item : null,
73    gaps: compiled.ok ? compiled.gaps : [],
74    errors: compiled.ok ? [] : compiled.errors,
75  };
76}
77
78/**
79 * Everything a save checks before it commits, and nothing it writes: the file by its kind's validator, then the set as it
80 * will stand, this item against every other, so a name two items of a kind claim is refused. The admin's Check runs it.
81 */
82export async function checkRegistryItem(env: RegistryEnv, slug: string, raw: string, kinds: Kinds = KINDS) {
83  const compiled = await compile(env, slug, raw, kinds);
84  if (!compiled.ok) return compiled;
85  const setErrors = registrySetErrors([...(await otherItems(env, slug)), compiled.item], kinds);
86  return setErrors.length > 0 ? { ok: false as const, errors: setErrors } : compiled;
87}
88
89/** Every other item, from D1: CI re-checks the whole set on the commit a save makes. */
90async function otherItems(env: RegistryEnv, slug: string): Promise<RegistryItem[]> {
91  return (await listRegistry(env)).filter((item) => registrySlug(item.kind, item.id) !== slug);
92}
93
94/** The statements that rebuild the two full-text indexes after a change to search_docs, as every kind's write does. */
95function rebuildStatements(db: D1Database) {
96  return [
97    db.prepare(`INSERT INTO search_identity (search_identity) VALUES ('rebuild')`),
98    db.prepare(`INSERT INTO search_prose (search_prose) VALUES ('rebuild')`),
99  ];
100}
101
102/**
103 * The item's D1 row, the derived store the drift check reads, and its search record (search-inputs.mjs), in one batch so
104 * they move together: a search for a primer's sequence finds the primer's own page as well as the protocols that use it. A
105 * draft has a row and no record.
106 */
107export async function writeRegistryRow(env: RegistryEnv, compiled: Compiled) {
108  const { item } = compiled;
109  const db = env.DB;
110  const input = registrySearchInput(item);
111  const records = item.status === "draft" || !input ? [] : recordsForPapers([input]);
112  await db.batch([
113    db
114      .prepare(
115        `INSERT INTO registry (kind, id, name, status, record, source_path, source_blob_sha, synced_at)
116         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, unixepoch())
117         ON CONFLICT(kind, id) DO UPDATE SET name = excluded.name, status = excluded.status, record = excluded.record,
118           source_path = excluded.source_path, source_blob_sha = excluded.source_blob_sha,
119           synced_at = excluded.synced_at`,
120      )
121      .bind(item.kind, item.id, item.name, item.status, compiled.record, compiled.sourcePath, compiled.sourceBlobSha),
122    db.prepare(`DELETE FROM search_docs WHERE doc_uid = ?1`).bind(registrySearchUid(item.kind, item.id)),
123    ...records.map((r) =>
124      db
125        .prepare(
126          `INSERT INTO search_docs (uid, url, type, title, body, tags, doc_tags, doc_uid,
127             doc_title, doc_url, anchor, ordinal, status, publish_at)
128           VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)`,
129        )
130        .bind(r.uid, r.url, r.type, r.title, r.body, r.tags, r.docTags, r.docUid, r.docTitle, r.docUrl, r.anchor, r.ordinal, r.status, null),
131    ),
132    ...rebuildStatements(db),
133  ]);
134}
135
136/** An item whose file is gone: its row and its search record go (the row is the drift marker). */
137export async function deleteRegistryRow(env: RegistryEnv, row: { slug: string }) {
138  const address = parseRegistrySlug(row.slug);
139  if (!address) throw new Error(`"${row.slug}" is not a registry address, so its row cannot be named`);
140  await env.DB.batch([
141    env.DB.prepare(`DELETE FROM registry WHERE kind = ?1 AND id = ?2`).bind(address.kind, address.id),
142    env.DB.prepare(`DELETE FROM search_docs WHERE doc_uid = ?1`).bind(registrySearchUid(address.kind, address.id)),
143    ...rebuildStatements(env.DB),
144  ]);
145}
146
147/** True when D1's row for the item was compiled from exactly this file (the git blob sha matches). */
148async function rowIsCurrent(env: RegistryEnv, kind: string, id: string, blobSha: string) {
149  const row = await env.DB.prepare("SELECT source_blob_sha FROM registry WHERE kind = ?1 AND id = ?2")
150    .bind(kind, id)
151    .first<{ source_blob_sha: string | null }>();
152  return row?.source_blob_sha === blobSha;
153}
154
155export type SavedRegistryItem = {
156  slug: string;
157  commitSha: string;
158  /** The git blob sha of the committed file: Carrel's version of the item. */
159  sourceBlobSha: string;
160  unchanged: boolean;
161  created: boolean;
162  purged: PurgeOutcome;
163  note?: string;
164};
165
166/** validate -> policy -> commitUnlessUnchanged -> blob verify -> derived write (convergeWithRetry) -> purge by tag. */
167export async function saveRegistryItem(
168  env: RegistryEnv,
169  options: {
170    slug: string;
171    raw: string;
172    expectedHeadSha?: string | null;
173    /** The blob sha of this file as Carrel loaded it; undefined means no check. */
174    expectedBlobSha?: string | null;
175    isNew: boolean;
176    actor: Actor;
177    /** The kinds to judge by; the registry's own unless a test brings its own. */
178    kinds?: Kinds;
179  },
180): Promise<SavedRegistryItem> {
181  const { raw, actor } = options;
182  const kinds = options.kinds ?? KINDS;
183  const { kind, id, slug, file } = registered(options.slug, kinds);
184  const existing = await readFile(env, file);
185  if (options.isNew && existing) throw new RegistryInvalid(slug, [`the item ${slug} already has a file (${file}); pass isNew false to edit it`]);
186  if (!options.isNew && !existing) throw new RegistryInvalid(slug, [`the item ${slug} has no file yet; pass isNew to create it`]);
187
188  const compiled = await checkRegistryItem(env, slug, raw, kinds);
189  if (!compiled.ok) throw new RegistryInvalid(slug, compiled.errors);
190  const isDraft = (text: string) => matter(text, {}).data.draft === true;
191  decideFileWrite({ actor, noun: kinds[kind]?.singular ?? "registry item", incomingDraft: compiled.item.status === "draft", priorRaw: existing?.content ?? null, isDraft });
192
193  const tag = actor.kind === "operator" ? ` [operator:${actor.id}]` : actor.kind === "carrel" ? ` [carrel:${actor.changeId}]` : "";
194  // After the gate, so a file that fails validation is still refused even when it is unchanged.
195  const written = await commitUnlessUnchanged(env, {
196    existing,
197    raw,
198    rowIsCurrent: () => rowIsCurrent(env, kind, id, compiled.sourceBlobSha),
199    commit: () =>
200      commitFiles(env, {
201        expectedHeadSha: options.expectedHeadSha,
202        expectedBlobs: blobGuard(file, options.expectedBlobSha),
203        message: `${options.isNew ? "Add" : "Update"} ${kinds[kind]?.singular ?? "registry item"}: ${compiled.item.name}${tag}`,
204        changes: [{ path: file, content: raw }],
205      }),
206  });
207  if (written.action === "noop") {
208    return { slug, commitSha: written.commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: true, created: false, purged: null, note: UNCHANGED_NOTE };
209  }
210  const { commitSha } = written;
211  const blobSha = written.blobShas[file];
212  if (blobSha && blobSha !== compiled.sourceBlobSha) {
213    throw new Error(
214      `The registry item "${slug}" WAS committed as ${commitSha}, but the committed bytes (${blobSha}) are not ` +
215        `the bytes compiled (${compiled.sourceBlobSha}), so its row was not updated. Run sync_registry to ` +
216        `rebuild the row from the repository.`,
217    );
218  }
219
220  let purged: PurgeOutcome = false;
221  await convergeWithRetry({
222    write: async () => {
223      await writeRegistryRow(env, compiled);
224      // The protocols that print this primer carry its facts in their record, so they are recompiled from their files.
225      if (kind === "primer" || kind === "strain" || kind === "reagent" || kind === "equipment") await refreshProceduresNaming(env, [id], [kind]);
226      purged = await purgeRegistry(`save registry item ${slug}`);
227    },
228    // Nothing kept in KV for an item: the thrown error names the commit and the repair (sync_registry), and the
229    // next ship's sync converges the rows from the repository anyway.
230    recordDivergence: async () => undefined,
231    slug,
232    commitSha,
233  });
234
235  return { slug, commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: written.action === "repair", created: options.isNew, purged };
236}