app/lib/llms/save.server.ts

Functions

Imports

It imports app/db/pages.ts, app/db/procedures.ts, app/db/publications.ts, app/db/settings.ts, app/lib/cache-purge.server.ts, app/lib/carrel/errors.server.ts, app/lib/content-pages.mjs, app/lib/content/hashes.mjs, app/lib/content/load-pipeline.server.ts, app/lib/editor/converge.mjs, app/lib/editor/github.server.ts, app/lib/editor/publish-policy.mjs, app/lib/editor/write-path.server.ts, app/lib/llms/validate.mjs, app/lib/publications/paths.mjs, app/lib/seo.ts. Imported by app/lib/carrel/document-handler.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1// The one llms.txt save, called by the site-api adapter for Carrel (app/lib/carrel/document-handler.server.ts),
2// and the reads the operator's get_llms tool and the llms-drift check answer (docs/LLMS.md). The document is
3// content/llms.txt, the source; /llms.txt is served from the `settings` row derived from it. A save judges the
4// file with the rules CI runs (validate.mjs), refuses it with every message if it fails, commits it to the
5// repository (unless it is byte-identical, commitUnlessUnchanged), then writes the row and purges /llms.txt,
6// so the change is live on the next request with no build or deploy. The commit is never reverted when the D1
7// write fails; the error says the commit landed and how to repair the row. A document has no
8// draft state: it is always the published file, and a save edits it live.
9
10import { listPublishedPageTitles } from "~/db/pages";
11import { listPublishedPublicationTwins } from "~/db/publications";
12import { listPublishedProcedures } from "~/db/procedures";
13import { getSetting, setSetting } from "~/db/settings";
14import { ContentInvalid } from "~/lib/carrel/errors.server";
15import { purgeLlms, type PurgeOutcome } from "~/lib/cache-purge.server";
16import { gitBlobSha } from "~/lib/content/hashes.mjs";
17import { loadPipeline } from "~/lib/content/load-pipeline.server";
18import { draftContentPagePaths, publishedContentPagePaths } from "~/lib/content-pages.mjs";
19import { convergeWithRetry } from "~/lib/editor/converge.mjs";
20import { blobGuard, commitFiles, readFile } from "~/lib/editor/github.server";
21import { decideFileWrite, type Actor } from "~/lib/editor/publish-policy.mjs";
22import { commitUnlessUnchanged, UNCHANGED_NOTE } from "~/lib/editor/write-path.server";
23import { paperMarkdownPath } from "~/lib/publications/paths.mjs";
24import { SITE_ORIGIN } from "~/lib/seo";
25
26import { LLMS_PATH, LLMS_SETTING_KEY, llmsChecks, llmsErrors } from "./validate.mjs";
27
28type LlmsEnv = Env & { GITHUB_TOKEN?: string };
29
30/** A file the validator refused: 422, with every message, so the caller can fix its own edit. */
31class LlmsInvalid extends ContentInvalid {
32  constructor(errors: string[]) {
33    super(`${LLMS_PATH} was not saved: it fails ${errors.length} check(s). Nothing was committed.`, errors);
34    this.name = "LlmsInvalid";
35  }
36}
37
38/**
39 * Every rule against what the site holds now: the registered pages and the published procedures (the
40 * protocols are pages, drawn from D1), and the papers the publications table serves.
41 */
42async function judgeLlms(env: LlmsEnv, raw: string) {
43  const [{ findWideDashes }, procedures, twins, pageTitles] = await Promise.all([
44    loadPipeline(),
45    listPublishedProcedures(env),
46    listPublishedPublicationTwins(env),
47    // The page rows the route serves from: a page published through Carrel can have its line saved at once.
48    listPublishedPageTitles(env),
49  ]);
50  const published = [...pageTitles.keys()];
51  return llmsChecks(raw, {
52    pagePaths: [...publishedContentPagePaths(published), ...procedures.map((p) => p.path)],
53    optionalPagePaths: draftContentPagePaths(published),
54    origin: SITE_ORIGIN,
55    findWideDashes,
56    twinUrls: twins.map((t) => paperMarkdownPath(t.slug)),
57  });
58}
59
60/**
61 * The one compile door: a save and sync_llms both read the file through it. The row carries the bytes, so
62 * "compiled" is the bytes judged, with the blob sha the drift check compares.
63 */
64export async function compileLlms(env: LlmsEnv, raw: string) {
65  const errors = llmsErrors(await judgeLlms(env, raw));
66  if (errors.length > 0) return { ok: false as const, errors };
67  return { ok: true as const, errors: [] as string[], raw, sourceBlobSha: await gitBlobSha(raw) };
68}
69
70/** get_llms: the file, the head it was read at, the rules it fails now, and whether the row serves it. */
71export async function readLlms(env: LlmsEnv) {
72  const file = await readFile(env, LLMS_PATH);
73  if (!file) return null;
74  const row = await getSetting(env, LLMS_SETTING_KEY);
75  return {
76    path: LLMS_PATH,
77    raw: file.content,
78    bytes: new TextEncoder().encode(file.content).length,
79    errors: llmsErrors(await judgeLlms(env, file.content)),
80    servedFromRepository: row === file.content,
81  };
82}
83
84/** The one write door for the row /llms.txt is served from: a save and sync_llms both end here, read back. */
85export async function writeLlmsRow(env: LlmsEnv, raw: string) {
86  await setSetting(env, LLMS_SETTING_KEY, raw);
87  const stored = await getSetting(env, LLMS_SETTING_KEY);
88  if (stored !== raw) {
89    throw new Error(
90      `The llms.txt row was written but the row read back is not the file (${stored === null ? "missing" : `${stored.length} characters`}, ` +
91        `expected ${raw.length}). Run the llms sync (sync_llms) to rebuild it from the repository.`,
92    );
93  }
94}
95
96/** True when the served row is exactly this file. */
97async function rowIsCurrent(env: LlmsEnv, raw: string) {
98  return (await getSetting(env, LLMS_SETTING_KEY)) === raw;
99}
100
101/** The llms.txt save. */
102export async function saveLlms(
103  env: LlmsEnv,
104  options: { raw: string; expectedHeadSha?: string | null;
105    /** The blob sha of this file as Carrel loaded it; undefined means no check. */
106    expectedBlobSha?: string | null; actor: Actor },
107) {
108  const { raw, actor } = options;
109  const existing = await readFile(env, LLMS_PATH);
110  if (!existing) {
111    throw new LlmsInvalid([`${LLMS_PATH} is missing from the repository, so there is nothing to edit. A save does not create it.`]);
112  }
113
114  const compiled = await compileLlms(env, raw);
115  if (!compiled.ok) throw new LlmsInvalid(compiled.errors);
116  decideFileWrite({ actor, noun: "llms.txt", incomingDraft: false, priorRaw: existing.content, isDraft: () => false });
117
118  const tag = actor.kind === "operator" ? ` [operator:${actor.id}]` : actor.kind === "carrel" ? ` [carrel:${actor.changeId}]` : "";
119  // After the gate, so a file that fails validation is still refused even when it is unchanged.
120  const written = await commitUnlessUnchanged(env, {
121    existing,
122    raw,
123    rowIsCurrent: () => rowIsCurrent(env, raw),
124    commit: () =>
125      commitFiles(env, {
126        expectedHeadSha: options.expectedHeadSha,
127        expectedBlobs: blobGuard(LLMS_PATH, options.expectedBlobSha),
128        message: `Update llms.txt${tag}`,
129        changes: [{ path: LLMS_PATH, content: raw }],
130      }),
131  });
132  if (written.action === "noop") {
133    return { commitSha: written.commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: true, note: UNCHANGED_NOTE, purged: null as PurgeOutcome };
134  }
135  const { commitSha, blobShas } = written;
136  if (written.action === "commit" && blobShas[LLMS_PATH] && blobShas[LLMS_PATH] !== compiled.sourceBlobSha) {
137    throw new Error(
138      `${LLMS_PATH} WAS committed as ${commitSha}, but the committed bytes (${blobShas[LLMS_PATH]}) are not ` +
139        `the bytes judged (${compiled.sourceBlobSha}), so the row was not updated. Run sync_llms to rebuild it ` +
140        "from the repository.",
141    );
142  }
143
144  let purged: PurgeOutcome = false;
145  await convergeWithRetry({
146    write: async () => {
147      await writeLlmsRow(env, raw);
148      purged = await purgeLlms("save llms.txt");
149    },
150    // Nothing kept in KV for a document: the thrown error names the commit and the repair (sync_llms), and
151    // the watchdog's llms-drift check repairs the row from the repository anyway.
152    recordDivergence: async () => undefined,
153    slug: "llms.txt",
154    commitSha,
155  });
156
157  return { commitSha, sourceBlobSha: compiled.sourceBlobSha, unchanged: written.action !== "commit", purged };
158}