app/lib/llms/validate.mjs

Functions

Imports

It imports no other module of this repo. Imported by app/lib/cache-purge.server.ts, app/lib/carrel/document-handler.server.ts, app/lib/health/checks.server.ts, app/lib/llms/save.server.ts, app/routes/llms.ts.

Source

1// The one judgement of content/llms.txt, shared by check:machine-readable (scripts/machine-readable/llms.mjs)
2// and the save (app/lib/llms/save.server.ts), so a file CI passes is the file a save accepts
3// (docs/LLMS.md). Pure: the caller supplies what the file is judged against.
4
5/** The repository file, which is the source. */
6export const LLMS_PATH = "content/llms.txt";
7
8/** The `settings` row /llms.txt is served from: derived from the file, never edited by hand. */
9export const LLMS_SETTING_KEY = "llms.txt";
10
11/** Every /llms.txt response carries it, and a save purges it (app/routes/llms.ts). */
12export const LLMS_CACHE_TAG = "llms";
13
14/** The file's whole size, far above today's and far below anything a crawler would read to the end. */
15export const LLMS_MAX_BYTES = 65_536;
16
17/** A file shorter than this is the retired 247-byte virology seed, not the real document. */
18const LLMS_MIN_BYTES = 200;
19
20/**
21 * What the file must document, as the exact text an agent is told. Each is a URL pattern or a header an
22 * agent acts on, so a rewrite that drops one has taken a capability away from every agent that read it.
23 */
24export const LLMS_REQUIRED_MENTIONS = [
25  "/llms-full.txt",
26  "/writing/rss.xml",
27  "/writing/feed.json",
28  "/writing/<slug>.md",
29  "Accept: text/markdown",
30  "/research/publications/<doi-slug>/",
31  "/research/publications/<doi-slug>.md",
32  "/research/publications.bib",
33  "/research/publications.ris",
34  "/research/publications.json",
35  "/colophon",
36];
37
38/*
39 * The llmstxt.org shape: a page is a list item whose link carries an absolute URL, `- [Title](https://host/path): note`.
40 * The collection hub /research/publications and the lab registry under /research/lab are routes and not content pages, so
41 * they are not read as ones.
42 */
43const LISTED_PAGE =
44  /^- \[[^\]]+\]\(https?:\/\/[^/\s)]+(?!\/research\/(?:publications|lab)\b)(\/(?:research|teaching|software|cv|privacy|terms|accessibility)(?:\/[a-z0-9-]+)*)\)/gm;
45const LISTED_TWIN = /^- \[[^\]]+\]\(https?:\/\/[^/\s)]+(\/research\/publications\/[a-z0-9-]+\.md)\)/gm;
46
47/** The page paths the file lists, one per list link. @param {string} text */
48function listedPagePaths(text) {
49  return new Set([...text.matchAll(LISTED_PAGE)].map((m) => m[1] ?? ""));
50}
51
52/** The paper twin URLs the file lists, as paths, one per list link. @param {string} text */
53export function listedPaperTwins(text) {
54  return new Set([...text.matchAll(LISTED_TWIN)].map((m) => m[1] ?? ""));
55}
56
57/**
58 * Listed twins no paper produces: the one rule check:machine-readable's twin gate and a save share. A paper
59 * added later is not in the list until it is written there, so the reverse is not a rule.
60 *
61 * @param {string} text
62 * @param {Iterable<string>} twinUrls the `/research/publications/<slug>.md` URLs the corpus produces
63 */
64export function overAdvertisedTwins(text, twinUrls) {
65  const known = new Set(twinUrls);
66  return [...listedPaperTwins(text)].filter((url) => !known.has(url));
67}
68
69/**
70 * @typedef {{ name: string, ok: boolean, detail?: string }} LlmsCheck
71 *
72 * @param {string} text the whole file
73 * @param {{
74 *   pagePaths: readonly string[],
75 *   origin: string,
76 *   findWideDashes: (text: string) => Array<{ line: number, char: string, excerpt: string }>,
77 *   twinUrls?: Iterable<string>,
78 *   optionalPagePaths?: readonly string[],
79 * }} options pagePaths is every Research, Teaching and Software path a reader can open, the procedures'
80 *   included, and the file must list each; optionalPagePaths are registered pages that are drafts, which the
81 *   file may list (so its line can land before the page is published) but need not; twinUrls is every paper
82 *   twin the corpus produces, left out where the corpus is judged elsewhere
83 * @returns {LlmsCheck[]} every rule, passed or failed, in the order they read
84 */
85export function llmsChecks(text, { pagePaths, origin, findWideDashes, twinUrls, optionalPagePaths = [] }) {
86  /** @type {LlmsCheck[]} */
87  const checks = [];
88  /** @param {string} name @param {boolean} ok @param {string} [detail] */
89  const check = (name, ok, detail) => checks.push({ name, ok, detail: ok ? undefined : detail });
90
91  const bytes = new TextEncoder().encode(text).length;
92  check("llms.txt is not empty", bytes > 0, "the file is empty");
93  check(
94    "llms.txt is long enough to be the real document",
95    bytes > LLMS_MIN_BYTES,
96    `Only ${bytes} bytes. The retired virology seed was 247 bytes; the real document is far longer, so a ` +
97      "short file here is the stale copy.",
98  );
99  check(
100    `llms.txt is at most ${LLMS_MAX_BYTES} bytes`,
101    bytes <= LLMS_MAX_BYTES,
102    `${bytes} bytes. An agent reads this file first and whole; a file this long is no longer a map.`,
103  );
104  check(
105    "llms.txt is LF-only",
106    !text.includes("\r"),
107    "It is pinned to LF in .gitattributes. CR here would sync CRLF into D1.",
108  );
109  check("llms.txt ends with a newline", text.endsWith("\n"), "the last byte is not a newline");
110  check(
111    "llms.txt opens with its H1 title",
112    text.startsWith("# dustinedwards.info\n"),
113    'the first line must be "# dustinedwards.info", as agents read the first heading as the site name',
114  );
115
116  // What PageSpeed's agent-discoverability audit reads: the summary, and links it can follow.
117  check(
118    "llms.txt has a blockquote summary under its title",
119    /^# [^\n]+\n\n> \S/.test(text),
120    'the llmstxt.org format puts a "> summary" line after the H1 and one blank line',
121  );
122  const relative = [...text.matchAll(/\]\((\/[^)\s]*)\)/g)].map((m) => m[1]);
123  check(
124    "every markdown link in llms.txt is absolute",
125    relative.length === 0,
126    `relative link(s): ${relative.slice(0, 5).join(", ")}. A crawler reading the file alone has no origin to resolve them against.`,
127  );
128
129  const dashes = findWideDashes(text);
130  check(
131    "llms.txt has no wide dash",
132    dashes.length === 0,
133    dashes
134      .slice(0, 5)
135      .map((d) => `line ${d.line} ${d.char}: ${d.excerpt}`)
136      .join("; ") + "; use a comma, period, parentheses or colon",
137  );
138
139  const missing = LLMS_REQUIRED_MENTIONS.filter((mention) => !text.includes(mention));
140  check(
141    `llms.txt documents the ${LLMS_REQUIRED_MENTIONS.length} URL patterns and headers an agent acts on`,
142    missing.length === 0,
143    `absent: ${missing.join(", ")}`,
144  );
145
146  // Both ways, derived from the list the routes and the sitemap read: a page missing here is a page an
147  // agent is not told about, twin or not.
148  const listed = listedPagePaths(text);
149  const unlisted = pagePaths.filter((path) => !listed.has(path));
150  check(
151    `llms.txt lists every Research, Teaching and Software page (${pagePaths.length})`,
152    unlisted.length === 0,
153    `absent from ${LLMS_PATH}: ${unlisted.join(", ")}`,
154  );
155  const known = new Set([...pagePaths, ...optionalPagePaths]);
156  const stray = [...listed].filter((path) => !known.has(path));
157  check(
158    "llms.txt lists no Research, Teaching or Software page that is neither in CONTENT_PAGE_PATHS nor a procedure",
159    stray.length === 0,
160    `listed but not a page: ${stray.join(", ")}`,
161  );
162
163  if (twinUrls) {
164    const over = overAdvertisedTwins(text, twinUrls);
165    check(
166      "llms.txt lists no paper twin the corpus does not produce",
167      over.length === 0,
168      `advertised with no paper: ${over.join(", ")}`,
169    );
170  }
171
172  // A tracked text file cannot import SITE_ORIGIN, so this binds its contact line to it. At DNS cutover
173  // it goes red until llms.txt follows, which is the point.
174  const contact = (text.match(/## Contact\s*\n\s*\n(?:- \[[^\]]*\]\()?(https?:\/\/[^\s)]+)/) ?? [])[1] ?? "";
175  check(
176    "llms.txt has a contact URL to compare",
177    contact.startsWith("https://"),
178    `parsed ${JSON.stringify(contact)} from the Contact section`,
179  );
180  check(
181    "the llms.txt contact URL is SITE_ORIGIN",
182    contact === origin,
183    `llms.txt says ${contact} and SITE_ORIGIN is ${origin}. The file crawlers read points somewhere this ` +
184      "site is not served from.",
185  );
186
187  return checks;
188}
189
190/**
191 * The messages of every failed rule, or an empty list: what a save refuses with.
192 *
193 * @param {LlmsCheck[]} checks
194 */
195export function llmsErrors(checks) {
196  return checks.filter((c) => !c.ok).map((c) => `${c.name}: ${c.detail}`);
197}