app/lib/pages/compile.mjs

Functions

Imports

It imports app/lib/content-pages.mjs, app/lib/content/hashes.mjs, app/lib/identity.generated.mjs, app/lib/pages/invariants.mjs, app/lib/pages/protected.mjs, app/lib/phages/compile.mjs. Imported by app/db/pages.ts, app/lib/carrel/page-handler.server.ts, app/lib/cv/compile.mjs, app/lib/dictionary/save.server.ts, app/lib/health/checks.server.ts, app/lib/operator/sync-tools.server.ts, app/lib/pages/page-json-ld.ts, app/lib/pages/save.server.ts, app/lib/phages/save.server.ts, app/routes/content-page.tsx.

Source

1// One door from a page file to everything derived from it: parse, validate, render, the markdown twin and
2// the search input. build:content, sync:content and the adapter's page save all call compilePage, so
3// a file CI passes is the file the save tool accepts and the page draws (docs/PAGES.md).
4
5import matter from "gray-matter";
6
7import { gitBlobSha } from "../content/hashes.mjs";
8import {
9  CONTENT_PAGE_SECTIONS,
10  DESCRIPTION_MAX,
11  LEGAL_BANNER,
12  LEGAL_PAGE_PATHS,
13  PAGE_FILE_PATHS,
14  SEO_TITLE_MAX,
15  contentPageFile,
16  contentPageMarkdownBody,
17  contentPageSearchInputs,
18  markdownTableFacts,
19} from "../content-pages.mjs";
20import { IDENTITY } from "../identity.generated.mjs";
21import { expandPhagePage, expandPhageTokens } from "../phages/compile.mjs";
22import { pageInvariantErrors } from "./invariants.mjs";
23import { protectedPageReason } from "./protected.mjs";
24
25export const PAGES_DIR = "content/pages";
26
27/** The identity facts a page may state by token ({{identity.role}}): text, from the CV (app/lib/identity.mjs). */
28export const IDENTITY_TOKEN_KEYS = /** @type {const} */ ([
29  "name", "degree", "discipline", "disciplineLower", "rank", "role", "adminTitle", "jobTitle", "cvTitle", "department", "departmentSubject", "affiliation",
30]);
31const IDENTITY_TOKEN = /\{\{\s*identity\.([A-Za-z]+)\s*\}\}/g;
32
33/**
34 * Every token a page can carry, filled: the owner's identity from the CV, then the phage facts from the rows. A
35 * title, a role or a headship a page states is read, never typed, so a CV change updates the page. A token that
36 * names nothing is an error, never an empty string.
37 *
38 * @param {string} text
39 * @param {import("../phages/compile.mjs").Phage[] | undefined} phages
40 * @returns {{ ok: true, text: string } | { ok: false, errors: string[] }}
41 */
42export function expandPageTokens(text, phages) {
43  /** @type {string[]} */
44  const errors = [];
45  const withIdentity = text.replace(IDENTITY_TOKEN, (whole, key) => {
46    if (!(/** @type {readonly string[]} */ (IDENTITY_TOKEN_KEYS).includes(key))) {
47      errors.push(`${whole} is not an identity fact (${IDENTITY_TOKEN_KEYS.join(", ")})`);
48      return whole;
49    }
50    return String(/** @type {Record<string, unknown>} */ (IDENTITY)[key]);
51  });
52  if (errors.length > 0) return { ok: false, errors };
53  return expandPhageTokens(withIdentity, phages);
54}
55
56/** The repository path of a page's file, from its file key (`research-phages`). */
57export function pageSourcePath(/** @type {string} */ slug) {
58  return `${PAGES_DIR}/${slug}.md`;
59}
60
61/** The file key of a registered path: `/research/phages` is `research-phages`. */
62export function pageSlug(/** @type {string} */ path) {
63  return contentPageFile(path).slice(0, -3);
64}
65
66/** The registered path a file key stands for, or undefined when no page file may have that key. */
67export function pagePathForSlug(/** @type {string} */ slug) {
68  return PAGE_FILE_PATHS.find((path) => pageSlug(path) === slug);
69}
70
71/** The schema.org types a page may declare; app/routes/content-page.tsx shapes a node for each. */
72const SCHEMA_TYPES = ["SoftwareApplication", "SoftwareSourceCode", "WebSite", "WebPage", "Dataset"];
73
74/**
75 * @typedef {{
76 *   schemaType?: string,
77 *   productUrl?: string,
78 *   codeRepository?: string,
79 *   applicationCategory?: string,
80 *   license?: string,
81 *   programmingLanguage?: string,
82 *   runtimePlatform?: string,
83 *   spatialCoverage?: string,
84 *   dataset?: { variableMeasured: string[], temporalCoverage: string | null, rows: number },
85 * }} PageSchema
86 *
87 * What only a legal page states (LEGAL_PAGE_PATHS): `banner` is the one word "Draft", `lastUpdated` an ISO date.
88 * @typedef {{ banner?: string, lastUpdated?: string }} PageLegal
89 *
90 * @typedef {{ depth: number, id: string, text: string }} TocEntry
91 *
92 * The page as the routes, the search index and the gates read it. `markdown` is the body without front
93 * matter; the twin a machine reads is built from it (contentPageMarkdownBody).
94 * @typedef {{
95 *   path: string, title: string, seoTitle: string, description: string, html: string,
96 *   markdown: string, toc: TocEntry[],
97 * } & PageSchema & PageLegal} CompiledPage
98 *
99 * What D1's `record` column holds: the page less its markdown, with its file key and draft flag.
100 * @typedef {Omit<CompiledPage, "markdown"> & { slug: string, draft: boolean }} PageRecord
101 */
102
103/**
104 * The structured-data facts a page's front matter states, each only when it is written there: the JSON-LD
105 * says what the page holds and never fills a gap. A Dataset also carries what its first table says.
106 *
107 * @param {Record<string, unknown>} fm
108 * @param {string} markdown
109 * @param {string[]} errors
110 * @returns {PageSchema}
111 */
112function pageSchema(fm, markdown, errors) {
113  const text = (/** @type {unknown} */ value) => (value == null || value === "" ? "" : String(value));
114  const schemaType = text(fm.schema_type);
115  if (schemaType && !SCHEMA_TYPES.includes(schemaType)) {
116    errors.push(`schema_type "${schemaType}" is not one of ${SCHEMA_TYPES.join(", ")}`);
117  }
118  const codeRepository = text(fm.code_repository);
119  if (codeRepository && !["SoftwareApplication", "SoftwareSourceCode"].includes(schemaType)) {
120    errors.push("code_repository is only stated for software (schema_type SoftwareApplication or SoftwareSourceCode)");
121  }
122  const dataset = schemaType === "Dataset" ? markdownTableFacts(markdown) : null;
123  if (schemaType === "Dataset" && !dataset) {
124    errors.push("schema_type is Dataset but the page has no markdown table for it to describe");
125  }
126  /** @type {Array<[Exclude<keyof PageSchema, "dataset">, string]>} */
127  const fields = [
128    ["schemaType", schemaType],
129    ["productUrl", text(fm.product_url)],
130    ["codeRepository", codeRepository],
131    ["applicationCategory", text(fm.application_category)],
132    ["license", text(fm.license)],
133    ["programmingLanguage", text(fm.programming_language)],
134    ["runtimePlatform", text(fm.runtime_platform)],
135    ["spatialCoverage", text(fm.spatial_coverage)],
136  ];
137  /** @type {PageSchema} */
138  const schema = dataset ? { dataset } : {};
139  for (const [key, value] of fields) if (value !== "") schema[key] = value;
140  return schema;
141}
142
143/**
144 * The banner and the last-updated date a legal page states, each only when it is written there. The banner is
145 * the single word "Draft" and nothing else; the date is a real calendar day. Neither belongs on any other page.
146 * YAML reads an unquoted `2026-10-06` as a Date and rolls an impossible `2026-02-30` over to March, so the
147 * day is read from the front matter's own text, the way it was written, and the parsed value only when the
148 * line cannot be found.
149 *
150 * @param {Record<string, unknown>} fm
151 * @param {string} frontMatter the front matter's text, as written
152 * @param {string} path
153 * @param {string[]} errors
154 * @returns {PageLegal}
155 */
156function pageLegal(fm, frontMatter, path, errors) {
157  /** @type {PageLegal} */
158  const legal = {};
159  const banner = fm.banner == null ? "" : String(fm.banner).trim();
160  if (banner !== "") {
161    if (banner !== LEGAL_BANNER) errors.push(`banner is ${JSON.stringify(banner)}; the one word it may say is "${LEGAL_BANNER}"`);
162    else legal.banner = banner;
163  }
164  const written = /^last_updated:[ \t]*["']?([^"'#\r\n]*?)["']?[ \t]*(?:#.*)?$/m.exec(frontMatter)?.[1];
165  const raw = written ?? fm.last_updated;
166  if (raw != null && raw !== "") {
167    const day = raw instanceof Date ? raw.toISOString().slice(0, 10) : String(raw).trim();
168    const real = /^\d{4}-\d{2}-\d{2}$/.test(day) && new Date(`${day}T00:00:00.000Z`).toISOString().slice(0, 10) === day;
169    if (real) legal.lastUpdated = day;
170    else errors.push(`last_updated is ${JSON.stringify(day)}; it is a real day written YYYY-MM-DD`);
171  }
172  if ((legal.banner || legal.lastUpdated || fm.banner != null || fm.last_updated != null) && !LEGAL_PAGE_PATHS.includes(path)) {
173    errors.push(`banner and last_updated belong to the legal pages (${LEGAL_PAGE_PATHS.join(", ")}), not to ${path}`);
174  }
175  return legal;
176}
177
178/**
179 * The page's dictionary entry (docs/DICTIONARY.md), when it has a published one, leads its twin and its search
180 * record, so `entry` is the caller's to give: the build reads it from content/dictionary, the Worker from D1.
181 *
182 * @param {{
183 *   slug: string,
184 *   raw: string,
185 *   pipeline: { renderBody: (input: { file: string, body: string, resolveImage: (src: string) => Promise<never> }) => Promise<{ html: string, toc: TocEntry[], blockedUrls: Array<{ url: string }> }>,
186 *               findWideDashes: (text: string) => Array<{ line: number, column: number, char: string, excerpt: string }> },
187 *   sourcePath?: string,
188 *   generated?: boolean,
189 *   entry?: import("../dictionary-entries.mjs").DictionaryEntry,
190 *   phages?: import("../phages/compile.mjs").Phage[],
191 * }} input
192 * @returns {Promise<
193 *   | { ok: false, errors: string[] }
194 *   | { ok: true, errors: [], page: CompiledPage, record: PageRecord, draft: boolean, markdown: string,
195 *       searchInput: ReturnType<typeof contentPageSearchInputs>[number], sourcePath: string, sourceBlobSha: string }
196 * >}
197 */
198export async function compilePage({ slug, raw, pipeline, sourcePath, generated = false, entry, phages }) {
199  const expectedPath = pagePathForSlug(slug);
200  if (!expectedPath) {
201    return {
202      ok: false,
203      errors: [
204        `"${slug}" is not a registered page. A page's address is structure: add the path to CONTENT_PAGE_PATHS ` +
205          "(app/lib/content-pages.mjs) in code, and a save can then write its file; a save cannot create a new path",
206      ],
207    };
208  }
209  const file = sourcePath ?? pageSourcePath(slug);
210  /** @type {string[]} */
211  const errors = (generated ? [] : pipeline.findWideDashes(raw)).map((hit) => `line ${hit.line}, column ${hit.column}: wide dash ${hit.char}; house style uses commas, periods, parentheses or colons ("${hit.excerpt}")`);
212
213  let parsed;
214  try {
215    parsed = matter(raw);
216  } catch (error) {
217    return { ok: false, errors: [...errors, `the front matter is not valid YAML: ${error instanceof Error ? error.message : String(error)}`] };
218  }
219  const fm = /** @type {Record<string, unknown>} */ (parsed.data);
220  // A phage fact a page states (the count, the span, a genome's size) is read from the phage rows by a token,
221  // never typed, so the title and description fill the same way the body does.
222  /** @param {unknown} value @returns {string} */
223  const filled = (value) => {
224    const result = expandPageTokens(String(value ?? ""), phages);
225    if (!result.ok) {
226      errors.push(...result.errors);
227      return String(value ?? "");
228    }
229    return result.text;
230  };
231  const page = {
232    path: String(fm.path ?? ""),
233    title: filled(fm.title),
234    seoTitle: filled(fm.seo_title),
235    description: filled(fm.description),
236  };
237  if (page.path !== expectedPath) {
238    errors.push(
239      `path is "${page.path}", but ${contentPageFile(expectedPath)} is the file for ${expectedPath}: every page ` +
240        "must be listed in CONTENT_PAGE_PATHS (app/lib/content-pages.mjs) and its file must say so",
241    );
242  }
243  for (const [field, value] of [["title", page.title], ["seo_title", page.seoTitle], ["description", page.description]]) {
244    if (!value) errors.push(`${field} is required`);
245  }
246  if (page.seoTitle.length > SEO_TITLE_MAX) {
247    errors.push(`seo_title is ${page.seoTitle.length} characters (at most ${SEO_TITLE_MAX}); Google would clip it`);
248  }
249  if (page.description.length > DESCRIPTION_MAX) {
250    errors.push(`description is ${page.description.length} characters (at most ${DESCRIPTION_MAX}); Google would clip it`);
251  }
252  if (fm.draft !== undefined && typeof fm.draft !== "boolean") {
253    errors.push(`draft is ${JSON.stringify(fm.draft)}; it is true or false`);
254  }
255  // The phage page's table and sections are drawn from the phage rows where the file's markers say
256  // (docs/PHAGES.md), before anything reads the body, so its HTML, twin, dataset facts and search record all come
257  // from one text. Every other page passes through unchanged.
258  const drawn = expandPhagePage(page.path, parsed.content, phages);
259  if (!drawn.ok) errors.push(...drawn.errors);
260  const drawnBody = drawn.ok ? expandPageTokens(drawn.markdown, phages) : /** @type {const} */ ({ ok: true, text: parsed.content });
261  if (!drawnBody.ok) errors.push(...drawnBody.errors);
262  const body = drawnBody.ok ? drawnBody.text : parsed.content;
263  const schema = pageSchema(fm, body, errors);
264  const legal = pageLegal(fm, parsed.matter, page.path, errors);
265  const keep = protectedPageReason(page.path);
266  if (keep && fm.draft === true) errors.push(`draft is true, but ${keep}`);
267  if (errors.length > 0) return { ok: false, errors };
268
269  let rendered;
270  try {
271    rendered = await pipeline.renderBody({
272      file,
273      body,
274      resolveImage: async (src) => {
275        throw new Error(`the page references an image ("${src}") and these pages have no image pipeline`);
276      },
277    });
278  } catch (error) {
279    return { ok: false, errors: [error instanceof Error ? error.message : String(error)] };
280  }
281  if (rendered.blockedUrls.length > 0) {
282    errors.push(`links the URL allowlist refused: ${rendered.blockedUrls.map((b) => b.url).join(", ")}`);
283  }
284  for (const section of CONTENT_PAGE_SECTIONS) {
285    const [sectionPath, id] = section.split("#");
286    if (sectionPath === page.path && !rendered.toc.some((heading) => heading.id === id)) {
287      errors.push(`no heading has the id "${id}", which CONTENT_PAGE_SECTIONS (app/lib/content-pages.mjs) lists for the header menu`);
288    }
289  }
290
291  /** @type {CompiledPage} */
292  const compiled = { ...page, html: rendered.html, markdown: body, toc: rendered.toc, ...schema, ...legal };
293  errors.push(...pageInvariantErrors(compiled, { phages }));
294  if (errors.length > 0) return { ok: false, errors };
295
296  const draft = fm.draft === true;
297  const { markdown: _body, ...rest } = compiled;
298  const [searchInput] = contentPageSearchInputs([compiled], () => entry);
299  if (!searchInput) throw new Error(`no search input was made for ${page.path}`);
300  return {
301    ok: true,
302    errors: [],
303    page: compiled,
304    record: { slug, draft, ...rest },
305    draft,
306    markdown: contentPageMarkdownBody(compiled, entry),
307    searchInput,
308    sourcePath: file,
309    sourceBlobSha: await gitBlobSha(raw),
310  };
311}