app/lib/phages/compile.mjs

Functions

Imports

It imports app/lib/content/hashes.mjs, app/lib/cv/view.mjs, app/lib/phage-table.mjs. Imported by app/db/phages.ts, app/kb/procedures/proof.server.ts, app/kb/registry/catalog.mjs, app/lib/carrel/phage-handler.server.ts, app/lib/health/checks.server.ts, app/lib/operator/sync-tools.server.ts, app/lib/pages/compile.mjs, app/lib/pages/invariants.mjs, app/lib/pages/save.server.ts, app/lib/phages/catalog.mjs, app/lib/phages/save.server.ts, app/routes/phages.tsx.

Source

1// The one door from a phage file to the row the phage table on /research/phages is drawn from (docs/PHAGES.md):
2// parse, validate, hash, and the markdown a row becomes. build:content, sync:content, the gates and the adapter's
3// phage save all call compilePhage, and the page compile (app/lib/pages/compile.mjs) calls expandPhagePage with
4// the rows, so a file CI passes is the file the save accepts and the page draws. Pure and dependency-light, so
5// the Worker and the Node scripts import the same module.
6//
7// A phage is ONE record: the row's fields (name, year, host, county, notes) and the PhagesDB link fields that
8// used to be a second table in code (PHAGESDB_RECORDS). The validator REFUSES any other field, so nothing a
9// visitor cannot already read in the table can enter the repository or D1: no sample, location beyond the county
10// the table shows, or personal data.
11
12import matter from "gray-matter";
13
14import { gitBlobSha } from "../content/hashes.mjs";
15import { foldText } from "../cv/view.mjs";
16import { phagesDbUrl } from "../phage-table.mjs";
17
18export const PHAGES_DIR = "content/phages";
19
20/** The repository path of a phage's file, from its file key (the name in lower case). */
21export function phagePath(/** @type {string} */ slug) {
22  return `${PHAGES_DIR}/${slug}.md`;
23}
24
25/** The one page the table and the sections are drawn into. */
26export const PHAGES_PAGE_PATH = "/research/phages";
27
28/**
29 * Where the page file says the generated markdown goes. Each stands alone on its line in the page's file, and the
30 * page compile replaces it BEFORE rendering, so the HTML, the twin, the table of contents, the dataset facts and
31 * the search record all come from one text. The page's prose, its headings and the table's caption stay in the
32 * page file.
33 */
34export const TABLE_MARKER = "<!-- phages:table -->";
35export const SECTIONS_MARKER = "<!-- phages:sections -->";
36
37/**
38 * The pages, besides the table's, whose text carries {{phage...}} tokens. A phage write re-derives each of them
39 * (app/lib/phages/save.server.ts), so this list and the pages must agree: test/phages.test.mjs fails when a page
40 * file uses a token and is not here, or is here and uses none.
41 */
42export const PHAGE_FACT_PAGES = /** @type {const} */ (["/research", "/research/bacteriophages", "/research/science-education"]);
43
44/** The two hosts the table names, as the table's cell and a section's line write them. A new host is code. */
45export const HOSTS = Object.freeze({
46  smegmatis: { short: "*M. smegmatis* mc²155", long: "*Mycobacterium smegmatis* mc²155" },
47  foliorum: { short: "*M. foliorum*", long: "*Microbacterium foliorum*" },
48});
49
50/** The table's columns, in order: the header the page has always had. */
51const TABLE_HEADER = "| Phage | Year | Host | County | PhagesDB | Paper |\n|---|---|---|---|---|---|";
52
53const STATE = "Texas";
54const FIELDS = ["name", "year", "host", "county", "phagesdb", "paper", "formerly", "note", "genome_bp", "genes"];
55const REQUIRED = ["name", "year", "host", "county", "phagesdb"];
56
57/** A phage's name: letters and digits, so its lower-case form is the file key, the heading id and the old address. */
58const NAME_PATTERN = /^[A-Za-z][A-Za-z0-9]*$/;
59const SLUG_PATTERN = /^[a-z][a-z0-9]*$/;
60/** A PhagesDB record name, as the PhagesDB address spells it. */
61const PHAGESDB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
62const PAPER_PATTERN = /^[0-9a-z][0-9a-z-]*$/;
63const COUNTY_PATTERN = /^(?:Texas|[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)* County)$/;
64/** A note is a plain sentence: no character that markdown would read as a link, a table cell or emphasis. */
65const NOTE_PATTERN = /^[A-Z][A-Za-z0-9 ,.'()%:;-]*\.$/;
66const NOTE_MAX = 160;
67
68/**
69 * @typedef {"smegmatis" | "foliorum"} HostKey
70 * @typedef {{
71 *   name: string, year: number, host: HostKey | null, county: string | null, phagesdb: string | null,
72 *   paper: string | null, formerly: string | null, note: string | null,
73 *   genomeBp?: number | null, genes?: number | null,
74 * }} Phage
75 * @typedef {{ paper: (slug: string) => Promise<boolean> }} PhageHost
76 * `findWideDashes` is injected, as the page compile injects the pipeline's: the Worker loads it lazily.
77 * @typedef {{ findWideDashes: (text: string) => Array<{ line: number, column: number, char: string, excerpt: string }> }} PhagePipeline
78 */
79
80/** The file key of a phage: its name in lower case, which is also its heading's id on the page. */
81export function phageSlug(/** @type {string} */ name) {
82  return name.toLowerCase();
83}
84
85/** The page's own order: year, then name, folded so case and accents do not matter. */
86export function sortPhages(/** @type {Phage[]} */ phages) {
87  return [...phages].sort(
88    (a, b) => a.year - b.year || foldText(a.name).localeCompare(foldText(b.name), "en", { numeric: true }),
89  );
90}
91
92/** @param {unknown} value @param {RegExp} pattern @returns {value is string} */
93const matches = (value, pattern) => typeof value === "string" && pattern.test(value);
94
95/**
96 * Judges one file and, when it passes, builds the phage. Every message the file has is returned, never the
97 * first alone, so a writer fixes all of them at once.
98 *
99 * @param {{ slug: string, raw: string, host: PhageHost, pipeline: PhagePipeline, sourcePath?: string }} input
100 * @returns {Promise<
101 *   | { ok: false, errors: string[] }
102 *   | { ok: true, errors: [], phage: Phage, record: string, sourcePath: string, sourceBlobSha: string }
103 * >}
104 */
105export async function compilePhage({ slug, raw, host, pipeline, sourcePath }) {
106  /** @type {string[]} */
107  const errors = pipeline
108    .findWideDashes(raw)
109    .map((hit) => `line ${hit.line}, column ${hit.column}: wide dash ${hit.char}; house style uses commas, periods, parentheses or colons ("${hit.excerpt}")`);
110  if (!raw.startsWith("---")) errors.push("the file must start with a --- front matter block");
111
112  let parsed;
113  try {
114    // An options object, even an empty one: gray-matter caches by input when it is omitted.
115    parsed = matter(raw, {});
116  } catch (error) {
117    return { ok: false, errors: [...errors, `the front matter is not valid YAML: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`] };
118  }
119  const data = parsed.data;
120  if (parsed.content.trim() !== "") errors.push("the body must be empty: a phage is its front matter and nothing else");
121  for (const key of Object.keys(data)) {
122    if (!FIELDS.includes(key)) {
123      errors.push(`${key} is not a phage field; a phage carries ${FIELDS.join(", ")} and nothing else (no samples, locations beyond the county, or other personal data)`);
124    }
125  }
126  for (const key of REQUIRED) {
127    if (data[key] === undefined) errors.push(`${key} is required${key === "name" || key === "year" ? "" : " (null where there is none)"}`);
128  }
129
130  const { name, year, host: hostKey, county, phagesdb, paper, formerly, note, genome_bp: genomeBp, genes } = data;
131  if (!matches(name, NAME_PATTERN)) errors.push(`name is ${JSON.stringify(name)}; it is the phage's name: letters and digits, starting with a letter`);
132  else if (slug !== phageSlug(name)) errors.push(`name is ${name}, but the file is ${phagePath(slug)}: a phage's file is named for its name in lower case (${phageSlug(name)}.md)`);
133  if (!SLUG_PATTERN.test(slug)) errors.push(`"${slug}" is not a phage file key; the file is the name in lower case, such as acorn15.md`);
134  if (typeof year !== "number" || !Number.isInteger(year) || year < 2000 || year > 2099) {
135    errors.push(`year is ${JSON.stringify(year)}; it is the year the phage was found, four digits`);
136  }
137  if (hostKey !== undefined && hostKey !== null && !Object.hasOwn(HOSTS, String(hostKey))) {
138    errors.push(`host is ${JSON.stringify(hostKey)}; it is ${Object.keys(HOSTS).join(" or ")}, or null where no host is on record`);
139  }
140  if (county !== undefined && county !== null && !matches(county, COUNTY_PATTERN)) {
141    errors.push(`county is ${JSON.stringify(county)}; it is a Texas county written "Erath County" (or "${STATE}" where only the state is on record), or null`);
142  }
143  if (phagesdb !== undefined && phagesdb !== null) {
144    if (!matches(phagesdb, PHAGESDB_PATTERN)) {
145      errors.push(`phagesdb is ${JSON.stringify(phagesdb)}; it is the PhagesDB record's name as PhagesDB spells it, or null where this phage has no verified record`);
146    } else if (typeof name === "string" && phagesdb.toLowerCase() !== name.toLowerCase()) {
147      errors.push(`phagesdb is ${phagesdb}, which is not ${name}'s record: a record is this phage's own, and a same-named phage of another lab is not it (set null)`);
148    }
149  }
150  if (paper !== undefined && paper !== null) {
151    if (!matches(paper, PAPER_PATTERN)) errors.push(`paper is ${JSON.stringify(paper)}; it is a publication's slug, such as 10-1128-mra-01242-18, or absent`);
152    else if (!(await host.paper(paper))) errors.push(`paper ${paper} is not a publication of this site (/research/publications/${paper}/); the genome announcement is a paper the site holds`);
153  }
154  if (formerly !== undefined && formerly !== null && !matches(formerly, NAME_PATTERN)) {
155    errors.push(`formerly is ${JSON.stringify(formerly)}; it is the phage's earlier name, letters and digits, or absent`);
156  }
157  if (note !== undefined && note !== null) {
158    if (!matches(note, NOTE_PATTERN) || note.length > NOTE_MAX) {
159      errors.push(`note is ${JSON.stringify(note)}; it is one plain sentence of at most ${NOTE_MAX} characters, starting with a capital and ending with a period, with no links or markdown`);
160    }
161  }
162  // The genome announcement's own numbers, so a page states them from the record instead of typing them.
163  for (const [field, value, low, high] of /** @type {const} */ ([["genome_bp", genomeBp, 1_000, 10_000_000], ["genes", genes, 1, 5_000]])) {
164    if (value !== undefined && value !== null && (typeof value !== "number" || !Number.isInteger(value) || value < low || value > high)) {
165      errors.push(`${field} is ${JSON.stringify(value)}; it is a whole number from ${low} to ${high}, or absent`);
166    }
167  }
168  if (errors.length > 0) return { ok: false, errors };
169
170  /** @type {Phage} */
171  const phage = {
172    name: /** @type {string} */ (name),
173    year: /** @type {number} */ (year),
174    host: /** @type {HostKey | null} */ (hostKey ?? null),
175    county: /** @type {string | null} */ (county ?? null),
176    phagesdb: /** @type {string | null} */ (phagesdb ?? null),
177    paper: /** @type {string | null} */ (paper ?? null),
178    formerly: /** @type {string | null} */ (formerly ?? null),
179    note: /** @type {string | null} */ (note ?? null),
180    genomeBp: /** @type {number | null} */ (genomeBp ?? null),
181    genes: /** @type {number | null} */ (genes ?? null),
182  };
183  return {
184    ok: true,
185    errors: [],
186    phage,
187    record: JSON.stringify(phage),
188    sourcePath: sourcePath ?? phagePath(slug),
189    sourceBlobSha: await gitBlobSha(raw),
190  };
191}
192
193/**
194 * What only the whole set can say: the files are the table, so an empty set is a fault, two files cannot be one
195 * phage, and two phages cannot claim one PhagesDB record.
196 *
197 * @param {Phage[]} phages
198 * @returns {string[]}
199 */
200export function phageSetErrors(phages) {
201  /** @type {string[]} */
202  const errors = [];
203  if (phages.length === 0) errors.push(`${PHAGES_DIR} holds no phage files; the table cannot be empty`);
204  const names = new Set();
205  const records = new Map();
206  for (const { name, phagesdb } of phages) {
207    if (names.has(phageSlug(name))) errors.push(`two files are the phage ${name}`);
208    names.add(phageSlug(name));
209    if (phagesdb !== null) {
210      const key = phagesdb.toLowerCase();
211      if (records.has(key)) errors.push(`${name} and ${records.get(key)} claim the same PhagesDB record, ${phagesdb}`);
212      records.set(key, name);
213    }
214  }
215  return errors;
216}
217
218/** The place a phage was found, as a section writes it after the year: ", Hood County, Texas", ", Texas" or "". */
219function foundPlace(/** @type {Phage} */ { county }) {
220  if (county === null) return "";
221  return county === STATE ? `, ${STATE}` : `, ${county}, ${STATE}`;
222}
223
224/** One phage's row in the table, in the table's columns. */
225export function phageTableRow(/** @type {Phage} */ phage) {
226  const cells = [
227    `[${phage.name}](#${phageSlug(phage.name)})`,
228    String(phage.year),
229    phage.host ? HOSTS[phage.host].short : "",
230    phage.county ?? "",
231    phage.phagesdb ? `[PhagesDB](${phagesDbUrl(phage.phagesdb)})` : "",
232    phage.paper ? `[Paper](/research/publications/${phage.paper}/)` : "",
233  ];
234  return `| ${cells.join(" | ")} |`;
235}
236
237/** One phage's `###` section: what the row says in a sentence, and its links, as the page has always worded them. */
238export function phageSection(/** @type {Phage} */ phage) {
239  const found = `Found in ${phage.year}${foundPlace(phage)}.${phage.formerly ? ` Formerly named ${phage.formerly}.` : ""}`;
240  const first = phage.host ? `Host: ${HOSTS[phage.host].long}. ${found}` : found;
241  const links = [
242    phage.phagesdb ? `[${phage.name} on PhagesDB](${phagesDbUrl(phage.phagesdb)}).` : "",
243    phage.paper ? `[Genome announcement](/research/publications/${phage.paper}/).` : "",
244    phage.note ?? "",
245  ].filter(Boolean);
246  return [`### ${phage.name}`, first, ...(links.length > 0 ? [links.join(" ")] : [])].join("\n\n");
247}
248
249/** The table's text: its header and one row per phage, in the page's order, no trailing newline. */
250export function phageTableMarkdown(/** @type {Phage[]} */ phages) {
251  return [TABLE_HEADER, ...sortPhages(phages).map(phageTableRow)].join("\n");
252}
253
254/** The sections' text: one per phage, in the page's order, no trailing newline. */
255function phageSectionsMarkdown(/** @type {Phage[]} */ phages) {
256  return sortPhages(phages).map(phageSection).join("\n\n");
257}
258
259const countFormat = new Intl.NumberFormat("en-US");
260
261/** @param {number[]} years the years a set of phages was found in, as prose: "2017" or "2017 to 2018". */
262function yearsText(years) {
263  const low = Math.min(...years);
264  const high = Math.max(...years);
265  return low === high ? String(low) : `${low} to ${high}`;
266}
267
268/**
269 * What the whole set states, computed from the rows so a page never types it: the count, the span of years, and
270 * for each host its count and the years it was used. The page text that used to carry these ("80 phages") drifted
271 * from the records; a token reads them instead.
272 *
273 * @param {Phage[]} phages
274 */
275function phageFacts(phages) {
276  const years = phages.map((p) => p.year);
277  /** @type {Record<string, { count: number, firstYear: number, lastYear: number, years: string }>} */
278  const host = {};
279  for (const key of Object.keys(HOSTS)) {
280    const mine = phages.filter((p) => p.host === key);
281    if (mine.length > 0) {
282      const own = mine.map((p) => p.year);
283      host[key] = { count: mine.length, firstYear: Math.min(...own), lastYear: Math.max(...own), years: yearsText(own) };
284    }
285  }
286  return {
287    count: phages.length,
288    firstYear: Math.min(...years),
289    lastYear: Math.max(...years),
290    noHost: phages.filter((p) => p.host === null).length,
291    host,
292  };
293}
294
295const TOKEN = /\{\{\s*([^{}]*?)\s*\}\}/g;
296const COMMON_COUNTY = /^phages\.commonCounty\(([a-z0-9,\s]+)\)$/;
297
298/**
299 * Replaces the tokens in a page's text with values read from the phage rows. A token that names nothing is an
300 * error, never an empty string, so a renamed phage or a missing field cannot blank a sentence quietly.
301 *
302 *   {{phages.count}}  {{phages.firstYear}}  {{phages.lastYear}}
303 *   {{phages.host.smegmatis.count}}  .firstYear  .lastYear  .years   (a host key from HOSTS)
304 *   {{phage.loca.bp}}  .genes  .county  .year                          (a phage's file key)
305 *   {{phages.commonCounty(godfather,fizzles)}}                         (the county they share; an error if they differ)
306 *
307 * @param {string} text
308 * @param {Phage[] | undefined} phages
309 * @returns {{ ok: true, text: string } | { ok: false, errors: string[] }}
310 */
311export function expandPhageTokens(text, phages) {
312  if (!text.includes("{{")) return { ok: true, text };
313  if (!phages || phages.length === 0) {
314    return { ok: false, errors: ["the page uses {{phage...}} tokens and the phage rows are not loaded: run the content sync (npm run sync:content -- --remote), or sync_phages"] };
315  }
316  const facts = phageFacts(phages);
317  const bySlug = new Map(phages.map((p) => [phageSlug(p.name), p]));
318  /** @type {string[]} */
319  const errors = [];
320  /** Only a genome's size takes thousands separators; a year or a count is written plain. @param {string} token @returns {string | number | null} */
321  const read = (token) => {
322    const common = COMMON_COUNTY.exec(token);
323    if (common) {
324      const counties = new Set((common[1] ?? "").split(",").map((k) => bySlug.get(k.trim())?.county ?? null));
325      const only = [...counties];
326      return only.length === 1 && only[0] !== null && only[0] !== undefined ? only[0] : null;
327    }
328    const parts = token.split(".");
329    if (parts[0] === "phages" && parts.length === 2) {
330      const value = /** @type {Record<string, unknown>} */ (facts)[parts[1] ?? ""];
331      return typeof value === "number" ? value : null;
332    }
333    if (parts[0] === "phages" && parts[1] === "host" && parts.length === 4) {
334      const entry = facts.host[parts[2] ?? ""];
335      const value = entry ? /** @type {Record<string, unknown>} */ (entry)[parts[3] ?? ""] : undefined;
336      return typeof value === "number" || typeof value === "string" ? value : null;
337    }
338    if (parts[0] === "phage" && parts.length === 3) {
339      const phage = bySlug.get(parts[1] ?? "");
340      if (!phage) return null;
341      const value = /** @type {Record<string, number | string | null>} */ ({ bp: phage.genomeBp ?? null, genes: phage.genes ?? null, county: phage.county, year: phage.year })[parts[2] ?? ""];
342      if (value === null || value === undefined) return null;
343      return parts[2] === "bp" && typeof value === "number" ? countFormat.format(value) : value;
344    }
345    return null;
346  };
347  const out = text.replace(TOKEN, (whole, inner) => {
348    const value = read(String(inner).trim());
349    if (value === null) {
350      errors.push(`${whole} names no phage fact (check the token, the phage's file key, and that its record carries the field)`);
351      return whole;
352    }
353    return String(value);
354  });
355  return errors.length > 0 ? { ok: false, errors } : { ok: true, text: out };
356}
357
358const markerLine = (/** @type {string} */ marker) => new RegExp(`^${marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, "gm");
359
360/**
361 * The text a page body is judged and rendered as. The phage page must carry each marker once, alone on its line,
362 * and every other page must carry none: a marker left in prose would render as nothing and hide a missing table.
363 *
364 * @param {string} path the page's registered path
365 * @param {string} body the page's markdown, front matter removed
366 * @param {Phage[] | undefined} phages the rows to draw; required for the phage page
367 * @returns {{ ok: false, errors: string[] } | { ok: true, markdown: string }}
368 */
369export function expandPhagePage(path, body, phages) {
370  /** @type {string[]} */
371  const errors = [];
372  const markers = /** @type {const} */ ([
373    [TABLE_MARKER, () => phageTableMarkdown(phages ?? [])],
374    [SECTIONS_MARKER, () => phageSectionsMarkdown(phages ?? [])],
375  ]);
376  if (path !== PHAGES_PAGE_PATH) {
377    for (const [marker] of markers) {
378      if (body.includes(marker)) errors.push(`${marker} belongs on ${PHAGES_PAGE_PATH} only`);
379    }
380    return errors.length > 0 ? { ok: false, errors } : { ok: true, markdown: body };
381  }
382  if (!phages || phages.length === 0) {
383    return { ok: false, errors: ["the phage table has no rows to draw: the phages table is empty. Run the content sync (npm run sync:content -- --remote), or sync_phages."] };
384  }
385  for (const [marker] of markers) {
386    const alone = (body.match(markerLine(marker)) ?? []).length;
387    const anywhere = body.split(marker).length - 1;
388    if (alone !== 1 || anywhere !== 1) {
389      errors.push(`the page must carry ${marker} exactly once, alone on its line (it has ${anywhere}); the phage rows are drawn there`);
390    }
391  }
392  if (errors.length > 0) return { ok: false, errors };
393  let markdown = body;
394  for (const [marker, draw] of markers) markdown = markdown.replace(markerLine(marker), () => draw());
395  return { ok: true, markdown };
396}