app/lib/publications/compile.mjs

Functions

Imports

It imports app/lib/content/hashes.mjs, app/lib/publications/citation-tags.mjs, app/lib/publications/cited-by.mjs, app/lib/publications/entities.mjs, app/lib/publications/exports.mjs, app/lib/publications/parse.mjs, app/lib/publications/paths.mjs, app/lib/publications/record.mjs, app/lib/publications/search-inputs.mjs, app/lib/publications/twin.mjs, app/lib/publications/validate.mjs, app/lib/search/ask-keys.mjs. Imported by app/lib/publications/save.server.ts.

Source

1// One door from a publication file to everything derived from it: parse, validate, the record every
2// surface reads, the markdown twin and the search input. check:machine-readable, sync:content, the
3// build and the Carrel adapter's save all call compilePublication, so a file CI passes is the file the
4// save accepts and the page draws, and the twin a machine reads is the one a person's page agrees with.
5
6import { gitBlobSha } from "../content/hashes.mjs";
7import { keyForUrl, urlForKey } from "../search/ask-keys.mjs";
8import { buildCitationTags, REQUIRED_CITATION_TAGS } from "./citation-tags.mjs";
9import { citedByFetchedAt, citedByFor } from "./cited-by.mjs";
10import { decodeEntities } from "./entities.mjs";
11import { organismsIn, toBibtex, toRis } from "./exports.mjs";
12import { parsePublication, publicationPath } from "./parse.mjs";
13import { paperAskUrl, paperMarkdownPath, paperPath, paperPdfPath } from "./paths.mjs";
14import { recordFromData } from "./record.mjs";
15import { paperSearchInputs } from "./search-inputs.mjs";
16import { paperTwin } from "./twin.mjs";
17import { ASK_URL_MAX, validatePublicationFile } from "./validate.mjs";
18
19const ORIGIN = "https://example.invalid";
20const REFERENCE = /&(?:amp|lt|gt|quot|apos|#\d+);/;
21
22/**
23 * What a record promises other surfaces, checked once it is built: the Scholar tag set, the exports,
24 * the twin, the search record and the Ask key. Each is a function of the record alone, so a file that
25 * passes here cannot fail the page at render time.
26 *
27 * @param {import("./types.ts").Publication} record
28 * @param {{ twin: string, searchBody: string, fullText: string | null }} built
29 * @returns {string[]}
30 */
31function derivedProblems(record, built) {
32  /** @type {string[]} */
33  const errors = [];
34  const { slug } = record;
35  const hosted = record.access === "self-hosted" && record.pdfPath !== null;
36
37  if (record.doi !== null) {
38    try {
39      const tags = buildCitationTags(record, {
40        abstractUrl: `${ORIGIN}${paperPath(slug)}`,
41        pdfUrl: hosted ? `${ORIGIN}${paperPdfPath(slug)}` : null,
42      });
43      const names = tags.map((t) => t.name);
44      for (const required of REQUIRED_CITATION_TAGS) {
45        if (!names.includes(required)) errors.push(`citation tags: no ${required}`);
46      }
47      const authorTags = tags.filter((t) => t.name === "citation_author").length;
48      if (authorTags !== record.authors.length) {
49        errors.push(`citation tags: ${authorTags} citation_author tags for ${record.authors.length} authors; one per author, or Scholar reads a single author whose name is the whole list`);
50      }
51      const pdf = tags.find((t) => t.name === "citation_pdf_url");
52      if (hosted && !pdf) errors.push("citation tags: hosted but no citation_pdf_url");
53      if (!hosted && pdf) errors.push("citation tags: not hosted but has citation_pdf_url");
54      if (pdf && !pdf.content.startsWith(`${ORIGIN}${paperPath(slug)}`)) {
55        errors.push(`citation tags: citation_pdf_url is not under ${paperPath(slug)}`);
56      }
57    } catch (error) {
58      errors.push(error instanceof Error ? error.message : String(error));
59    }
60
61    const bib = toBibtex(record);
62    const ris = toRis(record);
63    if (!bib.includes(record.doi) || !ris.includes(record.doi)) errors.push("exports: the DOI is missing from the BibTeX or RIS entry");
64    if (record.doi !== record.doi.toLowerCase() && !bib.includes(`doi = {${record.doi}}`)) {
65      errors.push("exports: a mixed-case DOI must be exported as deposited");
66    }
67    if (REFERENCE.test(bib) || REFERENCE.test(ris)) errors.push("exports: a character reference survives into BibTeX or RIS");
68    const unprotected = organismsIn(record.title).filter((o) => !bib.includes(`{${o}}`));
69    if (unprotected.length > 0) errors.push(`exports: organism name(s) not brace-protected in BibTeX: ${unprotected.join(", ")}`);
70  }
71
72  if (REFERENCE.test(built.twin)) errors.push("twin: carries an undecoded character reference");
73  if (hosted && built.fullText !== null && !built.twin.includes("## Full text")) errors.push("twin: a hosted paper's twin lacks its Full text section");
74  // Classic search shows the line it matched, so two-column text would snippet a mangled one.
75  if (hosted && built.fullText && built.searchBody.length > built.fullText.length / 2) {
76    errors.push("search record: the body carries the PDF text; only the abstract, summary and citation belong in it");
77  }
78
79  const key = keyForUrl(paperPath(slug));
80  if (key !== `research/publications/${slug}.md` || urlForKey(key) !== paperPath(slug)) {
81    errors.push(`Ask key: ${key} does not map back to ${paperPath(slug)}`);
82  }
83  if (`/${key}` !== paperMarkdownPath(slug)) errors.push(`Ask key: ${key} is not the twin's path ${paperMarkdownPath(slug)}`);
84  const askUrl = paperAskUrl(decodeEntities(record.title));
85  if (!askUrl.startsWith("/search?q=") || askUrl.length > ASK_URL_MAX) errors.push("Ask link: the quoted title does not build a /search URL");
86  return errors;
87}
88
89/**
90 * @param {object} input
91 * @param {string} input.slug
92 * @param {string} input.raw the whole file
93 * @param {import("./validate.mjs").PublicationHost} input.host
94 * @param {import("./validate.mjs").OtherPublication[]} [input.others]
95 * @param {unknown} input.citedByArtifact the parsed data/publications.cited-by.json, which the twin's
96 *   citedBy line is a pure function of, so a build and a save write the same twin
97 * @returns {Promise<
98 *   | { ok: false, errors: string[] }
99 *   | { ok: true, errors: [], record: import("./types.ts").Publication, draft: boolean, csl: Record<string, unknown> | null,
100 *       fullText: string | null, pdfSha256: string | null, twin: string, searchInput: { uid: string, url: string, title: string, body: string },
101 *       sourcePath: string, sourceBlobSha: string }
102 * >}
103 */
104export async function compilePublication({ slug, raw, host, others, citedByArtifact }) {
105  const file = publicationPath(slug);
106  const parsed = parsePublication({ file, raw });
107  const errors = await validatePublicationFile({ parsed, slug, host, others });
108  if (errors.length > 0) return { ok: false, errors };
109
110  const record = recordFromData(/** @type {any} */ (parsed.data), slug);
111  const hosted = record.access === "self-hosted" && record.pdfPath !== null;
112  const twin = paperTwin(record, {
113    pages: hosted ? [parsed.fullText ?? ""] : null,
114    citedBy: record.doi ? citedByFor(citedByArtifact, record.doi) : null,
115    citedByFetchedAt: citedByFetchedAt(citedByArtifact),
116    pagePath: paperPath(slug),
117    pdfPath: hosted ? paperPdfPath(slug) : null,
118  });
119  const searchInput = /** @type {NonNullable<ReturnType<typeof paperSearchInputs>[number]>} */ (paperSearchInputs([record])[0]);
120  const derived = derivedProblems(record, {
121    twin,
122    searchBody: searchInput.body,
123    fullText: parsed.fullText,
124  });
125  if (derived.length > 0) return { ok: false, errors: derived };
126
127  const csl = parsed.data.csl && typeof parsed.data.csl === "object" ? /** @type {Record<string, unknown>} */ (parsed.data.csl) : null;
128  return {
129    ok: true,
130    errors: [],
131    record,
132    draft: parsed.data.draft === true,
133    csl,
134    fullText: parsed.fullText,
135    pdfSha256: typeof parsed.data.pdfSha256 === "string" ? parsed.data.pdfSha256 : null,
136    twin,
137    searchInput,
138    sourcePath: file,
139    sourceBlobSha: await gitBlobSha(raw),
140  };
141}