app/lib/publications/twin.mjs

Functions

Imports

It imports app/lib/publications/entities.mjs. Imported by app/lib/publications/compile.mjs.

Source

1// Builds a paper's markdown twin. compile.mjs calls it, and the twin is stored in D1 and served by
2// app/routes/publications.$slug[.md].ts, so a paper edited through Carrel changes its twin with no build.
3// It was a static asset until publications moved to files (an asset wins over a route for a .md path, measured).
4
5import { decodeEntities } from "./entities.mjs";
6
7/**
8 * JSON.stringify is correct here: YAML's double-quoted style is a superset of JSON strings.
9 *
10 * @param {unknown} value
11 * @returns {string}
12 */
13const yamlString = (value) => JSON.stringify(String(value ?? ""));
14
15/**
16 * Absent rather than null or empty: an empty value reads as a value.
17 *
18 * @param {string} key
19 * @param {string | number | boolean | null | undefined} value
20 * @returns {string[]}
21 */
22const line = (key, value) =>
23  value === null || value === undefined || value === ""
24    ? []
25    : [`${key}: ${typeof value === "string" ? yamlString(value) : String(value)}`];
26
27/**
28 * A code-point test, not a regex: lint's no-control-regex refuses the character class.
29 *
30 * @param {string} value
31 * @returns {string}
32 */
33function stripControls(value) {
34  let out = "";
35  for (const character of value) {
36    const point = character.codePointAt(0) ?? 0;
37    const isControl = point < 0x20 && point !== 0x09 && point !== 0x0a && point !== 0x0d;
38    if (!isControl) out += character;
39  }
40  return out;
41}
42
43/**
44 * Its own shape for the reason article-json-ld.mjs gives at PaperFacts.
45 *
46 * @typedef {object} TwinFacts
47 * @property {string} id
48 * @property {string} title
49 * @property {string[]} authors
50 * @property {number | null} year
51 * @property {string | null} [publishedDate]
52 * @property {string | null} [journal]
53 * @property {string | null} [abstract]
54 * @property {string | null} [summary]
55 * @property {string | null} [pmcUrl]
56 * @property {string | null} [preprintDoi]
57 * @property {string | null} [externalUrl]
58 * @property {string | null} [doi]
59 * @property {boolean} [isOpenAccess]
60 * @property {string | null} [license]
61 * @property {{ kind: string, id: string }[]} [accessions]
62 */
63
64/**
65 * Pure: check:machine-readable regenerates every twin and compares bytes.
66 *
67 * @param {TwinFacts} paper a compiled publication record (compile.mjs)
68 * @param {object} options
69 * @param {string[] | null} options.pages extracted PDF text, one string per
70 *   page, or null when this site does not host the PDF
71 * @param {{ total: number } | null} options.citedBy from the cited-by artifact
72 * @param {string | null} options.citedByFetchedAt the date that artifact was read
73 * @param {string} options.pagePath the paper's HTML page, with its trailing slash
74 * @param {string | null} options.pdfPath the hosted PDF, or null
75 * @returns {string}
76 */
77export function paperTwin(paper, { pages, citedBy, citedByFetchedAt, pagePath, pdfPath }) {
78  // Decoded here: the stored corpus keeps character references escaped on purpose (see entities.mjs).
79  const title = decodeEntities(paper.title);
80  const frontmatter = [
81    "---",
82    ...line("id", paper.id),
83    ...line("title", title),
84    "authors:",
85    ...paper.authors.map((name) => `  - ${yamlString(decodeEntities(name))}`),
86    ...line("venue", paper.journal ? decodeEntities(paper.journal) : null),
87    ...line("year", paper.year),
88    ...line("date", paper.publishedDate),
89    ...line("doi", paper.doi),
90    ...line("url", pagePath),
91    ...line("pdf", pdfPath),
92    ...line("pmc", paper.pmcUrl),
93    ...line("preprintDoi", paper.preprintDoi),
94    ...line("resource", paper.externalUrl),
95    `openAccess: ${paper.isOpenAccess ? "true" : "false"}`,
96    ...line("license", paper.license),
97    // citedBy comes from the committed artifact, not live KV: the build stays a pure function of the
98    // repo, and the artifact carries the date it was read.
99    // accessions: absent, not empty, when there are none: the journal may simply have required no statement.
100    ...(paper.accessions && paper.accessions.length > 0
101      ? [
102          "accessions:",
103          ...paper.accessions.map(
104            (a) => `  - ${yamlString(`${a.kind}:${a.id}`)}`,
105          ),
106        ]
107      : []),
108    ...line("citedBy", citedBy ? citedBy.total : null),
109    ...line("citedBySource", citedBy && citedByFetchedAt ? `OpenAlex, read ${citedByFetchedAt}` : null),
110    "---",
111    "",
112  ];
113
114  const body = [`# ${title}`, ""];
115
116  if (paper.summary) {
117    body.push(paper.summary, "");
118  }
119
120  if (paper.abstract) {
121    body.push("## Abstract", "", decodeEntities(paper.abstract), "");
122  }
123
124  // The provenance sentence decides whether a quotation from machine-extracted text can be trusted.
125  if (pages !== null) {
126    // C0 controls (symbol fonts mapped to low code points) are stripped only at this presentation
127    // boundary; the artifact stays faithful to the PDF.
128    const text = pages
129      .map((page) => stripControls(page).trim())
130      .filter(Boolean)
131      .join("\n\n");
132    body.push("## Full text", "");
133    if (text.length === 0) {
134      body.push(
135        "Text extraction produced nothing for this PDF. It is most likely a scan " +
136          "with no text layer. The PDF itself is linked above.",
137        "",
138      );
139    } else {
140      body.push(
141        "Machine-extracted from the PDF linked above. It carries the artifacts " +
142          "that come with reading a typeset two-column page: running heads, " +
143          "figure captions in the flow of the prose, and words broken across " +
144          "line ends. The abstract above is the registry's deposit and is the " +
145          "authoritative text.",
146        "",
147        text,
148        "",
149      );
150    }
151  }
152
153  return `${frontmatter.join("\n")}${body.join("\n")}`;
154}