app/lib/publications/validate.mjs

Functions

Imports

It imports app/lib/carrel/content-id.mjs, app/lib/publications/accessions.mjs, app/lib/publications/hosting.mjs, app/lib/publications/paths.mjs, app/lib/publications/topics.mjs, app/lib/publications/update-notice.mjs. Imported by app/lib/publications/compile.mjs, app/lib/publications/save.server.ts, app/lib/publications/sources.server.ts.

Source

1// Judges a parsed publication file (docs/PUBLICATIONS.md). check:machine-readable and the Carrel adapter's
2// save both call this, so a file CI refuses is a file the save refuses, for the same reason in the same
3// words. Every message starts with the field path, with no leading dot (`authors[2]: ...`).
4//
5// The checks that need the repository (does the PDF exist, does its hash match) and the other papers
6// (is this id or DOI taken) arrive as `host` and `others`, so CI reads a clone and the Worker asks GitHub
7// and D1 without this module knowing which.
8
9import { contentIdFits } from "../carrel/content-id.mjs";
10import { accessionUrl, accessionsInText } from "./accessions.mjs";
11import { HOSTED_WITHOUT_LICENCE, permitsRedistribution } from "./hosting.mjs";
12import { doiSlug, paperPdfPath } from "./paths.mjs";
13import { updateNoticeProblem } from "./update-notice.mjs";
14import { TOPIC_IDS } from "./topics.mjs";
15
16const PUBLICATION_KIND = "publication";
17const TYPES = /** @type {const} */ (["article", "review", "chapter", "abstract", "teaching-resource"]);
18const ACCESS = /** @type {const} */ (["self-hosted", "external"]);
19const STATUSES = /** @type {const} */ (["published", "submitted"]);
20
21/** Every key a file may carry. Anything else is a typo that would be silently ignored. */
22const FRONT_MATTER_FIELDS = new Set([
23  "doi", "slug", "id", "status", "draft", "type", "title", "authors", "journal", "publishedDate", "year",
24  "volume", "issue", "pages", "topics", "access", "pdfPath", "pdfSha256", "externalUrl", "pmid", "pmcid",
25  "pmcUrl", "preprintDoi", "isOpenAccess", "license", "licenseSource", "summary", "selected",
26  "accessions", "updateNotice", "abstract", "csl",
27]);
28
29const SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
30const DOI_PATTERN = /^10\.\d{4,9}\/\S+$/;
31const PMC_URL = /^https:\/\/pmc\.ncbi\.nlm\.nih\.gov\/articles\/PMC\d+\/$/;
32const DATE_PATTERN = /^\d{4}(-\d{2}){0,2}$/;
33const SHA256 = /^[0-9a-f]{64}$/;
34const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
35/** A character reference: titles are stored decoded, so one left in is a value that would render twice-decoded. */
36const CHARACTER_REFERENCE = /&(?:#\d+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/;
37/** The PreToolUse hook cannot reach data files; escapes keep this file free of the dashes. */
38const WIDE_DASH = new RegExp("[\\u2013\\u2014]");
39/** A scanned PDF extracts to nothing and everything else still passes. */
40const MIN_FULL_TEXT_CHARS = 500;
41const SUMMARY_MAX = 200;
42/** What `paperAskUrl` may produce before it is a URL no client will send. */
43export const ASK_URL_MAX = 600;
44
45/**
46 * @typedef {object} PublicationHost
47 * @property {(path: string) => Promise<{ size: number, sha256?: string } | null>} pdf the repository file at a
48 *   site-absolute public path (`/research/...`), or null when it is not there. `sha256` is present where the
49 *   host can hash the bytes (CI); the Worker's GitHub reads cannot, so the hash is checked in CI only.
50 *
51 * @typedef {object} OtherPublication
52 * @property {string} slug
53 * @property {string} id
54 * @property {string | null} doi
55 */
56
57/** @param {unknown} value @returns {value is string} */
58const isString = (value) => typeof value === "string";
59/** @param {unknown} value @returns {value is Record<string, unknown>} */
60const isObject = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
61/** @param {unknown} value @returns {value is string} */
62const isFilled = (value) => isString(value) && value.trim() !== "";
63
64/**
65 * DOI names are case-insensitive: the one key the corpus joins on.
66 *
67 * @param {string | null | undefined} doi
68 */
69export const doiKey = (doi) => (doi ?? "").trim().toLowerCase();
70
71/**
72 * @param {object} input
73 * @param {import("./parse.mjs").ParsedPublication} input.parsed
74 * @param {string} input.slug the file's name without `.md`
75 * @param {PublicationHost} input.host
76 * @param {OtherPublication[]} [input.others] every OTHER publication, for uniqueness
77 * @returns {Promise<string[]>}
78 */
79export async function validatePublicationFile({ parsed, slug, host, others = [] }) {
80  /** @type {string[]} */
81  const errors = [...parsed.problems];
82  const { data, fullText } = parsed;
83  /** @param {string} field @param {string} message */
84  const bad = (field, message) => errors.push(`${field}: ${message}`);
85
86  for (const key of Object.keys(data)) {
87    if (!FRONT_MATTER_FIELDS.has(key)) bad(key, "is not a publication field");
88  }
89
90  // ---- identity: slug, id, DOI, status
91  const status = data.status ?? "published";
92  if (!STATUSES.includes(/** @type {any} */ (status))) {
93    bad("status", `${JSON.stringify(status)} is not one of ${STATUSES.join(", ")}`);
94  }
95  const submitted = status === "submitted";
96
97  if (!SLUG_PATTERN.test(slug)) {
98    bad("slug", `the file name "${slug}" is not a lowercase kebab-case slug`);
99  } else if (!contentIdFits(PUBLICATION_KIND, slug)) {
100    bad("slug", `"${slug}" is too long: ${PUBLICATION_KIND}.<slug> must fit the 200 characters of a Carrel content id`);
101  }
102  if (data.slug !== undefined && data.slug !== null && data.slug !== slug) {
103    bad("slug", `${JSON.stringify(data.slug)} is not the file's name "${slug}"`);
104  }
105
106  const doi = data.doi;
107  if (doi === undefined || doi === null) {
108    if (!submitted) bad("doi", "is required for a published paper; a manuscript without one is status: submitted");
109    if (submitted && data.slug !== slug) bad("slug", "a record with no DOI names its slug in the file (slug: ...), since there is no DOI to derive it from");
110  } else if (!isString(doi) || !DOI_PATTERN.test(doi)) {
111    bad("doi", `${JSON.stringify(doi)} is not a DOI name (10.<registrant>/<suffix>)`);
112  } else if (SLUG_PATTERN.test(slug) && doiSlug(doi) !== slug) {
113    bad("doi", `its page slug would be "${doiSlug(doi)}", but the file is named "${slug}". The file name is the DOI's slug and never changes after Google Scholar has indexed it`);
114  }
115
116  if (!isFilled(data.id)) {
117    bad("id", "is required (the curated id, which is also the BibTeX key)");
118  } else if (!isString(data.id) || !ID_PATTERN.test(data.id)) {
119    bad("id", `${JSON.stringify(data.id)} must be letters, digits, hyphens and underscores`);
120  }
121
122  for (const other of others) {
123    if (other.slug === slug) continue;
124    if (isString(data.id) && other.id === data.id) bad("id", `"${data.id}" is already the id of ${other.slug}`);
125    if (isString(doi) && other.doi && doiKey(other.doi) === doiKey(doi)) {
126      bad("doi", `${doi} is already ${other.slug}'s DOI (DOI names are case-insensitive, so two casings are one work)`);
127    }
128    if (doiSlug(other.doi ?? "") === slug && other.slug !== slug) bad("slug", `the page slug "${slug}" is already ${other.slug}'s`);
129  }
130
131  if (data.draft !== undefined && typeof data.draft !== "boolean") bad("draft", "must be true or false");
132
133  // ---- the citation
134  if (!isFilled(data.type) || !TYPES.includes(/** @type {any} */ (data.type))) {
135    bad("type", `${JSON.stringify(data.type)} is not one of ${TYPES.join(", ")}`);
136  }
137
138  const title = data.title;
139  if (!isFilled(title)) {
140    bad("title", "is required");
141  } else if (isString(title)) {
142    // paperAskUrl quotes the title and the search reads a quoted run as a phrase, so a quote splits it.
143    if (/["']/.test(title)) bad("title", "contains a quotation mark; the paper's Ask link quotes the title, so a quote inside splits the phrase");
144    if (CHARACTER_REFERENCE.test(title)) bad("title", "carries a character reference (&amp; and the like); write the character itself");
145    if (title.includes("<")) bad("title", "contains \"<\", which would end a JSON-LD script block early");
146    if (title !== title.trim() || /\s{2,}/.test(title)) bad("title", "has leading, trailing or repeated whitespace");
147  }
148
149  if (!Array.isArray(data.authors) || data.authors.length === 0) {
150    bad("authors", "must list at least one author; a shortened list is a wrong citation");
151  } else {
152    data.authors.forEach((name, i) => {
153      if (!isFilled(name)) bad(`authors[${i}]`, "is empty; every author has a name");
154    });
155  }
156
157  for (const key of ["journal", "volume", "issue", "pages"]) {
158    const value = data[key];
159    if (value !== undefined && value !== null && !isString(value)) bad(key, "must be text");
160    if (isString(value) && value.trim() === "") bad(key, "is blank; leave it out instead");
161  }
162
163  const date = data.publishedDate;
164  if (date !== undefined && date !== null && !(isString(date) && DATE_PATTERN.test(date))) {
165    bad("publishedDate", `${JSON.stringify(date)} is not YYYY-MM-DD, YYYY-MM or YYYY`);
166  }
167  if (data.year !== undefined && data.year !== null) {
168    if (!Number.isInteger(data.year) || /** @type {number} */ (data.year) < 1900 || /** @type {number} */ (data.year) > 2200) {
169      bad("year", `${JSON.stringify(data.year)} is not a year`);
170    } else if (isString(date) && String(data.year) !== date.slice(0, 4)) {
171      bad("year", `${data.year} disagrees with publishedDate ${date}; leave year out, it comes from the date`);
172    }
173  }
174  if ((date === undefined || date === null) && !(Number.isInteger(data.year))) {
175    bad("publishedDate", "is required (or year, for a manuscript with no date yet)");
176  }
177
178  if (!Array.isArray(data.topics) || data.topics.length === 0) {
179    bad("topics", "must carry at least one topic");
180  } else {
181    data.topics.forEach((topic, i) => {
182      if (!isString(topic) || !TOPIC_IDS.has(topic)) bad(`topics[${i}]`, `${JSON.stringify(topic)} is not a declared topic (${[...TOPIC_IDS].join(", ")})`);
183    });
184  }
185
186  // ---- identifiers
187  if (data.pmid !== undefined && data.pmid !== null && !(isString(data.pmid) && /^\d+$/.test(data.pmid))) {
188    bad("pmid", `${JSON.stringify(data.pmid)} is not digits`);
189  }
190  if (data.pmcUrl !== undefined && data.pmcUrl !== null && !(isString(data.pmcUrl) && PMC_URL.test(data.pmcUrl))) {
191    bad("pmcUrl", `${JSON.stringify(data.pmcUrl)} is not in landing-page form (https://pmc.ncbi.nlm.nih.gov/articles/PMC<digits>/)`);
192  }
193  if (Boolean(data.pmcid) !== Boolean(data.pmcUrl)) bad("pmcid", "and pmcUrl are present together or absent together");
194  if (data.pmcid !== undefined && data.pmcid !== null && !(isString(data.pmcid) && /^PMC\d+$/.test(data.pmcid))) {
195    bad("pmcid", `${JSON.stringify(data.pmcid)} is not PMC<digits>`);
196  }
197  if (data.preprintDoi !== undefined && data.preprintDoi !== null) {
198    if (!isString(data.preprintDoi) || !DOI_PATTERN.test(data.preprintDoi)) bad("preprintDoi", `${JSON.stringify(data.preprintDoi)} is not a DOI name`);
199    else if (doiKey(data.preprintDoi) === doiKey(isString(doi) ? doi : "")) bad("preprintDoi", "is the paper's own DOI");
200  }
201  if (data.isOpenAccess !== undefined && typeof data.isOpenAccess !== "boolean") bad("isOpenAccess", "must be true or false");
202  if (data.selected !== undefined && typeof data.selected !== "boolean") bad("selected", "must be true or false");
203
204  // ---- access: where the reader gets the paper
205  const access = data.access;
206  if (!ACCESS.includes(/** @type {any} */ (access))) {
207    bad("access", `${JSON.stringify(access)} is not one of ${ACCESS.join(", ")}`);
208  }
209  const hasPdfPath = data.pdfPath !== undefined && data.pdfPath !== null;
210  if (access === "self-hosted") {
211    if (submitted) bad("access", "a submitted manuscript has no PDF on this site; use external");
212    const derived = SLUG_PATTERN.test(slug) ? paperPdfPath(slug) : null;
213    if (!hasPdfPath) {
214      bad("pdfPath", `is required for a self-hosted paper (${derived ?? "the derived path"})`);
215    } else if (derived && data.pdfPath !== derived) {
216      bad("pdfPath", `${JSON.stringify(data.pdfPath)} should be ${derived}: the PDF lives in its own page's directory, which is what Google Scholar needs`);
217    }
218    if (data.externalUrl !== undefined && data.externalUrl !== null) bad("externalUrl", "is for external papers; a self-hosted one links its own PDF");
219    if (derived && hasPdfPath && data.pdfPath === derived) {
220      const file = await host.pdf(derived);
221      if (!file) {
222        bad("pdfPath", `${derived} is not in the repository at public${derived}. Commit the PDF first; adding one is a git push, never an upload through this tool`);
223      } else {
224        if (file.size === 0) bad("pdfPath", `${derived} is a zero-byte file`);
225        if (!isString(data.pdfSha256) || !SHA256.test(data.pdfSha256)) {
226          bad("pdfSha256", "is required for a self-hosted paper: the sha256 of the PDF the full text was extracted from");
227        } else if (file.sha256 !== undefined && file.sha256 !== data.pdfSha256) {
228          bad("pdfSha256", `${data.pdfSha256.slice(0, 12)} does not match the PDF in the repository (${file.sha256.slice(0, 12)}); re-extract the full text from the current PDF`);
229        }
230      }
231    }
232    if (fullText === null) {
233      bad("Full text", `a self-hosted paper carries its extracted text under a "## Full text" heading`);
234    } else if (fullText.length < MIN_FULL_TEXT_CHARS) {
235      bad("Full text", `is ${fullText.length} characters. Extraction produced almost nothing; a scanned PDF with no text layer looks exactly like this`);
236    }
237  } else if (access === "external") {
238    if (hasPdfPath) bad("pdfPath", "is for self-hosted papers; an external one has none");
239    if (data.pdfSha256 !== undefined && data.pdfSha256 !== null) bad("pdfSha256", "is for self-hosted papers");
240    if (fullText !== null) bad("Full text", "a paper this site does not host has no extracted text");
241    const external = data.externalUrl;
242    if (external !== undefined && external !== null && !(isString(external) && /^https:\/\/\S+$/.test(external))) {
243      bad("externalUrl", `${JSON.stringify(external)} is not an https URL`);
244    }
245    if (!submitted && !data.externalUrl && !isString(doi)) bad("externalUrl", "an external published paper links out through a DOI or an externalUrl");
246  }
247
248  // ---- rights: hosting a paper whose licence does not permit it is Dustin's call, per DOI
249  if (access === "self-hosted") {
250    const licence = isString(data.license) ? data.license : null;
251    if (!permitsRedistribution(licence) && !(isString(doi) && HOSTED_WITHOUT_LICENCE.has(doi))) {
252      bad(
253        "license",
254        `${licence ?? "none"} does not permit redistribution, and ${isString(doi) ? doi : "this paper"} is not one Dustin has chosen to host without one. ` +
255          `Hosting a paper whose license does not permit it is Dustin's call, recorded per DOI in app/lib/publications/hosting.mjs, so a new one is a new decision and not a precedent`,
256      );
257    }
258    if (!permitsRedistribution(licence) && !data.licenseSource && licence === null) {
259      bad("licenseSource", "a hosted paper with no license records what the registries said (crossref:tdm-only, none-deposited, ...), so a closed paper is not mistaken for an unchecked one");
260    }
261  }
262
263  // ---- the plain-language line
264  if (data.summary !== undefined && data.summary !== null) {
265    const summary = data.summary;
266    if (!isString(summary) || summary.trim() === "") {
267      bad("summary", "is present but blank; absent is a state, empty is a mistake");
268    } else {
269      if (summary.length > SUMMARY_MAX) bad("summary", `is ${summary.length} characters; the limit is ${SUMMARY_MAX}`);
270      if (/[.!?]\s+[A-Z]/.test(summary)) bad("summary", "is more than one sentence");
271      if (WIDE_DASH.test(summary)) bad("summary", "carries an em dash or en dash; use a comma, period, parentheses or colon");
272    }
273  }
274
275  // ---- the abstract and the raw record
276  if (data.abstract !== undefined && data.abstract !== null) {
277    if (!isString(data.abstract)) bad("abstract", "must be text");
278    else if (data.abstract.includes("<")) bad("abstract", `contains "<", which would end a JSON-LD script block early; store it as &lt;`);
279  }
280  if (data.csl !== undefined && data.csl !== null) {
281    if (!isObject(data.csl)) {
282      bad("csl", "must be the Crossref record as a mapping");
283    } else {
284      const cslDoi = data.csl.DOI;
285      if (isString(doi) && doiKey(isString(cslDoi) ? cslDoi : "") !== doiKey(doi)) {
286        bad("csl.DOI", `${JSON.stringify(cslDoi)} is not this paper's DOI ${doi}`);
287      }
288      if (isString(data.csl.abstract) && data.csl.abstract.includes("<")) {
289        bad("csl.abstract", `contains "<"`);
290      }
291    }
292  }
293
294  // ---- update notice and accessions
295  const noticeProblem = updateNoticeProblem(data.updateNotice);
296  if (noticeProblem) bad("updateNotice", noticeProblem);
297
298  if (data.accessions !== undefined && data.accessions !== null && !Array.isArray(data.accessions)) {
299    bad("accessions", "must be a list of { kind, id }");
300  }
301  const declared = Array.isArray(data.accessions) ? data.accessions : [];
302  declared.forEach((entry, i) => {
303    if (!isObject(entry) || !isFilled(entry.kind) || !isFilled(entry.id)) {
304      bad(`accessions[${i}]`, "needs a kind and an id");
305      return;
306    }
307    try {
308      const url = accessionUrl({ kind: String(entry.kind), id: String(entry.id) });
309      if (!url.startsWith("https://www.ncbi.nlm.nih.gov/") || !url.endsWith(String(entry.id))) {
310        bad(`accessions[${i}]`, `builds ${url}, which is not an NCBI URL ending in its own id`);
311      }
312    } catch (error) {
313      bad(`accessions[${i}].kind`, error instanceof Error ? error.message : String(error));
314    }
315  });
316  // Both directions: the statement in the paper, not a bare regex, decides what this paper deposited.
317  if (access === "self-hosted" && fullText !== null) {
318    const flat = (/** @type {Array<{ kind?: unknown, id?: unknown }>} */ list) =>
319      list.map((a) => `${a.kind}:${a.id}`).sort().join(",");
320    const found = accessionsInText([fullText]);
321    if (flat(found) !== flat(declared.filter(isObject))) {
322      bad(
323        "accessions",
324        `the data-availability statement in the PDF names [${flat(found)}] but the file lists [${flat(declared.filter(isObject))}]. ` +
325          `The text decides; a bare accession regex returns the comparison organisms' deposits, so only the statement counts`,
326      );
327    }
328  } else if (declared.length > 0) {
329    bad("accessions", "an accession is read from the paper's own data-availability statement, which only a hosted PDF's text carries");
330  }
331
332  return errors;
333}
334
335/**
336 * The corpus-level checks the per-file rules cannot see: duplicates across files. CI calls this over
337 * every file; the Worker's save passes `others` to the per-file rule instead.
338 *
339 * @param {Array<{ file: string, slug: string, id: unknown, doi: unknown }>} files
340 * @returns {string[]}
341 */
342export function validateCorpus(files) {
343  /** @type {string[]} */
344  const errors = [];
345  /** @param {string} label @param {(f: typeof files[number]) => string | null} key */
346  const unique = (label, key) => {
347    /** @type {Map<string, string>} */
348    const seen = new Map();
349    for (const f of files) {
350      const k = key(f);
351      if (!k) continue;
352      const first = seen.get(k);
353      if (first) errors.push(`${f.file}: ${label} "${k}" is also ${first}'s`);
354      else seen.set(k, f.file);
355    }
356  };
357  unique("id", (f) => (isString(f.id) ? f.id : null));
358  unique("DOI (case-folded)", (f) => (isString(f.doi) ? doiKey(f.doi) : null));
359  unique("page slug", (f) => f.slug);
360  if (files.length === 0) errors.push("no publication files: the corpus is empty, which would publish an empty page");
361  return errors;
362}