app/lib/publications/sources.server.ts

Functions

Imports

It imports app/db/publications.ts, app/lib/error-message.mjs, app/lib/publications/authors.mjs, app/lib/publications/validate.mjs, app/lib/seo.ts, app/lib/site-origin.mjs. Imported by app/lib/operator/api.server.ts.

Source

1import { listPublicationIdentities } from "~/db/publications";
2import { errorMessage } from "~/lib/error-message.mjs";
3import { OWNER_ORCID, SITE } from "~/lib/seo";
4
5import { canonicalAuthor } from "./authors.mjs";
6import { doiKey } from "./validate.mjs";
7import { SITE_USER_AGENT } from "~/lib/site-origin.mjs";
8
9// Checks the publications list against ORCID and Crossref, so a DOI is caught the day one appears. It
10// reads both and writes nothing: a publication is written through Carrel, never from here.
11//
12// ORCID's public API lists the works on the record. Crossref's `orcid:` filter lists the works whose
13// deposit names the ORCID. A publisher that leaves the ORCID out of its deposit is caught only by name, so
14// Crossref is also read for the last year's works under the owner's name, and those come back apart as
15// matches to confirm: a name is not an identity (the reason citations.server.ts reads per DOI only).
16
17// A name, not an address, for the reason citations.server.ts gives.
18const ORCID_ID = OWNER_ORCID.replace(/^https?:\/\/orcid\.org\//, "");
19const NAME_WINDOW_DAYS = 400;
20
21type SourceWork = { doi: string; title: string; year: number | null; sources: Array<"orcid" | "crossref"> };
22type SourceFailure = { source: "orcid" | "crossref" | "crossref-name"; detail: string };
23
24export type PublicationCheck = {
25  orcid: string;
26  /** DOIs ORCID or Crossref ties to the ORCID that no publication file carries, as its DOI or preprint DOI. */
27  notOnSite: SourceWork[];
28  /** Crossref works under the owner's name, created within NAME_WINDOW_DAYS, on neither list above nor the site. */
29  nameMatches: SourceWork[];
30  /** Published papers whose DOI is missing from the ORCID record. */
31  notOnOrcid: Array<{ slug: string; doi: string }>;
32  /** Files with no DOI yet (status submitted), to match the lists above against by title. */
33  manuscripts: Array<{ slug: string; title: string }>;
34  failures: SourceFailure[];
35};
36
37type CrossrefItem = {
38  DOI?: string;
39  title?: string[];
40  issued?: { "date-parts"?: Array<Array<number | null>> };
41  author?: Array<{ given?: string; family?: string; ORCID?: string }>;
42};
43
44async function getJson(url: string, accept = "application/json"): Promise<unknown> {
45  const res = await fetch(url, { headers: { accept, "user-agent": SITE_USER_AGENT } });
46  if (!res.ok) throw new Error(`HTTP ${res.status}`);
47  return res.json();
48}
49
50/** The works on the ORCID record that carry a DOI, one per DOI. */
51async function readOrcid(): Promise<SourceWork[]> {
52  const body = (await getJson(`https://pub.orcid.org/v3.0/${ORCID_ID}/works`)) as {
53    group?: Array<{
54      "external-ids"?: { "external-id"?: Array<{ "external-id-type"?: string; "external-id-value"?: string }> };
55      "work-summary"?: Array<{
56        title?: { title?: { value?: string } };
57        "publication-date"?: { year?: { value?: string } } | null;
58      }>;
59    }>;
60  };
61  if (!Array.isArray(body.group)) throw new Error("the response carried no group list");
62  const out: SourceWork[] = [];
63  for (const group of body.group) {
64    const summary = group["work-summary"]?.[0];
65    const year = Number(summary?.["publication-date"]?.year?.value);
66    for (const id of group["external-ids"]?.["external-id"] ?? []) {
67      if (id["external-id-type"] !== "doi" || !id["external-id-value"]) continue;
68      out.push({
69        doi: id["external-id-value"].trim(),
70        title: summary?.title?.title?.value ?? "",
71        year: Number.isFinite(year) && year > 0 ? year : null,
72        sources: ["orcid"],
73      });
74    }
75  }
76  return out;
77}
78
79function fromCrossref(item: CrossrefItem): SourceWork | null {
80  if (!item.DOI) return null;
81  const year = item.issued?.["date-parts"]?.[0]?.[0];
82  return { doi: item.DOI, title: item.title?.[0] ?? "", year: typeof year === "number" ? year : null, sources: ["crossref"] };
83}
84
85async function readCrossref(params: Record<string, string>): Promise<CrossrefItem[]> {
86  const url = new URL("https://api.crossref.org/works");
87  for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
88  url.searchParams.set("rows", "200");
89  url.searchParams.set("select", "DOI,title,issued,author");
90  const body = (await getJson(url.href)) as { message?: { items?: CrossrefItem[] } };
91  if (!Array.isArray(body.message?.items)) throw new Error("the response carried no item list");
92  return body.message.items;
93}
94
95const isOwner = (author: { given?: string; family?: string }) =>
96  canonicalAuthor(`${author.given ?? ""} ${author.family ?? ""}`) === SITE.name;
97
98/** Reads every source, keeps what each one failed on, and compares the answers with the publication rows. */
99export async function checkPublicationSources(env: Env, now = Date.now()): Promise<PublicationCheck> {
100  const failures: SourceFailure[] = [];
101  const attempt = async <T>(source: SourceFailure["source"], read: () => Promise<T>): Promise<T | null> => {
102    try {
103      return await read();
104    } catch (error) {
105      failures.push({ source, detail: errorMessage(error) });
106      return null;
107    }
108  };
109  const since = new Date(now - NAME_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
110  const [orcid, crossref, byName, rows] = await Promise.all([
111    attempt("orcid", readOrcid),
112    attempt("crossref", () => readCrossref({ filter: `orcid:${ORCID_ID}` })),
113    attempt("crossref-name", () => readCrossref({ "query.author": SITE.name, filter: `from-created-date:${since}` })),
114    listPublicationIdentities(env),
115  ]);
116
117  const known = new Set<string>();
118  const published: Array<{ slug: string; doi: string }> = [];
119  const manuscripts: PublicationCheck["manuscripts"] = [];
120  for (const row of rows) {
121    const record = JSON.parse(row.record) as { doi?: string | null; preprintDoi?: string | null };
122    if (record.doi) {
123      known.add(doiKey(record.doi));
124      published.push({ slug: row.slug, doi: record.doi });
125    } else {
126      manuscripts.push({ slug: row.slug, title: row.title });
127    }
128    if (record.preprintDoi) known.add(doiKey(record.preprintDoi));
129  }
130
131  const tied = new Map<string, SourceWork>();
132  for (const work of [...(orcid ?? []), ...(crossref ?? []).flatMap((i) => fromCrossref(i) ?? [])]) {
133    const key = doiKey(work.doi);
134    const seen = tied.get(key);
135    if (!seen) tied.set(key, { ...work });
136    else for (const source of work.sources) if (!seen.sources.includes(source)) seen.sources.push(source);
137  }
138
139  const nameMatches = new Map<string, SourceWork>();
140  for (const item of byName ?? []) {
141    const work = fromCrossref(item);
142    if (!work || !(item.author ?? []).some(isOwner)) continue;
143    const key = doiKey(work.doi);
144    if (!known.has(key) && !tied.has(key)) nameMatches.set(key, work);
145  }
146
147  const onOrcid = new Set((orcid ?? []).map((w) => doiKey(w.doi)));
148  return {
149    orcid: OWNER_ORCID,
150    notOnSite: [...tied.entries()].filter(([key]) => !known.has(key)).map(([, work]) => work),
151    nameMatches: [...nameMatches.values()],
152    // Unknowable when ORCID could not be read, so empty rather than every paper.
153    notOnOrcid: orcid ? published.filter((p) => !onOrcid.has(doiKey(p.doi))) : [],
154    manuscripts,
155    failures,
156  };
157}