app/lib/citations.server.ts

Functions

Imports

It imports app/db/client.ts, app/db/schema.ts, app/lib/context.ts, app/lib/error-message.mjs, app/lib/site-origin.mjs. Imported by app/lib/operator/api.server.ts, app/routes/publications.$slug.tsx, app/routes/publications.tsx.

Source

1import type { RouterContextProvider } from "react-router";
2import { inArray } from "drizzle-orm";
3
4import { getDb } from "~/db/client";
5import { publicationCitations } from "~/db/schema";
6import { getEnv, getExecutionContext } from "~/lib/context";
7import { errorMessage } from "~/lib/error-message.mjs";
8import { SITE_USER_AGENT } from "~/lib/site-origin.mjs";
9
10// Per DOI only: the OpenAlex author endpoint has works by other people merged into it.
11//
12// The counts live in D1 with no expiry (drizzle/0020_publications.sql). A row is replaced by a refresh
13// and never deleted, so a page never shows a blank for a paper that once had a count: it serves the last
14// count and refreshes behind the response. The refresh also runs weekly, from the watchdog, through the
15// operator API's refresh_citations.
16
17/** A row older than this is refreshed after the response that reads it. The count is shown regardless. */
18const STALE_AFTER_DAYS = 7;
19// A name, not an address: OpenAlex dropped the `mailto` pool, and a secrets gate reads this whole
20// file, comments included, so do not quote the address here either.
21
22export type CitationEntry = {
23  count: number;
24  /** The day OpenAlex was read, YYYY-MM-DD. */
25  fetchedAt: string;
26  /** Taken from the response. Never constructed. */
27  url: string | null;
28};
29
30/** DOIs are case-insensitive per spec: the table is keyed by the lower-cased name. */
31const citationKey = (doi: string) => doi.trim().toLowerCase();
32
33// The page renders the last count on any failure, but every failure is logged under one alert key, so
34// a revoked key or an outage is visible rather than looking like a cold cache.
35function logFailure(stage: string, doi: string | null, detail: unknown) {
36  console.error(
37    JSON.stringify({
38      alert: "citation-count-failed",
39      stage,
40      doi,
41      detail: errorMessage(detail),
42    }),
43  );
44}
45
46export type CitationFailure = { doi: string; stage: string; detail: string };
47
48/**
49 * The key is mandatory: keyless requests share a budget of about 100 credits, after which every
50 * one is refused and the count stops moving.
51 */
52async function fetchOne(
53  apiKey: string,
54  doi: string,
55): Promise<{ entry: CitationEntry } | { failure: CitationFailure }> {
56  const url = new URL(`https://api.openalex.org/works/doi:${encodeURIComponent(doi)}`);
57  url.searchParams.set("api_key", apiKey);
58  const fail = (stage: string, detail: unknown) => {
59    logFailure(stage, doi, detail);
60    return { failure: { doi, stage, detail: errorMessage(detail) } };
61  };
62
63  try {
64    const res = await fetch(url, { headers: { "user-agent": SITE_USER_AGENT } });
65    if (!res.ok) {
66      // 401 and 403 are configuration (a revoked or wrong key), and name it so; a 404 is a DOI
67      // OpenAlex does not hold.
68      return fail(res.status === 401 || res.status === 403 ? "openalex-key-refused" : "openalex-status", `HTTP ${res.status}`);
69    }
70    const body = (await res.json()) as { cited_by_count?: number; id?: string };
71    if (typeof body.cited_by_count !== "number") {
72      return fail("openalex-shape", "the response carried no cited_by_count");
73    }
74    return {
75      entry: {
76        count: body.cited_by_count,
77        fetchedAt: new Date().toISOString().slice(0, 10),
78        url: typeof body.id === "string" ? body.id : null,
79      },
80    };
81  } catch (error) {
82    return fail("openalex-fetch", error);
83  }
84}
85
86async function readRows(env: Env, dois: string[]): Promise<Map<string, CitationEntry>> {
87  const keys = [...new Set(dois.map(citationKey))];
88  const out = new Map<string, CitationEntry>();
89  if (keys.length === 0) return out;
90  const rows = await getDb(env)
91    .select()
92    .from(publicationCitations)
93    .where(inArray(publicationCitations.doi, keys));
94  for (const row of rows) out.set(row.doi, { count: row.count, fetchedAt: row.fetchedAt, url: row.url });
95  return out;
96}
97
98/** Replaces a row; a read older than the stored one never overwrites it, and nothing ever deletes one. */
99async function writeCitation(env: Env, doi: string, entry: CitationEntry) {
100  await env.DB.prepare(
101    `INSERT INTO publication_citations (doi, count, url, fetched_at) VALUES (?1, ?2, ?3, ?4)
102     ON CONFLICT(doi) DO UPDATE SET count = excluded.count, url = excluded.url, fetched_at = excluded.fetched_at
103     WHERE excluded.fetched_at >= publication_citations.fetched_at`,
104  )
105    .bind(citationKey(doi), entry.count, entry.url, entry.fetchedAt)
106    .run();
107}
108
109/**
110 * Reads each DOI from OpenAlex and replaces its row. A DOI that fails keeps the count it had, and the
111 * failure is returned, so the caller (the operator tool the watchdog calls) can say which and why.
112 */
113export async function refreshCitations(
114  env: Env,
115  dois: string[],
116): Promise<{ refreshed: number; failures: CitationFailure[] }> {
117  const apiKey = env.OPENALEX_API_KEY;
118  if (!apiKey) {
119    const detail = "OPENALEX_API_KEY is not set, so no count can refresh";
120    logFailure("config", null, detail);
121    return { refreshed: 0, failures: dois.map((doi) => ({ doi, stage: "config", detail })) };
122  }
123  let refreshed = 0;
124  const failures: CitationFailure[] = [];
125  for (const doi of dois) {
126    const result = await fetchOne(apiKey, doi);
127    if ("failure" in result) {
128      failures.push(result.failure);
129      continue;
130    }
131    try {
132      await writeCitation(env, doi, result.entry);
133      refreshed += 1;
134    } catch (error) {
135      logFailure("d1-write", doi, error);
136      failures.push({ doi, stage: "d1-write", detail: errorMessage(error) });
137    }
138  }
139  return { refreshed, failures };
140}
141
142/** Whether a row's read is old enough to refresh. A date the table cannot parse counts as stale. */
143function isStale(entry: CitationEntry, now = Date.now()) {
144  const read = Date.parse(`${entry.fetchedAt}T00:00:00Z`);
145  return Number.isNaN(read) || now - read > STALE_AFTER_DAYS * 24 * 60 * 60 * 1000;
146}
147
148/**
149 * Reads D1 only and never blocks on OpenAlex. A row that is stale, or missing, is refreshed after the
150 * response, and the last count is shown meanwhile: a count once seen is never replaced by a blank.
151 */
152export async function getCitationCounts(
153  context: Readonly<RouterContextProvider>,
154  dois: string[],
155): Promise<Record<string, CitationEntry>> {
156  const env = getEnv(context);
157  const ctx = getExecutionContext(context);
158  const rows = await readRows(env, dois);
159
160  const out: Record<string, CitationEntry> = {};
161  const due: string[] = [];
162  for (const doi of dois) {
163    const entry = rows.get(citationKey(doi));
164    if (entry) out[doi] = entry;
165    if (!entry || isStale(entry)) due.push(doi);
166  }
167
168  if (due.length > 0) {
169    if (!env.OPENALEX_API_KEY) {
170      logFailure("config", null, `OPENALEX_API_KEY is not set, so ${due.length} stale or missing DOI(s) never refresh`);
171    } else {
172      ctx.waitUntil(refreshCitations(env, due).then(() => undefined));
173    }
174  }
175
176  return out;
177}