app/lib/webmention/verify.server.ts

Functions

Imports

It imports app/db/index.ts, app/lib/error-message.mjs, app/lib/seo.ts, app/lib/webmention/urls.mjs. Imported by workers/webmention-verify.ts.

Source

1import { parseHTML } from "linkedom";
2
3import { recordWebmentionVerdict, type WebmentionVerdict } from "~/db";
4import { readCapped } from "@dustinedwards/site-runtime/body";
5import { SITE_ORIGIN } from "~/lib/seo";
6import { collapseExcerpt, sameDocument, sourceVerdict } from "~/lib/webmention/urls.mjs";
7import { errorMessage } from "~/lib/error-message.mjs";
8
9// linkedom's document, not the DOM's.
10type ParsedDocument = ReturnType<typeof parseHTML>["document"];
11
12// The row is written `unverified` and moves to `pending` only if an anchor on the source resolves to the
13// target. Runs in the queue consumer, workers/webmention-verify.ts, which retries a run whose verdict did
14// not land. Nothing from the source is stored as markup.
15
16const MAX_SOURCE_BYTES = 1024 * 1024;
17
18const SOURCE_TIMEOUT_MS = 5000;
19
20// Redirects are followed by hand, each hop re-checked: an automatic follow would let a public source
21// bounce the fetch to a loopback name or an IP literal the route refused on the first URL.
22const MAX_REDIRECTS = 5;
23
24// A fixed set, not a message: free text would put a remote server's prose into the admin plane.
25export const FAILURE_REASONS = {
26  fetchError: "fetch-error",
27  timeout: "timeout",
28  tooLarge: "too-large",
29  notHtml: "not-html",
30  noLink: "no-link",
31  redirectRefused: "redirect-refused",
32} as const;
33
34// No h-card: the name is the source's hostname, a fact rather than a guess. `u-url` is kept only if it
35// resolves to absolute http(s).
36function readAuthor(
37  document: ParsedDocument,
38  sourceUrl: string,
39): { authorName: string | null; authorUrl: string | null } {
40  const hostname = (() => {
41    try {
42      return new URL(sourceUrl).hostname;
43    } catch {
44      return null;
45    }
46  })();
47
48  const card = document.querySelector(".h-card");
49  if (!card) return { authorName: hostname, authorUrl: null };
50
51  const nameNode = card.querySelector(".p-name");
52  const name = collapseExcerpt(nameNode?.textContent ?? card.textContent ?? "");
53
54  const urlNode = card.querySelector(".u-url");
55  const rawUrl = urlNode?.getAttribute("href") ?? null;
56  let authorUrl: string | null = null;
57  if (rawUrl) {
58    try {
59      const resolved = new URL(rawUrl, sourceUrl);
60      if (resolved.protocol === "https:" || resolved.protocol === "http:") {
61        authorUrl = resolved.href;
62      }
63    } catch {
64      authorUrl = null;
65    }
66  }
67
68  return { authorName: name.length > 0 ? name : hostname, authorUrl };
69}
70
71// Pure of the database, so a test can drive the decision over a stubbed fetch.
72export async function inspectSource(
73  sourceUrl: string,
74  targetUrl: string,
75): Promise<WebmentionVerdict> {
76  // One signal, read at both sites: a timeout during the body read makes the read throw, which would
77  // otherwise be misreported as a fetch error.
78  const signal = AbortSignal.timeout(SOURCE_TIMEOUT_MS);
79
80  // The same two origins the route refused on the first URL: the canonical one and the target's host,
81  // which differ before the cutover. A hop to either is a redirect back onto this site.
82  const siteOrigins = [...new Set([SITE_ORIGIN, new URL(targetUrl).origin])];
83  let response: Response;
84  let at = sourceUrl;
85  for (let hop = 0; ; hop += 1) {
86    try {
87      response = await fetch(at, {
88        method: "GET",
89        headers: { accept: "text/html, application/xhtml+xml" },
90        redirect: "manual",
91        signal,
92      });
93    } catch {
94      return {
95        status: "failed",
96        failureReason: signal.aborted ? FAILURE_REASONS.timeout : FAILURE_REASONS.fetchError,
97      };
98    }
99    const location = response.headers.get("location");
100    if (response.status < 300 || response.status > 399 || !location) break;
101    await response.body?.cancel();
102    if (hop >= MAX_REDIRECTS) {
103      return { status: "failed", failureReason: FAILURE_REASONS.redirectRefused };
104    }
105    let next: string;
106    try {
107      next = new URL(location, at).href;
108    } catch {
109      return { status: "failed", failureReason: FAILURE_REASONS.redirectRefused };
110    }
111    if (!sourceVerdict(next, targetUrl, siteOrigins).ok) {
112      return { status: "failed", failureReason: FAILURE_REASONS.redirectRefused };
113    }
114    at = next;
115  }
116
117  if (!response.ok) {
118    return { status: "failed", failureReason: FAILURE_REASONS.fetchError };
119  }
120
121  // Checked before the body is read, so a video costs nothing to refuse. An absent content-type is not HTML.
122  const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
123  if (!contentType.includes("text/html") && !contentType.includes("application/xhtml+xml")) {
124    return { status: "failed", failureReason: FAILURE_REASONS.notHtml };
125  }
126
127  let body: string | null;
128  try {
129    body = await readCapped(response, MAX_SOURCE_BYTES);
130  } catch {
131    return {
132      status: "failed",
133      failureReason: signal.aborted ? FAILURE_REASONS.timeout : FAILURE_REASONS.fetchError,
134    };
135  }
136  if (body === null) {
137    return { status: "failed", failureReason: FAILURE_REASONS.tooLarge };
138  }
139  if (signal.aborted) {
140    return { status: "failed", failureReason: FAILURE_REASONS.timeout };
141  }
142
143  const { document } = parseHTML(body);
144
145  // Resolved against the source before comparing. The first match wins: a page linking here twice is one mention.
146  const anchors = [...document.querySelectorAll("a[href]")];
147  const match = anchors.find((a) => {
148    const href = a.getAttribute("href");
149    if (!href) return false;
150    try {
151      return sameDocument(new URL(href, sourceUrl).href, targetUrl);
152    } catch {
153      return false;
154    }
155  });
156
157  if (!match) {
158    return { status: "failed", failureReason: FAILURE_REASONS.noLink };
159  }
160
161  const { authorName, authorUrl } = readAuthor(document, sourceUrl);
162
163  // The containing element's text: the anchor alone is usually the title; the parent is the sentence.
164  const container = match.parentElement ?? match;
165  const excerpt = collapseExcerpt(container.textContent ?? "");
166
167  return {
168    status: "pending",
169    authorName,
170    authorUrl,
171    excerpt: excerpt.length > 0 ? excerpt : null,
172  };
173}
174
175// NEVER THROWS: every source failure is a recorded verdict, so the consumer retries only a failed write.
176export async function verifyWebmention(
177  env: Env,
178  id: number,
179  sourceUrl: string,
180  targetUrl: string,
181): Promise<void> {
182  let verdict: WebmentionVerdict;
183  try {
184    verdict = await inspectSource(sourceUrl, targetUrl);
185  } catch (error) {
186    console.error(
187      JSON.stringify({
188        alert: "webmention-verify-threw",
189        id,
190        detail: errorMessage(error),
191      }),
192    );
193    verdict = { status: "failed", failureReason: FAILURE_REASONS.fetchError };
194  }
195
196  try {
197    await recordWebmentionVerdict(env, id, verdict);
198  } catch (error) {
199    // The row stays `unverified`, a visible state rather than a fabricated verdict.
200    console.error(
201      JSON.stringify({
202        alert: "webmention-verdict-write-failed",
203        id,
204        detail: errorMessage(error),
205      }),
206    );
207  }
208}