audit.mjs

Functions

Imports

It imports head.mjs. No module of this repo imports it.

Source

1import { SERP_DESCRIPTION_LIMIT, SERP_TITLE_LIMIT } from "./head.mjs";
2
3/** How many redirects `crawlSite` follows from one URL, so a chain is seen hop by hop and a loop ends. */
4const MAX_HOPS = 5;
5
6/**
7 * @typedef {object} AuditPage
8 * @property {string} url absolute
9 * @property {number} status the HTTP status the crawl got; 0 when the fetch itself failed
10 * @property {Record<string, string> | Headers} [headers] the response headers (`X-Robots-Tag` is read from them)
11 * @property {string} [html] the body, for a 200 HTML page; a page without one gets no head checks
12 * @property {string} [location] where a 3xx pointed
13 * @property {string[]} [redirectedFrom] the URLs that redirected to this one, first to last
14 * @property {boolean} [inSitemap]
15 * @property {string} [error] why the fetch failed, for status 0
16 */
17
18/**
19 * @typedef {object} AuditFinding
20 * @property {string} url
21 * @property {string} check
22 * @property {"error" | "warning"} severity error: search engines get it wrong; warning: worse results
23 * @property {string} detail
24 */
25
26/**
27 * @typedef {object} AuditReport
28 * @property {AuditFinding[]} findings
29 * @property {number} pages how many pages were audited
30 * @property {Record<string, number>} summary findings per check
31 */
32
33/**
34 * The checks a crawler-based site audit runs (the ones Semrush and Ahrefs lead with), over pages a crawl already fetched:
35 * `crawlSite`, Cloudflare Browser Run's `/crawl`, or a site's own test fetching its sitemap. It reads only the tags `headTags`
36 * writes and the links in the page, so it needs no HTML parser. Pure: it fetches nothing.
37 *
38 * @param {AuditPage[]} pages
39 * @param {{ origin: string, sitemap?: string[] }} site `origin` is the production origin every canonical must be on;
40 *   `sitemap` the URLs the sitemap lists (a page's own `inSitemap` counts too), and an empty list is itself a finding
41 * @returns {AuditReport}
42 */
43export function auditPages(pages, { origin, sitemap }) {
44  /** @type {AuditFinding[]} */
45  const findings = [];
46  const add = (/** @type {string} */ url, /** @type {string} */ check, /** @type {"error" | "warning"} */ severity, /** @type {string} */ detail) =>
47    findings.push({ url, check, severity, detail });
48  const byUrl = new Map(pages.map((page) => [normalize(page.url), page]));
49  const listed = new Set((sitemap ?? []).map(normalize));
50  const inSitemap = (/** @type {AuditPage} */ page) => Boolean(page.inSitemap) || listed.has(normalize(page.url));
51  const redirectTargets = new Set(pages.filter(isRedirect).map((page) => resolve(page.location, page.url)).filter(Boolean).map((url) => normalize(/** @type {string} */ (url))));
52  /** @type {Map<string, string[]>} */
53  const titles = new Map();
54  /** @type {Map<string, string[]>} */
55  const descriptions = new Map();
56
57  if (sitemap && sitemap.length === 0) add(origin, "sitemap-empty", "error", "the sitemap lists no pages, or did not load");
58
59  for (const page of pages) {
60    const { url } = page;
61    const from = page.redirectedFrom ?? [];
62    if (from.length > 1 && !byUrl.has(normalize(from[0]))) add(from[0], "redirect-chain", "warning", [...from, url].join(" -> "));
63    if (isRedirect(page)) {
64      if (!redirectTargets.has(normalize(url))) {
65        const hops = chainFrom(page, byUrl);
66        if (hops.length > 2) add(url, "redirect-chain", "warning", hops.join(" -> "));
67      }
68      if (inSitemap(page)) add(url, "sitemap-redirect", "error", "the sitemap lists a URL that redirects");
69      continue;
70    }
71    if (page.status === 0) add(url, "fetch-failed", "error", page.error ?? "no answer");
72    if (page.status !== 200) {
73      if (inSitemap(page)) add(url, "sitemap-broken", "error", `the sitemap lists a URL that answers ${page.status}`);
74      continue;
75    }
76    if (page.html === undefined) continue;
77    const { html } = page;
78    const head = headOf(html);
79
80    const title = textOf(head, "title");
81    if (!title) add(url, "title-missing", "error", "no <title>");
82    else {
83      if (title.length > SERP_TITLE_LIMIT) add(url, "title-long", "warning", `${title.length} characters; results clip at about ${SERP_TITLE_LIMIT}`);
84      push(titles, title, url);
85    }
86
87    const description = meta(head, "name", "description");
88    if (!description) add(url, "description-missing", "error", "no meta description");
89    else {
90      if (description.length > SERP_DESCRIPTION_LIMIT) {
91        add(url, "description-long", "warning", `${description.length} characters; results clip at about ${SERP_DESCRIPTION_LIMIT}`);
92      }
93      push(descriptions, description, url);
94    }
95
96    const href = tags(head, "link").find((attrs) => attrs.rel?.toLowerCase() === "canonical")?.href;
97    const canonical = href ? resolve(href, url) : null;
98    if (!canonical) add(url, "canonical-missing", "error", "no canonical link");
99    else if (new URL(canonical).origin !== origin) add(url, "canonical-off-site", "error", `the canonical is ${canonical}, not on ${origin}`);
100    else if (inSitemap(page) && normalize(canonical) !== normalize(url)) {
101      add(url, "canonical-elsewhere", "error", `the sitemap lists this page, but its canonical is ${canonical}`);
102    }
103
104    const robots = `${meta(head, "name", "robots") ?? ""} ${header(page.headers, "x-robots-tag")}`;
105    if (inSitemap(page) && /noindex/i.test(robots)) add(url, "sitemap-noindex", "error", "the sitemap lists a noindex page");
106    if (!meta(head, "property", "og:title")) add(url, "og-title-missing", "warning", "no og:title, so a shared link's card falls back to a guess");
107    if (!meta(head, "property", "og:image")) add(url, "card-missing", "warning", "no og:image, so a shared link has no card");
108
109    for (const script of jsonLdBlocks(html)) {
110      try {
111        JSON.parse(script);
112      } catch {
113        add(url, "jsonld-invalid", "error", "a JSON-LD block does not parse");
114      }
115    }
116    for (const img of tags(html, "img")) {
117      if (img.alt === undefined) add(url, "image-alt-missing", "warning", `no alt on ${img.src ?? "an image"}`);
118    }
119    for (const target of new Set(internalLinks(html, url, origin).map(normalize))) {
120      const linked = byUrl.get(target);
121      if (!linked) continue;
122      if (linked.status >= 400) add(url, "link-broken", "error", `links to ${linked.url}, which answers ${linked.status}`);
123      else if (isRedirect(linked)) add(url, "link-redirect", "warning", `links to ${linked.url}, which redirects to ${linked.location ?? "somewhere"}`);
124    }
125  }
126
127  for (const [title, urls] of titles) if (urls.length > 1) for (const url of urls) add(url, "title-duplicate", "warning", `"${title}" is on ${urls.length} pages`);
128  for (const [, urls] of descriptions) if (urls.length > 1) for (const url of urls) add(url, "description-duplicate", "warning", `the same description is on ${urls.length} pages`);
129
130  /** @type {Record<string, number>} */
131  const summary = {};
132  for (const { check } of findings) summary[check] = (summary[check] ?? 0) + 1;
133  return { findings, pages: pages.length, summary };
134}
135
136/**
137 * @typedef {object} CrawlInput
138 * @property {string} origin the production origin; only its URLs are fetched
139 * @property {typeof fetch} fetch every request goes through it, so a scheduled job can pass its own (and a test a fake)
140 * @property {number} [maxPages] the most pages fetched in all, redirect hops and link checks included (200)
141 * @property {string} [sitemapPath] where the sitemap is (`/sitemap.xml`); a sitemap index is followed one level
142 */
143
144/**
145 * Fetch a site the way an audit crawler does, for `auditPages`: every sitemap URL with `redirect: "manual"`, following up to
146 * five redirects itself so each hop is a page and a chain shows; then, status only (HEAD, or GET where HEAD is refused), each
147 * same-origin link target those pages name that the sitemap did not. Sequential, and never throws: a fetch that fails is a
148 * page with status 0 and its error. A browser's fetch hides a manual redirect's status, so run it in a Worker or Node.
149 *
150 * @param {CrawlInput} input
151 * @returns {Promise<{ pages: AuditPage[], sitemap: string[] }>}
152 */
153export async function crawlSite({ origin, fetch: fetcher, maxPages = 200, sitemapPath = "/sitemap.xml" }) {
154  const sitemap = [...new Set(await readSitemap(fetcher, new URL(sitemapPath, origin).href, origin, true))];
155  /** @type {AuditPage[]} */
156  const pages = [];
157  const seen = new Set();
158
159  for (let url of sitemap) {
160    /** @type {string[]} */
161    const chain = [];
162    for (let hop = 0; hop <= MAX_HOPS && pages.length < maxPages && !seen.has(normalize(url)); hop++) {
163      seen.add(normalize(url));
164      const page = await fetchPage(fetcher, url, "GET", true);
165      if (chain.length > 0) page.redirectedFrom = [...chain];
166      pages.push(page);
167      const next = isRedirect(page) ? resolve(page.location, url) : null;
168      if (!next || new URL(next).origin !== origin) break;
169      chain.push(url);
170      url = next;
171    }
172  }
173
174  for (const page of pages.filter((each) => each.html !== undefined)) {
175    for (const url of internalLinks(/** @type {string} */ (page.html), page.url, origin)) {
176      if (pages.length >= maxPages) return { pages, sitemap };
177      if (seen.has(normalize(url))) continue;
178      seen.add(normalize(url));
179      const head = await fetchPage(fetcher, url, "HEAD", false);
180      pages.push(head.status === 405 || head.status === 501 ? await fetchPage(fetcher, url, "GET", false) : head);
181    }
182  }
183  return { pages, sitemap };
184}
185
186/** @param {typeof fetch} fetcher @param {string} url @param {string} origin @param {boolean} index @returns {Promise<string[]>} */
187async function readSitemap(fetcher, url, origin, index) {
188  let xml;
189  try {
190    const response = await fetcher(url);
191    if (!response.ok) {
192      await response.body?.cancel();
193      return [];
194    }
195    xml = await response.text();
196  } catch {
197    return [];
198  }
199  const locs = [...xml.matchAll(/<loc>\s*([^<]*?)\s*<\/loc>/gi)].map((match) => decodeEntities(match[1])).filter((loc) => sameOrigin(loc, origin));
200  if (!/<sitemapindex\b/i.test(xml)) return locs;
201  if (!index) return [];
202  const urls = [];
203  for (const loc of locs) urls.push(...(await readSitemap(fetcher, loc, origin, false)));
204  return urls;
205}
206
207/** @param {typeof fetch} fetcher @param {string} url @param {"GET" | "HEAD"} method @param {boolean} body @returns {Promise<AuditPage>} */
208async function fetchPage(fetcher, url, method, body) {
209  try {
210    const response = await fetcher(url, { method, redirect: "manual" });
211    /** @type {Record<string, string>} */
212    const headers = {};
213    response.headers.forEach((value, name) => {
214      headers[name] = value;
215    });
216    /** @type {AuditPage} */
217    const page = { url, status: response.status, headers };
218    if (headers.location) page.location = headers.location;
219    if (body && response.status === 200 && /html/i.test(headers["content-type"] ?? "text/html")) page.html = await response.text();
220    else await response.body?.cancel();
221    return page;
222  } catch (error) {
223    return { url, status: 0, error: error instanceof Error ? error.message : String(error) };
224  }
225}
226
227/** @param {AuditPage} page */
228function isRedirect(page) {
229  return page.status >= 300 && page.status < 400;
230}
231
232/** The URLs a redirect passes through, from `page` to where the crawl stops, a loop cut at its first repeat. @param {AuditPage} page @param {Map<string, AuditPage>} byUrl */
233function chainFrom(page, byUrl) {
234  const hops = [page.url];
235  /** @type {AuditPage | undefined} */
236  let current = page;
237  while (current && isRedirect(current)) {
238    const next = resolve(current.location, current.url);
239    if (!next || hops.some((url) => normalize(url) === normalize(next))) break;
240    hops.push(next);
241    current = byUrl.get(normalize(next));
242  }
243  return hops;
244}
245
246/** @param {Map<string, string[]>} map @param {string} key @param {string} url */
247function push(map, key, url) {
248  map.set(key, [...(map.get(key) ?? []), url]);
249}
250
251/** A URL without its fragment or a trailing slash, so `/a/` and `/a#top` name the page `/a` names. @param {string} url */
252function normalize(url) {
253  const parsed = new URL(url);
254  parsed.hash = "";
255  return parsed.href.replace(/\/$/, "");
256}
257
258/** `href` made absolute against `base`, without its fragment, or null when it is no URL. @param {string | undefined} href @param {string} base */
259function resolve(href, base) {
260  if (!href) return null;
261  try {
262    const url = new URL(href, base);
263    url.hash = "";
264    return url.href;
265  } catch {
266    return null;
267  }
268}
269
270/** @param {string} url @param {string} origin */
271function sameOrigin(url, origin) {
272  const resolved = resolve(url, origin);
273  return resolved !== null && new URL(resolved).origin === origin;
274}
275
276/** The same-origin `<a href>` targets in a page, absolute and without fragments, as written (not normalized). @param {string} html @param {string} base @param {string} origin */
277function internalLinks(html, base, origin) {
278  /** @type {string[]} */
279  const urls = [];
280  for (const { href } of tags(html, "a")) {
281    if (!href || href.startsWith("#") || /^(mailto|tel|javascript|data):/i.test(href)) continue;
282    const url = resolve(href, base);
283    if (url && new URL(url).origin === origin && !urls.includes(url)) urls.push(url);
284  }
285  return urls;
286}
287
288/** @param {Record<string, string> | Headers | undefined} headers @param {string} name */
289function header(headers, name) {
290  if (!headers) return "";
291  if (typeof headers.get === "function") return /** @type {Headers} */ (headers).get(name) ?? "";
292  const key = Object.keys(headers).find((each) => each.toLowerCase() === name);
293  return key ? /** @type {Record<string, string>} */ (headers)[key] : "";
294}
295
296/** @param {string} html */
297function headOf(html) {
298  const end = html.search(/<\/head>/i);
299  return end === -1 ? html : html.slice(0, end);
300}
301
302/** @param {string} html @param {string} name */
303function textOf(html, name) {
304  const match = html.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`, "i"));
305  return match ? decodeEntities(match[1]).trim() : null;
306}
307
308/** @param {string} html @param {"name" | "property"} key @param {string} value */
309function meta(html, key, value) {
310  return tags(html, "meta").find((attrs) => attrs[key]?.toLowerCase() === value)?.content ?? null;
311}
312
313/** @param {string} html */
314function jsonLdBlocks(html) {
315  return [...html.matchAll(/<script\b[^>]*type=["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi)].map((match) => match[1]);
316}
317
318/**
319 * Every `<name ...>` tag's attributes, values entity-decoded; an attribute with no value is the empty string.
320 *
321 * @param {string} html
322 * @param {string} name
323 * @returns {Array<Record<string, string>>}
324 */
325function tags(html, name) {
326  return [...html.matchAll(new RegExp(`<${name}\\b([^>]*)>`, "gi"))].map((match) => {
327    /** @type {Record<string, string>} */
328    const attrs = {};
329    for (const attr of match[1].matchAll(/([^\s=/"']+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) {
330      attrs[attr[1].toLowerCase()] = decodeEntities(attr[2] ?? attr[3] ?? attr[4] ?? "");
331    }
332    return attrs;
333  });
334}
335
336/** @param {string} text */
337function decodeEntities(text) {
338  return text
339    .replace(/&quot;/g, '"')
340    .replace(/&apos;|&#39;/g, "'")
341    .replace(/&lt;/g, "<")
342    .replace(/&gt;/g, ">")
343    .replace(/&amp;/g, "&");
344}