render-nomath.mjs

Functions

Imports

It imports render-directives.mjs, render-error.mjs, render-safe.mjs, render-url.mjs. Imported by render.mjs.

Source

1// The markdown pipeline: markdown in, a safe HTML fragment out. It is dustinedwards-info's renderer (app/lib/content/pipeline.mjs) moved
2// here, so every site that publishes Carrel's writing renders it the same way (carrel/design-markdown-pipeline.md). Runs in a Worker, so
3// no node:fs.
4//
5// This module is the pipeline without math, and imports no KaTeX: a site with no math imports it as `render/nomath`, `$3,000` stays
6// text, and KaTeX (about 1.6 MB raw) never enters its bundle. render.mjs is the same pipeline with KaTeX passed in.
7//
8// What the core does not hold: a site's layout, its frontmatter, its media rules, its code highlighting and its extra directives. The
9// site injects those through `options` (media, highlight, packs), so nothing here imports site code and shiki is not bundled unless a
10// site asks for it.
11//
12// Safe by construction, and the three options that keep it so are the ones a single edit could flip. Each is named at its use, and
13// test/render-safe.test.mjs plants each one and sees the hostile corpus go red:
14//   1. remark-rehype is used WITHOUT allowDangerousHtml, so raw HTML in the source is dropped before it becomes markup.
15//   2. rehypeUrlProtocols runs LAST, so it sees every href and src any plugin emits.
16//   3. KaTeX runs with its default `trust` off, so \href{javascript:...} and \url do not become links (render.mjs).
17
18import { toString as hastToString } from "hast-util-to-string";
19import rehypeAutolinkHeadings from "rehype-autolink-headings";
20import rehypeSlug from "rehype-slug";
21import rehypeStringify from "rehype-stringify";
22import remarkDirective from "remark-directive";
23import remarkGfm from "remark-gfm";
24import remarkParse from "remark-parse";
25import remarkRehype from "remark-rehype";
26import { unified } from "unified";
27import { visit } from "unist-util-visit";
28
29import {
30  CORE_DIRECTIVES,
31  remarkDetails,
32  remarkFigure,
33  remarkNumericTextDirectives,
34  remarkUnknownDirectives,
35} from "./render-directives.mjs";
36import { ContentError } from "./render-error.mjs";
37import { LQIP_STYLE, assertSafe, srcsetUrls } from "./render-safe.mjs";
38import { isAllowedUrl } from "./render-url.mjs";
39
40export { ContentError, isAllowedUrl };
41
42/**
43 * Bumped whenever the output changes for the same input. A site stores it beside its rendered HTML and re-renders on a mismatch; Carrel's
44 * health check reads it from meta.capabilities.render. test/render-golden.test.mjs fails when the output moves and this does not.
45 */
46export const RENDER_VERSION = "3";
47
48/** A source over this is refused before it is parsed: a 200,000-character bracket run took 2.8 s, which a Worker cannot be sent. */
49export const DEFAULT_MAX_BYTES = 512 * 1024;
50
51/** The deepest nesting of markdown nodes allowed (lists in quotes in lists), so a hostile source cannot recurse the stack. */
52export const DEFAULT_MAX_DEPTH = 100;
53
54const encoder = new TextEncoder();
55
56/**
57 * @typedef {import("./render-packs.mjs").RenderPack} RenderPack
58 *
59 * @typedef {object} MediaOptions What a site decides about its images; the core knows none of it.
60 * @property {(raw: string) => string | null} [keyOf] The key a URL cites, or null when it is not media. Collected into mediaRefs.
61 * @property {(src: string) => { srcSet?: string, sizes?: string, href?: string } | null} [imageProps] Responsive attributes for an
62 *   image, and the href of the link that wraps it (the unsized original). Null leaves the image as written.
63 *
64 * @typedef {object} HighlightOptions Code highlighting, injected so the core does not bundle a grammar set.
65 * @property {() => Promise<[unknown, ...unknown[]]>} plugin The rehype plugin and its arguments, as render-highlight.mjs builds them.
66 *
67 * @typedef {object} RenderOptions
68 * @property {readonly RenderPack[]} [packs] Extra directives, in the order they run.
69 * @property {MediaOptions} [media]
70 * @property {HighlightOptions} [highlight] Without it a code block is plain <pre><code class="language-x">.
71 * @property {number} [maxBytes] Refuse a source longer than this many UTF-8 bytes. Default 512 KB.
72 * @property {number} [maxDepth] Refuse a source nested deeper than this. Default 100.
73 * @property {boolean | { tags: ReadonlySet<string>, attributes: ReadonlySet<string> }} [assertSafe] Run the output scan on the result and
74 *   throw when it finds an element or attribute outside the allowlist. Off by default; `true` uses the allowlist derived from the
75 *   conformance corpus, an object supplies a site's own.
76 */
77
78/**
79 * @param {import("mdast").Root} tree
80 * @param {string} file
81 * @param {number} maxDepth
82 */
83function assertDepth(tree, file, maxDepth) {
84  /** @type {Array<[any, number]>} */
85  const stack = [[tree, 1]];
86  while (stack.length > 0) {
87    const [node, depth] = /** @type {[any, number]} */ (stack.pop());
88    if (depth > maxDepth) {
89      const line = node.position?.start?.line;
90      throw new ContentError(
91        file,
92        `markdown is nested more than ${maxDepth} levels deep${line ? ` at line ${line} of the body` : ""}`,
93      );
94    }
95    for (const child of node.children ?? []) stack.push([child, depth + 1]);
96  }
97}
98
99/**
100 * @param {string} file
101 * @param {number} maxDepth
102 */
103function remarkLimitDepth(file, maxDepth) {
104  return (/** @type {import("mdast").Root} */ tree) => assertDepth(tree, file, maxDepth);
105}
106
107/**
108 * Runs at render time so every citation the renderer emits is indexed by construction. Must run BEFORE remarkFigure, which erases the
109 * directive and would turn figure-directive into markdown-image.
110 *
111 * @param {any[]} sink
112 * @param {(raw: string) => string | null} keyOf
113 */
114function remarkCollectMedia(sink, keyOf) {
115  return (/** @type {import("mdast").Root} */ tree) => {
116    /** @param {string | null | undefined} url @param {string} form @param {any} node */
117    const add = (url, form, node) => {
118      const key = keyOf(url ?? "");
119      if (!key) return;
120      sink.push({
121        key,
122        form,
123        detail: node?.position?.start?.line ? `line ${node.position.start.line}` : null,
124      });
125    };
126
127    visit(tree, (node) => {
128      switch (node.type) {
129        case "image":
130          add(node.url, "markdown-image", node);
131          break;
132        case "link":
133          add(node.url, "link", node);
134          break;
135        // A reference definition: the link that uses it carries no URL of its own.
136        case "definition":
137          add(node.url, "link", node);
138          break;
139        case "containerDirective":
140          if (node.name === "figure") add(node.attributes?.src, "figure-directive", node);
141          break;
142        // Regex, not a parser, so every URL-carrying attribute is caught; over-detection only refuses a delete.
143        case "html": {
144          const text = String(node.value ?? "");
145          for (const match of text.matchAll(/["'\s(]((?:https?:)?\/[^"'\s)>]+)/g)) {
146            add(match[1], "html", node);
147          }
148          break;
149        }
150        default:
151          break;
152      }
153    });
154  };
155}
156
157/**
158 * A wrapper, not display: block on the table, which can strip the table role from a screen reader. tabindex so keyboard users can
159 * scroll it; no role=region, because an unnamed region is noise.
160 */
161function rehypeTableScroll() {
162  return (/** @type {import("hast").Root} */ tree) => {
163    visit(tree, "element", (node, index, parent) => {
164      if (node.tagName !== "table" || !parent || index === undefined) return;
165      if (parent.type === "element" && parent.properties?.className?.includes?.("table-scroll")) return;
166      parent.children[index] = {
167        type: "element",
168        tagName: "div",
169        properties: { className: ["table-scroll"], tabIndex: 0 },
170        children: [node],
171      };
172    });
173  };
174}
175
176/**
177 * remark-gfm emits a task-list checkbox with no name, which axe fails as critical. A <label> wraps the checkbox and the item's first
178 * line, the inline content after it up to the first block (a nested list, a paragraph, a table). The name is then what the reader
179 * sees once rendered (math, links, an image's alt), read once, and a loose item's later paragraphs stay out of it.
180 */
181const BLOCK_TAGS = new Set(["blockquote", "details", "div", "figure", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "ol", "p", "pre", "table", "ul"]);
182
183function rehypeTaskListLabels() {
184  return (/** @type {import("hast").Root} */ tree) => {
185    visit(tree, "element", (item) => {
186      if (item.tagName !== "li" || !item.properties?.className?.includes?.("task-list-item")) return;
187      // The checkbox is first in the item, or first in its paragraph when the list is loose.
188      const first = item.children.find((child) => child.type === "element");
189      const line = first?.tagName === "p" ? first : item;
190      const at = line.children.findIndex((child) => child.type === "element");
191      const checkbox = line.children[at];
192      if (checkbox?.type !== "element" || checkbox.tagName !== "input" || checkbox.properties?.type !== "checkbox") return;
193      let end = at + 1;
194      while (end < line.children.length) {
195        const child = line.children[end];
196        if (child.type === "element" && BLOCK_TAGS.has(child.tagName)) break;
197        end += 1;
198      }
199      // Trailing whitespace before a nested list stays outside the label.
200      while (end > at + 1) {
201        const last = line.children[end - 1];
202        if (last.type !== "text" || last.value.trim() !== "") break;
203        end -= 1;
204      }
205      const label = { type: "element", tagName: "label", properties: {}, children: line.children.slice(at, end) };
206      line.children.splice(at, end - at, /** @type {import("hast").Element} */ (label));
207    });
208  };
209}
210
211/**
212 * Renders as the original markdown so a blocked link visibly fails, as a text node so it is escaped. Runs LAST so it sees every href
213 * and src any plugin emits (option 2 in the header).
214 *
215 * @param {string} file
216 * @param {Array<{ file: string, tag: string, url: string }>} sink
217 */
218function rehypeUrlProtocols(file, sink) {
219  return (/** @type {import("hast").Root} */ tree) => {
220    visit(tree, "element", (node, index, parent) => {
221      const isLink = node.tagName === "a";
222      const isImage = node.tagName === "img";
223      if (!isLink && !isImage) return;
224
225      const attribute = isLink ? "href" : "src";
226      const raw = node.properties?.[attribute];
227      if (raw === undefined || raw === null) return;
228
229      const url = String(raw);
230      if (isAllowedUrl(url)) return;
231
232      sink.push({ file, tag: node.tagName, url });
233
234      const text = isLink
235        ? `[${textOf(node)}](${url})`
236        : `![${String(node.properties?.alt ?? "")}](${url})`;
237
238      if (parent && typeof index === "number") {
239        parent.children[index] = { type: "text", value: text };
240      }
241    });
242  };
243}
244
245/**
246 * @param {import("hast").Nodes} node
247 */
248function textOf(node) {
249  let out = "";
250  visit(node, "text", (child) => {
251    out += child.value;
252  });
253  return out;
254}
255
256/**
257 * The anchor and srcset come from one decision in one visitor so they cannot disagree about a storage tier. The first image is eager
258 * with fetchpriority=high: it is usually the LCP. The site decides, through media.imageProps, which images are responsive; an image it
259 * returns null for keeps the attributes the resolver gives it and nothing more.
260 *
261 * @param {string} file
262 * @param {(src: string) => Promise<{ width: number, height: number, placeholder?: string }>} resolveImage
263 * @param {Array<Promise<void>>} pending
264 * @param {(src: string) => { srcSet?: string, sizes?: string, href?: string } | null} imageProps
265 */
266function rehypeImageSources(file, resolveImage, pending, imageProps) {
267  let seen = 0;
268  return (/** @type {import("hast").Root} */ tree) => {
269    visit(tree, "element", (node, index, parent) => {
270      if (node.tagName !== "img") return;
271      // A diagram's asset is rendered by a later build step and may not exist yet. Checked by class, not plugin order, so moving this
272      // plugin cannot re-enable it.
273      const classes = node.properties?.className ?? [];
274      if (Array.isArray(classes) && classes.includes("diagram-image")) return;
275      const src = String(node.properties?.src ?? "");
276      // Left for rehypeUrlProtocols to show as text: no wrapper link, and the site's resolver is never handed a hostile URL.
277      if (!isAllowedUrl(src)) return;
278
279      const props = imageProps(src);
280      // srcset and the placeholder are written after rehypeUrlProtocols has run, so they are checked here: a candidate URL outside the
281      // policy drops the srcset, and a placeholder that is not the base64 form drops the placeholder.
282      const responsive =
283        props?.srcSet && srcsetUrls(props.srcSet).every(isAllowedUrl) ? { srcSet: props.srcSet, sizes: props.sizes } : {};
284
285      // Links to the unsized original, which srcset never serves. Diagrams are excluded: an anchor around the display: none half would
286      // be a focusable link with no name. An author's own link wins.
287      const wrappable =
288        src.length > 0 &&
289        parent &&
290        typeof index === "number" &&
291        parent.children[index] === node &&
292        !(parent.type === "element" && parent.tagName === "a");
293      if (wrappable) {
294        parent.children[index] = {
295          type: "element",
296          tagName: "a",
297          properties: { className: ["image-link"], href: props?.href ?? src },
298          children: [node],
299        };
300      }
301
302      // Counted after every exclusion, so the first image that gets attributes gets the eager pair.
303      seen += 1;
304      const first = seen === 1;
305
306      pending.push(
307        resolveImage(src).then(({ width, height, placeholder }) => {
308          const lqip = placeholder ? `background-image:url("${placeholder}")` : "";
309          node.properties = {
310            ...node.properties,
311            ...responsive,
312            width,
313            height,
314            loading: first ? "eager" : "lazy",
315            decoding: "async",
316            // fetchpriority=high is a budget: several images marked high is the same as none.
317            ...(first ? { fetchpriority: "high" } : {}),
318            // The inline style carries only the per-image URL.
319            ...(LQIP_STYLE.test(lqip)
320              ? {
321                  className: [
322                    ...(Array.isArray(node.properties?.className) ? node.properties.className : []),
323                    "has-lqip",
324                  ],
325                  // Only the form render-safe.mjs allows: base64 holds no quote, parenthesis or space.
326                  style: lqip,
327                }
328              : {}),
329          };
330        }),
331      );
332    });
333  };
334}
335
336/**
337 * @param {Array<{ depth: number, id: string, text: string }>} sink
338 */
339function rehypeCollectToc(sink) {
340  return (/** @type {import("hast").Root} */ tree) => {
341    visit(tree, "element", (node) => {
342      if (node.tagName !== "h2" && node.tagName !== "h3") return;
343      const id = String(node.properties?.id ?? "");
344      if (!id) return;
345      // remark-gfm emits a visually hidden "Footnotes" heading to label its section. It is not part of the outline a reader navigates.
346      const classes = node.properties?.className ?? [];
347      if (Array.isArray(classes) && classes.includes("sr-only")) return;
348      sink.push({
349        depth: node.tagName === "h2" ? 2 : 3,
350        id,
351        text: hastToString(node).replace(/#$/, "").trim(),
352      });
353    });
354  };
355}
356
357/**
358 * @typedef {object} RenderInput
359 * @property {string} file label used in error messages
360 * @property {string} body markdown with any frontmatter already stripped
361 * @property {(src: string) => Promise<{ width: number, height: number, placeholder?: string }>} resolveImage
362 * @property {RenderOptions} [options]
363 *
364 * @typedef {object} MathPlugins What render.mjs passes in; without it there is no math syntax and `$` is text.
365 * @property {any} parse remark-math
366 * @property {(file: string, sink: { value: boolean }) => (tree: import("mdast").Root) => void} validate
367 * @property {any} html rehype-katex
368 * @property {object} options KATEX_OPTIONS
369 */
370
371/** @param {RenderInput} input */
372export function render(input) {
373  return renderWith(input, null);
374}
375
376/**
377 * @param {RenderInput} input
378 * @param {MathPlugins | null} math
379 */
380export async function renderWith({ file, body, resolveImage, options = {} }, math) {
381  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
382  const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
383  const size = encoder.encode(body).byteLength;
384  if (size > maxBytes) {
385    throw new ContentError(file, `source is ${size} bytes and the limit is ${maxBytes}`);
386  }
387
388  const packs = options.packs ?? [];
389  const known = [...CORE_DIRECTIVES, ...packs.flatMap((pack) => pack.directives)];
390  const runs = packs.map((pack) => pack.setup({ file, body }));
391
392  /** @type {Array<{ depth: number, id: string, text: string }>} */
393  const toc = [];
394  /** @type {Array<Promise<void>>} */
395  const pending = [];
396  /** @type {Array<{ key: string, form: string, detail: string | null }>} */
397  const mediaRefs = [];
398  // Collected, not thrown: one bad href must not fail the whole build; the preview shows it.
399  /** @type {Array<{ file: string, tag: string, url: string }>} */
400  const blockedUrls = [];
401  const hasMath = { value: false };
402
403  const keyOf = options.media?.keyOf ?? (() => null);
404  const imageProps = options.media?.imageProps ?? (() => null);
405
406  /** @type {any} */
407  const processor = unified().use(remarkParse).use(remarkGfm);
408  if (math) processor.use(math.parse);
409  processor
410    .use(remarkDirective)
411    .use(remarkLimitDepth, file, maxDepth)
412    // Before any directive is interpreted, so prose such as 12:30 is text again.
413    .use(remarkNumericTextDirectives, body)
414    .use(remarkUnknownDirectives, file, known);
415  if (math) processor.use(math.validate, file, hasMath);
416  processor
417    // Before remarkFigure, which erases the directive the media form is read from.
418    .use(remarkCollectMedia, mediaRefs, keyOf)
419    .use(remarkFigure, file)
420    .use(remarkDetails, file);
421
422  for (const run of runs) {
423    for (const [plugin, ...args] of run.remark) processor.use(plugin, ...args);
424  }
425
426  // Option 1 in the header: no allowDangerousHtml, so raw HTML in the source never becomes markup.
427  processor.use(remarkRehype).use(rehypeTableScroll).use(rehypeTaskListLabels);
428
429  for (const run of runs) {
430    for (const [plugin, ...args] of run.rehype) processor.use(plugin, ...args);
431  }
432
433  processor
434    .use(rehypeSlug)
435    .use(rehypeCollectToc, toc)
436    .use(rehypeAutolinkHeadings, {
437      // Keyboard-reachable on purpose; the aria-label is the name, because # is not one. It carries the heading, so a links list is
438      // not a column of identical "Link to this section" entries. Called before the # is appended, so the text is the heading's alone.
439      properties: (/** @type {any} */ heading) => ({
440        className: ["heading-anchor"],
441        "aria-label": `Link to section: ${hastToString(heading).trim()}`,
442      }),
443      behavior: "append",
444      content: { type: "text", value: "#" },
445    });
446  // After rehypeSlug, rehypeCollectToc and the autolinks: they read heading text, and after KaTeX that text is the expression twice
447  // over (MathML plus HTML), ruining ids and the toc. Option 3 in the header: KATEX_OPTIONS leaves `trust` off.
448  if (math) processor.use(math.html, math.options);
449  processor.use(rehypeImageSources, file, resolveImage, pending, imageProps);
450
451  if (options.highlight) {
452    const [plugin, ...args] = await options.highlight.plugin();
453    processor.use(plugin, ...args);
454  }
455
456  // LAST (option 2 in the header), so it sees every href and src any plugin emits.
457  processor.use(rehypeUrlProtocols, file, blockedUrls).use(rehypeStringify);
458
459  const tree = processor.parse(body);
460  const transformed = await processor.run(tree);
461  await Promise.all(pending);
462  const html = processor.stringify(transformed);
463
464  if (options.assertSafe) {
465    assertSafe(html, options.assertSafe === true ? undefined : options.assertSafe);
466  }
467
468  /** @type {Record<string, unknown>} */
469  const extras = Object.assign({}, ...runs.map((run) => run.result?.() ?? {}));
470
471  return {
472    // A pack may add fields and may fill diagrams; it may not replace the fields below, which are the contract.
473    diagrams: /** @type {Array<{ key: string, source: string }>} */ ([]),
474    ...extras,
475    // A fragment: no <html>, no layout. The site wraps it in its own.
476    html,
477    toc,
478    hasMath: hasMath.value,
479    mediaRefs,
480    // Not a durable field: a fact about one render, and hashing it would churn on a blocked URL.
481    blockedUrls,
482    renderVersion: RENDER_VERSION,
483  };
484}