app/lib/editor/markdown.mjs

Functions

Imports

It imports no other module of this repo. Imported by app/components/editor/rich-editor.tsx, app/components/editor/writing-surface.tsx, app/lib/editor/schema.mjs.

Source

1// Markdown in and out of the formatted editor (carrel/research/editor-spike.md).
2// The source stays the truth. Parsing splits a file into its top-level blocks with the parser family
3// the site renders with (micromark and mdast, as remark), and every block remembers its exact bytes
4// and the bytes before it. Saving writes each block the editor did not change as those bytes, and
5// only a changed or new block is spelled again, by the writer below, which is checked by parsing what
6// it wrote. So a save changes the bytes of the blocks a person edited and no others.
7//
8// Prose (paragraphs, headings, lists, block quotes, breaks, and bold, italic, strikethrough, code and
9// links inside them) becomes formatted text. Everything else becomes a source block, or inside a
10// paragraph a source atom, that holds its Markdown as written: front matter, fenced and indented
11// code, tables, directives, math, HTML, footnotes, images, references and bare links.
12//
13// Plain JavaScript with JSDoc types, so the app, the tests and scripts/check-roundtrip.mjs run the
14// same code with no build step. It touches no DOM.
15
16import { fromMarkdown } from "mdast-util-from-markdown";
17import { directiveFromMarkdown } from "mdast-util-directive";
18import { frontmatterFromMarkdown } from "mdast-util-frontmatter";
19import { gfmFromMarkdown } from "mdast-util-gfm";
20import { mathFromMarkdown } from "mdast-util-math";
21import { directive } from "micromark-extension-directive";
22import { frontmatter } from "micromark-extension-frontmatter";
23import { gfm } from "micromark-extension-gfm";
24import { math } from "micromark-extension-math";
25
26/**
27 * "site": a site's Markdown, with the directives and math dustinedwards.info renders.
28 * "plain": a book file, which is CommonMark with GFM and a header.
29 * @typedef {"site" | "plain"} Dialect
30 * @typedef {{ id: number, source: string, before: string }} MdOrigin
31 * @typedef {{ type: string, attrs?: Record<string, any>, content?: JsonNode[], text?: string, marks?: JsonMark[] }} JsonNode
32 * @typedef {{ type: string, attrs?: Record<string, any> }} JsonMark
33 * @typedef {{ doc: JsonNode, tail: string }} Loaded
34 */
35
36/** The node types that may hold a block's origin. */
37export const BLOCK_TYPES = ["paragraph", "heading", "bulletList", "orderedList", "blockquote", "horizontalRule", "sourceBlock"];
38
39/**
40 * @param {string} source
41 * @param {Dialect} dialect
42 */
43function parseTree(source, dialect) {
44  const site = dialect === "site";
45  return fromMarkdown(source, {
46    extensions: [frontmatter(["yaml"]), gfm(), ...(site ? [directive(), math()] : [])],
47    mdastExtensions: [frontmatterFromMarkdown(["yaml"]), gfmFromMarkdown(), ...(site ? [directiveFromMarkdown(), mathFromMarkdown()] : [])],
48  });
49}
50
51// ---------- reading: mdast to the editor's JSON
52
53/** @param {any} node */
54const span = (node) => [node.position.start.offset, node.position.end.offset];
55
56/**
57 * A node's bytes as written. A link GFM finds in bare text (www.example.com) is made after parsing
58 * and has no position, so its bytes are its children's.
59 * @param {any} node
60 * @param {string} source
61 * @returns {string}
62 */
63function sliceOf(node, source) {
64  if (node.position?.start?.offset !== undefined && node.position?.end?.offset !== undefined) return source.slice(node.position.start.offset, node.position.end.offset);
65  if (Array.isArray(node.children)) return node.children.map((/** @type {any} */ c) => sliceOf(c, source)).join("");
66  return typeof node.value === "string" ? node.value : "";
67}
68
69/**
70 * @param {any[]} children
71 * @param {string} source
72 * @param {JsonMark[]} marks
73 * @returns {JsonNode[] | null} null when something here cannot be formatted text
74 */
75function inlines(children, source, marks = []) {
76  /** @type {JsonNode[]} */
77  const out = [];
78  /** @param {JsonNode} n */
79  const push = (n) => {
80    const last = out.at(-1);
81    // Adjacent text with the same marks is one text node, as ProseMirror keeps it.
82    if (n.type === "text" && last?.type === "text" && JSON.stringify(last.marks ?? []) === JSON.stringify(n.marks ?? [])) last.text = (last.text ?? "") + (n.text ?? "");
83    else out.push(n);
84  };
85  /** @param {any} node */
86  const atom = (node) => {
87    push({ type: "sourceInline", attrs: { text: sliceOf(node, source) }, ...(marks.length ? { marks } : {}) });
88  };
89  for (const node of children) {
90    switch (node.type) {
91      case "text":
92        if (node.value) push({ type: "text", text: node.value, ...(marks.length ? { marks } : {}) });
93        break;
94      case "emphasis":
95      case "strong":
96      case "delete": {
97        const type = node.type === "emphasis" ? "italic" : node.type === "strong" ? "bold" : "strike";
98        const inner = inlines(node.children, source, [...marks, { type }]);
99        if (!inner || inner.length === 0) return null;
100        inner.forEach(push);
101        break;
102      }
103      case "inlineCode":
104        // Code excludes every other mark in the editor, so code inside bold or a link stays as written.
105        if (marks.length) atom(node);
106        else push({ type: "text", text: node.value, marks: [{ type: "code" }] });
107        break;
108      case "link": {
109        // Only [text](url) is formatted; a bare URL, <url> or an e-mail stays exactly as typed.
110        if (!node.position || source[node.position.start.offset] !== "[" || node.children.length === 0 || marks.some((m) => m.type === "link")) {
111          atom(node);
112          break;
113        }
114        const inner = inlines(node.children, source, [...marks, { type: "link", attrs: { href: node.url, title: node.title ?? null } }]);
115        if (!inner || inner.length === 0) atom(node);
116        else inner.forEach(push);
117        break;
118      }
119      case "break":
120        push({ type: "hardBreak" });
121        break;
122      case "textDirective":
123        // The site reads 4.5:1 and localhost:8080 as text, not as directives named "1" or "8080".
124        if (/^\d/.test(node.name ?? "")) push({ type: "text", text: sliceOf(node, source), ...(marks.length ? { marks } : {}) });
125        else atom(node);
126        break;
127      default:
128        atom(node);
129    }
130  }
131  return out;
132}
133
134/**
135 * @param {any} node
136 * @param {string} source
137 * @returns {JsonNode | null} null when the block cannot be formatted text
138 */
139function blockJson(node, source) {
140  switch (node.type) {
141    case "paragraph": {
142      const content = inlines(node.children, source);
143      return content && content.length ? { type: "paragraph", content } : null;
144    }
145    case "heading": {
146      const content = inlines(node.children, source);
147      if (!content) return null;
148      return { type: "heading", attrs: { level: node.depth }, ...(content.length ? { content } : {}) };
149    }
150    case "thematicBreak":
151      return { type: "horizontalRule" };
152    case "blockquote": {
153      const content = node.children.map((/** @type {any} */ c) => blockJson(c, source));
154      if (content.length === 0 || content.some((/** @type {any} */ c) => !c)) return null;
155      return { type: "blockquote", content };
156    }
157    case "list": {
158      const items = [];
159      for (const item of node.children) {
160        // A task item has a checkbox the editor does not draw; it stays Markdown.
161        if (item.checked !== null && item.checked !== undefined) return null;
162        const content = item.children.map((/** @type {any} */ c) => blockJson(c, source));
163        if (content.length === 0 || content[0]?.type !== "paragraph" || content.some((/** @type {any} */ c) => !c)) return null;
164        items.push({ type: "listItem", content });
165      }
166      if (items.length === 0) return null;
167      return node.ordered ? { type: "orderedList", attrs: { start: node.start ?? 1 }, content: items } : { type: "bulletList", content: items };
168    }
169    default:
170      return null;
171  }
172}
173
174/** What a source block is, for its label: "Front matter", "Code", "Table", "Chart" and so on. */
175/** @param {any} node */
176function kindOf(node) {
177  if (node.type === "yaml") return "Front matter";
178  if (node.type === "code") return node.lang ? `Code (${node.lang})` : "Code";
179  if (node.type === "table") return "Table";
180  if (node.type === "math") return "Math";
181  if (node.type === "html") return "HTML";
182  if (node.type === "footnoteDefinition") return "Footnote";
183  if (node.type === "definition") return "Link reference";
184  if (node.type === "containerDirective" || node.type === "leafDirective") return node.name ? node.name.charAt(0).toUpperCase() + node.name.slice(1) : "Directive";
185  if (node.type === "list") return "List";
186  if (node.type === "blockquote") return "Quote";
187  return "Markdown";
188}
189
190/**
191 * A file as the editor's document: one top-level node per block, each with its origin.
192 * @param {string} source
193 * @param {Dialect} dialect
194 * @returns {Loaded}
195 */
196export function toDoc(source, dialect) {
197  const tree = parseTree(source, dialect);
198  /** @type {JsonNode[]} */
199  const content = [];
200  let at = 0;
201  tree.children.forEach((/** @type {any} */ node, /** @type {number} */ id) => {
202    const [start, end] = span(node);
203    const text = source.slice(start, end);
204    const json = blockJson(node, source) ?? { type: "sourceBlock", attrs: { text, kind: kindOf(node) } };
205    json.attrs = { ...json.attrs, md: { id, source: text, before: source.slice(at, start) } };
206    content.push(json);
207    at = end;
208  });
209  if (content.length === 0) {
210    // An empty or blank file: one empty paragraph that writes nothing until something is typed.
211    return { doc: { type: "doc", content: [{ type: "paragraph", attrs: { md: { id: 0, source: "", before: "" } } }] }, tail: source };
212  }
213  return { doc: { type: "doc", content }, tail: source.slice(at) };
214}
215
216// ---------- comparing
217
218/**
219 * A node with its origins taken out, as a string: two blocks with the same print are the same text.
220 * @param {JsonNode} node
221 */
222function print(node) {
223  return JSON.stringify(node, (key, value) => (key === "md" ? undefined : value));
224}
225
226/**
227 * The prints of a freshly loaded document's blocks, by origin id. Taken from the editor's own nodes
228 * after loading, so whatever the editor's schema fills in is in both sides of the comparison.
229 * @param {JsonNode} doc
230 */
231export function fingerprints(doc) {
232  /** @type {Map<number, string>} */
233  const prints = new Map();
234  for (const block of doc.content ?? []) {
235    const md = /** @type {MdOrigin | null | undefined} */ (block.attrs?.md);
236    if (md) prints.set(md.id, print(block));
237  }
238  return prints;
239}
240
241/** The fields that carry meaning, so a check by parsing ignores the defaults the editor adds. */
242/** @param {JsonNode} node @returns {any} */
243function canon(node) {
244  const keep = ["level", "start", "href", "title", "text"];
245  /** @param {Record<string, any> | undefined} attrs */
246  const attrs = (attrs) => {
247    /** @type {Record<string, any>} */
248    const out = {};
249    for (const k of keep) if (attrs && attrs[k] !== undefined && attrs[k] !== null && !(k === "start" && attrs[k] === 1)) out[k] = attrs[k];
250    return out;
251  };
252  const order = ["link", "bold", "italic", "strike", "code"];
253  /** @type {any[]} */
254  const content = [];
255  /** @param {any} c */
256  const add = (c) => {
257    if (c.type === "text" && c.text === "") return;
258    const last = content.at(-1);
259    if (c.type === "text" && last?.type === "text" && JSON.stringify(last.marks) === JSON.stringify(c.marks)) last.text += c.text;
260    else content.push(c);
261  };
262  for (const child of node.content ?? []) {
263    // A source atom is written as its bytes. Checked alone, a block can read it as plain text (a
264    // reference whose definition is in another block), so both count as the same characters.
265    const c = child.type === "sourceInline" ? canon({ type: "text", text: child.attrs?.text ?? "", marks: child.marks }) : canon(child);
266    // Whitespace at the edge of bold or italic cannot be written inside the delimiters, so the writer
267    // puts it outside; both readings count as the same text.
268    if (c.type === "text" && c.marks.length && !c.marks.some((/** @type {any} */ m) => m.type === "code")) {
269      const [, lead = "", body = "", trail = ""] = /^(\s*)([\s\S]*?)(\s*)$/.exec(c.text) ?? [];
270      add({ ...c, text: lead, marks: [] });
271      add({ ...c, text: body });
272      add({ ...c, text: trail, marks: [] });
273    } else add(c);
274  }
275  // Markdown trims the spaces at the start and end of a paragraph or heading.
276  if ((node.type === "paragraph" || node.type === "heading") && content.length) {
277    const first = content[0];
278    const last = content.at(-1);
279    if (first.type === "text") first.text = first.text.replace(/^[ \t]+/, "");
280    if (last.type === "text") last.text = last.text.replace(/[ \t]+$/, "");
281    while (content.length && content[0].type === "text" && content[0].text === "") content.shift();
282    while (content.length && content.at(-1).type === "text" && content.at(-1).text === "") content.pop();
283  }
284  return {
285    type: node.type,
286    attrs: attrs(node.attrs),
287    text: node.text,
288    marks: (node.marks ?? []).map((m) => ({ type: m.type, attrs: attrs(m.attrs) })).sort((a, b) => order.indexOf(a.type) - order.indexOf(b.type)),
289    content,
290  };
291}
292
293/**
294 * Whether two blocks hold the same text and formatting, ignoring origins, the defaults the editor
295 * fills in, and whitespace moved out of bold or italic.
296 * @param {JsonNode} a
297 * @param {JsonNode} b
298 */
299export function sameBlock(a, b) {
300  return JSON.stringify(canon(a)) === JSON.stringify(canon(b));
301}
302
303// ---------- writing
304
305const MARK_ORDER = ["link", "bold", "italic", "strike", "code"];
306
307/**
308 * @param {string} text
309 * @param {{ all: boolean, dialect: Dialect, lineStart: boolean, afterAtom: boolean }} o
310 * @returns {string}
311 */
312function escapeText(text, o) {
313  // Right after a bare URL, a backslash would become part of the URL, so its first mark stays as is.
314  const keep = o.afterAtom && /^[_*~]/.test(text) ? text[0] : "";
315  if (keep) return keep + escapeText(text.slice(1), { ...o, afterAtom: false });
316  if (o.all) return text.replace(/[!-/:-@[-`{-~]/g, (c) => `\\${c}`);
317  let out = "";
318  for (let i = 0; i < text.length; i++) {
319    const c = text[i] ?? "";
320    const prev = i === 0 ? "" : (text[i - 1] ?? "");
321    const next = text[i + 1] ?? "";
322    const word = (/** @type {string} */ ch) => /[\p{L}\p{N}]/u.test(ch);
323    if (c === "\\" || c === "*" || c === "`") out += `\\${c}`;
324    else if (c === "_" && !(word(prev) && word(next))) out += "\\_";
325    else if (c === "~" && (next === "~" || prev === "~")) out += "\\~";
326    else if (c === "<" && /[A-Za-z/!?]/.test(next)) out += "\\<";
327    else if (c === "&" && /^&(#?[A-Za-z0-9]+);/.test(text.slice(i))) out += "\\&";
328    else if (c === "[" && /\]\s*[([:]/.test(text.slice(i))) out += "\\[";
329    else if (o.dialect === "site" && c === ":" && /[A-Za-z]/.test(next) && prev !== ":") out += "\\:";
330    else if (o.dialect === "site" && c === "$") out += "\\$";
331    else out += c;
332  }
333  // What would start another block at the start of a line.
334  return out.replace(/(^|\n)([ \t]*)(#{1,6}(?=[ \t]|$)|>|[-+](?=[ \t])|\d{1,9}(?=[.)][ \t]|[.)]$)|=+[ \t]*$|-+[ \t]*$)/g, (m, nl, ws, mark) => {
335    if (!o.lineStart && nl === "") return m;
336    return /^\d/.test(mark) ? `${nl}${ws}${mark}\\` : `${nl}${ws}\\${mark}`;
337  });
338}
339
340/** @param {string} code */
341function codeSpan(code) {
342  const longest = Math.max(0, ...(code.match(/`+/g) ?? []).map((r) => r.length));
343  const fence = "`".repeat(longest + 1);
344  const pad = /^`|`$|^ .* $/.test(code) || code.startsWith("`") || code.endsWith("`") ? " " : "";
345  return `${fence}${pad}${code}${pad}${fence}`;
346}
347
348/** @param {string} href @param {string | null} title */
349function linkTail(href, title) {
350  const url = /[\s()<>]/.test(href) ? `<${href.replace(/[<>]/g, (c) => encodeURIComponent(c))}>` : href;
351  return `](${url}${title ? ` "${title.replace(/["\\]/g, (c) => `\\${c}`)}"` : ""})`;
352}
353
354/**
355 * @param {JsonNode[]} nodes
356 * @param {{ all: boolean, dialect: Dialect, emphasis: string }} o
357 */
358function writeInlines(nodes, o) {
359  /** @type {JsonMark[]} */
360  let open = [];
361  let out = "";
362  /** @param {JsonMark} m @param {boolean} opening */
363  const delim = (m, opening) => {
364    if (m.type === "bold") return "**";
365    if (m.type === "italic") return o.emphasis;
366    if (m.type === "strike") return "~~";
367    if (m.type === "link") return opening ? "[" : linkTail(m.attrs?.href ?? "", m.attrs?.title ?? null);
368    return "";
369  };
370  const sorted = (/** @type {JsonNode} */ n) => [...(n.marks ?? [])].filter((m) => m.type !== "code").sort((a, b) => MARK_ORDER.indexOf(a.type) - MARK_ORDER.indexOf(b.type));
371  const same = (/** @type {JsonMark} */ a, /** @type {JsonMark} */ b) => a.type === b.type && JSON.stringify(a.attrs ?? {}) === JSON.stringify(b.attrs ?? {});
372  /** @param {JsonMark[]} wanted @param {string} lead */
373  const moveTo = (wanted, lead = "") => {
374    let keep = 0;
375    while (keep < open.length && keep < wanted.length && same(/** @type {JsonMark} */ (open[keep]), /** @type {JsonMark} */ (wanted[keep]))) keep++;
376    for (let i = open.length - 1; i >= keep; i--) out += delim(/** @type {JsonMark} */ (open[i]), false);
377    out += lead;
378    for (let i = keep; i < wanted.length; i++) out += delim(/** @type {JsonMark} */ (wanted[i]), true);
379    open = wanted;
380  };
381  nodes.forEach((node, i) => {
382    const marks = sorted(node);
383    if (node.type === "text") {
384      const code = (node.marks ?? []).some((m) => m.type === "code");
385      let text = node.text ?? "";
386      // Delimiters cannot sit against a space: whitespace at the edges of a marked run goes outside.
387      const prefix = (/** @type {JsonMark[]} */ a, /** @type {JsonMark[]} */ b) => {
388        let k = 0;
389        while (k < a.length && k < b.length && same(/** @type {JsonMark} */ (a[k]), /** @type {JsonMark} */ (b[k]))) k++;
390        return k;
391      };
392      // A mark opens or closes wherever the stack changes below it, not only where it starts or ends.
393      const opens = prefix(open, marks) < marks.length;
394      const lead = opens && !code ? (/^\s+/.exec(text)?.[0] ?? "") : "";
395      text = text.slice(lead.length);
396      const after = sorted(nodes[i + 1] ?? { type: "end" });
397      const closes = prefix(marks, after) < marks.length;
398      const trail = closes && !code ? (/\s+$/.exec(text)?.[0] ?? "") : "";
399      text = text.slice(0, text.length - trail.length);
400      moveTo(marks, lead);
401      const lineStart = out === "" || out.endsWith("\n");
402      out += code ? codeSpan(text) : escapeText(text, { all: o.all, dialect: o.dialect, lineStart, afterAtom: !lead && nodes[i - 1]?.type === "sourceInline" });
403      if (trail) {
404        moveTo(marks.slice(0, prefix(marks, after)));
405        out += trail;
406      }
407    } else if (node.type === "hardBreak") {
408      moveTo(marks);
409      out += "\\\n";
410    } else if (node.type === "sourceInline") {
411      moveTo(marks);
412      out += node.attrs?.text ?? "";
413    }
414  });
415  moveTo([]);
416  return out;
417}
418
419/** @param {string} text @param {string} first @param {string} rest */
420const indent = (text, first, rest) =>
421  text
422    .split("\n")
423    .map((line, i) => (i === 0 ? first : line === "" ? "" : rest) + line)
424    .join("\n");
425
426/**
427 * One block, spelled in the house style.
428 * @param {JsonNode} node
429 * @param {{ all: boolean, dialect: Dialect, emphasis: string }} o
430 * @returns {string}
431 */
432function writeBlock(node, o) {
433  switch (node.type) {
434    case "paragraph":
435      return writeInlines(node.content ?? [], o);
436    case "heading":
437      return `${"#".repeat(node.attrs?.level ?? 1)}${node.content?.length ? ` ${writeInlines(node.content, o)}` : ""}`;
438    case "horizontalRule":
439      return "***";
440    case "sourceBlock":
441      return node.attrs?.text ?? "";
442    case "blockquote":
443      return (node.content ?? [])
444        .map((c) => writeBlock(c, o))
445        .join("\n\n")
446        .split("\n")
447        .map((line) => (line === "" ? ">" : `> ${line}`))
448        .join("\n");
449    case "bulletList":
450    case "orderedList": {
451      const start = node.attrs?.start ?? 1;
452      const items = node.content ?? [];
453      const loose = items.some((item) => (item.content ?? []).filter((c) => c.type !== "bulletList" && c.type !== "orderedList").length > 1);
454      return items
455        .map((item, i) => {
456          const marker = node.type === "bulletList" ? "- " : `${start + i}. `;
457          const body = (item.content ?? []).map((c) => writeBlock(c, o)).join(loose ? "\n\n" : "\n");
458          return indent(body, marker, " ".repeat(marker.length));
459        })
460        .join(loose ? "\n\n" : "\n");
461    }
462    default:
463      return "";
464  }
465}
466
467/** The emphasis delimiter a block already uses, so an edit keeps its spelling. */
468/** @param {JsonNode} block */
469function emphasisOf(block) {
470  const source = /** @type {MdOrigin | null | undefined} */ (block.attrs?.md)?.source ?? "";
471  return /(^|[^\p{L}\p{N}_])_[^_\s]/u.test(source) && !/(^|[^*\\])\*[^*\s]/.test(source) ? "_" : "*";
472}
473
474/**
475 * A changed block, spelled and then checked: what was written must parse back to the same block.
476 * When the light escaping does not survive, every punctuation mark in the text is escaped.
477 * @param {JsonNode} block
478 * @param {Dialect} dialect
479 */
480export function writeChecked(block, dialect) {
481  if (block.type === "sourceBlock") return block.attrs?.text ?? "";
482  const emphasis = emphasisOf(block);
483  const want = JSON.stringify(canon(block));
484  let text = "";
485  for (const all of [false, true]) {
486    text = writeBlock(block, { all, dialect, emphasis });
487    if (block.type === "paragraph" && !(block.content ?? []).length) return "";
488    const back = toDoc(text, dialect).doc.content ?? [];
489    if (back.length === 1 && JSON.stringify(canon(/** @type {JsonNode} */ (back[0]))) === want) return text;
490  }
491  return text;
492}
493
494/**
495 * The file the editor's document stands for. A block the editor did not change is written as the
496 * bytes it was read from, with the bytes that came before it; a changed one keeps the bytes before
497 * it and is spelled again; a new one follows a blank line.
498 * @param {JsonNode} doc the editor's document as JSON
499 * @param {string} tail what followed the last block when the file was read
500 * @param {Map<number, string>} prints the blocks as they were loaded (fingerprints)
501 * @param {Dialect} dialect
502 */
503export function fromDoc(doc, tail, prints, dialect) {
504  let out = "";
505  for (const block of doc.content ?? []) {
506    const md = /** @type {MdOrigin | null | undefined} */ (block.attrs?.md);
507    if (md && prints.get(md.id) === print(block)) {
508      out += md.before + md.source;
509      continue;
510    }
511    const text = writeChecked(block, dialect);
512    if (md) out += md.before + text;
513    else if (text !== "") out += (out === "" ? "" : "\n\n") + text;
514  }
515  return out + tail;
516}
517
518/**
519 * Markdown pasted as plain text, as blocks with no origin: they are new, and are spelled on save.
520 * @param {string} text
521 * @param {Dialect} dialect
522 * @returns {JsonNode[]}
523 */
524export function pastedBlocks(text, dialect) {
525  return JSON.parse(JSON.stringify(toDoc(text, dialect).doc.content ?? []), (key, value) => (key === "md" ? null : value));
526}