app/lib/search/records.mjs

Functions

Imports

It imports app/lib/search/visibility.mjs. Imported by app/kb/procedures/render.mjs, app/kb/procedures/save.server.ts, app/kb/registry/save.server.ts, app/lib/content-pages.mjs, app/lib/cv/save.server.ts, app/lib/editor/post-rows.server.ts, app/lib/pages/save.server.ts, app/lib/publications/save.server.ts, app/lib/search/ask.server.ts, app/lib/search/replace.server.ts.

Source

1// Section-grained: a reader lands on the matching heading, and fusion needs more than one hit per
2// index to do anything. Pure: no clock, filesystem or git, so a record never carries a revision date.
3
4import { PUBLISHED_STATUS, statusForDraft } from "./visibility.mjs";
5
6/**
7 * Fenced code is kept: the identifiers readers search for live in code blocks as often as in prose.
8 *
9 * @param {string} markdown
10 * @returns {string}
11 */
12export function plainText(markdown) {
13  return (
14    markdown
15      // Fence markers, keeping the code between them.
16      .replace(/^\s{0,3}(?:```|~~~).*$/gm, "")
17      // Container directives (:::figure ... :::) and their attribute lines.
18      .replace(/^\s{0,3}:::.*$/gm, "")
19      // Images first, so the alt text of an image does not survive as prose.
20      .replace(/!\[[^\]]*\]\([^)]*\)/g, "")
21      // Links keep their text, drop their target.
22      .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
23      // Reference-style link labels and footnote markers.
24      .replace(/\[\^([^\]]+)\]/g, "")
25      .replace(/^\s{0,3}\[[^\]]+\]:.*$/gm, "")
26      // Leading block markers: heading hashes, quotes, list bullets.
27      .replace(/^\s{0,3}#{1,6}\s+/gm, "")
28      .replace(/^\s{0,3}>\s?/gm, "")
29      .replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
30      // Table pipes and separator rows.
31      .replace(/^\s{0,3}\|?[\s:|-]+\|[\s:|-]*$/gm, "")
32      .replace(/\|/g, " ")
33      // Inline code and emphasis markers; the words they wrap stay.
34      .replace(/`+/g, "")
35      .replace(/\*+/g, "")
36      // Underscores only as emphasis: intra-word ones are identifiers (posts_fts), as in CommonMark.
37      .replace(/(?<![A-Za-z0-9])_+|_+(?![A-Za-z0-9])/g, "")
38      // Whitespace last, so every rule above can leave blank lines behind.
39      .replace(/\s+/g, " ")
40      .trim()
41  );
42}
43
44/**
45 * Headings inside fences are skipped. Zipped with toc BY INDEX, since toc carries the ids rehype-slug
46 * generated and re-deriving them here would drift on punctuation.
47 *
48 * @param {string} markdown post body with frontmatter already stripped
49 * @param {Array<{ depth: number, id: string, text: string }>} toc
50 * @returns {{ intro: string, sections: Array<{ anchor: string, title: string, depth: number, body: string }> }}
51 */
52export function splitSections(markdown, toc) {
53  const lines = markdown.split(/\r?\n/);
54  /*
55   * @type {Array<{ line: number, depth: number }>}
56   */
57  const headings = [];
58
59  let fence = "";
60  for (const [i, line] of lines.entries()) {
61    const fenceMatch = line.match(/^\s{0,3}(```+|~~~+)/);
62    if (fenceMatch) {
63      const marker = fenceMatch[1]?.[0];
64      if (marker) {
65        if (!fence) fence = marker;
66        else if (fence === marker) fence = "";
67      }
68      continue;
69    }
70    if (fence) continue;
71    const heading = line.match(/^\s{0,3}(#{2,3})\s+\S/);
72    if (heading?.[1]) headings.push({ line: i, depth: heading[1].length });
73  }
74
75  // Fail closed: if this slicer and the renderer disagree on heading count, every later anchor is wrong.
76  if (headings.length !== toc.length) {
77    throw new Error(
78      `heading count disagrees with the rendered outline: markdown has ${headings.length}, ` +
79        `the pipeline collected ${toc.length}. Records would carry wrong anchors.`,
80    );
81  }
82
83  const introEnd = headings[0]?.line ?? lines.length;
84  const intro = plainText(lines.slice(0, introEnd).join("\n"));
85
86  /** @type {Array<{ anchor: string, title: string, depth: number, body: string }>} */
87  const sections = [];
88  for (const [s, heading] of headings.entries()) {
89    const entry = toc[s];
90    if (!entry) throw new Error(`no toc entry for heading ${s}`);
91    const start = heading.line + 1;
92    const end = headings[s + 1]?.line ?? lines.length;
93    sections.push({
94      anchor: entry.id,
95      title: entry.text,
96      depth: heading.depth,
97      body: plainText(lines.slice(start, end).join("\n")),
98    });
99  }
100
101  return { intro, sections };
102}
103
104/**
105 * One index record. A document's own record has no anchor and ordinal 0; a section's record takes
106 * the document's uid and url with its anchor appended. Key order is fixed: the output is compared
107 * across runs.
108 *
109 * @param {{ uid: string, url: string, type: "post" | "page", title: string, docTags: string,
110 *           status: string, publishAt: any }} doc
111 * @param {{ title: string, body: string, tags?: string,
112 *           section?: { anchor: string, ordinal: number } }} part
113 * @returns {Record<string, any>}
114 */
115function record(doc, { title, body, tags = "", section }) {
116  return {
117    uid: section ? `${doc.uid}#${section.anchor}` : doc.uid,
118    url: section ? `${doc.url}#${section.anchor}` : doc.url,
119    type: doc.type,
120    title,
121    body,
122    tags,
123    docTags: doc.docTags,
124    docUid: doc.uid,
125    docTitle: doc.title,
126    docUrl: doc.url,
127    anchor: section ? section.anchor : null,
128    ordinal: section ? section.ordinal : 0,
129    status: doc.status,
130    publishAt: doc.publishAt,
131  };
132}
133
134/**
135 * @param {any} post a rendered post record from the shared pipeline
136 * @returns {Array<Record<string, any>>}
137 */
138export function recordsForPost(post) {
139  const tags = post.tags.join(" ");
140  // Pipe-delimited on both ends so a filter can test for an exact tag with
141  // instr(doc_tags, '|d1|') and never match `d1` inside `d10`.
142  const docTags = post.tags.length > 0 ? `|${post.tags.join("|")}|` : "";
143  const { intro, sections } = splitSections(post.markdown, post.toc);
144  const doc = {
145    uid: `post:${post.slug}`,
146    url: `/writing/${post.slug}`,
147    type: /** @type {const} */ ("post"),
148    title: post.title,
149    docTags,
150    status: statusForDraft(post.draft),
151    publishAt: post.publishAt,
152  };
153
154  /** @type {Array<Record<string, any>>} */
155  const records = [
156    record(doc, {
157      title: post.title,
158      body: [post.description, intro].filter(Boolean).join(" "),
159      tags,
160    }),
161  ];
162
163  sections.forEach((section, i) => {
164    // A heading with no prose of its own would match nothing and only pad the facet counts.
165    if (!section.body) return;
166    // Tags stay on the document: repeated per section, section count would drive identity ranking.
167    records.push(
168      record(doc, {
169        title: section.title,
170        body: section.body,
171        section: { anchor: section.anchor, ordinal: i + 1 },
172      }),
173    );
174  });
175
176  return records;
177}
178
179/**
180 * @param {any[]} posts
181 * @returns {Array<Record<string, any>>}
182 */
183export function recordsForPosts(posts) {
184  return [...posts]
185    .sort((a, b) => a.slug.localeCompare(b.slug))
186    .flatMap((post) => recordsForPost(post));
187}
188
189/**
190 * publishAt null and status published by design: the visibility predicate admits a null date, and
191 * inventing one would put an untrue fact in the index.
192 *
193 * @param {{ url: string, uid: string, title: string, description: string,
194 *           intro: string,
195 *           sections: Array<{ anchor: string, title: string, body: string }> }} page
196 * @returns {Array<Record<string, any>>}
197 */
198function recordsForPage(page) {
199  // Fail closed: no sections loses every deep link, and a duplicate anchor collides on uid.
200  if (page.sections.length === 0) {
201    throw new Error(
202      `page ${page.url} produced no section records. A page indexed without ` +
203        `its sections loses every deep link, which is not an empty result, it ` +
204        `is a wrong one.`,
205    );
206  }
207  const anchors = new Set(page.sections.map((s) => s.anchor));
208  if (anchors.size !== page.sections.length) {
209    throw new Error(
210      `page ${page.url} has duplicate section anchors, so two records would ` +
211        `share a uid.`,
212    );
213  }
214
215  // No tags: a page would appear under a tag nothing else on the site shares.
216  const doc = {
217    uid: page.uid,
218    url: page.url,
219    type: /** @type {const} */ ("page"),
220    title: page.title,
221    docTags: "",
222    status: PUBLISHED_STATUS,
223    publishAt: null,
224  };
225
226  /** @type {Array<Record<string, any>>} */
227  const records = [
228    record(doc, { title: page.title, body: [page.description, page.intro].filter(Boolean).join(" ") }),
229  ];
230
231  page.sections.forEach((section, i) => {
232    if (!section.body) return;
233    records.push(
234      record(doc, {
235        title: section.title,
236        body: section.body,
237        section: { anchor: section.anchor, ordinal: i + 1 },
238      }),
239    );
240  });
241
242  return records;
243}
244
245/**
246 * Sorted by uid: the output is compared across runs, so call order must not leak into it.
247 *
248 * @param {any[]} pages page inputs, as `recordsForPage` takes them
249 * @returns {Array<Record<string, any>>}
250 */
251export function recordsForPages(pages) {
252  return [...pages]
253    .sort((a, b) => String(a.uid).localeCompare(String(b.uid)))
254    .flatMap((page) => recordsForPage(page));
255}
256
257/**
258 * One record per paper, not recordsForPage: a paper has no deep links to lose, and some have no abstract.
259 * type "page" because a CHECK constraint allows only post or page, and a facet label is not worth a migration.
260 *
261 * @param {Array<{ uid: string, url: string, title: string, body: string }>} papers
262 * @returns {Array<Record<string, any>>}
263 */
264export function recordsForPapers(papers) {
265  for (const paper of papers) {
266    // Fail closed: an empty body would match only its own title and still look healthy.
267    if (!paper.body || paper.body.trim().length === 0) {
268      throw new Error(
269        `paper ${paper.url} produced an empty search body. A record with no ` +
270          `text is not a small record, it is one that cannot be found by ` +
271          `anything except its own title.`,
272      );
273    }
274  }
275  return [...papers]
276    .sort((a, b) => String(a.uid).localeCompare(String(b.uid)))
277    .map((paper) =>
278      // No tags: the tag facet is the blog's, and its archive would not list a paper.
279      record(
280        {
281          uid: paper.uid,
282          url: paper.url,
283          type: "page",
284          title: paper.title,
285          docTags: "",
286          status: PUBLISHED_STATUS,
287          publishAt: null,
288        },
289        { title: paper.title, body: paper.body },
290      ),
291    );
292}