app/lib/search/search.server.ts

Functions

Imports

It imports app/kb/search-bases.mjs, app/lib/content/og-card-text.mjs, app/lib/search/query.mjs, app/lib/search/sort.mjs, app/lib/search/visibility.mjs. Imported by app/lib/search/ask.server.ts, app/routes/search.tsx.

Source

1// Raw D1 rather than drizzle: bm25(), snippet() and MATCH have no drizzle surface. Nothing here
2// awaits a third party, so the AI layer cannot slow or break classic search.
3
4import {
5  RRF_K,
6  fuse,
7  hasFilters,
8  parseQuery,
9  toMatchExpression,
10} from "./query.mjs";
11import { applySort, parseSort, type SearchSort } from "./sort.mjs";
12import { PUBLISHED_STATUS } from "./visibility.mjs";
13import { clampWords } from "~/lib/content/og-card-text.mjs";
14import { escapeXml } from "@drdustinedwards/site-helpers";
15import { SEARCH_BASES, baseOfUrl, basePrefix } from "~/kb/search-bases.mjs";
16
17type ParsedQuery = ReturnType<typeof parseQuery>;
18
19/** Facet counts are computed over the fused set, so this bounds facet accuracy too. */
20const CANDIDATE_LIMIT = 100;
21
22const IDENTITY_WEIGHTS = { title: 10.0, tags: 4.0 };
23
24/**
25 * snippet() does not escape, so it splices control-character markers that the corpus cannot contain;
26 * the snippet is escaped first and the markers become <mark> after. Built at runtime so no control
27 * byte is ever written into this file.
28 */
29const MARK_START = String.fromCharCode(1);
30const MARK_END = String.fromCharCode(2);
31const MARK_START_SQL = "char(1)";
32const MARK_END_SQL = "char(2)";
33
34/** "filter" is not an index match: it labels a browse-path record, where there was no text to match. */
35export type MatchReason = "title" | "tag" | "body" | "filter";
36
37export interface SearchHit {
38  uid: string;
39  url: string;
40  type: string;
41  title: string;
42  docTitle: string;
43  docUrl: string;
44  anchor: string | null;
45  tags: string[];
46  publishAt: number | null;
47  /** Escaped HTML with <mark> around matched terms. Safe to render. */
48  snippet: string;
49  why: MatchReason[];
50  score: number;
51}
52
53interface SearchFacets {
54  types: Array<{ value: string; count: number }>;
55  /** The knowledge bases the hits belong to (app/kb/search-bases.mjs), each with its words. */
56  bases: Array<{ value: string; label: string; count: number }>;
57  tags: Array<{ value: string; count: number }>;
58  years: Array<{ value: number; count: number }>;
59}
60
61/** No score field: bm25 across differently tokenized indexes is not comparable, so it never leaves SQL. */
62interface SearchExplainRow {
63  uid: string;
64  title: string;
65  /** Records are section-grained, so a row's own title is often a bare heading. */
66  docTitle: string;
67  url: string;
68  identityRank: number | null;
69  proseRank: number | null;
70  identityContribution: number | null;
71  proseContribution: number | null;
72  score: number;
73}
74
75interface SearchExplain {
76  k: number;
77  layers: string[];
78  identityCount: number;
79  proseCount: number;
80  rows: SearchExplainRow[];
81}
82
83export interface SearchResult {
84  parsed: ParsedQuery;
85  hits: SearchHit[];
86  total: number;
87  page: number;
88  pageSize: number;
89  facets: SearchFacets;
90  /** True when a list hit CANDIDATE_LIMIT, so counts are a floor not a total. */
91  truncated: boolean;
92  tookMs: number;
93  explain?: SearchExplain;
94}
95
96interface RawRow {
97  uid: string;
98  url: string;
99  type: string;
100  title: string;
101  doc_title: string;
102  doc_url: string;
103  anchor: string | null;
104  doc_tags: string;
105  publish_at: number | null;
106  snippet: string;
107}
108
109function renderSnippet(raw: string): string {
110  return escapeXml(raw)
111    .split(MARK_START)
112    .join("<mark>")
113    .split(MARK_END)
114    .join("</mark>");
115}
116
117/** The pipe-delimited `doc_tags` column as a list. */
118function splitDocTags(docTags: string): string[] {
119  return docTags.split("|").filter(Boolean);
120}
121
122/** One index row as a result; the caller decides the snippet, the reasons and the rank. */
123function toHit(row: RawRow, snippet: string, why: MatchReason[], score: number): SearchHit {
124  return {
125    uid: row.uid,
126    url: row.url,
127    type: row.type,
128    title: row.title,
129    docTitle: row.doc_title,
130    docUrl: row.doc_url,
131    anchor: row.anchor,
132    tags: splitDocTags(row.doc_tags),
133    publishAt: row.publish_at,
134    snippet,
135    why,
136    score,
137  };
138}
139
140/**
141 * The SQL twin of publiclyVisible() in app/db/client.ts: they cannot be one function, so
142 * test/visibility-invariants.test.mjs asserts they admit the same rows.
143 * @public read by that test
144 */
145export function visibilityClause(alias = "d"): string {
146  const q = alias ? `${alias}.` : "";
147  return `${q}status = '${PUBLISHED_STATUS}' AND (${q}publish_at IS NULL OR ${q}publish_at <= ?)`;
148}
149
150/** Named, not a bare "": an empty literal between SQL literals misleads the raw-SQL scans. DO NOT INLINE. */
151const NO_ALIAS = "";
152
153interface FilterSql {
154  clause: string;
155  params: unknown[];
156}
157
158function buildFilters(parsed: ParsedQuery, nowSeconds: number): FilterSql {
159  const clauses: string[] = [visibilityClause()];
160  const params: unknown[] = [nowSeconds];
161
162  if (parsed.types.length > 0) {
163    clauses.push(`d.type IN (${parsed.types.map(() => "?").join(", ")})`);
164    params.push(...parsed.types);
165  }
166
167  // A record is in one knowledge base, read from its address, so base:a base:b widens to either. An id no base has
168  // matches nothing, as an unknown type does.
169  if (parsed.bases.length > 0) {
170    const prefixes = parsed.bases.map(basePrefix);
171    clauses.push(`(${prefixes.map(() => "d.doc_url LIKE ?").join(" OR ")})`);
172    params.push(...prefixes.map((prefix) => (prefix === null ? "\u0000" : `${prefix}%`)));
173  }
174
175  // Every tag must be present, so tag:a tag:b narrows rather than widens.
176  for (const tag of parsed.tags) {
177    clauses.push(`instr(d.doc_tags, ?) > 0`);
178    params.push(`|${tag}|`);
179  }
180
181  if (parsed.year !== null) {
182    const start = Math.floor(Date.UTC(parsed.year, 0, 1) / 1000);
183    const end = Math.floor(Date.UTC(parsed.year + 1, 0, 1) / 1000);
184    clauses.push(`d.publish_at >= ? AND d.publish_at < ?`);
185    params.push(start, end);
186  }
187
188  return { clause: clauses.join(" AND "), params };
189}
190
191async function runIndex(
192  db: D1Database,
193  table: "search_identity" | "search_prose",
194  snippetColumn: number,
195  order: string,
196  match: string,
197  filters: FilterSql,
198): Promise<RawRow[]> {
199  const sql = `
200    SELECT d.uid, d.url, d.type, d.title, d.doc_title, d.doc_url, d.anchor,
201           d.doc_tags, d.publish_at,
202           snippet(${table}, ${snippetColumn}, ${MARK_START_SQL}, ${MARK_END_SQL}, '…', 14) AS snippet
203    FROM ${table}
204    JOIN search_docs d ON d.id = ${table}.rowid
205    WHERE ${table} MATCH ?
206      AND ${filters.clause}
207    ORDER BY ${order}
208    LIMIT ${CANDIDATE_LIMIT}
209  `;
210  const result = await db
211    .prepare(sql)
212    .bind(match, ...filters.params)
213    .all<RawRow>();
214  return result.results ?? [];
215}
216
217/**
218 * The posts' listing order for the search index (app/lib/post-order.mjs, app/db/posts.ts newestFirstOrder): publish
219 * instant, then a series kept together by the newest id among its posts of that instant, then part, then the
220 * later row. The index has its own row ids, so a post reads its series, part and id from the posts table through
221 * its address; a page has none of those and falls back to its index id.
222 */
223const NEWEST_JOIN = `LEFT JOIN posts p ON d.type = 'post' AND p.kind = 'post' AND d.url = '/writing/' || p.slug`;
224const NEWEST_ORDER = `d.publish_at DESC,
225      CASE WHEN p.series IS NULL THEN COALESCE(p.id, d.id)
226           ELSE COALESCE((SELECT MAX(g.id) FROM posts g WHERE g.kind = 'post' AND g.status = '${PUBLISHED_STATUS}' AND g.publish_at = p.publish_at AND g.series = p.series), p.id) END DESC,
227      COALESCE(p.part, 0) DESC,
228      COALESCE(p.id, d.id) DESC`;
229
230/**
231 * Document records only: returning each section separately would present the corpus as a multiple of
232 * its size. Date-ordered, since with no relevance signal recency is the only defensible order.
233 */
234async function runBrowse(db: D1Database, filters: FilterSql): Promise<RawRow[]> {
235  const sql = `
236    SELECT d.uid, d.url, d.type, d.title, d.doc_title, d.doc_url, d.anchor,
237           d.doc_tags, d.publish_at,
238           substr(d.body, 1, 240) AS snippet
239    FROM search_docs d
240    ${NEWEST_JOIN}
241    WHERE d.anchor IS NULL
242      AND ${filters.clause}
243    ORDER BY ${NEWEST_ORDER}
244    LIMIT ${CANDIDATE_LIMIT}
245  `;
246  const result = await db.prepare(sql).bind(...filters.params).all<RawRow>();
247  return result.results ?? [];
248}
249
250/** Computed in app code: fts5 reports that a row matched, not which column carried it. */
251function whyMatched(row: RawRow, parsed: ParsedQuery, inProse: boolean): MatchReason[] {
252  const needles = [...parsed.terms, ...parsed.phrases].map((t) => t.toLowerCase());
253  const why: MatchReason[] = [];
254  const title = row.title.toLowerCase();
255  const tags = row.doc_tags.toLowerCase();
256
257  if (needles.some((n) => title.includes(n))) why.push("title");
258  if (needles.some((n) => tags.includes(n))) why.push("tag");
259  if (inProse) why.push("body");
260  // An identity-index match can be stemmed or diacritic-folded, which substring testing does not see.
261  if (why.length === 0) why.push(inProse ? "body" : "title");
262  return why;
263}
264
265interface SearchOptions {
266  q: string;
267  type?: string | null;
268  /** A knowledge base's id (app/kb/search-bases.mjs). */
269  base?: string | null;
270  tag?: string | null;
271  year?: string | null;
272  page?: number;
273  pageSize?: number;
274  prefix?: boolean;
275  now?: Date;
276  /** Read off what fuse() recorded, so the playground demo and real search cannot disagree. */
277  explain?: boolean;
278  /**
279   * Re-sorts the FUSED list, never the SQL: sorting in SQL would change the candidate set per sort, so
280   * pages would disagree. The browse path ignores it.
281   */
282  sort?: SearchSort;
283}
284
285export async function search(env: Env, options: SearchOptions): Promise<SearchResult> {
286  const pageSize = options.pageSize ?? 10;
287  const page = Math.max(1, options.page ?? 1);
288  const sort: SearchSort = options.sort ?? "relevance";
289  const now = options.now ?? new Date();
290  const nowSeconds = Math.floor(now.getTime() / 1000);
291
292  // ?tag=d1 and a typed tag:d1 merge into one parse, so facet links can be plain hrefs.
293  const parsed = parseQuery(options.q ?? "");
294  if (options.type) parsed.types.push(options.type.toLowerCase());
295  if (options.base) parsed.bases.push(options.base.toLowerCase());
296  if (options.tag) parsed.tags.push(options.tag.toLowerCase());
297  if (options.year && /^\d{4}$/.test(options.year)) parsed.year = Number(options.year);
298
299  const empty: SearchResult = {
300    parsed,
301    hits: [],
302    total: 0,
303    page,
304    pageSize,
305    facets: { types: [], bases: [], tags: [], years: [] },
306    truncated: false,
307    tookMs: 0,
308  };
309
310  const match = toMatchExpression(parsed, options.prefix ?? false);
311  if (!match && !hasFilters(parsed)) return empty;
312
313  const filters = buildFilters(parsed, nowSeconds);
314  if (!match) return browse(env, parsed, filters, page, pageSize);
315
316  const started = Date.now();
317
318  const [identityRows, proseRows] = await Promise.all([
319    runIndex(
320      env.DB,
321      "search_identity",
322      0,
323      `bm25(search_identity, ${IDENTITY_WEIGHTS.title}, ${IDENTITY_WEIGHTS.tags}) ASC`,
324      match,
325      filters,
326    ),
327    runIndex(env.DB, "search_prose", 0, `bm25(search_prose) ASC`, match, filters),
328  ]);
329
330  const tookMs = Date.now() - started;
331  const truncated =
332    identityRows.length >= CANDIDATE_LIMIT || proseRows.length >= CANDIDATE_LIMIT;
333
334  const proseUids = new Set(proseRows.map((r) => r.uid));
335  const fused = fuse<RawRow & { uid: string }>([identityRows, proseRows]);
336
337  const fusedHits: SearchHit[] = fused.map(({ item, score }) =>
338    toHit(
339      item,
340      // The prose snippet has sentences; the identity one is just the title, already shown.
341      renderSnippet(
342        (proseUids.has(item.uid)
343          ? proseRows.find((r) => r.uid === item.uid)?.snippet
344          : item.snippet) ?? item.snippet,
345      ),
346      whyMatched(item, parsed, proseUids.has(item.uid)),
347      score,
348    ),
349  );
350
351  const hits = applySort(fusedHits, sort);
352
353  return {
354    parsed,
355    hits: hits.slice((page - 1) * pageSize, page * pageSize),
356    total: hits.length,
357    page,
358    pageSize,
359    facets: buildFacets(hits),
360    truncated,
361    tookMs,
362    ...(options.explain ? { explain: explainFor(fused, identityRows, proseRows) } : {}),
363  };
364}
365
366/** No-match browse: the filters alone, newest first, with no relevance to fuse. */
367async function browse(
368  env: Env,
369  parsed: ParsedQuery,
370  filters: FilterSql,
371  page: number,
372  pageSize: number,
373): Promise<SearchResult> {
374  const started = Date.now();
375  const rows = await runBrowse(env.DB, filters);
376  const browseTook = Date.now() - started;
377  const browseHits: SearchHit[] = rows.map((item, index) =>
378    // Rank is the date order this came back in, not a relevance score.
379    toHit(item, renderSnippet(clampWords(item.snippet, 200)), ["filter"], rows.length - index),
380  );
381  return {
382    parsed,
383    hits: browseHits.slice((page - 1) * pageSize, page * pageSize),
384    total: browseHits.length,
385    page,
386    pageSize,
387    facets: buildFacets(browseHits),
388    truncated: rows.length >= CANDIDATE_LIMIT,
389    tookMs: browseTook,
390  };
391}
392
393/** Each fused row's rank and RRF contribution in each layer, for `?explain`. */
394function explainFor(
395  fused: ReturnType<typeof fuse<RawRow>>,
396  identityRows: RawRow[],
397  proseRows: RawRow[],
398): SearchExplain {
399  return {
400    k: RRF_K,
401    layers: ["search_identity", "search_prose"],
402    identityCount: identityRows.length,
403    proseCount: proseRows.length,
404    rows: fused.map(({ item, score, sources, ranks, contributions }) => {
405      const at = (list: number) => sources.indexOf(list);
406      const i = at(0);
407      const p = at(1);
408      const rank = (n: number) => (n === -1 ? null : (ranks[n] ?? null));
409      const contribution = (n: number) =>
410        n === -1 ? null : (contributions[n] ?? null);
411      return {
412        uid: item.uid,
413        title: item.title,
414        docTitle: item.doc_title,
415        url: item.url,
416        identityRank: rank(i),
417        proseRank: rank(p),
418        identityContribution: contribution(i),
419        proseContribution: contribution(p),
420        score,
421      };
422    }),
423  };
424}
425
426/** Over the whole match set, not the page. If truncated, the caller must present these as a floor. */
427function buildFacets(hits: SearchHit[]): SearchFacets {
428  const types = new Map<string, number>();
429  const bases = new Map<string, number>();
430  const tags = new Map<string, number>();
431  const years = new Map<number, number>();
432
433  for (const hit of hits) {
434    types.set(hit.type, (types.get(hit.type) ?? 0) + 1);
435    const base = baseOfUrl(hit.docUrl);
436    if (base) bases.set(base, (bases.get(base) ?? 0) + 1);
437    for (const tag of hit.tags) tags.set(tag, (tags.get(tag) ?? 0) + 1);
438    if (hit.publishAt !== null) {
439      const year = new Date(hit.publishAt * 1000).getUTCFullYear();
440      years.set(year, (years.get(year) ?? 0) + 1);
441    }
442  }
443
444  return {
445    types: [...types].map(([value, count]) => ({ value, count })).sort((a, b) => b.count - a.count),
446    // In the bases' own order, not by count, so the list reads the same from one search to the next.
447    bases: SEARCH_BASES.filter((b) => bases.has(b.id)).map((b) => ({ value: b.id, label: b.label, count: bases.get(b.id) ?? 0 })),
448    tags: [...tags].map(([value, count]) => ({ value, count })).sort((a, b) => b.count - a.count),
449    years: [...years].map(([value, count]) => ({ value, count })).sort((a, b) => b.value - a.value),
450  };
451}
452
453export async function askCorpusRecords(
454  env: Env,
455  now = new Date(),
456): Promise<Array<{ url: string; title: string; body: string }>> {
457  const nowSeconds = Math.floor(now.getTime() / 1000);
458  const rows = await env.DB.prepare(
459    // CONCATENATED, not interpolated, for the reason stated at NO_ALIAS.
460    `SELECT url, title, body FROM search_docs WHERE type = 'post' AND ` +
461      visibilityClause(NO_ALIAS),
462  )
463    .bind(nowSeconds)
464    .all<{ url: string; title: string; body: string }>();
465  return rows.results ?? [];
466}
467
468export async function askExpectedUrls(env: Env, now = new Date()): Promise<string[]> {
469  const nowSeconds = Math.floor(now.getTime() / 1000);
470  const rows = await env.DB.prepare(
471    // Concatenated, not interpolated (see NO_ALIAS). Posts only: pages are never uploaded to Ask.
472    `SELECT url FROM search_docs WHERE type = 'post' AND ` + visibilityClause(NO_ALIAS),
473  )
474    .bind(nowSeconds)
475    .all<{ url: string }>();
476  return (rows.results ?? []).map((r) => r.url);
477}
478
479export async function zeroState(env: Env, parsed: ParsedQuery, now = new Date()) {
480  const nowSeconds = Math.floor(now.getTime() / 1000);
481  const needles = [...parsed.terms, ...parsed.phrases].map((t) => t.toLowerCase());
482
483  const tagRows = await env.DB.prepare(
484    // Concatenated, not interpolated: a ${...} truncates the literal the raw-SQL scans can see.
485    `SELECT DISTINCT doc_tags FROM search_docs WHERE doc_tags <> '' AND ` +
486      visibilityClause(NO_ALIAS),
487  )
488    .bind(nowSeconds)
489    .all<{ doc_tags: string }>();
490
491  const allTags = new Set<string>();
492  for (const row of tagRows.results ?? []) {
493    for (const tag of splitDocTags(row.doc_tags)) allTags.add(tag);
494  }
495
496  // Prefix or substring, not edit distance: on a handful of tags, fuzzy matching surfaces confident nonsense.
497  const nearest = [...allTags]
498    .filter((tag) => needles.some((n) => n.length >= 2 && (tag.includes(n) || n.includes(tag))))
499    .sort()
500    .slice(0, 6);
501
502  const recent = await env.DB.prepare(
503    `SELECT d.uid, d.url, d.title, d.publish_at FROM search_docs d ` +
504      NEWEST_JOIN +
505      ` WHERE d.anchor IS NULL AND d.type = 'post' AND ` +
506      visibilityClause() +
507      ` ORDER BY ` +
508      NEWEST_ORDER +
509      ` LIMIT 5`,
510  )
511    .bind(nowSeconds)
512    .all<{ uid: string; url: string; title: string; publish_at: number | null }>();
513
514  return {
515    nearestTags: nearest,
516    recentPosts: (recent.results ?? []).map((r) => ({
517      url: r.url,
518      title: r.title,
519      publishAt: r.publish_at,
520    })),
521  };
522}
523
524export { parseSort };