app/lib/content/chunks.mjs

Functions

Imports

It imports no other module of this repo. Imported by app/lib/publications/save.server.ts.

Source

1// D1 refuses a statement over 100 KB (SQLITE_TOOBIG), and a publication's twin with the PDF's full text
2// passes it. A long text is therefore written in chunks, the first in the INSERT and the rest appended.
3// scripts/sync-content.mjs and the Worker's publication write both split through here.
4
5/** Characters per chunk: even at three UTF-8 bytes each, a chunk and its escaping stay well under 100 KB. */
6export const CHUNK_CHARS = 25_000;
7
8/**
9 * @param {string} text
10 * @returns {string[]} the text in order, never split inside a surrogate pair; one empty string for none
11 */
12export function textChunks(text) {
13  /** @type {string[]} */
14  const out = [];
15  let at = 0;
16  while (at < text.length) {
17    let end = Math.min(at + CHUNK_CHARS, text.length);
18    const last = text.charCodeAt(end - 1);
19    if (end < text.length && last >= 0xd800 && last <= 0xdbff) end -= 1;
20    out.push(text.slice(at, end));
21    at = end;
22  }
23  return out.length > 0 ? out : [""];
24}