app/lib/voice.server.ts

Functions

Imports

It imports app/db/schema.ts, app/lib/content.server.ts, app/lib/projects.server.ts, app/lib/roles.ts, app/lib/sites.server.ts. Imported by app/lib/content.server.ts, app/lib/mcp/tools.ts, app/routes/ai-draft.tsx, app/routes/project.voice.tsx.

Source

1// The voice pack of a site project: what an AI session reads before it drafts a post, and what its draft
2// is checked against. Real examples copy a voice better than a description of it (Wang et al., Findings
3// of EMNLP 2025), so the pack is built from the author's own published posts: a measured profile and the
4// excerpts nearest it. The guide and the word lists are the Owner's, on the project's Voice page. The
5// checks are site-runtime's ./prose, the one check Carrel's books and Foxhound's emails also run.
6//
7// Learning from edits: when the author publishes a post an AI session drafted, the words the latest AI
8// draft used and the published post does not are counted. One cut in three posts, and never kept in one,
9// is offered on the Voice page for the phrase list. The author decides; nothing is added on its own.
10//
11// Nothing here scores a draft against an AI detector. The aim is a post that reads like its author.
12
13import {
14  checkProse,
15  describeFeature,
16  distanceFrom,
17  nearestPassages,
18  profileOf,
19  VOICE_THRESHOLD,
20  WRITING_RULES,
21  type Profile,
22  type ProseIssue,
23} from "@dustinedwards/site-runtime/prose";
24import { and, desc, eq, isNotNull } from "drizzle-orm";
25import { drizzle } from "drizzle-orm/d1";
26
27import { aiDrafts, siteItems, voiceCuts, voicePacks } from "~/db/schema";
28import type { ProjectRole } from "~/lib/content.server";
29import type { SiteProject } from "~/lib/projects.server";
30import { can } from "~/lib/roles";
31import { siteClient } from "~/lib/sites.server";
32
33/** The newest published posts a pack is built from: enough for a profile, few enough for one request's subrequests. */
34export const PACK_POSTS = 40;
35/** Excerpts handed to a session: the research's few-shot range, and short enough to read in full. */
36const PACK_EXCERPTS = 6;
37/** A cut offered for the list once it has been cut from this many posts and kept in none. */
38export const CUT_SUGGEST_AT = 3;
39/** The most new cut candidates one publish records, so one heavy rewrite cannot flood the table. */
40const CUTS_PER_PUBLISH = 150;
41
42export type VoicePack = {
43  guide: string;
44  phrases: string[];
45  allow: string[];
46  profile: Profile | null;
47  excerpts: string[];
48  posts: number;
49  builtAt: string | null;
50};
51
52export type DraftCheck = {
53  issues: ProseIssue[];
54  /** How far the draft reads from the published posts, and the three measures furthest off; null without a profile or under 250 words. */
55  voice: { distance: number; threshold: number; unlike: boolean; furthest: string[] } | null;
56};
57
58function forbid(): never {
59  throw new Response("Forbidden", { status: 403 });
60}
61
62function strings(json: string): string[] {
63  try {
64    const value: unknown = JSON.parse(json);
65    return Array.isArray(value) ? value.filter((v): v is string => typeof v === "string") : [];
66  } catch {
67    return [];
68  }
69}
70
71/**
72 * A post's prose for measuring: the frontmatter, code, HTML, images and Markdown marks taken out, link
73 * text kept. Measured on the raw source, every post would read as heavy on colons and brackets.
74 */
75export function proseOf(source: string): string {
76  return source
77    .replace(/\r\n/g, "\n")
78    .replace(/^---\n[\s\S]*?\n---\n?/, "")
79    .replace(/```[\s\S]*?```/g, "")
80    .replace(/<!--[\s\S]*?-->/g, "")
81    .replace(/!\[[^\]]*\]\([^)]*\)/g, "")
82    .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
83    .replace(/<[^>]+>/g, "")
84    .replace(/`([^`]*)`/g, "$1")
85    .replace(/^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+|\d+\.\s+)/gm, "")
86    .replace(/(\*\*|__|\*|_)(?=\S)([^*_]+?)\1/g, "$2")
87    .replace(/\n{3,}/g, "\n\n")
88    .trim();
89}
90
91export async function readVoicePack(db: D1Database, project: ProjectRole): Promise<VoicePack> {
92  if (!can(project.role, "read")) forbid();
93  const row = await drizzle(db).select().from(voicePacks).where(eq(voicePacks.projectId, project.id)).get();
94  if (!row) return { guide: "", phrases: [], allow: [], profile: null, excerpts: [], posts: 0, builtAt: null };
95  return {
96    guide: row.guide,
97    phrases: strings(row.phrases),
98    allow: strings(row.allow),
99    profile: row.profile ? (JSON.parse(row.profile) as Profile) : null,
100    excerpts: strings(row.excerpts),
101    posts: row.posts,
102    builtAt: row.builtAt,
103  };
104}
105
106/**
107 * Rebuilds the profile and excerpts from the newest published posts the index knows, read from the site.
108 * Anyone who may read the project may rebuild: it reads only what is published.
109 */
110export async function buildVoicePack(env: Env, project: SiteProject, fetcher?: typeof fetch, now = new Date()): Promise<VoicePack> {
111  if (!can(project.role, "read")) forbid();
112  const rows = await drizzle(env.DB)
113    .select({ itemId: siteItems.itemId })
114    .from(siteItems)
115    .where(and(eq(siteItems.projectId, project.id), eq(siteItems.status, "published"), isNotNull(siteItems.publishedAt)))
116    .orderBy(desc(siteItems.publishedAt))
117    .limit(PACK_POSTS)
118    .all();
119  const client = siteClient(env, project.site, fetcher);
120  const texts: string[] = [];
121  // One at a time: a site answers its own API, and a burst of forty is no faster for it.
122  for (const { itemId } of rows) {
123    try {
124      const doc = await client.get(itemId);
125      const prose = proseOf(doc.source);
126      if (prose) texts.push(prose);
127    } catch (error) {
128      console.error(JSON.stringify({ voice: "post-unreadable", itemId, error: String(error) }));
129    }
130  }
131  const profile = profileOf(texts);
132  const excerpts = profile ? nearestPassages(profile, texts, PACK_EXCERPTS) : [];
133  const values = { profile: profile ? JSON.stringify(profile) : null, excerpts: JSON.stringify(excerpts), posts: texts.length, builtAt: now.toISOString(), updatedAt: now.toISOString() };
134  await drizzle(env.DB)
135    .insert(voicePacks)
136    .values({ projectId: project.id, ...values })
137    .onConflictDoUpdate({ target: voicePacks.projectId, set: values });
138  return readVoicePack(env.DB, project);
139}
140
141/** One entry per line, trimmed, blank lines and repeats dropped. */
142export function listFrom(text: string): string[] {
143  return [...new Set(text.split("\n").map((l) => l.trim()).filter(Boolean))];
144}
145
146/** The Owner's part of the pack: the guide and the two word lists. */
147export async function saveVoiceGuide(db: D1Database, project: ProjectRole, input: { guide: string; phrases: string[]; allow: string[] }, now = new Date()) {
148  if (!can(project.role, "manage")) forbid();
149  const values = { guide: input.guide.trim(), phrases: JSON.stringify(input.phrases), allow: JSON.stringify(input.allow), updatedAt: now.toISOString() };
150  await drizzle(db)
151    .insert(voicePacks)
152    .values({ projectId: project.id, ...values })
153    .onConflictDoUpdate({ target: voicePacks.projectId, set: values });
154}
155
156/** Dashes are flagged unless the author's own posts use them, one in 1,000 words or more. */
157function dashes(pack: VoicePack): "flag" | "allow" {
158  return (pack.profile?.mean.dashes ?? 0) >= 1 ? "allow" : "flag";
159}
160
161/** The checks on a post's text against this project's pack. Flags only; the author decides. */
162export function checkAgainstPack(pack: VoicePack, source: string): DraftCheck {
163  const front = /^---\n[\s\S]*?\n---\n?/.exec(source.replace(/\r\n/g, "\n"));
164  const firstLine = front ? front[0].split("\n").length - (front[0].endsWith("\n") ? 1 : 0) + 1 : 1;
165  const body = front ? source.replace(/\r\n/g, "\n").slice(front[0].length) : source;
166  const issues = checkProse(body, { phrases: pack.phrases, allow: pack.allow, dashes: dashes(pack), firstLine });
167  const reading = pack.profile ? distanceFrom(pack.profile, proseOf(source)) : null;
168  return {
169    issues,
170    voice: reading
171      ? { distance: Math.round(reading.distance * 100) / 100, threshold: VOICE_THRESHOLD, unlike: reading.distance >= VOICE_THRESHOLD, furthest: reading.furthest.map(describeFeature) }
172      : null,
173  };
174}
175
176/** What the published posts measure, in words a session can write to. */
177function measures(profile: Profile | null): string[] {
178  if (!profile) return [];
179  const m = profile.mean;
180  const round = (n: number) => Math.round(n * 10) / 10;
181  return [
182    `Sentences average ${round(m["sentence length"] ?? 0)} words.`,
183    `${round(m.contractions ?? 0)} contractions per 1,000 words.`,
184    `${round(m.dashes ?? 0)} dashes, ${round(m.semicolons ?? 0)} semicolons, ${round(m.questions ?? 0)} questions and ${round(m.exclamations ?? 0)} exclamation marks per 1,000 words.`,
185    `"I" ${round(m['"i"'] ?? 0)} and "you" ${round(m['"you"'] ?? 0)} times per 1,000 words.`,
186  ];
187}
188
189export type VoiceBrief = {
190  rules: string;
191  guide: string;
192  phrases: string[];
193  allow: string[];
194  measures: string[];
195  excerpts: string[];
196  posts: number;
197  builtAt: string | null;
198};
199
200/** Everything a session reads before it drafts: the rules, the author's guide and lists, the measures and the excerpts. */
201export function briefOf(pack: VoicePack): VoiceBrief {
202  return {
203    rules: WRITING_RULES,
204    guide: pack.guide,
205    phrases: pack.phrases,
206    allow: pack.allow,
207    measures: measures(pack.profile),
208    excerpts: pack.excerpts,
209    posts: pack.posts,
210    builtAt: pack.builtAt,
211  };
212}
213
214// ---------- learning from edits
215
216/** Words too common to say anything about a voice. */
217const COMMON = new Set(
218  (
219    "the a an and or but if then than so of to in on at by for with from as into onto over under about after before " +
220    "is are was were be been being am do does did done have has had having will would can could should may might must " +
221    "it its it's this that these those there here what which who whom whose when where why how not no yes all any some " +
222    "each every more most less many much few very just also only even still too own same other such both either neither " +
223    "i me my we us our you your he him his she her they them their one two first new out up down off again once"
224  ).split(" "),
225);
226
227/** The words and two-word phrases of a text worth counting: lower case, no common-word-only phrases. */
228export function phrasesOf(text: string): Set<string> {
229  const words = proseOf(text).toLowerCase().match(/[\p{L}]+(?:['’][\p{L}]+)*/gu) ?? [];
230  const out = new Set<string>();
231  words.forEach((w, i) => {
232    if (w.length >= 4 && !COMMON.has(w)) out.add(w);
233    const next = words[i + 1];
234    if (next && (!COMMON.has(w) || !COMMON.has(next)) && w.length + next.length >= 7) out.add(`${w} ${next}`);
235  });
236  return out;
237}
238
239/**
240 * After the author publishes `itemId`: counts what the latest AI draft of it had and the published text
241 * does not, and marks every listed cut the published text kept. A post counts once per phrase however
242 * often it is republished. Never throws: learning is not worth a failed publish.
243 */
244export async function recordCuts(db: D1Database, projectId: number, personId: number, itemId: string, published: string, now = new Date()): Promise<number> {
245  try {
246    const draft = await drizzle(db)
247      .select({ source: aiDrafts.source })
248      .from(aiDrafts)
249      .where(and(eq(aiDrafts.projectId, projectId), eq(aiDrafts.itemId, itemId), eq(aiDrafts.personId, personId)))
250      .orderBy(desc(aiDrafts.createdAt), desc(aiDrafts.id))
251      .get();
252    if (!draft) return 0;
253    const kept = phrasesOf(published);
254    const cut = [...phrasesOf(draft.source)].filter((p) => !kept.has(p)).slice(0, CUTS_PER_PUBLISH);
255    const known = await drizzle(db).select().from(voiceCuts).where(eq(voiceCuts.projectId, projectId)).all();
256    const byPhrase = new Map(known.map((row) => [row.phrase, row]));
257    const writes = [];
258    for (const row of known) {
259      if (kept.has(row.phrase)) writes.push(drizzle(db).update(voiceCuts).set({ kept: row.kept + 1 }).where(and(eq(voiceCuts.projectId, projectId), eq(voiceCuts.phrase, row.phrase))));
260    }
261    for (const phrase of cut) {
262      const row = byPhrase.get(phrase);
263      const items = row ? strings(row.items) : [];
264      if (items.includes(itemId)) continue;
265      const set = { items: JSON.stringify([...items, itemId]), lastCutAt: now.toISOString() };
266      writes.push(drizzle(db).insert(voiceCuts).values({ projectId, phrase, ...set }).onConflictDoUpdate({ target: [voiceCuts.projectId, voiceCuts.phrase], set }));
267    }
268    for (const w of writes) await w;
269    return cut.length;
270  } catch (error) {
271    console.error(JSON.stringify({ voice: "cuts-failed", itemId, error: String(error) }));
272    return 0;
273  }
274}
275
276export type CutSuggestion = { phrase: string; posts: number; lastCutAt: string };
277
278/** Cuts seen in CUT_SUGGEST_AT posts or more, kept in none, not set aside and not already listed: most posts first. */
279export async function cutSuggestions(db: D1Database, project: ProjectRole, pack: VoicePack): Promise<CutSuggestion[]> {
280  if (!can(project.role, "read")) forbid();
281  const listed = new Set([...pack.phrases, ...pack.allow].map((p) => p.toLowerCase()));
282  const rows = await drizzle(db)
283    .select()
284    .from(voiceCuts)
285    .where(and(eq(voiceCuts.projectId, project.id), eq(voiceCuts.kept, 0), eq(voiceCuts.setAside, false)))
286    .all();
287  return rows
288    .map((r) => ({ phrase: r.phrase, posts: strings(r.items).length, lastCutAt: r.lastCutAt }))
289    .filter((r) => r.posts >= CUT_SUGGEST_AT && !listed.has(r.phrase))
290    .sort((a, b) => b.posts - a.posts || a.phrase.localeCompare(b.phrase))
291    .slice(0, 50);
292}
293
294/** The Owner's answer to a suggestion: add it to the phrase list, or set it aside for good. */
295export async function decideCut(db: D1Database, project: ProjectRole, phrase: string, decision: "add" | "set-aside") {
296  if (!can(project.role, "manage")) forbid();
297  if (decision === "add") {
298    const pack = await readVoicePack(db, project);
299    await saveVoiceGuide(db, project, { guide: pack.guide, phrases: [...new Set([...pack.phrases, phrase])], allow: pack.allow });
300  }
301  await drizzle(db).update(voiceCuts).set({ setAside: true }).where(and(eq(voiceCuts.projectId, project.id), eq(voiceCuts.phrase, phrase)));
302}