src/truth-report.ts

Functions

Imports

It imports src/rule-docs.ts, src/time.ts. Imported by src/status.ts, src/tools/lint.ts.

Source

1import { ruleDocCap, ruleDocProblem, staleSummary, type RuleDocProblem } from "./rule-docs";
2import { DAY_MS } from "./time";
3interface TruthFinding {
4  // What was checked, and what did not pass.
5  check: string;
6  subject: string;
7  detail: string;
8}
9
10interface TruthCheck {
11  check: string;
12  subjects: number;
13  ok: number;
14  findings: TruthFinding[];
15}
16
17export interface TruthReport {
18  namespace: string;
19  generated: string;
20  documents: {
21    total: number;
22    by_type: Record<string, number>;
23    unconsolidated: number;
24    archived: number;
25  };
26  checks: TruthCheck[];
27  // Good standing over subjects, across every check that had a subject to judge.
28  // Null when nothing was checkable, which is not the same as 100 percent.
29  integrity: number | null;
30  findings: TruthFinding[];
31}
32
33export interface ReportDoc {
34  path: string;
35  type: string | null;
36  status: string | null;
37  title: string | null;
38  body: string | null;
39  updated_at: string;
40}
41
42export interface ReportEdge {
43  from_ns: string;
44  from_path: string;
45  type: string;
46  to_ns: string;
47  to_path: string;
48  source_missing?: number;
49  target_missing?: number;
50}
51
52// The shape src/counts.ts returns, mirrored so this module stays a pure function
53// over plain data.
54interface ReportCountClaim {
55  path: string;
56  noun: string;
57  states: string;
58  authoritative: string;
59  quote: string;
60}
61
62export interface TruthInput {
63  namespace: string;
64  now: Date;
65  docs: ReportDoc[];
66  edges: ReportEdge[];
67  danglingEdges: ReportEdge[];
68  countClaims: ReportCountClaim[];
69  // Repo paths on the default branch. Undefined when the repo could not be read,
70  // which makes the drift check unrun rather than clean.
71  repoPaths?: Set<string>;
72}
73
74// A published decision older than this is worth re-reading. A prompt, not a defect.
75export const STALE_DECISION_DAYS = 180;
76
77// Consolidate when a namespace holds more than five unconsolidated documents. Capsid's own threshold; the portfolio conventions no longer state one.
78export const UNCONSOLIDATED_CADENCE = 5;
79
80// Matches a repo path in prose: at least one slash, a file extension, and no
81// spaces. `capsid/conventions.md` matches too and is excluded below, because a
82// Capsid document path is not a repo path.
83//
84// The lookbehind, not a word boundary. A word boundary does not fall between a space
85// and a dot, so a match would start one character late: `.github/workflows/ci.yml`
86// would be captured as `github/workflows/ci.yml` and reported absent from a repo that
87// has it, and the dotted paths are the ones canon names most. The lookbehind also
88// stops the same path matching again after the dot.
89//
90// The extension must start with a letter, which excludes section numbers such as
91// `2/2.1` and `5a-1/5a-3/1.5`: two slashes and a dot, and not a file.
92const REPO_PATH = /(?<![A-Za-z0-9_./-])(\.?(?:[A-Za-z0-9_.-]+\/)+[A-Za-z0-9_.-]+\.[A-Za-z][A-Za-z0-9]{0,4})\b/g;
93
94// Namespace prefixes that mean "a Capsid document", not "a file in the repo".
95// A path starting with one of these is a store address and is skipped.
96const NAMESPACE_PREFIXES = ["capsid/", "germomics/", "foxing/", "foxhound/", "dustinedwards/", "julieedwards/", "txasm/", "bsw/", "claude-skills/", "recova/"];
97
98// The stored report's path prefix.
99export const REPORTS_PREFIX = "reports/";
100
101// Prefixes that are not subjects of the report. `archive/` is consumed history.
102// `reports/` holds earlier reports: `report` stores its result as an ordinary
103// document, and the next run would read it back, so every finding it wrote becomes a
104// finding it finds, a quote describing a fixed contradiction is parsed as a fresh
105// one, and drift paths are attributed to the report that merely listed them. Left
106// alone it ratchets integrity down on every run while the store improves. A report is
107// an observation of the store, not a member of it.
108const UNSCANNED_PREFIXES = ["archive/", REPORTS_PREFIX];
109
110export function isUnscanned(path: string): boolean {
111  return UNSCANNED_PREFIXES.some((prefix) => path.startsWith(prefix));
112}
113
114function daysBetween(a: Date, b: Date): number {
115  return Math.abs(a.getTime() - b.getTime()) / DAY_MS;
116}
117
118function parseStamp(value: string): Date | null {
119  // D1 writes `YYYY-MM-DD HH:MM:SS` with no zone. Read as UTC rather than local,
120  // or the same document is stale on one machine and fresh on another.
121  const normalized = value.includes("T") ? value : `${value.replace(" ", "T")}Z`;
122  const date = new Date(normalized);
123  return Number.isNaN(date.getTime()) ? null : date;
124}
125
126// A document is "standing" if it is part of the canon rather than raw material.
127// archive/ is consumed history and is not held to any of this.
128const isArchived = (doc: ReportDoc) => doc.path.startsWith("archive/");
129
130export function buildTruthReport(input: TruthInput): TruthReport {
131  const { namespace, now, docs } = input;
132  // `archived` is counted on its own, not as "everything not standing", so the
133  // reports/ exclusion does not inflate it.
134  const standing = docs.filter((d) => !isUnscanned(d.path));
135  const archived = docs.filter(isArchived).length;
136
137  const byType: Record<string, number> = {};
138  for (const doc of docs) {
139    const type = doc.type ?? "(untyped)";
140    byType[type] = (byType[type] ?? 0) + 1;
141  }
142
143  const checks: TruthCheck[] = [];
144
145  // 1. Contradictions: the count claim, prose asserting a number the artifact
146  //    disagrees with (src/counts.ts). A claim in an excluded document is dropped
147  //    here as well as by the caller.
148  const countClaims = input.countClaims.filter((claim) => !isUnscanned(claim.path));
149  checks.push({
150    check: "contradictions",
151    subjects: countClaims.length,
152    ok: 0,
153    findings: countClaims.map((claim) => ({
154      check: "contradictions",
155      subject: `${namespace}/${claim.path}`,
156      detail: `states ${claim.states} ${claim.noun}; the artifact says ${claim.authoritative}. In: ${claim.quote}`,
157    })),
158  });
159  // Every claim returned is a mismatch, so the population is the standing documents:
160  // what fraction carry a wrong claim.
161  const contradicting = new Set(countClaims.map((c) => c.path));
162  checks[0] = {
163    check: "contradictions",
164    subjects: standing.length,
165    ok: standing.length - [...contradicting].filter((p) => standing.some((d) => d.path === p)).length,
166    findings: checks[0].findings,
167  };
168
169  // 2. Stale decisions: published and untouched in half a year. A state claim is
170  //    re-verified before it is acted on, and a reader cannot apply that rule to a
171  //    document whose age they cannot see.
172  const decisions = standing.filter((d) => d.type === "decision" && (d.status ?? "published") === "published");
173  const staleDecisions = decisions.filter((d) => {
174    const stamp = parseStamp(d.updated_at);
175    return stamp !== null && daysBetween(now, stamp) > STALE_DECISION_DAYS;
176  });
177  checks.push({
178    check: "stale_decisions",
179    subjects: decisions.length,
180    ok: decisions.length - staleDecisions.length,
181    findings: staleDecisions.map((d) => ({
182      check: "stale_decisions",
183      subject: `${namespace}/${d.path}`,
184      detail: `last written ${d.updated_at}, over ${STALE_DECISION_DAYS} days ago, and still published`,
185    })),
186  });
187
188  // 3. Unbound specs: no typed edge in either direction, so a reader cannot find
189  //    from the store whether the work happened.
190  const specs = standing.filter((d) => d.type === "spec" || d.type === "protocol" || d.type === "procedural");
191  const bound = new Set<string>();
192  for (const edge of input.edges) {
193    if (edge.from_ns === namespace) bound.add(edge.from_path);
194    if (edge.to_ns === namespace) bound.add(edge.to_path);
195  }
196  const unbound = specs.filter((d) => !bound.has(d.path));
197  checks.push({
198    check: "unbound_specs",
199    subjects: specs.length,
200    ok: specs.length - unbound.length,
201    findings: unbound.map((d) => ({
202      check: "unbound_specs",
203      subject: `${namespace}/${d.path}`,
204      detail: `type '${d.type}' with no typed edge in either direction; nothing in the store points at it and it points at nothing`,
205    })),
206  });
207
208  // 4. Broken links, from gather, counted against the whole edge population.
209  const namespaceEdges = input.edges.filter((e) => e.from_ns === namespace || e.to_ns === namespace);
210  checks.push({
211    check: "broken_links",
212    subjects: namespaceEdges.length,
213    ok: namespaceEdges.length - input.danglingEdges.length,
214    findings: input.danglingEdges.map((e) => ({
215      check: "broken_links",
216      subject: `${e.from_ns}/${e.from_path} -[${e.type}]-> ${e.to_ns}/${e.to_path}`,
217      detail: e.source_missing ? "the source document no longer exists" : "the target document no longer exists",
218    })),
219  });
220
221  // 5. Doc-vs-code drift: a repo path named in canon that is no longer in the repo.
222  //    The repo is gated and the store is not, so the store is where a path rots.
223  //    Unrun is not clean: when the repo could not be read the check has zero
224  //    subjects and is excluded from integrity, rather than reporting zero findings,
225  //    which would read as perfect.
226  if (input.repoPaths) {
227    // A candidate counts only when its first segment is a top-level entry of this
228    // namespace's mapped repo. That separates `src/gone.ts`, which this repo could
229    // really be missing, from `apps/web/...` in another portfolio repo or a path in
230    // another repo entirely, which are not drift here.
231    const repoRoots = new Set([...input.repoPaths].map((p) => p.split("/")[0]));
232    // A document path in this namespace is a store address, whatever it looks like.
233    // The prefix list cannot reach these: the Worker writes `jobs/<id>.md` itself, and
234    // `improve/` is both a store prefix and a real directory, so the root check would
235    // pass it through. Every document counts, excluded ones included, because an
236    // address does not stop being an address.
237    const documentPaths = new Set(docs.map((d) => d.path));
238    const cited = new Map<string, Set<string>>();
239    for (const doc of standing) {
240      for (const match of (doc.body ?? "").matchAll(REPO_PATH)) {
241        const path = match[1];
242        if (NAMESPACE_PREFIXES.some((p) => path.startsWith(p))) continue;
243        if (documentPaths.has(path)) continue;
244        if (!repoRoots.has(path.split("/")[0])) continue;
245        if (!cited.has(path)) cited.set(path, new Set());
246        cited.get(path)!.add(doc.path);
247      }
248    }
249    const missing = [...cited.entries()].filter(([path]) => !input.repoPaths!.has(path));
250    checks.push({
251      check: "doc_vs_code_drift",
252      subjects: cited.size,
253      ok: cited.size - missing.length,
254      findings: missing.map(([path, docPaths]) => ({
255        check: "doc_vs_code_drift",
256        subject: path,
257        detail: `named in ${[...docPaths].sort().join(", ")} and absent from the repo`,
258      })),
259    });
260  } else {
261    checks.push({
262      check: "doc_vs_code_drift",
263      subjects: 0,
264      ok: 0,
265      findings: [
266        {
267          check: "doc_vs_code_drift",
268          subject: namespace,
269          detail: "NOT RUN: the repo tree could not be read, so this check is excluded from integrity rather than counted as clean",
270        },
271      ],
272    });
273  }
274
275  // 6. Unconsolidated: a backlog, not a defect. At or under the cadence it is not a
276  //    finding and does not lower integrity, so integrity always traces to a finding.
277  const unconsolidated = standing.filter((d) => d.type === "episodic" || d.type === "source");
278  const overCadence = unconsolidated.length > UNCONSOLIDATED_CADENCE;
279  checks.push({
280    check: "unconsolidated",
281    subjects: standing.length,
282    ok: overCadence ? standing.length - unconsolidated.length : standing.length,
283    findings:
284      overCadence
285        ? [
286            {
287              check: "unconsolidated",
288              subject: namespace,
289              detail: `${unconsolidated.length} unconsolidated documents, over the consolidation threshold of 5`,
290            },
291          ]
292        : [],
293  });
294
295  // 7 and 8. The rule documents (src/rule-docs.ts): each under its cap, and naming only
296  //    tools and actions the Worker serves. The subjects are the rule documents in this
297  //    namespace, so a namespace with none has nothing judged here.
298  const ruleDocs = standing.filter((d) => ruleDocCap(namespace, d.path) !== null);
299  const problems = ruleDocs.map((d) => ruleDocProblem(namespace, d.path, d.body ?? "")).filter((p): p is RuleDocProblem => p !== null);
300  const oversize = problems.filter((p) => p.oversize);
301  checks.push({
302    check: "rule_doc_caps",
303    subjects: ruleDocs.length,
304    ok: ruleDocs.length - oversize.length,
305    findings: oversize.map((p) => ({
306      check: "rule_doc_caps",
307      subject: `${namespace}/${p.path}`,
308      detail: `${p.bytes} bytes, over its ${p.cap}-byte cap`,
309    })),
310  });
311  const stale = problems.filter((p) => p.stale.length > 0);
312  checks.push({
313    check: "stale_tool_names",
314    subjects: ruleDocs.length,
315    ok: ruleDocs.length - stale.length,
316    findings: stale.map((p) => ({
317      check: "stale_tool_names",
318      subject: `${namespace}/${p.path}`,
319      detail: `names ${staleSummary(p.stale)}`,
320    })),
321  });
322
323  const judged = checks.filter((c) => c.subjects > 0);
324  const subjects = judged.reduce((sum, c) => sum + c.subjects, 0);
325  const good = judged.reduce((sum, c) => sum + c.ok, 0);
326  const integrity = subjects > 0 ? Math.round((good / subjects) * 1000) / 10 : null;
327
328  return {
329    namespace,
330    generated: input.now.toISOString(),
331    documents: {
332      total: docs.length,
333      by_type: byType,
334      unconsolidated: unconsolidated.length,
335      archived,
336    },
337    checks,
338    integrity,
339    findings: checks.flatMap((c) => c.findings),
340  };
341}
342
343// The integrity line is first and in a fixed shape, because `capsid_run_improve` action `status` parses
344// it back out.
345export const INTEGRITY_LINE = /^integrity:\s*([0-9]+(?:\.[0-9]+)?)%\s*$/m;
346
347export function renderTruthReport(report: TruthReport): string {
348  const lines: string[] = [];
349  lines.push(`# Truth report - ${report.namespace} - ${report.generated.slice(0, 10)}`);
350  lines.push("");
351  // Not measured is not zero. `integrity` is null when no check had a subject, and
352  // "0%" would read as a store in the worst state it can be in. INTEGRITY_LINE does
353  // not match the words, so integrityOf returns null and capsid_run_improve action status reports no
354  // report, the same answer a missing document gets.
355  lines.push(`integrity: ${report.integrity === null ? "not measured" : `${report.integrity}%`}`);
356  lines.push("");
357  lines.push(
358    "One number and the checks it is made of. Integrity is subjects in good standing over subjects judged, across every check that had a subject to judge. " +
359      "A check that could not run is excluded rather than counted as clean. Nothing here judges whether a ruling is CORRECT; it counts what a program can count without an opinion."
360  );
361  lines.push("");
362  lines.push("## Documents");
363  lines.push("");
364  lines.push(`- total: ${report.documents.total}`);
365  lines.push(`- archived: ${report.documents.archived}`);
366  lines.push(`- unconsolidated: ${report.documents.unconsolidated}`);
367  for (const [type, n] of Object.entries(report.documents.by_type).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
368    lines.push(`- type ${type}: ${n}`);
369  }
370  lines.push("");
371  lines.push("## Checks");
372  lines.push("");
373  lines.push("| check | subjects | ok | findings |");
374  lines.push("| --- | --- | --- | --- |");
375  for (const check of report.checks) {
376    lines.push(`| ${check.check} | ${check.subjects} | ${check.ok} | ${check.findings.length} |`);
377  }
378  lines.push("");
379  if (report.findings.length === 0) {
380    lines.push("## Findings");
381    lines.push("");
382    lines.push("None.");
383  } else {
384    lines.push("## Findings");
385    lines.push("");
386    for (const check of report.checks) {
387      if (check.findings.length === 0) continue;
388      lines.push(`### ${check.check}`);
389      lines.push("");
390      for (const finding of check.findings) {
391        lines.push(`- **${finding.subject}**: ${finding.detail}`);
392      }
393      lines.push("");
394    }
395  }
396  lines.push("Generated by `capsid_lint_namespace` mode `report`. The trend is the point: compare this against the previous dated report in this directory.");
397  lines.push("");
398  return lines.join("\n");
399}
400
401// One document per namespace per day; a second run the same day overwrites.
402export function reportPath(date: Date): string {
403  return `${REPORTS_PREFIX}lint-${date.toISOString().slice(0, 10)}.md`;
404}
405
406// Pull the integrity number back out of a stored report. Returns null on anything
407// unexpected, which `capsid_run_improve` action `status` reports as "no report" rather than as zero.
408export function integrityOf(body: string | null | undefined): number | null {
409  if (!body) return null;
410  const match = INTEGRITY_LINE.exec(body);
411  if (!match) return null;
412  const value = Number(match[1]);
413  return Number.isFinite(value) ? value : null;
414}