scripts/code-map.mjs

Functions

Imports

It imports no other module of this repo. No module of this repo imports it.

Source

1#!/usr/bin/env node
2// The code map: one JSON file per public repository saying,
3// for every module, how big it is, what it imports and whether it sits in an import cycle, and for every function its
4// cyclomatic complexity, its share of statements covered, its CRAP score and, when a Stryker report is given, its mutants
5// and mutation score. Capsid publishes it and the explorer at capsid.dustinedwards.info draws it.
6//
7// Everything is read through the repository's own TypeScript, so there is no dependency here and no second parser:
8// imports are resolved by ts.resolveModuleName with the repository's compiler options, and complexity is counted on the
9// syntax tree. Coverage is Vitest's (istanbul's) coverage-final.json, or the lcov file `node --test
10// --experimental-test-coverage --test-reporter=lcov` writes, and mutation is Stryker's mutation.json, each mapped onto the
11// innermost function whose lines hold a statement (an lcov line) or a mutant. A number nobody measured is null, never 0.
12//
13// TypeScript and JavaScript alike: .ts, .tsx, .js, .mjs and the rest, never a declaration file or a *.test.* or *.spec.*
14// file. A repository with no tsconfig is read as plain modules (allowJs, NodeNext). `--include .` maps every folder, and
15// `--exclude <prefix>` (repeatable) leaves a path out.
16//
17// Mutation runs less often than the map (weekly, design D11), so a run's reports are kept as one smaller file, mutation.json,
18// beside the map: the commit and day of the run, and per file its source then and each mutant's status and line. A later map
19// reads that file, places each mutant in the function it was made in from the source then, and carries the numbers to a
20// function whose lines are unchanged since. A function that changed, or is new, is `stale: true` with no numbers, and the
21// map names the run it read as `mutated`.
22//
23//   node code-map.mjs --out code-map.json [--tsconfig tsconfig.json] [--include src/]... [--exclude <prefix>]... [--coverage <file>]
24//     [--mutation <stryker report>]... [--mutation-kept <mutation.json>] [--mutation-out <mutation.json>]
25//
26// The repository and commit come from GITHUB_REPOSITORY and GITHUB_SHA, or --repo and --sha. code-map.yml carries this file
27// byte for byte; after an edit run `node scripts/inline-rollback-guard.mjs`.
28
29import { execFileSync } from "node:child_process";
30import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
31import { createRequire } from "node:module";
32import { join, relative, resolve, sep } from "node:path";
33import { pathToFileURL } from "node:url";
34
35/** CRAP = CC² × (1 − coverage)³ + CC (crap4j's formula), rounded to two places. Null when coverage was not measured. */
36export function crap(/** @type {number} */ cc, /** @type {number | null} */ coverage) {
37  if (coverage === null) return null;
38  return Math.round((cc * cc * (1 - coverage) ** 3 + cc) * 100) / 100;
39}
40
41/** The repository's TypeScript, from its own node_modules, so the map parses what its compiler parses. */
42export function loadTypeScript(/** @type {string} */ root) {
43  for (const base of [join(root, "package.json"), import.meta.url]) {
44    try {
45      return createRequire(base)("typescript");
46    } catch {
47      // Try the next place.
48    }
49  }
50  throw new Error("typescript is not installed in the repository; the code map reads its code through it");
51}
52
53/**
54 * Every function in a source file with its lines and cyclomatic complexity: 1, plus one for each branch (if, ?:, a loop,
55 * a case, a catch, &&, ||, ?? and their assignments) in its own body, not in a function nested inside it.
56 * @param {any} ts @param {any} sf
57 * @returns {{ name: string, line: number, end: number, cc: number }[]}
58 */
59export function functionsOf(ts, sf) {
60  /** @type {{ name: string, line: number, end: number, cc: number }[]} */
61  const out = [];
62  const K = ts.SyntaxKind;
63  const LOGICAL = new Set([K.AmpersandAmpersandToken, K.BarBarToken, K.QuestionQuestionToken, K.AmpersandAmpersandEqualsToken, K.BarBarEqualsToken, K.QuestionQuestionEqualsToken]);
64  const isFunction = (/** @type {any} */ n) =>
65    ts.isFunctionDeclaration(n) || ts.isFunctionExpression(n) || ts.isArrowFunction(n) || ts.isMethodDeclaration(n) ||
66    ts.isConstructorDeclaration(n) || ts.isGetAccessorDeclaration(n) || ts.isSetAccessorDeclaration(n);
67  const lineOf = (/** @type {number} */ pos) => sf.getLineAndCharacterOfPosition(pos).line + 1;
68  const nameOf = (/** @type {any} */ n) => {
69    if (ts.isConstructorDeclaration(n)) return "constructor";
70    if (n.name && (ts.isIdentifier(n.name) || ts.isStringLiteral(n.name) || ts.isPrivateIdentifier(n.name))) return n.name.text;
71    const p = n.parent;
72    if (p && (ts.isVariableDeclaration(p) || ts.isPropertyAssignment(p) || ts.isPropertyDeclaration(p)) && ts.isIdentifier(p.name)) return p.name.text;
73    return `<anonymous:${lineOf(n.getStart(sf))}>`;
74  };
75  const branches = (/** @type {any} */ n) => {
76    let count = 0;
77    const walk = (/** @type {any} */ c) => {
78      if (isFunction(c)) return;
79      if (ts.isIfStatement(c) || ts.isConditionalExpression(c) || ts.isForStatement(c) || ts.isForInStatement(c) ||
80        ts.isForOfStatement(c) || ts.isWhileStatement(c) || ts.isDoStatement(c) || ts.isCaseClause(c) || ts.isCatchClause(c)) count++;
81      else if (ts.isBinaryExpression(c) && LOGICAL.has(c.operatorToken.kind)) count++;
82      ts.forEachChild(c, walk);
83    };
84    if (ts.isBlock(n.body)) ts.forEachChild(n.body, walk);
85    else walk(n.body);
86    return count;
87  };
88  const visit = (/** @type {any} */ n) => {
89    if (isFunction(n) && n.body) out.push({ name: nameOf(n), line: lineOf(n.getStart(sf)), end: lineOf(n.getEnd()), cc: 1 + branches(n) });
90    ts.forEachChild(n, visit);
91  };
92  visit(sf);
93  return out;
94}
95
96/** The module specifiers a file imports or re-exports, static and dynamic. @param {any} ts @param {any} sf */
97export function specifiersOf(ts, sf) {
98  /** @type {string[]} */
99  const out = [];
100  const visit = (/** @type {any} */ n) => {
101    if ((ts.isImportDeclaration(n) || ts.isExportDeclaration(n)) && n.moduleSpecifier && ts.isStringLiteral(n.moduleSpecifier)) out.push(n.moduleSpecifier.text);
102    else if (ts.isCallExpression(n) && n.expression.kind === ts.SyntaxKind.ImportKeyword && n.arguments[0] && ts.isStringLiteral(n.arguments[0])) out.push(n.arguments[0].text);
103    ts.forEachChild(n, visit);
104  };
105  visit(sf);
106  return out;
107}
108
109/** Every import cycle (Tarjan's strongly connected components of two or more modules, or one that imports itself). */
110export function cyclesOf(/** @type {Map<string, string[]>} */ graph) {
111  let index = 0;
112  /** @type {Map<string, number>} */ const idx = new Map();
113  /** @type {Map<string, number>} */ const low = new Map();
114  /** @type {string[]} */ const stack = [];
115  const on = new Set();
116  /** @type {string[][]} */ const out = [];
117  const connect = (/** @type {string} */ v) => {
118    idx.set(v, index);
119    low.set(v, index++);
120    stack.push(v);
121    on.add(v);
122    for (const w of graph.get(v) ?? []) {
123      if (!idx.has(w)) {
124        connect(w);
125        low.set(v, Math.min(/** @type {number} */ (low.get(v)), /** @type {number} */ (low.get(w))));
126      } else if (on.has(w)) low.set(v, Math.min(/** @type {number} */ (low.get(v)), /** @type {number} */ (idx.get(w))));
127    }
128    if (low.get(v) === idx.get(v)) {
129      /** @type {string[]} */ const component = [];
130      let w;
131      do {
132        w = /** @type {string} */ (stack.pop());
133        on.delete(w);
134        component.push(w);
135      } while (w !== v);
136      if (component.length > 1 || (graph.get(v) ?? []).includes(v)) out.push(component.sort());
137    }
138  };
139  for (const v of [...graph.keys()].sort()) if (!idx.has(v)) connect(v);
140  return out.sort((a, b) => a[0].localeCompare(b[0]));
141}
142
143/**
144 * The innermost function whose lines hold `line`, or undefined.
145 * @template {{ line: number, end: number }} F
146 * @param {F[]} fns @param {number} line
147 */
148function innermost(fns, line) {
149  let best;
150  for (const f of fns) if (f.line <= line && line <= f.end && (!best || f.end - f.line < best.end - best.line)) best = f;
151  return best;
152}
153
154/** A module the map reads: source, not a declaration file and not a test. */
155const SOURCE = /\.[cm]?[jt]sx?$/;
156const DECLARATION = /\.d\.[cm]?ts$/;
157const TEST = /\.(test|spec)\.[cm]?[jt]sx?$/;
158
159/** The JavaScript source a hand-written declaration file describes, when it sits beside it: x.d.mts for x.mjs. */
160function sourceOf(/** @type {string} */ file) {
161  const m = /^(.*)\.d\.([cm]?)ts$/.exec(file);
162  if (!m) return file;
163  const js = `${m[1]}.${m[2]}js`;
164  return existsSync(js) ? js : file;
165}
166
167/** Every file under a directory, leaving out node_modules and dot-folders. @param {string} dir @returns {string[]} */
168function walk(dir) {
169  return readdirSync(dir).flatMap((name) => {
170    if (name === "node_modules" || name.startsWith(".")) return [];
171    const full = join(dir, name);
172    return statSync(full).isDirectory() ? walk(full) : [full];
173  });
174}
175
176/**
177 * lcov as istanbul's coverage-final.json shape: per file, each instrumented line (DA) a statement with its hit count.
178 * Paths are resolved against the repository's root. @param {string} text @param {string} root
179 */
180export function lcovCoverage(text, root) {
181  /** @type {Record<string, { statementMap: Record<string, { start: { line: number } }>, s: Record<string, number> }>} */
182  const out = {};
183  let file = null;
184  for (const raw of text.split("\n")) {
185    const line = raw.trim();
186    if (line.startsWith("SF:")) {
187      // Forward slashes, as TypeScript names its files, so the lookup also matches on Windows.
188      const abs = resolve(root, line.slice(3)).replaceAll("\\", "/");
189      file = out[abs] ??= { statementMap: {}, s: {} };
190    } else if (line.startsWith("DA:") && file) {
191      const [n, hits] = line.slice(3).split(",").map(Number);
192      const id = String(Object.keys(file.statementMap).length);
193      file.statementMap[id] = { start: { line: n } };
194      file.s[id] = hits;
195    } else if (line === "end_of_record") file = null;
196  }
197  return out;
198}
199
200/** Stryker's statuses: detected and valid, as its mutation score counts them. */
201const DETECTED = new Set(["Killed", "Timeout"]);
202const VALID = new Set(["Killed", "Timeout", "Survived", "NoCoverage"]);
203
204/**
205 * One kept mutation report from a run's Stryker reports (one per chunk or shard): per file its source and each valid
206 * mutant's status and line, with the run's commit and day.
207 * @param {any[]} reports @param {string} sha @param {string} date
208 */
209export function keptReport(reports, sha, date) {
210  /** @type {Record<string, { source: string | null, mutants: { status: string, location: { start: { line: number } } }[] }>} */
211  const files = {};
212  for (const r of reports) {
213    for (const [path, f] of Object.entries(r?.files ?? {})) {
214      const mutants = (/** @type {any} */ (f).mutants ?? [])
215        .filter((/** @type {any} */ m) => VALID.has(m.status))
216        .map((/** @type {any} */ m) => ({ status: m.status, location: { start: { line: m.location.start.line } } }));
217      const was = files[path];
218      files[path] = { source: was?.source ?? /** @type {any} */ (f).source ?? null, mutants: [...(was?.mutants ?? []), ...mutants] };
219    }
220  }
221  return { version: 1, sha, date, files };
222}
223
224/**
225 * The code map of a repository.
226 * @param {{ ts: any, root: string, tsconfig?: string, include?: string[], exclude?: string[], coverage?: any, mutation?: any, repo: string, sha: string, date: string }} o
227 */
228export function codeMap({ ts, root, tsconfig = "tsconfig.json", include = ["src/"], exclude = [], coverage = null, mutation = null, repo, sha, date }) {
229  const parsed = existsSync(resolve(root, tsconfig))
230    ? ts.getParsedCommandLineOfConfigFile(resolve(root, tsconfig), {}, { ...ts.sys, onUnRecoverableConfigFileDiagnostic: (/** @type {any} */ d) => { throw new Error(ts.flattenDiagnosticMessageText(d.messageText, "\n")); } })
231    : { fileNames: walk(root), options: { allowJs: true, module: ts.ModuleKind.NodeNext, moduleResolution: ts.ModuleResolutionKind.NodeNext, target: ts.ScriptTarget.ES2022 } };
232  const rel = (/** @type {string} */ abs) => relative(root, abs).split(sep).join("/");
233  // A package that ships hand-written .d.mts beside its .mjs: the compiler lists the declaration in the source's place.
234  const files = [...new Set(parsed.fileNames.map(sourceOf))]
235    .filter((f) => SOURCE.test(f) && !DECLARATION.test(f) && !TEST.test(f) && include.some((p) => p === "." || rel(f).startsWith(p)) && !exclude.some((p) => rel(f).startsWith(p)))
236    .sort();
237  const mapped = new Set(files.map(rel));
238  const host = ts.createCompilerHost(parsed.options);
239  /** @type {Map<string, string[]>} */
240  const graph = new Map();
241  const modules = files.map((abs) => {
242    const path = rel(abs);
243    const text = readFileSync(abs, "utf8");
244    const sf = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, true);
245    const imports = new Set();
246    for (const spec of specifiersOf(ts, sf)) {
247      const hit = ts.resolveModuleName(spec, abs, parsed.options, host).resolvedModule;
248      const to = hit && rel(sourceOf(hit.resolvedFileName));
249      if (to && mapped.has(to)) imports.add(to);
250    }
251    graph.set(path, [...imports].sort());
252    /** @type {{ name: string, line: number, end: number, cc: number, statements: number, covered: number, mutants: number, detected: number, stale?: true }[]} */
253    const fns = functionsOf(ts, sf).map((f) => ({ ...f, statements: 0, covered: 0, mutants: 0, detected: 0 }));
254
255    const cov = coverage?.[abs] ?? coverage?.[path];
256    if (cov) {
257      for (const [id, s] of Object.entries(cov.statementMap ?? {})) {
258        const f = innermost(fns, /** @type {any} */ (s).start.line);
259        if (!f) continue;
260        f.statements++;
261        if ((cov.s?.[id] ?? 0) > 0) f.covered++;
262      }
263    }
264    const mut = mutation?.files?.[path] ?? mutation?.files?.[abs];
265    if (mut) {
266      // Each mutant goes to the function it was made in, read from the file as the run saw it, so lines that moved since
267      // still meet their function.
268      const then = typeof mut.source === "string" ? functionsOf(ts, ts.createSourceFile(abs, mut.source, ts.ScriptTarget.Latest, true)).map((f) => ({ ...f, mutants: 0, detected: 0 })) : fns;
269      for (const m of mut.mutants ?? []) {
270        if (!VALID.has(m.status)) continue;
271        const f = innermost(then, m.location.start.line);
272        if (!f) continue;
273        f.mutants++;
274        if (DETECTED.has(m.status)) f.detected++;
275      }
276      if (then !== fns) {
277        // A function keeps the run's numbers only when its lines are the same as then: the nth function of a name now
278        // against the nth then.
279        const linesThen = mut.source.split("\n");
280        const linesNow = text.split("\n");
281        const body = (/** @type {string[]} */ lines, /** @type {{ line: number, end: number }} */ f) => lines.slice(f.line - 1, f.end).join("\n");
282        const keyed = (/** @type {{ name: string }[]} */ list) => {
283          /** @type {Map<string, number>} */ const n = new Map();
284          return list.map((f) => { const i = n.get(f.name) ?? 0; n.set(f.name, i + 1); return `${f.name}#${i}`; });
285        };
286        const keysThen = keyed(then);
287        const byKey = new Map(then.map((f, i) => [keysThen[i], f]));
288        keyed(fns).forEach((key, i) => {
289          const was = byKey.get(key);
290          if (was && body(linesThen, was) === body(linesNow, fns[i])) Object.assign(fns[i], { mutants: was.mutants, detected: was.detected });
291          else Object.assign(fns[i], { stale: true });
292        });
293      }
294    }
295
296    return {
297      path,
298      loc: text.split("\n").filter((l) => l.trim() !== "").length,
299      imports: graph.get(path),
300      functions: fns.map(({ name, line, end, cc, statements, covered, mutants, detected, stale }) => {
301        const c = cov ? (statements === 0 ? 1 : Math.round((covered / statements) * 1000) / 1000) : null;
302        const measured = mut && !stale;
303        return { name, line, end, cc, coverage: c, crap: crap(cc, c), mutants: measured ? mutants : null, score: measured && mutants > 0 ? Math.round((detected / mutants) * 1000) / 10 : null, ...(stale ? { stale } : {}) };
304      }),
305    };
306  });
307  const cycles = cyclesOf(graph);
308  const inCycle = new Set(cycles.flat());
309  for (const m of modules) if (inCycle.has(m.path)) Object.assign(m, { cycle: true });
310  return {
311    version: 1,
312    repo,
313    sha,
314    date,
315    tools: { typescript: ts.version, coverage: coverage ? "istanbul" : null, mutation: mutation ? "stryker" : null },
316    ...(mutation?.sha && mutation.sha !== sha ? { mutated: { sha: mutation.sha, date: mutation.date } } : {}),
317    modules,
318    cycles,
319  };
320}
321
322/** --name value pairs; a repeated name collects every value. @param {string[]} argv */
323function args(argv) {
324  /** @type {Record<string, string[]>} */
325  const out = {};
326  for (let i = 0; i < argv.length; i += 2) {
327    if (!argv[i].startsWith("--") || argv[i + 1] === undefined) throw new Error(`expected --name value, got ${argv[i]}`);
328    (out[argv[i].slice(2)] ??= []).push(argv[i + 1]);
329  }
330  return out;
331}
332
333if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
334  const a = args(process.argv.slice(2));
335  const root = process.cwd();
336  const json = (/** @type {string | undefined} */ f) => (f && existsSync(f) ? JSON.parse(readFileSync(f, "utf8")) : null);
337  for (const f of [a.coverage?.[0], ...(a.mutation ?? []), a["mutation-kept"]?.[0]]) if (f && !existsSync(f)) console.log(`::warning::${f} does not exist, so the map has no numbers from it`);
338  const sha = a.sha?.[0] ?? process.env.GITHUB_SHA ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
339  const date = new Date().toISOString().slice(0, 10);
340  // A fresh run's reports win over a kept one; with neither, the map has no mutation numbers.
341  const reports = (a.mutation ?? []).map(json).filter(Boolean);
342  const mutation = reports.length ? keptReport(reports, sha, date) : json(a["mutation-kept"]?.[0]);
343  if (mutation && a["mutation-out"]?.[0]) writeFileSync(a["mutation-out"][0], `${JSON.stringify(mutation)}\n`);
344  const map = codeMap({
345    ts: loadTypeScript(root),
346    root,
347    tsconfig: a.tsconfig?.[0],
348    include: a.include,
349    exclude: a.exclude,
350    coverage: a.coverage?.[0] && existsSync(a.coverage[0]) && !a.coverage[0].endsWith(".json") ? lcovCoverage(readFileSync(a.coverage[0], "utf8"), root) : json(a.coverage?.[0]),
351    mutation,
352    repo: a.repo?.[0] ?? process.env.GITHUB_REPOSITORY ?? "",
353    sha,
354    date,
355  });
356  writeFileSync(a.out?.[0] ?? "code-map.json", `${JSON.stringify(map, null, 2)}\n`);
357  const fns = map.modules.reduce((n, m) => n + m.functions.length, 0);
358  const stale = map.modules.reduce((n, m) => n + m.functions.filter((f) => f.stale).length, 0);
359  console.log(`code map: ${map.modules.length} modules, ${fns} functions, ${map.cycles.length} cycles${map.mutated ? `; mutation from ${map.mutated.sha.slice(0, 7)} (${map.mutated.date}), ${stale} functions stale` : ""}`);
360}