scripts/validate-skills.mjs

Functions

Imports

It imports no other module of this repo. No module of this repo imports it.

Source

1// The skill package format, as code.
2//
3// A skill is loaded and acted on, so the metadata that says what it is has to be
4// machine-checkable rather than a convention people remember. This refuses any
5// skill missing an L1 field and runs in CI, which is what makes the format a
6// format instead of a README section.
7//
8// FAIL CLOSED, the same way line-scope.mjs does. The frontmatter parser below
9// understands exactly the subset the format uses and refuses everything else,
10// because a parser that guesses at YAML it does not understand turns a malformed
11// header into a passing one.
12
13import { readdirSync, readFileSync, statSync } from 'node:fs';
14import { join, posix, sep } from 'node:path';
15import { pathToFileURL } from 'node:url';
16
17const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
18const STATUSES = ['candidate', 'live', 'retired'];
19
20// name is required on a skill and derived from the filename on a command, so it
21// is checked separately rather than listed here.
22const REQUIRED = ['description', 'trigger', 'namespaces', 'version', 'status', 'source', 'termination', 'interface'];
23
24const KEY = /^([A-Za-z][A-Za-z0-9_-]*):(?: (.*))?$/;
25const NESTED = /^ {2}([A-Za-z][A-Za-z0-9_-]*):(?: (.*))?$/;
26
27// A double-quoted scalar has to close, and \" and \\ are the only escapes.
28const QUOTED = /^"((?:[^"\\]|\\.)*)"$/;
29
30function scalar(raw, where) {
31  const v = raw.trim();
32  if (v === '|' || v === '>' || v.startsWith('|') || v.startsWith('>')) {
33    throw new Error(`${where}: block scalars are not part of this format`);
34  }
35  const q = v.match(QUOTED);
36  if (q) return q[1].replace(/\\(["\\])/g, '$1');
37  if (v.startsWith('"')) throw new Error(`${where}: unterminated quote`);
38  return v;
39}
40
41// Splits a flow list on its top-level commas, so a comma inside a quoted item
42// does not end the item.
43function flowList(raw, where) {
44  const inner = raw.trim().slice(1, -1).trim();
45  if (inner === '') return [];
46  const items = [];
47  let buf = '';
48  let quoted = false;
49  let escaped = false;
50  for (const ch of inner) {
51    if (escaped) { buf += ch; escaped = false; continue; }
52    if (ch === '\\' && quoted) { buf += ch; escaped = true; continue; }
53    if (ch === '"') { quoted = !quoted; buf += ch; continue; }
54    if (ch === ',' && !quoted) { items.push(buf); buf = ''; continue; }
55    buf += ch;
56  }
57  if (quoted) throw new Error(`${where}: unterminated quote in list`);
58  items.push(buf);
59  return items.map((i) => scalar(i, where));
60}
61
62function value(raw, where) {
63  if (raw === undefined || raw.trim() === '') return '';
64  if (raw.trim().startsWith('[')) {
65    if (!raw.trim().endsWith(']')) throw new Error(`${where}: unterminated list`);
66    return flowList(raw, where);
67  }
68  return scalar(raw, where);
69}
70
71export function parseFrontmatter(text) {
72  const src = text.replace(/\r\n/g, '\n');
73  if (!src.startsWith('---\n')) return { ok: false, error: 'no frontmatter block', fields: {} };
74  const end = src.indexOf('\n---', 3);
75  if (end === -1) return { ok: false, error: 'frontmatter is never closed', fields: {} };
76
77  const fields = {};
78  let lastKey = null;
79  let lineNo = 1;
80  for (const line of src.slice(4, end).split('\n')) {
81    lineNo++;
82    if (line.trim() === '' || line.startsWith('#')) continue;
83    const where = `line ${lineNo}`;
84    try {
85      const nested = line.match(NESTED);
86      if (nested) {
87        // Only a key that declared itself a map may hold nested keys. This is
88        // what catches an accidentally indented top-level field.
89        if (!lastKey || typeof fields[lastKey] !== 'object' || Array.isArray(fields[lastKey])) {
90          return { ok: false, error: `${where}: "${nested[1]}" is indented under a field that is not a map`, fields };
91        }
92        fields[lastKey][nested[1]] = value(nested[2], where);
93        continue;
94      }
95      const key = line.match(KEY);
96      if (!key) return { ok: false, error: `${where}: cannot read "${line}"`, fields };
97      const v = value(key[2], where);
98      // An empty value opens a nested map; a later nested line fills it in.
99      fields[key[1]] = v === '' && key[2] === undefined ? {} : v;
100      lastKey = key[1];
101    } catch (e) {
102      return { ok: false, error: e.message, fields };
103    }
104  }
105  return { ok: true, error: null, fields };
106}
107
108const empty = (v) => v === undefined || v === '' || (Array.isArray(v) && v.length === 0)
109  || (v && typeof v === 'object' && Object.keys(v).length === 0);
110
111/**
112 * @param {{path: string, text: string, kind: 'skill'|'candidate'|'command',
113 *          expectedName: string, vendored?: boolean}} entry
114 */
115export function validateEntry({ path, text, kind, expectedName, vendored = false }) {
116  const errors = [];
117  const parsed = parseFrontmatter(text);
118  if (!parsed.ok) {
119    return { path, kind, name: expectedName, vendored, ok: false, errors: [parsed.error], status: null, version: null };
120  }
121  const f = parsed.fields;
122
123  for (const field of REQUIRED) {
124    if (empty(f[field])) errors.push(`missing required field: ${field}`);
125  }
126
127  if (kind !== 'command') {
128    if (empty(f.name)) errors.push('missing required field: name');
129    else if (f.name !== expectedName) {
130      errors.push(`name "${f.name}" does not match its directory "${expectedName}"`);
131    }
132  }
133
134  if (!empty(f.namespaces) && !Array.isArray(f.namespaces)) {
135    errors.push('namespaces must be a list, for example ["*"] or ["my-project"]');
136  }
137
138  if (!empty(f.interface)) {
139    if (typeof f.interface !== 'object' || Array.isArray(f.interface)) {
140      errors.push('interface must be a map of inputs and outputs');
141    } else {
142      if (empty(f.interface.inputs)) errors.push('interface is missing inputs');
143      if (empty(f.interface.outputs)) errors.push('interface is missing outputs');
144    }
145  }
146
147  if (!empty(f.version) && !SEMVER.test(f.version)) {
148    errors.push(`version "${f.version}" is not semver`);
149  }
150
151  if (!empty(f.status) && !STATUSES.includes(f.status)) {
152    errors.push(`status "${f.status}" is not one of ${STATUSES.join(', ')}`);
153  }
154
155  // Where a skill sits and what its status says have to agree, or a status is
156  // just a word. The promotion PR moves the directory and flips the status in
157  // one commit precisely so these two never drift apart.
158  if (kind === 'candidate') {
159    if (f.status !== 'candidate') {
160      errors.push(`status "${f.status}" but the directory is skills/_candidates/: a candidate says status candidate`);
161    }
162    if (f.source === 'human') {
163      errors.push('source human is not available to a candidate: name the attempt or job id that produced it');
164    }
165  } else if (f.status === 'candidate') {
166    errors.push('status candidate outside skills/_candidates/: a candidate the loader can see is not a candidate');
167  }
168
169  return { path, kind, name: f.name ?? expectedName, vendored, ok: errors.length === 0, errors, status: f.status ?? null, version: f.version ?? null };
170}
171
172const rel = (root, p) => p.slice(root.length + 1).split(sep).join(posix.sep);
173
174function dirsIn(dir) {
175  try {
176    return readdirSync(dir).filter((n) => statSync(join(dir, n)).isDirectory());
177  } catch {
178    return [];
179  }
180}
181
182export function validateTree(root) {
183  let vendored = [];
184  try {
185    vendored = JSON.parse(readFileSync(join(root, 'MANIFEST.json'), 'utf8')).vendored?.skills ?? [];
186  } catch {
187    // A missing manifest is not a validation failure; it only means nothing is
188    // marked vendored, and the vendored flag is a report rather than a rule.
189  }
190
191  const entries = [];
192  const read = (p) => readFileSync(p, 'utf8');
193
194  for (const name of dirsIn(join(root, 'skills'))) {
195    if (name === '_candidates') continue;
196    const p = join(root, 'skills', name, 'SKILL.md');
197    try { statSync(p); } catch { continue; }
198    entries.push(validateEntry({
199      path: rel(root, p), text: read(p), kind: 'skill', expectedName: name, vendored: vendored.includes(name),
200    }));
201  }
202
203  for (const name of dirsIn(join(root, 'skills', '_candidates'))) {
204    const p = join(root, 'skills', '_candidates', name, 'SKILL.md');
205    try { statSync(p); } catch { continue; }
206    entries.push(validateEntry({
207      path: rel(root, p), text: read(p), kind: 'candidate', expectedName: name,
208    }));
209  }
210
211  for (const file of readdirSync(join(root, 'commands')).filter((n) => n.endsWith('.md'))) {
212    const p = join(root, 'commands', file);
213    entries.push(validateEntry({
214      path: rel(root, p), text: read(p), kind: 'command', expectedName: file.replace(/\.md$/, ''),
215    }));
216  }
217
218  return { ok: entries.every((e) => e.ok), entries };
219}
220
221function main() {
222  const report = validateTree(process.cwd());
223  const failures = report.entries.filter((e) => !e.ok);
224  for (const f of failures) {
225    console.error(`FAIL ${f.path}`);
226    for (const e of f.errors) console.error(`     ${e}`);
227  }
228  const counts = {
229    skills: report.entries.filter((e) => e.kind === 'skill').length,
230    candidates: report.entries.filter((e) => e.kind === 'candidate').length,
231    commands: report.entries.filter((e) => e.kind === 'command').length,
232    vendored: report.entries.filter((e) => e.vendored).length,
233  };
234  console.log(
235    `${report.entries.length} checked: ${counts.skills} skills (${counts.vendored} vendored), `
236    + `${counts.candidates} candidates, ${counts.commands} commands`,
237  );
238  if (!report.ok) {
239    console.error(`\n${failures.length} failed validation`);
240    process.exit(1);
241  }
242  console.log('every entry carries a complete L1 block');
243}
244
245if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();