app/kb/procedures/parse.mjs

Functions

Imports

It imports app/kb/procedures/marks.mjs, app/lib/yaml-plain.mjs. Imported by app/components/procedure-steps.tsx, app/kb/editor.server.ts, app/kb/procedures/compile.mjs, app/kb/procedures/primer-dependents.server.ts, app/kb/procedures/render.mjs, app/kb/procedures/save.server.ts, app/kb/procedures/validate.mjs, app/kb/registry/host.server.ts, app/lib/carrel/procedure-handler.server.ts, app/lib/health/checks.server.ts, app/lib/operator/api.server.ts, app/lib/operator/sync-tools.server.ts.

Source

1// Reads a procedure file (docs/PROCEDURES.md) into its structure: the front matter as written, then the
2// body as an intro and sections, each section a run of prose blocks and step lists. Nothing here renders
3// or judges; validate.mjs judges and render.mjs renders, so the check and the save tool read one parse.
4
5import { slug as githubSlug } from "github-slugger";
6import matter from "gray-matter";
7
8import { yamlPlain } from "../../lib/yaml-plain.mjs";
9
10import { readConditions, segmentText, tokenize } from "./marks.mjs";
11
12export const PROCEDURES_DIR = "content/procedures";
13
14/** The repository path of a procedure's file. */
15export function procedurePath(/** @type {string} */ slug) {
16  return `${PROCEDURES_DIR}/${slug}.md`;
17}
18
19/**
20 * The heading id rehype-slug gives the same words, by the same library, so an anchor the old pages
21 * published keeps working. A repeated heading's -1, -2 suffix is render.mjs's, which slugs the page in
22 * one pass as rehype-slug does.
23 *
24 * @param {string} text
25 */
26export function headingId(text) {
27  return githubSlug(text);
28}
29
30const FLAG_KEYS = /** @type {const} */ ({
31  CRITICAL: "critical",
32  "PAUSE POINT": "pause",
33  WHY: "why",
34  TROUBLESHOOTING: "troubleshooting",
35  EXPECT: "expect",
36  SPIN: "spin",
37  CALC: "calc",
38  CHALLENGE: "challenge",
39  SOLUTION: "solution",
40});
41
42/** The platforms a fenced block may name (docs/PROCEDURES.md, S4): a how-to's commands where they differ by system. */
43export const PLATFORMS = /** @type {const} */ ({ linux: "Linux", macos: "macOS", windows: "Windows" });
44
45/**
46 * @typedef {{ critical: string[], pause: string[], why: string[], troubleshooting: string[], expect: string[], spin: string[], calc: string[], challenge: string[], solution: string[] }} Flags
47 * @typedef {{ lang: string, code: string, output: string | null, platform: string | null }} Command
48 * @typedef {{
49 *   number: number,
50 *   line: number,
51 *   source: string,
52 *   segments: import("./marks.mjs").Segment[],
53 *   flags: Flags,
54 *   commands: Command[],
55 *   photos: Array<{ alt: string, src: string }>,
56 * }} Step
57 * @typedef {{ type: "prose", markdown: string } | { type: "steps", steps: Step[] }} Block
58 * @typedef {{ title: string, id: string, custom: boolean, line: number, blocks: Block[] }} Section
59 */
60
61/**
62 * The lines under a step, with its indentation removed, split into the step's own words, its flags,
63 * its commands and its photos.
64 *
65 * @param {string} first the text after "N. "
66 * @param {string[]} rest the indented lines under it, dedented
67 * @param {number} line
68 * @param {number} number
69 * @returns {{ step: Step, problems: string[] }}
70 */
71function readStep(first, rest, line, number) {
72  /** @type {string[]} */
73  const problems = [];
74  /** @type {string[]} */
75  const words = [first];
76  /** @type {Flags} */
77  const flags = { critical: [], pause: [], why: [], troubleshooting: [], expect: [], spin: [], calc: [], challenge: [], solution: [] };
78  /** @type {Command[]} */
79  const commands = [];
80  /** @type {Array<{ alt: string, src: string }>} */
81  const photos = [];
82  /** @type {keyof Flags | null} */
83  let openFlag = null;
84
85  for (let i = 0; i < rest.length; i += 1) {
86    const text = rest[i] ?? "";
87    // "```bash platform=linux": a language, then attributes; platform is the only one the format has.
88    const fence = /^```(\S*)((?:\s+[a-z]+=\S+)*)\s*$/.exec(text);
89    if (fence) {
90      openFlag = null;
91      const lang = fence[1] ?? "";
92      const attrs = Object.fromEntries([...(fence[2] ?? "").matchAll(/([a-z]+)=(\S+)/g)].map((m) => [m[1], m[2]]));
93      for (const key of Object.keys(attrs)) if (key !== "platform") problems.push(`line ${line}: step ${number} has a code block with "${key}=", and platform= is the only attribute a block takes`);
94      /** @type {string[]} */
95      const code = [];
96      i += 1;
97      while (i < rest.length && !/^```\s*$/.test(rest[i] ?? "")) {
98        code.push(rest[i] ?? "");
99        i += 1;
100      }
101      if (i >= rest.length) problems.push(`line ${line}: step ${number} has a code block that is never closed`);
102      if (lang === "output") {
103        const target = commands[commands.length - 1];
104        if (!target) problems.push(`line ${line}: step ${number} has an output block with no command before it`);
105        else target.output = code.join("\n");
106      } else {
107        commands.push({ lang, code: code.join("\n"), output: null, platform: attrs.platform ?? null });
108      }
109      continue;
110    }
111    const flag = /^>\s*(CRITICAL|PAUSE POINT|WHY|TROUBLESHOOTING|EXPECT|SPIN|CALC|CHALLENGE|SOLUTION)\s*:\s*(.*)$/.exec(text);
112    if (flag) {
113      openFlag = FLAG_KEYS[/** @type {keyof typeof FLAG_KEYS} */ (flag[1])];
114      const body = (flag[2] ?? "").trim();
115      // A solution answers the challenge before it (the Carpentries challenge with its nested solution), one each.
116      if (openFlag === "solution" && flags.solution.length >= flags.challenge.length) {
117        problems.push(`line ${line}: step ${number} has a SOLUTION with no CHALLENGE before it to answer`);
118      }
119      if (openFlag === "troubleshooting" || openFlag === "calc") {
120        flags[openFlag].push(...body.split(/[,\s]+/).filter(Boolean));
121      } else {
122        flags[openFlag].push(body);
123      }
124      continue;
125    }
126    const more = /^>\s?(.*)$/.exec(text);
127    if (more) {
128      if (!openFlag || openFlag === "troubleshooting" || openFlag === "calc") {
129        problems.push(`line ${line}: step ${number} has a quoted line that follows no flag: "${text.trim()}"`);
130      } else {
131        const list = flags[openFlag];
132        list[list.length - 1] = `${list[list.length - 1] ?? ""}\n${more[1] ?? ""}`.replace(/^\n/, "");
133      }
134      continue;
135    }
136    openFlag = null;
137    const photo = /^!\[([^\]]*)\]\(([^)\s]+)\)\s*$/.exec(text.trim());
138    if (photo) {
139      photos.push({ alt: photo[1] ?? "", src: photo[2] ?? "" });
140      continue;
141    }
142    if (text.trim()) words.push(text.trim());
143  }
144
145  const source = words.join(" ");
146  return {
147    step: {
148      number,
149      line,
150      source,
151      segments: tokenize(source),
152      flags: {
153        ...flags,
154        critical: flags.critical.map((s) => s.trim()),
155        pause: flags.pause.map((s) => s.trim()),
156        why: flags.why.map((s) => s.trim()),
157        expect: flags.expect.map((s) => s.trim()),
158        spin: flags.spin.map((s) => s.trim()),
159        challenge: flags.challenge.map((s) => s.trim()),
160        solution: flags.solution.map((s) => s.trim()),
161      },
162      commands,
163      photos,
164    },
165    problems,
166  };
167}
168
169/**
170 * A section's lines as blocks: numbered items become steps; everything else at the margin is prose.
171 *
172 * @param {string[]} lines
173 * @param {number} firstLine the file line of lines[0]
174 * @returns {{ blocks: Block[], problems: string[] }}
175 */
176function readBlocks(lines, firstLine) {
177  /** @type {Block[]} */
178  const blocks = [];
179  /** @type {string[]} */
180  const problems = [];
181  /** @type {string[]} */
182  let prose = [];
183  /** @type {Step[] | null} */
184  let steps = null;
185  let inFence = false;
186
187  const flushProse = () => {
188    const markdown = prose.join("\n").trim();
189    if (markdown) blocks.push({ type: "prose", markdown });
190    prose = [];
191  };
192
193  for (let i = 0; i < lines.length; i += 1) {
194    const text = lines[i] ?? "";
195    if (text.startsWith("```")) inFence = !inFence;
196    const item = inFence ? null : /^(\d+)\.\s+(.*)$/.exec(text);
197    if (!item) {
198      if (steps && (text.trim() === "" || /^\s{2,}/.test(text))) {
199        // A blank line inside a list is still the list, unless the margin comes next.
200        const next = lines.slice(i + 1).find((l) => l.trim() !== "");
201        if (text.trim() === "" && (next === undefined || !/^(\s{2,}|\d+\.\s)/.test(next))) steps = null;
202        continue;
203      }
204      steps = null;
205      prose.push(text);
206      continue;
207    }
208    flushProse();
209    const start = i;
210    /** @type {string[]} */
211    const rest = [];
212    while (i + 1 < lines.length) {
213      const next = lines[i + 1] ?? "";
214      if (/^\s{2,}\S/.test(next)) rest.push(next.replace(/^\s{2,4}/, ""));
215      else if (next.trim() === "" && /^\s{2,}\S/.test(lines[i + 2] ?? "")) rest.push("");
216      else break;
217      i += 1;
218    }
219    const read = readStep(item[2] ?? "", rest, firstLine + start, Number(item[1]));
220    problems.push(...read.problems);
221    if (!steps) {
222      steps = [];
223      blocks.push({ type: "steps", steps });
224    }
225    steps.push(read.step);
226  }
227  flushProse();
228  return { blocks, problems };
229}
230
231/**
232 * @param {{ file: string, raw: string }} input
233 */
234export function parseProcedure({ file, raw }) {
235  /** @type {string[]} */
236  const problems = [];
237  let parsed;
238  try {
239    parsed = matter(raw);
240  } catch (error) {
241    return {
242      file,
243      data: /** @type {Record<string, any>} */ ({}),
244      intro: "",
245      sections: /** @type {Section[]} */ ([]),
246      problems: [`the front matter is not valid YAML: ${error instanceof Error ? error.message : String(error)}`],
247    };
248  }
249  const data = /** @type {Record<string, any>} */ (yamlPlain(parsed.data));
250  const headerLines = raw.slice(0, raw.length - parsed.content.length).split("\n").length - 1;
251  const lines = parsed.content.split(/\r?\n/);
252
253  /** @type {Section[]} */
254  const sections = [];
255  /** @type {string[]} */
256  let intro = [];
257  let current = /** @type {{ title: string, id: string, custom: boolean, line: number, lines: string[] } | null} */ (null);
258  const close = () => {
259    if (!current) return;
260    const { blocks, problems: found } = readBlocks(current.lines, current.line + 1);
261    problems.push(...found);
262    sections.push({ title: current.title, id: current.id, custom: current.custom, line: current.line, blocks });
263  };
264
265  let inFence = false;
266  lines.forEach((text, index) => {
267    if (text.startsWith("```")) inFence = !inFence;
268    const heading = inFence ? null : /^## (.+?)(?:\s+\{#([a-z0-9-]+)\})?\s*$/.exec(text);
269    if (heading) {
270      close();
271      const title = (heading[1] ?? "").trim();
272      current = { title, id: heading[2] ?? headingId(title), custom: Boolean(heading[2]), line: headerLines + index + 1, lines: [] };
273      return;
274    }
275    if (current) current.lines.push(text);
276    else intro.push(text);
277  });
278  close();
279
280  // A step list before the first heading would have no section to belong to.
281  if (intro.some((line) => /^\d+\.\s/.test(line))) {
282    problems.push("the method starts before the first ## heading; put every step under a section");
283  }
284
285  return { file, data, intro: intro.join("\n").trim(), sections, problems };
286}
287
288/** @typedef {ReturnType<typeof parseProcedure>} ParsedProcedure */
289
290/**
291 * Every step in the procedure, in order.
292 *
293 * @param {ParsedProcedure} parsed
294 */
295export function allSteps(parsed) {
296  return parsed.sections.flatMap((section) =>
297    section.blocks.flatMap((block) => (block.type === "steps" ? block.steps : [])),
298  );
299}
300
301/**
302 * The calculators a step's CALC flag names, each with the values the step itself states for its fields:
303 * `> CALC: webbed-plate volume=10, titer` is webbed-plate with volume 10, then titer. A `name=value` belongs
304 * to the calculator before it; one with none before it is read as a calculator with no id, which validate.mjs refuses.
305 *
306 * @param {string[]} tokens the flag's words, as readStep split them
307 * @returns {Array<{ id: string, values: Record<string, string> }>}
308 */
309export function readCalcs(tokens) {
310  /** @type {Array<{ id: string, values: Record<string, string> }>} */
311  const calcs = [];
312  for (const token of tokens) {
313    const eq = token.indexOf("=");
314    if (eq === -1) {
315      calcs.push({ id: token, values: {} });
316      continue;
317    }
318    if (calcs.length === 0) calcs.push({ id: "", values: {} });
319    /** @type {{ id: string, values: Record<string, string> }} */ (calcs[calcs.length - 1]).values[token.slice(0, eq)] = token.slice(eq + 1);
320  }
321  return calcs;
322}
323
324/**
325 * A step's words with its marks read out, and the temperatures and spins those words state.
326 *
327 * @param {Step} step
328 */
329export function stepConditions(step) {
330  return readConditions(step.segments.map((s) => segmentText(s)).join(""));
331}