repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

perf(the-system): parse frontmatter without document bodies

70d9fe849413613abc6d6f0726b87b5000642ebc

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Indexing ran the full remark pipeline over every bundle body only to read
the YAML block; parseFrontmatter scans the fence directly with
micromark-frontmatter semantics (loadProjectIndex 48.5ms -> 2.9ms, -94%).
The renderer now harvests search text from the same parse as the HTML
pipeline instead of re-parsing each document (cold board render -7%).
Share references/dependencyReferences between documents and renderer and
drop the duplicated refsFor/dependencyIds logic. Verified by
differential fuzz and full-site artifact hashes.

Changed files (5)

extensions/the-system/CHANGELOG.md

Modified
2 2 # Changelog
4 4 ## Unreleased
6 +- Cut `/system` index and render cost: frontmatter extraction no longer full-parses document bodies (index load -94%), and the renderer harvests search text from the same parse as the HTML pipeline instead of re-parsing every document (cold board render -7%).
6 8 - Resolve the active System root as the nearest `.system` directory at or above cwd; nested systems override parents, `/system init` initializes at cwd, and split Git/JJ roots no longer conflict.
8 10 - Use rich Markdown bundles with YAML graph metadata, config-only machine state,

extensions/the-system/__tests__/markdown.test.ts

Modified
1 1 import assert from "node:assert/strict";
2 2 import { describe, it } from "vitest";
3 -import { MarkdownDocumentError, parseRichMarkdown } from "../markdown.ts";
3 +import {
4 + type FrontmatterMap,
5 + MarkdownDocumentError,
6 + parseFrontmatter,
7 +} from "../markdown.ts";
5 9 const rich = `---
6 10 id: PX-SPEC-001
15 19 # Rich documents
17 -> [!IMPORTANT]
18 -> Canonical source remains editable.
20 -Term
21 -: Definition
23 -| A | B |
24 -| - | - |
25 -| 1 | 2 |
27 -Inline $E = mc^2$.
29 -\`\`\`mermaid
30 -graph LR
31 - A --> B
32 -\`\`\`
34 -<span data-example="raw">raw HTML</span>
21 +Body after frontmatter stays unparsed.
37 -describe("rich Markdown parsing", () => {
38 - it("parses the complete profile and preserves frontmatter order", () => {
39 - const result = parseRichMarkdown(rich);
24 +const entry = (frontmatter: FrontmatterMap, key: string) =>
25 + frontmatter.get(key);
27 +describe("frontmatter parsing", () => {
28 + it("preserves frontmatter order and shapes", () => {
29 + const frontmatter = parseFrontmatter(rich);
40 30 assert.deepEqual(
41 - [...result.frontmatter.keys()],
31 + [...frontmatter.keys()],
42 32 ["id", "type", "title", "owners", "review"],
34 + assert.deepEqual(entry(frontmatter, "owners"), ["human", "the-system"]);
44 35 assert.deepEqual(
45 - result.frontmatter.get("review"),
36 + entry(frontmatter, "review"),
46 37 new Map([
47 38 ["approved", "2026-07-15"],
48 39 ["flags", ["visual", "editable"]],
51 - const tree = JSON.stringify(result.tree);
52 - for (const nodeType of [
53 - '"type":"table"',
54 - '"type":"defList"',
55 - '"type":"inlineMath"',
56 - '"lang":"mermaid"',
57 - '"type":"html"',
42 + });
44 + it("accepts fences with trailing spaces or tabs", () => {
45 + assert.equal(entry(parseFrontmatter("--- \nid: A\n---\t\n"), "id"), "A");
46 + });
48 + it("accepts CRLF and CR line endings", () => {
49 + assert.equal(entry(parseFrontmatter("---\r\nid: A\r\n---\r\n"), "id"), "A");
50 + assert.equal(entry(parseFrontmatter("---\rid: A\r---\r"), "id"), "A");
51 + });
53 + it("drops a byte order mark before the opening fence", () => {
54 + assert.equal(entry(parseFrontmatter("\ufeff---\nid: A\n---\n"), "id"), "A");
55 + });
57 + it("keeps lone carriage returns inside the value literal", () => {
58 + const frontmatter = parseFrontmatter("---\r\n \rk1: v1\r\n---\n");
59 + assert.deepEqual([...frontmatter.keys()], ["\rk1"]);
60 + });
62 + it("strips exactly one trailing line ending from the value", () => {
63 + const frontmatter = parseFrontmatter("---\nkey: |\n l1\n l2\n\n---\n");
64 + assert.equal(entry(frontmatter, "key"), "l1\nl2\n");
65 + });
67 + it("rejects thematic breaks and indented or unclosed fences", () => {
68 + for (const source of [
69 + "----\nid: A\n----\n",
70 + " ---\nid: A\n---\n",
71 + "\n---\nid: A\n---\n",
72 + "---\nid: A",
73 + "---id: A\n---\n",
59 - assert.ok(tree.includes(nodeType), nodeType);
75 + assert.throws(
76 + () => parseFrontmatter(source),
77 + (error: unknown) =>
78 + error instanceof MarkdownDocumentError &&
79 + error.code === "frontmatter.missing",
80 + source,
81 + );
82 + });
84 + it("treats a dotted line as content, not a closing fence", () => {
85 + assert.equal(entry(parseFrontmatter("---\nid: A\n...\n---\n"), "id"), "A");
86 + });
88 + it("keeps fence-like lines inside the value instead of closing", () => {
89 + assert.throws(
90 + () => parseFrontmatter("---\nid: A\n---x\n---\n"),
91 + (error: unknown) =>
92 + error instanceof MarkdownDocumentError &&
93 + error.code === "frontmatter.invalid" &&
94 + error.message.includes("---x"),
95 + );
62 98 it.each([
63 - ["missing", "# No frontmatter", "frontmatter.missing"],
64 99 ["duplicate", "---\nid: A\nid: B\n---\n", "frontmatter.invalid"],
65 100 ["alias", "---\na: &a [1]\nb: *a\n---\n", "frontmatter.unsupported"],
101 + ["empty", "---\n---\n", "frontmatter.shape"],
66 102 ])("rejects %s frontmatter", (_name, source, code) => {
67 103 assert.throws(
68 - () => parseRichMarkdown(source),
104 + () => parseFrontmatter(source),
69 105 (error: unknown) =>
70 106 error instanceof MarkdownDocumentError && error.code === code,

extensions/the-system/documents.ts

Modified
3 3 import { lstatSync, readdirSync, readFileSync } from "node:fs";
4 4 import path from "node:path";
5 5 import {
6 6 type FrontmatterMap,
7 7 type MarkdownDocumentError,
8 - parseRichMarkdown,
8 + parseFrontmatter,
9 9 } from "./markdown.ts";
10 10 import { SystemDocumentError } from "./project-root.ts";
11 11 import {
254 254 source: string,
255 255 prefix: string,
256 256 ): AuthoredDocument {
257 - const parsed = parseRichMarkdown(source);
258 - const metadata = projectMetadata(parsed.frontmatter, prefix);
257 + const frontmatter = parseFrontmatter(source);
258 + const metadata = projectMetadata(frontmatter, prefix);
260 260 metadata.type === "phase" &&
261 261 Buffer.byteLength(source, "utf8") > PHASE_BYTE_LIMIT
264 264 "document.phase-size",
265 265 `Phase exceeds the ${PHASE_BYTE_LIMIT}-byte limit`,
267 - return { metadata, frontmatter: parsed.frontmatter };
267 + return { metadata, frontmatter };
270 270 export type CurrentPhaseDocument = Readonly<{
476 -const references = (
476 +export const references = (
477 477 metadata: DocumentMetadata,
478 478 ): readonly (readonly [string, DocumentType])[] => {
479 479 switch (metadata.type) {
496 -const dependencyReferences = (metadata: DocumentMetadata): readonly string[] =>
496 +export const dependencyReferences = (
497 + metadata: DocumentMetadata,
498 +): readonly string[] =>
497 499 metadata.type === "plan" ? (metadata.depends_on ?? []) : [];
498 500 const diagnosticIdentity = (item: Diagnostic) =>
499 501 JSON.stringify([item.path, item.code, item.message]);

extensions/the-system/markdown.ts

Modified
1 -import { remarkDefinitionList } from "remark-definition-list";
2 -import remarkFrontmatter from "remark-frontmatter";
3 -import remarkGfm from "remark-gfm";
4 -import remarkGithubAlert from "remark-github-blockquote-alert";
5 -import remarkMath from "remark-math";
6 -import remarkParse from "remark-parse";
7 -import { unified } from "unified";
8 1 import {
9 2 Alias,
10 3 isMap,
23 16 | readonly FrontmatterValue[]
24 17 | FrontmatterMap;
25 18 export type FrontmatterMap = ReadonlyMap<string, FrontmatterValue>;
26 -type MarkdownRoot = Readonly<{
27 - type: "root";
28 - children: readonly Readonly<{ type: string; value?: string }>[];
29 -}>;
31 -export type RichMarkdownDocument = Readonly<{
32 - tree: MarkdownRoot;
33 - frontmatter: FrontmatterMap;
34 -}>;
36 20 export class MarkdownDocumentError extends Error {
37 21 readonly code: string;
46 -export const createMarkdownParser = () =>
47 - unified()
48 - .use(remarkParse)
49 - .use(remarkFrontmatter, ["yaml"])
50 - .use(remarkGfm)
51 - .use(remarkDefinitionList)
52 - .use(remarkMath)
53 - .use(remarkGithubAlert);
30 +const lineEndingLength = (source: string, at: number): number => {
31 + if (source[at] === "\r" && source[at + 1] === "\n") return 2;
32 + return source[at] === "\r" || source[at] === "\n" ? 1 : 0;
33 +};
55 -const markdownParser = createMarkdownParser();
35 +// Extracts the leading YAML frontmatter block exactly like remark-frontmatter
36 +// with ["yaml"] would: a `---` fence on line 1 (optional trailing spaces or
37 +// tabs) closed by a `---` line, with one trailing line ending removed from the
38 +// value. Lone carriage returns stay literal, matching the mdast value.
39 +const frontmatterBlock = (source: string): string | undefined => {
40 + let at = source.charCodeAt(0) === 0xfeff ? 1 : 0;
41 + if (source.slice(at, at + 3) !== "---") return undefined;
42 + at += 3;
43 + while (source[at] === " " || source[at] === "\t") at += 1;
44 + if (lineEndingLength(source, at) === 0) return undefined;
45 + at += lineEndingLength(source, at);
46 + const valueStart = at;
47 + for (;;) {
48 + if (source.startsWith("---", at)) {
49 + let cursor = at + 3;
50 + while (source[cursor] === " " || source[cursor] === "\t") cursor += 1;
51 + if (cursor === source.length || lineEndingLength(source, cursor) > 0) {
52 + const value = source.slice(valueStart, at);
53 + return value.endsWith("\r\n")
54 + ? value.slice(0, -2)
55 + : /\r|\n/.test(value.at(-1) ?? "")
56 + ? value.slice(0, -1)
57 + : value;
58 + }
59 + }
60 + const eol = source.slice(at).search(/[\r\n]/);
61 + if (eol < 0) return undefined;
62 + at += eol + lineEndingLength(source, at + eol);
63 + }
64 +};
57 66 const unsupported = (message: string): never => {
58 67 throw new MarkdownDocumentError("frontmatter.unsupported", message);
93 102 return result;
96 -export function parseRichMarkdown(source: string): RichMarkdownDocument {
97 - const tree = markdownParser.runSync(
98 - markdownParser.parse(source),
99 - ) as MarkdownRoot;
100 - const blocks = tree.children.filter((node) => node.type === "yaml");
101 - if (blocks.length === 0)
105 +export function parseFrontmatter(source: string): FrontmatterMap {
106 + const block = frontmatterBlock(source);
107 + if (block === undefined)
102 108 throw new MarkdownDocumentError(
103 109 "frontmatter.missing",
104 110 "Document requires one leading YAML frontmatter block",
106 - if (blocks.length !== 1)
107 - throw new MarkdownDocumentError(
108 - "frontmatter.multiple",
109 - `Document requires one YAML frontmatter block; received ${blocks.length}`,
110 - );
111 - const document = parseDocument(blocks[0]?.value ?? "", {
112 + const document = parseDocument(block, {
112 113 schema: "core",
113 114 strict: true,
114 115 uniqueKeys: true,
123 124 "frontmatter.shape",
124 125 "Frontmatter must be a YAML map",
126 - return { tree, frontmatter: convertMap(document.contents.items) };
127 + return convertMap(document.contents.items);

extensions/the-system/system-renderer.ts

Modified
43 43 type Diagnostic,
44 44 type DocumentMetadata,
45 45 type DocumentType,
46 + dependencyReferences,
46 47 parseSystemConfig,
48 + references,
47 49 } from "./documents.ts";
48 50 import type { FrontmatterMap, FrontmatterValue } from "./markdown.ts";
73 75 article: string;
74 76 text: string;
75 77 refs: readonly string[];
78 + deps: readonly string[];
76 79 label?: string;
79 -type MarkdownNode = Readonly<{
80 - type: string;
81 - value?: string;
82 - depth?: number;
83 - children?: readonly MarkdownNode[];
84 -}>;
85 -type MarkdownRoot = MarkdownNode & Readonly<{ type: "root" }>;
86 -type MarkdownText = MarkdownNode & Readonly<{ type: "text"; value: string }>;
82 +type MarkdownText = Readonly<{ value: string }>;
88 84 const GROUPS = ["law", "phase", "spec", "plan", "issue", "research"] as const;
89 85 const DOCUMENT_ORDER: readonly DocumentType[] = [
110 106 .use(rehypeStringify, { allowDangerousHtml: true });
112 108 // Board re-renders process unchanged documents again on every mutation; the
113 -// unified pipeline output only depends on the markdown source, so memoize it.
109 +// pipeline output only depends on the markdown source, so memoize it. The text
110 +// comes from the same parse the HTML pipeline consumes, because a separate
111 +// text-only parse doubles the cold-path parse cost.
114 112 const PROCESSED_CACHE_LIMIT = 512;
115 113 const processedCache = new Map<
116 114 string,
122 120 ): Readonly<{ html: string; text: string }> => {
123 121 const cached = processedCache.get(markdown);
124 122 if (cached) return cached;
123 + const tree = processor.parse(markdown);
124 + let text = "";
125 + visit(tree, ["text", "inlineMath", "math"] as const, (node: MarkdownText) => {
126 + text += `${node.value} `;
127 + });
125 128 const processed = {
126 - html: rewriteMermaidBlocks(String(processor.processSync(markdown))),
127 - text: textContent(markdown),
129 + html: rewriteMermaidBlocks(
130 + String(processor.stringify(processor.runSync(tree))),
131 + ),
132 + text: text.trim(),
129 134 if (processedCache.size >= PROCESSED_CACHE_LIMIT) processedCache.clear();
130 135 processedCache.set(markdown, processed);
207 212 return (
208 213 parseSystemConfig(
209 214 readFileSync(path.join(root, ".system/config.json"), "utf8"),
210 - ).title ??
211 - path.basename(root) ??
212 - "Project"
215 + ).title ?? path.basename(root)
214 217 } catch {
215 218 return path.basename(root) || "Project";
288 291 if (visited.has(id)) return;
289 292 if (visiting.has(id)) return;
290 293 visiting.add(id);
291 - for (const dependencyId of [...dependencyIds(document.metadata)].sort()) {
294 + for (const dependencyId of [
295 + ...dependencyReferences(document.metadata),
296 + ].sort()) {
292 297 const dependency = byId.get(dependencyId);
293 298 if (dependency) visit(dependency);
379 384 article,
380 385 text: processed.text,
381 386 refs: [],
387 + deps: [],
389 395 renderedSources: ReadonlySet<string>,
390 396 live?: LiveContext,
391 397 ): RenderedDocument => {
392 - const outputPath = document.path.replace(/index\.md$/, "index.html");
398 + const outputPath = htmlPath(document.path);
393 399 const article = articleHtml({
394 400 ...(live ? { live } : {}),
395 401 id: document.metadata.id,
414 420 article,
415 421 text: `${frontmatterText(document.frontmatter)} ${processed.text}`,
416 422 refs: refsFor(document.metadata),
423 + deps: dependencyReferences(document.metadata),
417 424 ...(document.metadata.type === "phase"
418 425 ? { label: document.metadata.label }
419 426 : {}),
466 473 const summaryHtml = (metadata?: DocumentMetadata): string => {
467 474 if (!metadata) return "";
468 - const dependencies = dependencyIds(metadata);
475 + const dependencies = dependencyReferences(metadata);
469 476 const refs = refsFor(metadata).filter((id) => !dependencies.includes(id));
470 477 return `<p class="summary">${esc(metadata.type)}${metadata.type === "phase" ? ` · ${esc(metadata.label)}` : ""}${"status" in metadata ? ` · ${esc(metadata.status)}` : ""}${refs.length ? ` · refs ${refs.map(esc).join(", ")}` : ""}${dependencies.length ? ` · after ${dependencies.map(esc).join(", ")}` : ""}</p>`;
485 492 return `<details class="doc-related"><summary>Referenced by</summary>${backlinks
487 494 (document) =>
488 - `<a href="${pathToFileURL(path.join(outputRoot, document.path.replace(/index\.md$/, "index.html"))).href}"><span>${esc(document.metadata.type)}</span>${esc(document.metadata.title)}<code>${esc(document.metadata.id)}</code></a>`,
495 + `<a href="${documentHref(outputRoot, document.path)}"><span>${esc(document.metadata.type)}</span>${esc(document.metadata.title)}<code>${esc(document.metadata.id)}</code></a>`,
490 497 .join("")}</details>`;
545 552 ): string => {
546 553 const target = documents.find((document) => document.metadata.id === value);
547 554 if (!target) return esc(value);
548 - return `<a href="${pathToFileURL(path.join(outputRoot, target.path.replace(/index\.md$/, "index.html"))).href}">${esc(value)}</a>`;
555 + return `<a href="${documentHref(outputRoot, target.path)}">${esc(value)}</a>`;
551 -const refsFor = (metadata: DocumentMetadata): readonly string[] => {
552 - switch (metadata.type) {
553 - case "spec":
554 - return metadata.research ?? [];
555 - case "phase":
556 - return [];
557 - case "plan":
558 - return [metadata.spec, ...dependencyIds(metadata)];
559 - case "issue":
560 - return metadata.specs;
561 - default:
562 - return [];
558 +const refsFor = (metadata: DocumentMetadata): readonly string[] =>
559 + references(metadata).map(([id]) => id);
566 -const dependencyIds = (metadata: DocumentMetadata): readonly string[] =>
567 - metadata.type === "plan" ? (metadata.depends_on ?? []) : [];
561 +const htmlPath = (sourcePath: string) =>
562 + sourcePath.replace(/index\.md$/, "index.html");
569 -const stripFrontmatter = (markdown: string) =>
570 - markdown.replace(/^---\n[\s\S]*?\n---\n?/, "");
564 +const documentHref = (outputRoot: string, sourcePath: string) =>
565 + pathToFileURL(path.join(outputRoot, htmlPath(sourcePath))).href;
572 567 const frontmatterText = (frontmatter: FrontmatterMap): string =>
573 568 [...frontmatter.entries()]
669 664 return undefined;
672 -const textContent = (markdown: string): string => {
673 - const tree = unified()
674 - .use(remarkParse)
675 - .parse(stripFrontmatter(markdown)) as MarkdownRoot;
676 - let value = "";
677 - visit(tree, "text", (node: MarkdownText) => {
678 - value += `${node.value} `;
679 - });
680 - return value.trim();
683 667 const GROUP_LABELS: Readonly<Record<(typeof GROUPS)[number], string>> = {
684 668 law: "Law",
685 669 spec: "Specs",
956 940 for (const ref of graphRefsForNode(node))
957 941 if (byId.has(ref))
958 942 graph.setEdge(ref, node.id, {
959 - weight: nodeDependencyIds(node).includes(ref) ? 3 : 1,
943 + weight: node.deps.includes(ref) ? 3 : 1,
961 945 dagreLayout(graph);
962 946 const positions = new Map<string, GraphPoint>();
987 971 height: NODE_HEIGHT,
990 -const nodeDependencyIds = (node: RenderedDocument): readonly string[] =>
991 - node.type === "plan" ? node.refs.filter((id) => id.includes("-PLAN-")) : [];
993 -const graphRefsForNode = (node: RenderedDocument): readonly string[] => {
994 - const dependencies = nodeDependencyIds(node);
995 - if (dependencies.length) return dependencies;
996 - return node.refs;
974 +const graphRefsForNode = (node: RenderedDocument): readonly string[] =>
975 + node.deps.length ? node.deps : node.refs;
999 977 const shortLabel = (document: RenderedDocument) => {
1000 978 if (document.type === "law") return document.title;