pi-ext bugabingas pi extensions
owned by admin
README Source History Refs Compare Notes Search perf(the-system): parse frontmatter without document bodies 70d9fe849413613abc6d6f0726b87b5000642ebc
Oliver Jan Krylow <oliver@bugabinga.net > · 22 Sep 2026
SSH signature · not checked
Browse files Compare with HEAD
Indexing ran the full remark pipeline over every bundle body only to read
the YAML block; parseFrontmatter scans the fence directly with
micromark-frontmatter semantics (loadProjectIndex 48.5ms -> 2.9ms, -94%).
The renderer now harvests search text from the same parse as the HTML
pipeline instead of re-parsing each document (cold board render -7%).
Share references/dependencyReferences between documents and renderer and
drop the duplicated refsFor/dependencyIds logic. Verified by
differential fuzz and full-site artifact hashes.
Changed files (5) extensions/the-system/CHANGELOG.md Modified 6 + - Cut `/system` index and render cost: frontmatter extraction no longer full-parses document bodies (index load -94%), and the renderer harvests search text from the same parse as the HTML pipeline instead of re-parsing every document (cold board render -7%).
6 8 - Resolve the active System root as the nearest `.system` directory at or above cwd; nested systems override parents, `/system init` initializes at cwd, and split Git/JJ roots no longer conflict.
8 10 - Use rich Markdown bundles with YAML graph metadata, config-only machine state,
extensions/the-system/__tests__/markdown.test.ts Modified 1 1 import assert
from "node:assert/strict" ;
2 2 import { describe, it }
from "vitest" ;
3 - import { MarkdownDocumentError, parseRichMarkdown }
from "../markdown.ts" ;
5 + MarkdownDocumentError,
7 + }
from "../markdown.ts" ;
18 - > Canonical source remains editable.
34 - <span data-example="raw">raw HTML</span> 21 + Body after frontmatter stays unparsed.
37 - describe (
"rich Markdown parsing" , ()
=> {
38 - it (
"parses the complete profile and preserves frontmatter order" , ()
=> {
39 - const result
= parseRichMarkdown (rich);
24 + const entry = (frontmatter: FrontmatterMap, key: string) => 25 + frontmatter.get(key); 27 + describe("frontmatter parsing", () => { 28 + it("preserves frontmatter order and shapes", () => { 29 + const frontmatter = parseFrontmatter(rich); 41 - [
... result.frontmatter.
keys ()],
31 + [...frontmatter.keys()], 42 32 ["id", "type", "title", "owners", "review"], 34 + assert.deepEqual(entry(frontmatter, "owners"), ["human", "the-system"]); 45 - result.frontmatter.
get (
"review" ),
36 + entry(frontmatter, "review"), 47 38 ["approved", "2026-07-15"], 48 39 ["flags", ["visual", "editable"]], 51 - const tree
= JSON .
stringify (result.tree);
52 - for (
const nodeType
of [
55 - '"type":"inlineMath"' ,
44 + it("accepts fences with trailing spaces or tabs", () => { 45 + assert.equal(entry(parseFrontmatter("--- \nid: A\n---\t\n"), "id"), "A"); 48 + it("accepts CRLF and CR line endings", () => { 49 + assert.equal(entry(parseFrontmatter("---\r\nid: A\r\n---\r\n"), "id"), "A"); 50 + assert.equal(entry(parseFrontmatter("---\rid: A\r---\r"), "id"), "A"); 53 + it("drops a byte order mark before the opening fence", () => { 54 + assert.equal(entry(parseFrontmatter("\ufeff---\nid: A\n---\n"), "id"), "A"); 57 + it("keeps lone carriage returns inside the value literal", () => { 58 + const frontmatter = parseFrontmatter("---\r\n \rk1: v1\r\n---\n"); 59 + assert.deepEqual([...frontmatter.keys()], ["\rk1"]); 62 + it("strips exactly one trailing line ending from the value", () => { 63 + const frontmatter = parseFrontmatter("---\nkey: |\n l1\n l2\n\n---\n"); 64 + assert.equal(entry(frontmatter, "key"), "l1\nl2\n"); 67 + it("rejects thematic breaks and indented or unclosed fences", () => { 68 + for (const source of [ 69 + "----\nid: A\n----\n", 70 + " ---\nid: A\n---\n", 71 + "\n---\nid: A\n---\n", 59 - assert.
ok (tree.
includes (nodeType), nodeType);
76 + () => parseFrontmatter(source), 78 + error instanceof MarkdownDocumentError && 79 + error.code === "frontmatter.missing", 84 + it("treats a dotted line as content, not a closing fence", () => { 85 + assert.equal(entry(parseFrontmatter("---\nid: A\n...\n---\n"), "id"), "A"); 88 + it("keeps fence-like lines inside the value instead of closing", () => { 90 + () => parseFrontmatter("---\nid: A\n---x\n---\n"), 92 + error instanceof MarkdownDocumentError && 93 + error.code === "frontmatter.invalid" && 94 + error.message.includes("---x"), 63 - [
"missing" ,
"# No frontmatter" ,
"frontmatter.missing" ],
64 99 ["duplicate", "---\nid: A\nid: B\n---\n", "frontmatter.invalid"], 65 100 ["alias", "---\na: &a [1]\nb: *a\n---\n", "frontmatter.unsupported"], 101 + ["empty", "---\n---\n", "frontmatter.shape"], 66 102 ])("rejects %s frontmatter", (_name, source, code) => { 68 - ()
=> parseRichMarkdown (source),
104 + () => parseFrontmatter(source), 69 105 (error: unknown) => 70 106 error instanceof MarkdownDocumentError && error.code === code, extensions/the-system/documents.ts Modified 3 3 import { lstatSync, readdirSync, readFileSync }
from "node:fs" ;
4 4 import path
from "node:path" ;
7 7 type MarkdownDocumentError,
9 9 }
from "./markdown.ts" ;
10 10 import { SystemDocumentError }
from "./project-root.ts" ;
256 256 ): AuthoredDocument {
257 - const parsed
= parseRichMarkdown (source);
258 - const metadata
= projectMetadata (parsed.frontmatter, prefix);
257 + const frontmatter
= parseFrontmatter (source);
258 + const metadata
= projectMetadata (frontmatter, prefix);
260 260 metadata.
type === "phase" && 261 261 Buffer .
byteLength (source,
"utf8" )
> PHASE_BYTE_LIMIT
264 264 "document.phase-size" ,
265 265 `Phase exceeds the ${PHASE_BYTE_LIMIT}-byte limit` ,
267 - return { metadata, frontmatter: parsed.frontmatter };
267 + return { metadata, frontmatter };
270 270 export type CurrentPhaseDocument = Readonly <{
476 - const references = (
476 + export const references = (
477 477 metadata
: DocumentMetadata ,
478 478 )
: readonly (
readonly [
string ,
DocumentType ])[]
=> {
479 479 switch (metadata.
type ) {
496 - const dependencyReferences = (metadata
: DocumentMetadata )
: readonly string []
=> 496 + export const dependencyReferences = (
497 + metadata
: DocumentMetadata ,
498 + )
: readonly string []
=> 497 499 metadata.
type === "plan" ? (metadata.depends_on
?? [])
: [];
498 500 const diagnosticIdentity = (item
: Diagnostic )
=> 499 501 JSON .
stringify ([item.path, item.
code , item.message]);
extensions/the-system/markdown.ts Modified 1 - import { remarkDefinitionList }
from "remark-definition-list" ;
2 - import remarkFrontmatter
from "remark-frontmatter" ;
3 - import remarkGfm
from "remark-gfm" ;
4 - import remarkGithubAlert
from "remark-github-blockquote-alert" ;
5 - import remarkMath
from "remark-math" ;
6 - import remarkParse
from "remark-parse" ;
7 - import { unified }
from "unified" ;
23 16 | readonly FrontmatterValue[]
25 18 export type FrontmatterMap = ReadonlyMap <
string ,
FrontmatterValue >;
26 - type MarkdownRoot = Readonly <{
28 - children
: readonly Readonly <{ type
: string ; value
?: string }>[];
31 - export type RichMarkdownDocument = Readonly <{
33 - frontmatter
: FrontmatterMap ;
36 20 export class MarkdownDocumentError extends Error {
37 21 readonly code
: string ;
46 - export const createMarkdownParser = ()
=> 49 - .
use (remarkFrontmatter, [
"yaml" ])
51 - .
use (remarkDefinitionList)
53 - .
use (remarkGithubAlert);
30 + const lineEndingLength = (source
: string , at
: number )
: number => {
31 + if (source[at]
=== "\r" && source[at
+ 1 ]
=== "\n" )
return 2 ;
32 + return source[at]
=== "\r" || source[at]
=== "\n" ? 1 : 0 ;
55 - const markdownParser
= createMarkdownParser ();
39 + const frontmatterBlock = (source
: string )
: string | undefined => {
40 + let at
= source.
charCodeAt (
0 )
=== 0xfeff ? 1 : 0 ;
41 + if (source.
slice (at, at
+ 3 )
!== "---" )
return undefined ;
43 + while (source[at]
=== " " || source[at]
=== "\t" ) at
+= 1 ;
44 + if (
lineEndingLength (source, at)
=== 0 )
return undefined ;
45 + at
+= lineEndingLength (source, at);
46 + const valueStart
= at;
48 + if (source.
startsWith (
"---" , at)) {
49 + let cursor
= at
+ 3 ;
50 + while (source[cursor]
=== " " || source[cursor]
=== "\t" ) cursor
+= 1 ;
51 + if (cursor
=== source.
length || lineEndingLength (source, cursor)
> 0 ) {
52 + const value
= source.
slice (valueStart, at);
53 + return value.
endsWith (
"\r\n" )
54 + ? value.
slice (
0 ,
- 2 )
55 + : /\r | \n/ .
test (value.
at (
- 1 )
?? "" )
56 + ? value.
slice (
0 ,
- 1 )
60 + const eol
= source.
slice (at).
search (
/[\r\n]/ );
61 + if (eol
< 0 )
return undefined ;
62 + at
+= eol
+ lineEndingLength (source, at
+ eol);
57 66 const unsupported = (message
: string )
: never => {
58 67 throw new MarkdownDocumentError (
"frontmatter.unsupported" , message);
96 - export function parseRichMarkdown (source
: string )
: RichMarkdownDocument {
97 - const tree
= markdownParser.
runSync (
98 - markdownParser.
parse (source),
100 - const blocks
= tree.children.
filter ((node)
=> node.
type === "yaml" );
101 - if (blocks.
length === 0 )
105 + export function parseFrontmatter (source
: string )
: FrontmatterMap {
106 + const block
= frontmatterBlock (source);
107 + if (block
=== undefined )
102 108 throw new MarkdownDocumentError (
103 109 "frontmatter.missing" ,
104 110 "Document requires one leading YAML frontmatter block" ,
106 - if (blocks.
length !== 1 )
107 - throw new MarkdownDocumentError (
108 - "frontmatter.multiple" ,
109 - `Document requires one YAML frontmatter block; received ${blocks. length }` ,
111 - const document
= parseDocument (blocks[
0 ]?.
value ?? "" , {
112 + const document
= parseDocument (block, {
114 115 uniqueKeys:
true ,
123 124 "frontmatter.shape" ,
124 125 "Frontmatter must be a YAML map" ,
126 - return { tree, frontmatter:
convertMap (
document .contents.items) };
127 + return convertMap (
document .contents.items);
extensions/the-system/system-renderer.ts Modified 44 44 type DocumentMetadata ,
46 + dependencyReferences,
47 49 } from
"./documents.ts" ;
48 50 import type { FrontmatterMap, FrontmatterValue }
from "./markdown.ts" ;
75 77 refs : readonly string[];
78 + deps : readonly string[];
79 - type MarkdownNode = Readonly <{
83 - children
?: readonly MarkdownNode [];
85 - type MarkdownRoot = MarkdownNode & Readonly <{ type
: "root" }>;
86 - type MarkdownText = MarkdownNode & Readonly <{ type
: "text" ; value
: string }>;
82 + type MarkdownText = Readonly <{ value
: string }>;
88 84 const GROUPS
= [
"law" ,
"phase" ,
"spec" ,
"plan" ,
"issue" ,
"research" ]
as const ;
89 85 const DOCUMENT_ORDER
: readonly DocumentType []
= [
110 106 .
use (rehypeStringify, { allowDangerousHtml:
true });
114 112 const PROCESSED_CACHE_LIMIT
= 512 ;
115 113 const processedCache
= new Map < 122 120 )
: Readonly <{ html
: string ; text
: string }>
=> {
123 121 const cached
= processedCache.
get (markdown);
124 122 if (cached)
return cached;
123 + const tree
= processor.
parse (markdown);
125 + visit (tree, [
"text" ,
"inlineMath" ,
"math" ]
as const , (node
: MarkdownText )
=> {
126 + text
+= `${node. value } ` ;
125 128 const processed
= {
126 - html:
rewriteMermaidBlocks (
String (processor.
processSync (markdown))),
127 - text:
textContent (markdown),
129 + html:
rewriteMermaidBlocks (
130 + String (processor.
stringify (processor.
runSync (tree))),
129 134 if (processedCache.
size >= PROCESSED_CACHE_LIMIT) processedCache.
clear ();
130 135 processedCache.
set (markdown, processed);
208 213 parseSystemConfig (
209 214 readFileSync (path.
join (
root ,
".system/config.json" ),
"utf8" ),
211 - path.
basename (
root )
?? 215 + ).
title ?? path.
basename (
root )
215 218 return path.
basename (
root )
|| "Project" ;
288 291 if (visited.
has (id))
return ;
289 292 if (visiting.
has (id))
return ;
290 293 visiting.
add (id);
291 - for (
const dependencyId
of [
... dependencyIds (
document .metadata)].
sort ()) {
294 + for (
const dependencyId
of [
295 + ... dependencyReferences (
document .metadata),
292 297 const dependency
= byId.
get (dependencyId);
293 298 if (dependency)
visit (dependency);
380 385 text : processed.
text ,
389 395 renderedSources : ReadonlySet
< string
> ,
390 396 live
?: LiveContext,
391 397 )
: RenderedDocument => {
392 - const outputPath
= document .path.
replace (
/index\.md $ / ,
"index.html" );
398 + const outputPath
= htmlPath (
document .path);
393 399 const article
= articleHtml ({
394 400 ... (live
? { live }
: {}),
395 401 id:
document .metadata.
id ,
415 421 text :
`${ frontmatterText ( document .frontmatter)} ${processed. text }` ,
416 422 refs :
refsFor (
document .metadata),
423 + deps :
dependencyReferences (
document .metadata),
417 424 ... (
document .metadata.
type === "phase" 418 425 ? { label:
document .metadata.
label }
466 473 const summaryHtml = (metadata
?: DocumentMetadata )
: string => {
467 474 if (
! metadata)
return "" ;
468 - const dependencies
= dependencyIds (metadata);
475 + const dependencies
= dependencyReferences (metadata);
469 476 const refs
= refsFor (metadata).
filter ((id)
=> ! dependencies.
includes (id));
470 477 return `<p class="summary">${ esc (metadata. type )}${metadata. type === "phase" ? ` · ${ esc (metadata. label )}` : ""}${"status" in metadata ? ` · ${ esc (metadata. status )}` : ""}${refs. length ? ` · refs ${refs. map (esc). join (", ")}` : ""}${dependencies. length ? ` · after ${dependencies. map (esc). join (", ")}` : ""}</p>` ;
485 492 return `<details class="doc-related"><summary>Referenced by</summary>${backlinks 488 - `<a href="${ pathToFileURL (path. join (outputRoot, document .path. replace (/index\.md $ /, "index.html"))). href }"><span>${ esc ( document .metadata. type )}</span>${ esc ( document .metadata. title )}<code>${ esc ( document .metadata. id )}</code></a>`, 495 + `<a href="${ documentHref (outputRoot, document .path)}"><span>${ esc ( document .metadata. type )}</span>${ esc ( document .metadata. title )}<code>${ esc ( document .metadata. id )}</code></a>`, 490 497 . join ("")}</details>` ;
546 553 const target
= documents.
find ((document)
=> document .metadata.
id === value);
547 554 if (
! target)
return esc (value);
548 - return `<a href="${ pathToFileURL (path. join (outputRoot, target.path. replace (/index\.md $ /, "index.html"))). href }">${ esc (value)}</a>` ;
555 + return `<a href="${ documentHref (outputRoot, target.path)}">${ esc (value)}</a>` ;
551 - const refsFor = (metadata
: DocumentMetadata )
: readonly string []
=> {
552 - switch (metadata.
type ) {
554 - return metadata.research
?? [];
558 - return [metadata.spec,
... dependencyIds (metadata)];
560 - return metadata.specs;
558 + const refsFor = (metadata
: DocumentMetadata )
: readonly string []
=> 559 + references (metadata).
map (([id])
=> id);
566 - const dependencyIds = (metadata
: DocumentMetadata )
: readonly string []
=> 567 - metadata.
type === "plan" ? (metadata.depends_on
?? [])
: [];
561 + const htmlPath = (sourcePath
: string )
=> 562 + sourcePath.
replace (
/index\.md $ / ,
"index.html" );
569 - const stripFrontmatter = (markdown
: string )
=> 570 - markdown.
replace (
/ ^ ---\n[\s\S] *? \n---\n ? / ,
"" );
564 + const documentHref = (outputRoot
: string , sourcePath
: string )
=> 565 + pathToFileURL (path.
join (outputRoot,
htmlPath (sourcePath))).
href ;
572 567 const frontmatterText = (frontmatter
: FrontmatterMap )
: string => 573 568 [
... frontmatter.
entries ()]
669 664 return undefined ;
672 - const textContent = (markdown
: string )
: string => {
673 - const tree
= unified ()
675 - .
parse (
stripFrontmatter (markdown))
as MarkdownRoot ;
677 - visit (tree,
"text" , (node
: MarkdownText )
=> {
678 - value
+= `${node. value } ` ;
680 - return value.
trim ();
683 667 const GROUP_LABELS
: Readonly <
Record <(
typeof GROUPS)[
number ],
string >>
= {
956 940 for (
const ref
of graphRefsForNode (node))
957 941 if (byId.
has (ref))
958 942 graph.
setEdge (ref, node.
id , {
959 - weight:
nodeDependencyIds (node).
includes (ref)
? 3 : 1 ,
943 + weight: node.deps.
includes (ref)
? 3 : 1 ,
961 945 dagreLayout (graph);
962 946 const positions
= new Map <
string ,
GraphPoint >();
987 971 height : NODE_HEIGHT,
990 - const nodeDependencyIds = (node
: RenderedDocument )
: readonly string []
=> 991 - node.
type === "plan" ? node.refs.
filter ((id)
=> id.
includes (
"-PLAN-" ))
: [];
993 - const graphRefsForNode = (node
: RenderedDocument )
: readonly string []
=> {
994 - const dependencies
= nodeDependencyIds (node);
995 - if (dependencies.
length )
return dependencies;
974 + const graphRefsForNode = (node
: RenderedDocument )
: readonly string []
=> 975 + node.deps.
length ? node.deps
: node.refs;
999 977 const shortLabel = (document
: RenderedDocument )
=> {
1000 978 if (
document .
type === "law" )
return document .
title ;