import { lstatSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import { type FrontmatterMap, type MarkdownDocumentError, parseFrontmatter, } from "./markdown.ts"; import { SystemDocumentError } from "./project-root.ts"; import { type DocumentType, parseDocumentId, type SystemConfig, } from "./system-config.ts"; export { discoverProjectRoot, requireProjectRoot, resolveGovernedPath, SystemDocumentError, } from "./project-root.ts"; export * from "./system-config.ts"; export const PHASE_BYTE_LIMIT = 32 * 1024; export type ResearchMetadata = Readonly<{ id: string; type: "research"; title: string; }>; export type SpecMetadata = Readonly<{ id: string; type: "spec"; title: string; research?: readonly string[]; }>; export type PhaseMetadata = Readonly<{ id: string; type: "phase"; title: string; label: string; }>; export type PlanMetadata = Readonly<{ id: string; type: "plan"; title: string; spec: string; status: "draft" | "approved"; depends_on?: readonly string[]; }>; export type IssueMetadata = Readonly<{ id: string; type: "issue"; title: string; specs: readonly string[]; }>; export type DocumentMetadata = | ResearchMetadata | SpecMetadata | PhaseMetadata | PlanMetadata | IssueMetadata; export type Diagnostic = Readonly<{ code: string; path: string; message: string; }>; export type AuthoredDocument = Readonly<{ metadata: DocumentMetadata; frontmatter: FrontmatterMap; }>; export type BundleDocumentSource = Readonly<{ path: string; markdown: string }>; export type BundleDocumentRecord = Readonly<{ path: string; metadata: DocumentMetadata; frontmatter: FrontmatterMap; }>; export type BundleDocumentIndex = Readonly<{ documents: readonly BundleDocumentRecord[]; diagnostics: readonly Diagnostic[]; }>; export type BundleDiscovery = Readonly<{ sources: readonly BundleDocumentSource[]; diagnostics: readonly Diagnostic[]; }>; const SLUG = "[a-z0-9]+(?:-[a-z0-9]+)*"; const DIRECTORY_BY_TYPE: Readonly> = { spec: "specs", phase: "phases", plan: "plans", issue: "issues", research: "research", }; export function directoryForType(type: DocumentType): string { return DIRECTORY_BY_TYPE[type]; } export function validateDocumentPath( documentPath: string, metadata: DocumentMetadata, prefix: string, ): void { const parsed = parseDocumentId(metadata.id, prefix); const expectedDirectory = DIRECTORY_BY_TYPE[metadata.type]; const pattern = new RegExp(`^${metadata.id}-${SLUG}$`); const parts = documentPath.split("/"); if ( parsed.type !== metadata.type || parts.length !== 4 || parts[0] !== ".system" || parts[1] !== expectedDirectory || !pattern.test(parts[2] ?? "") || parts[3] !== "index.md" ) throw new SystemDocumentError( "document.path", `Bundle path mismatch for ${metadata.id}: ${documentPath}`, ); } const requiredStringField = (fields: FrontmatterMap, key: string): string => { const value = fields.get(key); if (typeof value !== "string" || value.trim() === "") throw new SystemDocumentError( "metadata.value", `${key} must be a non-empty string`, ); return value; }; const optionalUniqueStrings = ( fields: FrontmatterMap, key: string, ): readonly string[] | undefined => { const value = fields.get(key); if (value === undefined) return undefined; if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) throw new SystemDocumentError( "metadata.value", `${key} must be a string array`, ); if (new Set(value).size !== value.length) throw new SystemDocumentError( "metadata.duplicate", `${key} must contain unique values`, ); return value as readonly string[]; }; const requiredUniqueStrings = ( fields: FrontmatterMap, key: string, ): readonly string[] => { const value = optionalUniqueStrings(fields, key); if (!value) throw new SystemDocumentError( "metadata.value", `${key} must be a string array`, ); return value; }; const statusField = ( fields: FrontmatterMap, allowed: readonly T[], message: string, ): T => { const status = requiredStringField(fields, "status"); if (!allowed.includes(status as T)) throw new SystemDocumentError("metadata.status", message); return status as T; }; const projectMetadata = ( fields: FrontmatterMap, prefix: string, ): DocumentMetadata => { const id = requiredStringField(fields, "id"); const type = requiredStringField(fields, "type"); const title = requiredStringField(fields, "title"); const own = parseDocumentId(id, prefix); if (own.type !== type) throw new SystemDocumentError( "metadata.identity", `Metadata ID/type mismatch: ${id} / ${type}`, ); const reference = (key: string, expected: DocumentType) => { const value = requiredStringField(fields, key); const parsed = parseDocumentId(value, prefix); if (parsed.type !== expected) throw new SystemDocumentError( "metadata.reference-type", `${key} must reference ${expected}: ${value}`, ); return value; }; const dependencies = (expected: "plan") => { const values = optionalUniqueStrings(fields, "depends_on"); if (!values) return undefined; for (const value of values) { const parsed = parseDocumentId(value, prefix); if (parsed.type !== expected) throw new SystemDocumentError( "metadata.reference-type", `depends_on must reference ${expected}: ${value}`, ); } return values; }; switch (type) { case "research": return { id, type, title }; case "spec": { const research = optionalUniqueStrings(fields, "research"); if (research) for (const item of research) parseDocumentId(item, prefix); return research ? { id, type, title, research } : { id, type, title }; } case "phase": { const label = requiredStringField(fields, "label"); if (!/^[A-Za-z0-9._-]{1,12}$/.test(label)) throw new SystemDocumentError( "metadata.phase-label", "Phase label must contain 1-12 ASCII letters, digits, dots, underscores, or hyphens", ); return { id, type, title, label }; } case "plan": { const depends_on = dependencies("plan"); return { id, type, title, spec: reference("spec", "spec"), status: statusField( fields, ["draft", "approved"], "Plan status must be draft or approved", ), ...(depends_on ? { depends_on } : {}), }; } case "issue": return { id, type, title, specs: requiredUniqueStrings(fields, "specs"), }; default: throw new SystemDocumentError( "metadata.type", `Unsupported metadata type: ${type}`, ); } }; export function parseAuthoredDocument( source: string, prefix: string, ): AuthoredDocument { const frontmatter = parseFrontmatter(source); const metadata = projectMetadata(frontmatter, prefix); if ( metadata.type === "phase" && Buffer.byteLength(source, "utf8") > PHASE_BYTE_LIMIT ) throw new SystemDocumentError( "document.phase-size", `Phase exceeds the ${PHASE_BYTE_LIMIT}-byte limit`, ); return { metadata, frontmatter }; } export type CurrentPhaseDocument = Readonly<{ path: string; metadata: PhaseMetadata; markdown: string; }>; export function readCurrentPhase( root: string, config: SystemConfig, ): CurrentPhaseDocument | undefined { if (!config.currentPhase) return undefined; const directory = path.join(root, ".system/phases"); let matches: string[]; try { matches = readdirSync(directory).filter((name) => name.startsWith(`${config.currentPhase}-`), ); } catch { throw new SystemDocumentError( "config.current-phase", `Current phase does not exist: ${config.currentPhase}`, ); } if (matches.length !== 1) throw new SystemDocumentError( "config.current-phase", matches.length === 0 ? `Current phase does not exist: ${config.currentPhase}` : `Current phase is ambiguous: ${config.currentPhase}`, ); const relative = `.system/phases/${matches[0]}/index.md`; const bundle = path.join(directory, matches[0] as string); const target = path.join(bundle, "index.md"); const targetStats = lstatSync(target, { throwIfNoEntry: false }); if ( lstatSync(bundle, { throwIfNoEntry: false })?.isSymbolicLink() || !targetStats?.isFile() || targetStats.isSymbolicLink() ) throw new SystemDocumentError( "config.current-phase", `Current phase is not a regular document: ${config.currentPhase}`, ); if (targetStats.size > PHASE_BYTE_LIMIT) throw new SystemDocumentError( "document.phase-size", `Current phase exceeds the ${PHASE_BYTE_LIMIT}-byte limit`, ); const markdown = readFileSync(target, "utf8"); const authored = parseAuthoredDocument(markdown, config.prefix); validateDocumentPath(relative, authored.metadata, config.prefix); if ( authored.metadata.type !== "phase" || authored.metadata.id !== config.currentPhase ) throw new SystemDocumentError( "config.current-phase", `Current phase document does not match ${config.currentPhase}`, ); return { path: relative, metadata: authored.metadata, markdown }; } const diagnostic = ( code: string, diagnosticPath: string, message: string, ): Diagnostic => ({ code, path: diagnosticPath, message }); const compareDiagnostics = (left: Diagnostic, right: Diagnostic) => left.path.localeCompare(right.path) || left.code.localeCompare(right.code) || left.message.localeCompare(right.message); const scanBundleAssets = ( absolute: string, relative: string, findings: Diagnostic[], ): void => { for (const entry of readdirSync(absolute, { withFileTypes: true }).sort( (left, right) => left.name.localeCompare(right.name), )) { if (relative.split("/").length === 3 && entry.name === "index.md") continue; const childAbsolute = path.join(absolute, entry.name); const childRelative = `${relative}/${entry.name}`; const stats = lstatSync(childAbsolute, { throwIfNoEntry: false }); if (!stats) continue; if (stats.isSymbolicLink()) findings.push( diagnostic( "bundle.symbolic-link", childRelative, `Bundle path is a symbolic link: ${childRelative}`, ), ); else if (stats.isDirectory()) scanBundleAssets(childAbsolute, childRelative, findings); } }; export function discoverDocumentBundles(root: string): BundleDiscovery { const sources: BundleDocumentSource[] = []; const diagnostics: Diagnostic[] = []; const systemRoot = path.join(root, ".system"); if (lstatSync(systemRoot, { throwIfNoEntry: false })?.isSymbolicLink()) return { sources, diagnostics: [ diagnostic( "bundle.symbolic-link", ".system", "Bundle path is a symbolic link: .system", ), ], }; for (const directory of Object.values(DIRECTORY_BY_TYPE)) { const relativeDirectory = `.system/${directory}`; const absoluteDirectory = path.join(root, relativeDirectory); const directoryStats = lstatSync(absoluteDirectory, { throwIfNoEntry: false, }); if (!directoryStats) continue; if (directoryStats.isSymbolicLink()) { diagnostics.push( diagnostic( "bundle.symbolic-link", relativeDirectory, `Bundle path is a symbolic link: ${relativeDirectory}`, ), ); continue; } if (!directoryStats.isDirectory()) { diagnostics.push( diagnostic( "bundle.expected-directory", relativeDirectory, `Expected a document collection directory: ${relativeDirectory}`, ), ); continue; } for (const entry of readdirSync(absoluteDirectory, { withFileTypes: true, }).sort((left, right) => left.name.localeCompare(right.name))) { const bundleRelative = `${relativeDirectory}/${entry.name}`; const bundleAbsolute = path.join(absoluteDirectory, entry.name); const bundleStats = lstatSync(bundleAbsolute, { throwIfNoEntry: false }); if (bundleStats?.isSymbolicLink()) { diagnostics.push( diagnostic( "bundle.symbolic-link", bundleRelative, `Bundle path is a symbolic link: ${bundleRelative}`, ), ); continue; } if (!bundleStats?.isDirectory()) { diagnostics.push( diagnostic( "bundle.expected-directory", bundleRelative, `Expected a document bundle directory: ${bundleRelative}`, ), ); continue; } const indexRelative = `${bundleRelative}/index.md`; const indexAbsolute = path.join(bundleAbsolute, "index.md"); const indexStats = lstatSync(indexAbsolute, { throwIfNoEntry: false }); if (!indexStats) diagnostics.push( diagnostic( "bundle.missing-index", bundleRelative, `Bundle is missing index.md: ${bundleRelative}`, ), ); else if (indexStats.isSymbolicLink()) diagnostics.push( diagnostic( "bundle.symbolic-link", indexRelative, `Bundle path is a symbolic link: ${indexRelative}`, ), ); else if (!indexStats.isFile()) diagnostics.push( diagnostic( "bundle.expected-file", indexRelative, `Expected a regular index.md: ${indexRelative}`, ), ); else sources.push({ path: indexRelative, markdown: readFileSync(indexAbsolute, "utf8"), }); scanBundleAssets(bundleAbsolute, bundleRelative, diagnostics); } } return { sources: sources.sort((left, right) => left.path.localeCompare(right.path)), diagnostics: diagnostics.sort(compareDiagnostics), }; } export const references = ( metadata: DocumentMetadata, ): readonly (readonly [string, DocumentType])[] => { switch (metadata.type) { case "spec": return (metadata.research ?? []).map((id) => [id, "research"] as const); case "phase": return []; case "plan": return [ [metadata.spec, "spec"], ...(metadata.depends_on ?? []).map((id) => [id, "plan"] as const), ]; case "issue": return metadata.specs.map((spec) => [spec, "spec"] as const); default: return []; } }; export const dependencyReferences = ( metadata: DocumentMetadata, ): readonly string[] => metadata.type === "plan" ? (metadata.depends_on ?? []) : []; const diagnosticIdentity = (item: Diagnostic) => JSON.stringify([item.path, item.code, item.message]); export function indexBundleDocuments( sources: readonly BundleDocumentSource[], prefix: string, ): BundleDocumentIndex { const documents: BundleDocumentRecord[] = []; const graphEligible: BundleDocumentRecord[] = []; const diagnostics: Diagnostic[] = []; for (const item of [...sources].sort((left, right) => left.path.localeCompare(right.path), )) { try { const parsed = parseAuthoredDocument(item.markdown, prefix); let pathValid = true; try { validateDocumentPath(item.path, parsed.metadata, prefix); } catch (error) { pathValid = false; diagnostics.push( diagnostic("document.path", item.path, (error as Error).message), ); } const record = { path: item.path, metadata: parsed.metadata, frontmatter: parsed.frontmatter, }; documents.push(record); if (pathValid) graphEligible.push(record); } catch (error) { const value = error as SystemDocumentError | MarkdownDocumentError; diagnostics.push( diagnostic( "document.metadata", item.path, `${value.code}: ${value.message}`, ), ); } } const groups = new Map(); for (const document of graphEligible) groups.set(document.metadata.id, [ ...(groups.get(document.metadata.id) ?? []), document, ]); for (const [id, records] of groups) { if (records.length < 2) continue; const paths = records.map((record) => record.path).sort(); for (const record of records) diagnostics.push( diagnostic( "graph.duplicate-id", record.path, `Duplicate ID ${id}: ${paths.join(", ")}`, ), ); } const unique = new Map( [...groups].flatMap(([id, records]) => { const [record] = records; return records.length === 1 && record ? [[id, record] as const] : []; }), ); for (const document of graphEligible) { for (const [targetId, expectedType] of references(document.metadata)) { const referencedType = parseDocumentId(targetId, prefix).type; const target = unique.get(targetId); if (referencedType !== expectedType) diagnostics.push( diagnostic( "graph.forbidden-edge", document.path, `${document.metadata.id} must reference ${expectedType}; ${targetId} is ${referencedType}`, ), ); else if (!target) diagnostics.push( diagnostic( "graph.dangling-reference", document.path, `${document.metadata.id} references missing ${targetId}`, ), ); else if (target.metadata.type !== expectedType) diagnostics.push( diagnostic( "graph.forbidden-edge", document.path, `${document.metadata.id} must reference ${expectedType}; ${targetId} is ${target.metadata.type}`, ), ); } if (document.metadata.type === "plan") { for (const dependencyId of dependencyReferences(document.metadata)) { const dependency = unique.get(dependencyId); if ( dependency?.metadata.type === document.metadata.type && dependency.metadata.spec !== document.metadata.spec ) diagnostics.push( diagnostic( "graph.forbidden-edge", document.path, `${dependencyId} belongs to ${dependency.metadata.spec}, not ${document.metadata.spec}`, ), ); } } } const visiting = new Set(); const visited = new Set(); const visit = (document: BundleDocumentRecord, stack: readonly string[]) => { const id = document.metadata.id; if (visited.has(id)) return; if (visiting.has(id)) { const start = stack.indexOf(id); diagnostics.push( diagnostic( "graph.dependency-cycle", document.path, `Dependency cycle: ${[...stack.slice(start), id].join(" -> ")}`, ), ); return; } visiting.add(id); for (const dependencyId of [ ...dependencyReferences(document.metadata), ].sort()) { const dependency = unique.get(dependencyId); if (dependency?.metadata.type === document.metadata.type) visit(dependency, [...stack, id]); } visiting.delete(id); visited.add(id); }; for (const document of graphEligible) if (document.metadata.type === "plan") visit(document, []); return { documents, diagnostics: diagnostics.sort(compareDiagnostics) }; } export function validateBundleCandidate( current: readonly BundleDocumentSource[], candidate: BundleDocumentSource, prefix: string, ): readonly Diagnostic[] { const baseline = new Set( indexBundleDocuments(current, prefix).diagnostics.map(diagnosticIdentity), ); const next = indexBundleDocuments( [...current.filter((source) => source.path !== candidate.path), candidate], prefix, ); return next.diagnostics.filter( (item) => item.path === candidate.path || (!baseline.has(diagnosticIdentity(item)) && item.code !== "graph.duplicate-id"), ); }