import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { fauxAssistantMessage } from "@earendil-works/pi-ai"; import { initTheme } from "@earendil-works/pi-coding-agent"; import { type Component, stripTerminalSequences, type TUI, } from "@earendil-works/pi-tui"; import { afterEach, beforeEach, expect, it, type MockInstance, vi, } from "vitest"; import { createPiSession, type PiSession, } from "../../../test/fixture/create-pi-session.ts"; import voice from "../index.ts"; import { SpeakClient } from "../src/speak-client.ts"; import { NativeSpeechProcess } from "../src/speech-backend.ts"; import type { NativeSpeechEvent, PreparedSpeech, } from "../src/speech-events.ts"; const fake = fileURLToPath( new URL("../__e2e__/fake-native.mjs", import.meta.url), ); const voicePath = fileURLToPath(new URL("../index.ts", import.meta.url)); const journeys = fileURLToPath( new URL("../__e2e__/fixtures/voice-journeys.json", import.meta.url), ); const recordedSpeech = fileURLToPath( new URL("../__e2e__/fixtures/recorded-speech.json", import.meta.url), ); const clients: SpeakClient[] = []; let session: PiSession | undefined; let network: MockInstance; beforeEach(() => { network = vi .spyOn(globalThis, "fetch") .mockRejectedValue( new Error("Network forbidden in fake-backed Voice acceptance"), ); }); afterEach(async () => { try { await Promise.all(clients.splice(0).map((client) => client.close())); await session?.dispose(); session = undefined; expect(network).not.toHaveBeenCalled(); } finally { network.mockRestore(); } }); function client(scenario: string): SpeakClient { const value = new SpeakClient([ process.execPath, fake, "--scenario", scenario, ]); clients.push(value); return value; } async function waitFor(check: () => boolean): Promise { for (let attempt = 0; attempt < 100; attempt++) { if (check()) return; await new Promise((resolve) => setTimeout(resolve, 10)); } throw new Error("Timed out waiting for fake speech event."); } function recordingUI(active: PiSession, width = 80, rows = 24) { initTheme("dark", false); const runner = active.session.extensionRunner; const base = runner.getUIContext(); const screens: string[] = []; let widget: Component | undefined; let queued = false; const render = () => { if (widget) screens.push(stripTerminalSequences(widget.render(width).join("\n"))); }; const tui = { terminal: { rows }, requestRender: () => { if (queued) return; queued = true; queueMicrotask(() => { queued = false; render(); }); }, } as unknown as TUI; const getEditorText = vi.fn(() => "unsent editor draft"); const setEditorText = vi.fn(); const setWidget: typeof base.setWidget = (_id, content) => { widget = typeof content === "function" ? content(tui, base.theme) : undefined; render(); }; runner.setUIContext( { ...base, setWidget, getEditorText, setEditorText }, "tui", ); return { screens, getEditorText, setEditorText, setWidget }; } it("loads Voice through Pi's public session harness with an explicit test executable", async () => { session = await createPiSession({ extensionPaths: [], extensionFactories: [voice], env: { PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "preview-race", ]), }, }); const [extension] = session.extensions; expect(session.errors).toEqual([]); expect(extension?.commands.has("voice")).toBe(true); expect(session.session.messages).toEqual([]); }); it("runs a complete fake-backed call through registered Pi commands without touching the editor", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-command-")); const trace = join(root, "wire.ndjson"); const previewRelease = join(root, "preview.release"); try { session = await createPiSession({ extensionPaths: [voicePath], responses: [ fauxAssistantMessage( JSON.stringify({ heard: "Hello.", reply: "Hello back.", hotwords: [], }), ), ], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "ui-preview", "--preview-release", previewRelease, "--wire-trace", trace, ]), }, }); const runner = session.session.extensionRunner; const { screens, getEditorText, setEditorText } = recordingUI(session); const wire = async () => (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); const inbound = async () => (await wire()) .filter((entry) => entry.direction === "inbound") .map((entry) => entry.value); const command = (args: string) => session!.session.prompt(`/voice ${args}`); await command(`start --root ${join(root, "history")}`); await vi.waitFor(() => expect(screens.join("\n")).toContain("hello world")); expect(session.providerState.callCount).toBe(0); expect( (await inbound()).some( (request) => request.method === "synthesis.append", ), ).toBe(false); await writeFile(previewRelease, "release\n", { flag: "wx" }); await vi.waitFor( async () => { expect( (await inbound()).some( (request) => request.method === "synthesis.append" && request.params.text.includes("Hello back."), ), JSON.stringify({ requests: await inbound(), screens: screens.slice(-2), calls: session!.providerState.callCount, errors: session!.errors, }), ).toBe(true); }, { timeout: 5000 }, ); await vi.waitFor(() => expect(screens.join("\n")).toContain("Hello back.")); expect(screens.join("\n")).toContain("hello world"); expect(session.providerState.callCount).toBe(1); expect(session.session.messages).toEqual([]); const output = (await wire()) .filter((entry) => entry.direction === "outbound") .map((entry) => entry.value); expect( output.some( (value) => value.method === "playback.progress" && value.params.data.played_text === "Hello back.", ), ).toBe(true); await command("mute"); await command("unmute"); expect( (await inbound()).filter( (request) => request.method === "recognition.start", ), ).toHaveLength(2); await command("stop"); const first = (await wire()).find( (entry) => entry.direction === "lifecycle" && entry.value.event === "start", ).value.pid; expect(() => process.kill(first, 0)).toThrow(); expect( (await inbound()).some((request) => request.method === "resource.close"), ).toBe(true); expect( (await inbound()).some( (request) => request.method === "service.shutdown", ), ).toBe(true); await command(`start --root ${join(root, "history")}`); await runner.emit({ type: "session_shutdown", reason: "reload" }); await command(`start --root ${join(root, "history")}`); await session.dispose(); session = undefined; const starts = (await wire()).filter( (entry) => entry.direction === "lifecycle" && entry.value.event === "start", ); expect(starts).toHaveLength(3); for (const entry of starts) expect(() => process.kill(entry.value.pid, 0)).toThrow(); expect(network).not.toHaveBeenCalled(); expect(getEditorText).not.toHaveBeenCalled(); expect(setEditorText).not.toHaveBeenCalled(); } finally { await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 10000); it("keeps unrelated Pi work streaming through Voice barge-in and hangup", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-barge-in-")); const trace = join(root, "wire.ndjson"); let releaseOrdinary!: () => void; const ordinaryResponse = new Promise((resolve) => { releaseOrdinary = resolve; }); let ordinaryPrompt: Promise | undefined; try { session = await createPiSession({ extensionPaths: [voicePath], responses: [ () => ordinaryResponse.then(() => fauxAssistantMessage("Unrelated work completed."), ), fauxAssistantMessage( JSON.stringify({ heard: "Hello.", reply: "An answer to interrupt.", hotwords: [], }), ), ], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "barge-in", "--wire-trace", trace, ]), }, }); const { screens } = recordingUI(session); const abort = vi.spyOn(session.session, "abort"); ordinaryPrompt = session.session.prompt("Harmless unrelated fixture work"); await vi.waitFor(() => expect(session!.providerState.callCount).toBe(1), { timeout: 5000, }); await vi.waitFor(() => expect(session!.session.isStreaming).toBe(true), { timeout: 5000, }); const command = (args: string) => session!.session.prompt(`/voice ${args}`); await command(`start --root ${join(root, "history")}`); await vi.waitFor( async () => { const entries = (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); expect( entries.some( (entry) => entry.direction === "outbound" && entry.value.method === "recognition.activity" && entry.value.params.data.active === true && entry.value.params.data.segment === "interrupt", ), ).toBe(true); expect( entries.some( (entry) => entry.direction === "outbound" && entry.value.method === "operation.ended" && entry.value.params.data.outcome === "cancelled" && entry.value.params.data.reason === "speech", ), ).toBe(true); expect( entries.some( (entry) => entry.direction === "inbound" && entry.value.method === "synthesis.append" && entry.value.params.text.includes("An answer to interrupt."), ), ).toBe(true); expect( entries.some( (entry) => entry.direction === "outbound" && entry.value.method === "recognition.final" && entry.value.params.data.text === "Hello.", ), ).toBe(true); }, { timeout: 5000 }, ); expect(screens.join("\n")).toContain("Hello."); expect(session.session.isStreaming).toBe(true); expect(abort).not.toHaveBeenCalled(); let ordinarySettled = false; void ordinaryPrompt.then(() => { ordinarySettled = true; }); await Promise.resolve(); expect(ordinarySettled).toBe(false); await command("stop"); expect(session.session.isStreaming).toBe(true); expect(abort).not.toHaveBeenCalled(); expect(ordinarySettled).toBe(false); releaseOrdinary(); await ordinaryPrompt; await vi.waitFor(() => expect(session!.session.messages).toEqual( expect.arrayContaining([ expect.objectContaining({ role: "assistant", content: expect.arrayContaining([ expect.objectContaining({ type: "text", text: "Unrelated work completed.", }), ]), }), ]), ), ); expect(abort).not.toHaveBeenCalled(); } finally { releaseOrdinary(); await ordinaryPrompt?.catch(() => undefined); await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 10000); it.runIf(process.env["PI_VOICE_REALTIME_ACCEPTANCE"] === "1")( "replays recorded speech through the public command with provisional UI, stable reply, and consumed-prefix certification", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-recorded-speech-")); const trace = join(root, "wire.ndjson"); try { const fixture = JSON.parse(await readFile(recordedSpeech, "utf8")) as { journeys: { "recorded-speech": { caption: string; reply: string } }; }; const { caption, reply } = fixture.journeys["recorded-speech"]; session = await createPiSession({ extensionPaths: [voicePath], responses: [ fauxAssistantMessage( JSON.stringify({ heard: caption, reply, hotwords: [] }), ), ], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "recorded-speech", "--scenario-file", recordedSpeech, "--wire-trace", trace, ]), }, }); const { screens, getEditorText, setEditorText } = recordingUI( session, 80, 24, ); const command = (args: string) => session!.session.prompt(`/voice ${args}`); await command(`start --root ${join(root, "history")}`); await vi.waitFor( () => expect(screens.join("\n").toLowerCase()).toContain( "he was in a fevered", ), { timeout: 5000 }, ); const provisionalScreen = screens.join("\n"); expect(provisionalScreen).toMatch(/[▁▂▃▄▅▆▇█]/u); expect(session.providerState.callCount).toBe(0); await vi.waitFor( () => expect(screens.join("\n").replace(/\s+/gu, " ")).toContain(reply), { timeout: 12000 }, ); await vi.waitFor( async () => { const wire = (await readFile(trace, "utf8")) .trim() .split("\n") .map( (line) => JSON.parse(line) as { direction: string; value: { method?: string; params?: unknown }; }, ); expect( wire.some((entry) => { if ( entry.direction !== "outbound" || entry.value.method !== "playback.progress" ) return false; return ( typeof ( entry.value.params as { data?: { played_text?: unknown } } ).data?.played_text === "string" && ((entry.value.params as { data?: { played_text?: string } }) .data?.played_text?.length ?? 0) > 0 ); }), ).toBe(true); }, { timeout: 5000 }, ); expect(screens.join("\n").replace(/\s+/gu, " ").toLowerCase()).toContain( caption.toLowerCase(), ); const traceEntries = (await readFile(trace, "utf8")) .trim() .split("\n") .map( (line) => JSON.parse(line) as { direction: string; value: { method?: string; params?: unknown }; }, ); const audio = traceEntries.filter( (entry) => entry.direction === "outbound" && entry.value.method === "audio.level", ); const waveforms = new Set( audio.map((entry) => JSON.stringify( (entry.value.params as { data?: { waveform?: number[] } }).data ?.waveform, ), ), ); expect(waveforms.size).toBeGreaterThan(1); expect( audio.some((entry) => { const params = entry.value.params as { data?: { waveform?: number[]; input?: number }; }; return ( (params.data?.waveform?.length ?? 0) > 0 && (params.data?.input ?? 0) > 0 ); }), ).toBe(true); expect(session.providerState.callCount).toBe(1); expect(session.session.messages).toEqual([]); expect(getEditorText).not.toHaveBeenCalled(); expect(setEditorText).not.toHaveBeenCalled(); const appendedTexts = traceEntries .filter( (entry) => entry.direction === "inbound" && entry.value.method === "synthesis.append", ) .map((entry) => (entry.value.params as { text?: string }).text ?? ""); expect(appendedTexts.join("")).toContain(reply); expect( traceEntries.some((entry) => { if ( entry.direction !== "outbound" || entry.value.method !== "playback.progress" ) return false; const data = ( entry.value.params as { data?: { played_text?: string; consumed_frames?: number }; } ).data; return ( data?.played_text === reply && typeof data.consumed_frames === "number" && data.consumed_frames > 0 ); }), ).toBe(true); await command("stop"); const stoppedTrace = (await readFile(trace, "utf8")) .trim() .split("\n") .map( (line) => JSON.parse(line) as { direction: string; value: { method?: string; params?: unknown; event?: string; pid?: number; }; }, ); const inbound = stoppedTrace .filter((entry) => entry.direction === "inbound") .map((entry) => entry.value.method); expect(inbound).toContain("resource.close"); expect(inbound).toContain("service.shutdown"); const pid = stoppedTrace.find( (entry) => entry.direction === "lifecycle" && entry.value["event"] === "start", )?.value["pid"] as number; expect(() => process.kill(pid, 0)).toThrow(); } finally { await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 20000, ); it("keeps preview provisional, accepts only a stable final, and certifies speech only from fake consumed progress", async () => { const service = client("preview-race"); const info = await service.start(); expect(info.io_mode).toBe("fake"); const profile = info.profiles[0]; if (!profile) throw new Error("Fake profile missing."); const events: NativeSpeechEvent[] = []; const speech = new NativeSpeechProcess(); const backend: PreparedSpeech = { client: service, command: [process.execPath, fake, "--scenario", "preview-race"], profile, preview: true, }; await speech.start({ backend, onEvent: (event) => events.push(event) }); await waitFor(() => events.some((event) => event.type === "final")); expect(events.filter((event) => event.type === "preview")).toEqual([ expect.objectContaining({ revision: 1, text: "hello" }), expect.objectContaining({ revision: 2, text: "hello world" }), ]); expect(events.find((event) => event.type === "final")).toMatchObject({ text: "Hello.", }); await speech.speak({ assistantTurnId: "assistant-local", userTurnId: "user-local", text: "A streamed reply.", }); await speech.complete("assistant-local"); await waitFor(() => events.some((event) => event.type === "playback")); expect( events.filter((event) => event.type === "playback").at(-1), ).toMatchObject({ playedText: "A streamed reply.", }); await speech.setMuted(true); await speech.setMuted(false); await speech.close(); }); it("captures only RPC-schema fields on the test-owned fake wire trace", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-wire-")); const trace = join(root, "wire.ndjson"); const service = new SpeakClient([ process.execPath, fake, "--wire-trace", trace, ]); clients.push(service); try { await service.start(); const opened = await service.request<{ resource: string; epoch: number; operation: string; }>("resource.open", { profile: "basic" }); await service.waitForOperation(opened.operation, { timeoutMs: 1_000 }); const synthesis = await service.request<{ operation: string }>( "synthesis.start", { resource: opened.resource, epoch: opened.epoch }, ); await service.request("synthesis.append", { operation: synthesis.operation, index: 0, text: "Arbitrary prose stays payload text.", }); const requests = (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line) as { direction: string; value: object }) .filter((entry) => entry.direction === "inbound") .map((entry) => entry.value as { method?: string; params?: object }); const append = requests.find( (request) => request.method === "synthesis.append", ); expect(Object.keys(append?.params ?? {}).sort()).toEqual([ "index", "operation", "text", ]); await service.close(); const stopped = (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); expect(stopped).toContainEqual({ direction: "lifecycle", value: { event: "exit", code: 0 }, }); } finally { await service.close(); await rm(root, { recursive: true, force: true }); } }); it("renders a long provenance-backed dictation through the public Voice command", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-long-caption-")); try { session = await createPiSession({ extensionPaths: [voicePath], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "long-caption", "--scenario-file", journeys, ]), }, }); const { screens } = recordingUI(session, 44, 8); await session.session.prompt( `/voice start --root ${join(root, "history")}`, ); await vi.waitFor(() => expect(screens.join("\n")).toContain("points of his situation"), ); expect(session.providerState.callCount).toBe(0); expect(session.session.messages).toEqual([]); } finally { await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 10000); it("treats silence as an observed empty boundary without a model turn", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-silence-")); const trace = join(root, "wire.ndjson"); try { session = await createPiSession({ extensionPaths: [voicePath], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "silence", "--wire-trace", trace, ]), }, }); const { getEditorText, setEditorText } = recordingUI(session, 44, 8); await session.session.prompt( `/voice start --root ${join(root, "history")}`, ); await vi.waitFor(async () => { const entries = (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); expect( entries.some( (entry) => entry.direction === "outbound" && entry.value.method === "recognition.empty", ), ).toBe(true); }); expect(session.providerState.callCount).toBe(0); expect(session.session.messages).toEqual([]); expect(getEditorText).not.toHaveBeenCalled(); expect(setEditorText).not.toHaveBeenCalled(); } finally { await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 10000); it("mutes an observed preview, then rejects its released final across unmute", async () => { const root = await mkdtemp(join(tmpdir(), "pi-voice-mute-fence-")); const trace = join(root, "wire.ndjson"); const release = join(root, "preview.release"); try { session = await createPiSession({ extensionPaths: [voicePath], env: { PI_VOICE_MODEL: "faux/faux-model", PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier", PI_VOICE_COMMAND: JSON.stringify([ process.execPath, fake, "--scenario", "ui-preview", "--preview-release", release, "--wire-trace", trace, ]), }, }); const { screens, getEditorText, setEditorText } = recordingUI( session, 44, 8, ); await session.session.prompt( `/voice start --root ${join(root, "history")}`, ); await vi.waitFor(() => expect(screens.join("\n")).toContain("hello world")); await session.session.prompt("/voice mute"); await writeFile(release, "release cancelled input\n", { flag: "wx" }); await session.session.prompt("/voice unmute"); await session.session.prompt("/voice stop"); const wire = (await readFile(trace, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); expect(session.providerState.callCount).toBe(0); expect(session.session.messages).toEqual([]); expect( wire.some( (entry) => entry.direction === "inbound" && entry.value.method === "resource.set_muted" && entry.value.params.muted === true, ), ).toBe(true); expect( wire.filter( (entry) => entry.direction === "inbound" && entry.value.method === "recognition.start", ), ).toHaveLength(2); expect( wire.some( (entry) => entry.direction === "outbound" && entry.value.method === "recognition.final", ), ).toBe(false); expect(getEditorText).not.toHaveBeenCalled(); expect(setEditorText).not.toHaveBeenCalled(); } finally { await session?.dispose(); session = undefined; await rm(root, { recursive: true, force: true }); } }, 10000); it("declines blocked fake setup without asking the service to install assets", async () => { const service = client("consent"); const info = await service.start(); const profile = info.profiles[0]; if (!profile) throw new Error("Fake profile missing."); const plan = await service.request<{ plan: { blockers: string[] } }>( "assets.plan", { profile: profile.id }, ); expect(plan.plan.blockers).toEqual(["approval required"]); await expect( service.request("assets.install", { plan: "plan", approved: true, accepted_licenses: [], }), ).rejects.toThrow("Consent required"); });