repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/voice/__serial__/voice-rpc-e2e.test.ts
Rawimport { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { fauxAssistantMessage } from "@earendil-works/pi-ai";
import { initTheme } from "@earendil-works/pi-coding-agent";
import {
type Component,
stripTerminalSequences,
type TUI,
} from "@earendil-works/pi-tui";
import {
afterEach,
beforeEach,
expect,
it,
type MockInstance,
vi,
} from "vitest";
import {
createPiSession,
type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import voice from "../index.ts";
import { SpeakClient } from "../src/speak-client.ts";
import { NativeSpeechProcess } from "../src/speech-backend.ts";
import type {
NativeSpeechEvent,
PreparedSpeech,
} from "../src/speech-events.ts";
const fake = fileURLToPath(
new URL("../__e2e__/fake-native.mjs", import.meta.url),
);
const voicePath = fileURLToPath(new URL("../index.ts", import.meta.url));
const journeys = fileURLToPath(
new URL("../__e2e__/fixtures/voice-journeys.json", import.meta.url),
);
const recordedSpeech = fileURLToPath(
new URL("../__e2e__/fixtures/recorded-speech.json", import.meta.url),
);
const clients: SpeakClient[] = [];
let session: PiSession | undefined;
let network: MockInstance<typeof fetch>;
beforeEach(() => {
network = vi
.spyOn(globalThis, "fetch")
.mockRejectedValue(
new Error("Network forbidden in fake-backed Voice acceptance"),
);
});
afterEach(async () => {
try {
await Promise.all(clients.splice(0).map((client) => client.close()));
await session?.dispose();
session = undefined;
expect(network).not.toHaveBeenCalled();
} finally {
network.mockRestore();
}
});
function client(scenario: string): SpeakClient {
const value = new SpeakClient([
process.execPath,
fake,
"--scenario",
scenario,
]);
clients.push(value);
return value;
}
async function waitFor(check: () => boolean): Promise<void> {
for (let attempt = 0; attempt < 100; attempt++) {
if (check()) return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
throw new Error("Timed out waiting for fake speech event.");
}
function recordingUI(active: PiSession, width = 80, rows = 24) {
initTheme("dark", false);
const runner = active.session.extensionRunner;
const base = runner.getUIContext();
const screens: string[] = [];
let widget: Component | undefined;
let queued = false;
const render = () => {
if (widget)
screens.push(stripTerminalSequences(widget.render(width).join("\n")));
};
const tui = {
terminal: { rows },
requestRender: () => {
if (queued) return;
queued = true;
queueMicrotask(() => {
queued = false;
render();
});
},
} as unknown as TUI;
const getEditorText = vi.fn(() => "unsent editor draft");
const setEditorText = vi.fn();
const setWidget: typeof base.setWidget = (_id, content) => {
widget =
typeof content === "function" ? content(tui, base.theme) : undefined;
render();
};
runner.setUIContext(
{ ...base, setWidget, getEditorText, setEditorText },
"tui",
);
return { screens, getEditorText, setEditorText, setWidget };
}
it("loads Voice through Pi's public session harness with an explicit test executable", async () => {
session = await createPiSession({
extensionPaths: [],
extensionFactories: [voice],
env: {
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"preview-race",
]),
},
});
const [extension] = session.extensions;
expect(session.errors).toEqual([]);
expect(extension?.commands.has("voice")).toBe(true);
expect(session.session.messages).toEqual([]);
});
it("runs a complete fake-backed call through registered Pi commands without touching the editor", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-command-"));
const trace = join(root, "wire.ndjson");
const previewRelease = join(root, "preview.release");
try {
session = await createPiSession({
extensionPaths: [voicePath],
responses: [
fauxAssistantMessage(
JSON.stringify({
heard: "Hello.",
reply: "Hello back.",
hotwords: [],
}),
),
],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"ui-preview",
"--preview-release",
previewRelease,
"--wire-trace",
trace,
]),
},
});
const runner = session.session.extensionRunner;
const { screens, getEditorText, setEditorText } = recordingUI(session);
const wire = async () =>
(await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
const inbound = async () =>
(await wire())
.filter((entry) => entry.direction === "inbound")
.map((entry) => entry.value);
const command = (args: string) => session!.session.prompt(`/voice ${args}`);
await command(`start --root ${join(root, "history")}`);
await vi.waitFor(() => expect(screens.join("\n")).toContain("hello world"));
expect(session.providerState.callCount).toBe(0);
expect(
(await inbound()).some(
(request) => request.method === "synthesis.append",
),
).toBe(false);
await writeFile(previewRelease, "release\n", { flag: "wx" });
await vi.waitFor(
async () => {
expect(
(await inbound()).some(
(request) =>
request.method === "synthesis.append" &&
request.params.text.includes("Hello back."),
),
JSON.stringify({
requests: await inbound(),
screens: screens.slice(-2),
calls: session!.providerState.callCount,
errors: session!.errors,
}),
).toBe(true);
},
{ timeout: 5000 },
);
await vi.waitFor(() => expect(screens.join("\n")).toContain("Hello back."));
expect(screens.join("\n")).toContain("hello world");
expect(session.providerState.callCount).toBe(1);
expect(session.session.messages).toEqual([]);
const output = (await wire())
.filter((entry) => entry.direction === "outbound")
.map((entry) => entry.value);
expect(
output.some(
(value) =>
value.method === "playback.progress" &&
value.params.data.played_text === "Hello back.",
),
).toBe(true);
await command("mute");
await command("unmute");
expect(
(await inbound()).filter(
(request) => request.method === "recognition.start",
),
).toHaveLength(2);
await command("stop");
const first = (await wire()).find(
(entry) =>
entry.direction === "lifecycle" && entry.value.event === "start",
).value.pid;
expect(() => process.kill(first, 0)).toThrow();
expect(
(await inbound()).some((request) => request.method === "resource.close"),
).toBe(true);
expect(
(await inbound()).some(
(request) => request.method === "service.shutdown",
),
).toBe(true);
await command(`start --root ${join(root, "history")}`);
await runner.emit({ type: "session_shutdown", reason: "reload" });
await command(`start --root ${join(root, "history")}`);
await session.dispose();
session = undefined;
const starts = (await wire()).filter(
(entry) =>
entry.direction === "lifecycle" && entry.value.event === "start",
);
expect(starts).toHaveLength(3);
for (const entry of starts)
expect(() => process.kill(entry.value.pid, 0)).toThrow();
expect(network).not.toHaveBeenCalled();
expect(getEditorText).not.toHaveBeenCalled();
expect(setEditorText).not.toHaveBeenCalled();
} finally {
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
}, 10000);
it("keeps unrelated Pi work streaming through Voice barge-in and hangup", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-barge-in-"));
const trace = join(root, "wire.ndjson");
let releaseOrdinary!: () => void;
const ordinaryResponse = new Promise<void>((resolve) => {
releaseOrdinary = resolve;
});
let ordinaryPrompt: Promise<void> | undefined;
try {
session = await createPiSession({
extensionPaths: [voicePath],
responses: [
() =>
ordinaryResponse.then(() =>
fauxAssistantMessage("Unrelated work completed."),
),
fauxAssistantMessage(
JSON.stringify({
heard: "Hello.",
reply: "An answer to interrupt.",
hotwords: [],
}),
),
],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"barge-in",
"--wire-trace",
trace,
]),
},
});
const { screens } = recordingUI(session);
const abort = vi.spyOn(session.session, "abort");
ordinaryPrompt = session.session.prompt("Harmless unrelated fixture work");
await vi.waitFor(() => expect(session!.providerState.callCount).toBe(1), {
timeout: 5000,
});
await vi.waitFor(() => expect(session!.session.isStreaming).toBe(true), {
timeout: 5000,
});
const command = (args: string) => session!.session.prompt(`/voice ${args}`);
await command(`start --root ${join(root, "history")}`);
await vi.waitFor(
async () => {
const entries = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(
entries.some(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "recognition.activity" &&
entry.value.params.data.active === true &&
entry.value.params.data.segment === "interrupt",
),
).toBe(true);
expect(
entries.some(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "operation.ended" &&
entry.value.params.data.outcome === "cancelled" &&
entry.value.params.data.reason === "speech",
),
).toBe(true);
expect(
entries.some(
(entry) =>
entry.direction === "inbound" &&
entry.value.method === "synthesis.append" &&
entry.value.params.text.includes("An answer to interrupt."),
),
).toBe(true);
expect(
entries.some(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "recognition.final" &&
entry.value.params.data.text === "Hello.",
),
).toBe(true);
},
{ timeout: 5000 },
);
expect(screens.join("\n")).toContain("Hello.");
expect(session.session.isStreaming).toBe(true);
expect(abort).not.toHaveBeenCalled();
let ordinarySettled = false;
void ordinaryPrompt.then(() => {
ordinarySettled = true;
});
await Promise.resolve();
expect(ordinarySettled).toBe(false);
await command("stop");
expect(session.session.isStreaming).toBe(true);
expect(abort).not.toHaveBeenCalled();
expect(ordinarySettled).toBe(false);
releaseOrdinary();
await ordinaryPrompt;
await vi.waitFor(() =>
expect(session!.session.messages).toEqual(
expect.arrayContaining([
expect.objectContaining({
role: "assistant",
content: expect.arrayContaining([
expect.objectContaining({
type: "text",
text: "Unrelated work completed.",
}),
]),
}),
]),
),
);
expect(abort).not.toHaveBeenCalled();
} finally {
releaseOrdinary();
await ordinaryPrompt?.catch(() => undefined);
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
}, 10000);
it.runIf(process.env["PI_VOICE_REALTIME_ACCEPTANCE"] === "1")(
"replays recorded speech through the public command with provisional UI, stable reply, and consumed-prefix certification",
async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-recorded-speech-"));
const trace = join(root, "wire.ndjson");
try {
const fixture = JSON.parse(await readFile(recordedSpeech, "utf8")) as {
journeys: { "recorded-speech": { caption: string; reply: string } };
};
const { caption, reply } = fixture.journeys["recorded-speech"];
session = await createPiSession({
extensionPaths: [voicePath],
responses: [
fauxAssistantMessage(
JSON.stringify({ heard: caption, reply, hotwords: [] }),
),
],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"recorded-speech",
"--scenario-file",
recordedSpeech,
"--wire-trace",
trace,
]),
},
});
const { screens, getEditorText, setEditorText } = recordingUI(
session,
80,
24,
);
const command = (args: string) =>
session!.session.prompt(`/voice ${args}`);
await command(`start --root ${join(root, "history")}`);
await vi.waitFor(
() =>
expect(screens.join("\n").toLowerCase()).toContain(
"he was in a fevered",
),
{ timeout: 5000 },
);
const provisionalScreen = screens.join("\n");
expect(provisionalScreen).toMatch(/[▁▂▃▄▅▆▇█]/u);
expect(session.providerState.callCount).toBe(0);
await vi.waitFor(
() => expect(screens.join("\n").replace(/\s+/gu, " ")).toContain(reply),
{ timeout: 12000 },
);
await vi.waitFor(
async () => {
const wire = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map(
(line) =>
JSON.parse(line) as {
direction: string;
value: { method?: string; params?: unknown };
},
);
expect(
wire.some((entry) => {
if (
entry.direction !== "outbound" ||
entry.value.method !== "playback.progress"
)
return false;
return (
typeof (
entry.value.params as { data?: { played_text?: unknown } }
).data?.played_text === "string" &&
((entry.value.params as { data?: { played_text?: string } })
.data?.played_text?.length ?? 0) > 0
);
}),
).toBe(true);
},
{ timeout: 5000 },
);
expect(screens.join("\n").replace(/\s+/gu, " ").toLowerCase()).toContain(
caption.toLowerCase(),
);
const traceEntries = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map(
(line) =>
JSON.parse(line) as {
direction: string;
value: { method?: string; params?: unknown };
},
);
const audio = traceEntries.filter(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "audio.level",
);
const waveforms = new Set(
audio.map((entry) =>
JSON.stringify(
(entry.value.params as { data?: { waveform?: number[] } }).data
?.waveform,
),
),
);
expect(waveforms.size).toBeGreaterThan(1);
expect(
audio.some((entry) => {
const params = entry.value.params as {
data?: { waveform?: number[]; input?: number };
};
return (
(params.data?.waveform?.length ?? 0) > 0 &&
(params.data?.input ?? 0) > 0
);
}),
).toBe(true);
expect(session.providerState.callCount).toBe(1);
expect(session.session.messages).toEqual([]);
expect(getEditorText).not.toHaveBeenCalled();
expect(setEditorText).not.toHaveBeenCalled();
const appendedTexts = traceEntries
.filter(
(entry) =>
entry.direction === "inbound" &&
entry.value.method === "synthesis.append",
)
.map((entry) => (entry.value.params as { text?: string }).text ?? "");
expect(appendedTexts.join("")).toContain(reply);
expect(
traceEntries.some((entry) => {
if (
entry.direction !== "outbound" ||
entry.value.method !== "playback.progress"
)
return false;
const data = (
entry.value.params as {
data?: { played_text?: string; consumed_frames?: number };
}
).data;
return (
data?.played_text === reply &&
typeof data.consumed_frames === "number" &&
data.consumed_frames > 0
);
}),
).toBe(true);
await command("stop");
const stoppedTrace = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map(
(line) =>
JSON.parse(line) as {
direction: string;
value: {
method?: string;
params?: unknown;
event?: string;
pid?: number;
};
},
);
const inbound = stoppedTrace
.filter((entry) => entry.direction === "inbound")
.map((entry) => entry.value.method);
expect(inbound).toContain("resource.close");
expect(inbound).toContain("service.shutdown");
const pid = stoppedTrace.find(
(entry) =>
entry.direction === "lifecycle" && entry.value["event"] === "start",
)?.value["pid"] as number;
expect(() => process.kill(pid, 0)).toThrow();
} finally {
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
},
20000,
);
it("keeps preview provisional, accepts only a stable final, and certifies speech only from fake consumed progress", async () => {
const service = client("preview-race");
const info = await service.start();
expect(info.io_mode).toBe("fake");
const profile = info.profiles[0];
if (!profile) throw new Error("Fake profile missing.");
const events: NativeSpeechEvent[] = [];
const speech = new NativeSpeechProcess();
const backend: PreparedSpeech = {
client: service,
command: [process.execPath, fake, "--scenario", "preview-race"],
profile,
preview: true,
};
await speech.start({ backend, onEvent: (event) => events.push(event) });
await waitFor(() => events.some((event) => event.type === "final"));
expect(events.filter((event) => event.type === "preview")).toEqual([
expect.objectContaining({ revision: 1, text: "hello" }),
expect.objectContaining({ revision: 2, text: "hello world" }),
]);
expect(events.find((event) => event.type === "final")).toMatchObject({
text: "Hello.",
});
await speech.speak({
assistantTurnId: "assistant-local",
userTurnId: "user-local",
text: "A streamed reply.",
});
await speech.complete("assistant-local");
await waitFor(() => events.some((event) => event.type === "playback"));
expect(
events.filter((event) => event.type === "playback").at(-1),
).toMatchObject({
playedText: "A streamed reply.",
});
await speech.setMuted(true);
await speech.setMuted(false);
await speech.close();
});
it("captures only RPC-schema fields on the test-owned fake wire trace", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-wire-"));
const trace = join(root, "wire.ndjson");
const service = new SpeakClient([
process.execPath,
fake,
"--wire-trace",
trace,
]);
clients.push(service);
try {
await service.start();
const opened = await service.request<{
resource: string;
epoch: number;
operation: string;
}>("resource.open", { profile: "basic" });
await service.waitForOperation(opened.operation, { timeoutMs: 1_000 });
const synthesis = await service.request<{ operation: string }>(
"synthesis.start",
{ resource: opened.resource, epoch: opened.epoch },
);
await service.request("synthesis.append", {
operation: synthesis.operation,
index: 0,
text: "Arbitrary prose stays payload text.",
});
const requests = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as { direction: string; value: object })
.filter((entry) => entry.direction === "inbound")
.map((entry) => entry.value as { method?: string; params?: object });
const append = requests.find(
(request) => request.method === "synthesis.append",
);
expect(Object.keys(append?.params ?? {}).sort()).toEqual([
"index",
"operation",
"text",
]);
await service.close();
const stopped = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(stopped).toContainEqual({
direction: "lifecycle",
value: { event: "exit", code: 0 },
});
} finally {
await service.close();
await rm(root, { recursive: true, force: true });
}
});
it("renders a long provenance-backed dictation through the public Voice command", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-long-caption-"));
try {
session = await createPiSession({
extensionPaths: [voicePath],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"long-caption",
"--scenario-file",
journeys,
]),
},
});
const { screens } = recordingUI(session, 44, 8);
await session.session.prompt(
`/voice start --root ${join(root, "history")}`,
);
await vi.waitFor(() =>
expect(screens.join("\n")).toContain("points of his situation"),
);
expect(session.providerState.callCount).toBe(0);
expect(session.session.messages).toEqual([]);
} finally {
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
}, 10000);
it("treats silence as an observed empty boundary without a model turn", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-silence-"));
const trace = join(root, "wire.ndjson");
try {
session = await createPiSession({
extensionPaths: [voicePath],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"silence",
"--wire-trace",
trace,
]),
},
});
const { getEditorText, setEditorText } = recordingUI(session, 44, 8);
await session.session.prompt(
`/voice start --root ${join(root, "history")}`,
);
await vi.waitFor(async () => {
const entries = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(
entries.some(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "recognition.empty",
),
).toBe(true);
});
expect(session.providerState.callCount).toBe(0);
expect(session.session.messages).toEqual([]);
expect(getEditorText).not.toHaveBeenCalled();
expect(setEditorText).not.toHaveBeenCalled();
} finally {
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
}, 10000);
it("mutes an observed preview, then rejects its released final across unmute", async () => {
const root = await mkdtemp(join(tmpdir(), "pi-voice-mute-fence-"));
const trace = join(root, "wire.ndjson");
const release = join(root, "preview.release");
try {
session = await createPiSession({
extensionPaths: [voicePath],
env: {
PI_VOICE_MODEL: "faux/faux-model",
PI_VOICE_CLASSIFIER_MODEL: "faux/unavailable-classifier",
PI_VOICE_COMMAND: JSON.stringify([
process.execPath,
fake,
"--scenario",
"ui-preview",
"--preview-release",
release,
"--wire-trace",
trace,
]),
},
});
const { screens, getEditorText, setEditorText } = recordingUI(
session,
44,
8,
);
await session.session.prompt(
`/voice start --root ${join(root, "history")}`,
);
await vi.waitFor(() => expect(screens.join("\n")).toContain("hello world"));
await session.session.prompt("/voice mute");
await writeFile(release, "release cancelled input\n", { flag: "wx" });
await session.session.prompt("/voice unmute");
await session.session.prompt("/voice stop");
const wire = (await readFile(trace, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(session.providerState.callCount).toBe(0);
expect(session.session.messages).toEqual([]);
expect(
wire.some(
(entry) =>
entry.direction === "inbound" &&
entry.value.method === "resource.set_muted" &&
entry.value.params.muted === true,
),
).toBe(true);
expect(
wire.filter(
(entry) =>
entry.direction === "inbound" &&
entry.value.method === "recognition.start",
),
).toHaveLength(2);
expect(
wire.some(
(entry) =>
entry.direction === "outbound" &&
entry.value.method === "recognition.final",
),
).toBe(false);
expect(getEditorText).not.toHaveBeenCalled();
expect(setEditorText).not.toHaveBeenCalled();
} finally {
await session?.dispose();
session = undefined;
await rm(root, { recursive: true, force: true });
}
}, 10000);
it("declines blocked fake setup without asking the service to install assets", async () => {
const service = client("consent");
const info = await service.start();
const profile = info.profiles[0];
if (!profile) throw new Error("Fake profile missing.");
const plan = await service.request<{ plan: { blockers: string[] } }>(
"assets.plan",
{ profile: profile.id },
);
expect(plan.plan.blockers).toEqual(["approval required"]);
await expect(
service.request("assets.install", {
plan: "plan",
approved: true,
accepted_licenses: [],
}),
).rejects.toThrow("Consent required");
});