repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/voice/__serial__/speech-backend.test.ts
Rawimport { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import {
buildSessionProjection,
createEventBus,
} from "@earendil-works/pi-coding-agent";
import { afterEach, expect, it, vi } from "vitest";
import { VoiceController } from "../src/controller.ts";
import { NativeVoiceCall } from "../src/native-call.ts";
import { SpeakClient } from "../src/speak-client.ts";
import { NativeSpeechProcess } from "../src/speech-backend.ts";
import type {
NativeSpeechEvent,
PreparedSpeech,
} from "../src/speech-events.ts";
const fake = fileURLToPath(
new URL("../__e2e__/fake-native.mjs", import.meta.url),
);
const peer = fileURLToPath(
new URL("./fixtures/speech-peer.mjs", import.meta.url),
);
const cleanups: Array<() => Promise<unknown>> = [];
afterEach(async () => {
for (const cleanup of cleanups.splice(0).reverse()) await cleanup();
});
async function directory() {
const path = await mkdtemp(join(tmpdir(), "speech-backend-"));
cleanups.push(() => rm(path, { recursive: true, force: true }));
return path;
}
async function releaseGate() {
const path = join(await directory(), "release");
await writeFile(path, "locked");
return { path, release: () => rm(path, { force: true }) };
}
async function launch(command: string[], trace: string) {
const client = new SpeakClient(command);
cleanups.push(() => client.close());
const info = await client.start();
const profile = info.profiles[0];
if (!profile) throw new Error("Missing test profile");
const backend: PreparedSpeech = { client, command, profile, preview: true };
const speech = new NativeSpeechProcess();
cleanups.push(() => speech.close());
const events: NativeSpeechEvent[] = [];
return {
speech,
backend,
events,
start: () =>
speech.start({ backend, onEvent: (event) => events.push(event) }),
wire: async () =>
(await readFile(trace, "utf8"))
.trim()
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line)),
};
}
async function logical(scenario = "quiet") {
const trace = join(await directory(), "wire.ndjson");
const value = await launch(
[process.execPath, fake, "--scenario", scenario, "--wire-trace", trace],
trace,
);
return {
...value,
requests: async () =>
(await value.wire())
.filter((entry) => entry.direction === "inbound")
.map((entry) => entry.value),
};
}
const capabilities = {
capture: true,
render: true,
aec: true,
recognition: true,
preview: true,
hotwords: true,
synthesis: true,
sounds: true,
test_io: true,
};
const info = {
instance: "scripted",
io_mode: "fake",
profiles: [
{
id: "profile",
label: "Scripted",
description: "Adversarial test peer",
installed: true,
capabilities,
},
],
devices: [],
sounds: [
{
id: "opaque",
label: "Cue",
installed: true,
sha256:
"9237a5856d97f7f1a2eadd7e6047b7a3357ea5273a63382fbeba64477f42722b",
},
],
capabilities,
limits: {
line_bytes: 65536,
pending_requests: 32,
active_operations: 8,
resources: 1,
text_chars: 8000,
queued_text_chars: 16000,
unacked_segments: 32,
hotwords: 20,
hotword_chars: 256,
waveform_bins: 32,
output_queue_bytes: 262144,
instance_requests: 16384,
instance_operations: 4096,
},
};
type Notice = { method: string; params: Record<string, unknown> };
type Step = {
method: string;
result?: object;
params?: object;
after?: Notice[];
before?: Notice[];
releaseFile?: { path: string; afterResult?: boolean };
error?: object;
};
function notice(
method: string,
operation: string | null,
data: object,
epoch = 1,
): Notice {
return { method, params: { operation, epoch, data } };
}
function ended(
operation: string,
outcome = "completed",
epoch = 1,
stopFrame: number | null = null,
): Notice {
return notice(
"operation.ended",
operation,
{
outcome,
reason: outcome,
stop_frame: stopFrame,
error:
outcome === "failed"
? {
kind: "internal",
retryable: false,
field: null,
resource: "device",
operation,
detail: "Failed test operation",
}
: null,
},
epoch,
);
}
const open: Step = {
method: "resource.open",
result: { resource: "device", epoch: 1, operation: "open" },
after: [
notice("resource.ready", "open", {
capture: true,
render: true,
speech: true,
capture_rate: 16000,
render_rate: 24000,
}),
ended("open"),
],
};
const final = (segment = "segment", operation = "recognition", epoch = 1) =>
notice(
"recognition.final",
operation,
{ segment, text: "Stable words.", start_frame: 0, end_frame: 16000 },
epoch,
);
async function scripted(
steps: Step[],
loading = false,
opening: Step = open,
mode = "fake",
) {
const dir = await directory();
const trace = join(dir, "wire.ndjson");
const script = join(dir, "script.json");
await writeFile(
script,
JSON.stringify({
trace,
steps: [
{ method: "service.info", result: { ...info, io_mode: mode } },
{ method: "service.info", result: { ...info, io_mode: mode } },
opening,
...steps,
],
}),
);
const value = await launch([process.execPath, peer, script], trace);
if (!loading) await value.speech.setCueState("off");
return { ...value, requests: value.wire };
}
it.each(["health", "recognition"])(
"refuses automatic recovery for fixture-mode %s failure",
async (failure) => {
const after =
failure === "health"
? notice(
"resource.health",
null,
{
status: "unavailable",
capture_frame: 0,
render_frame: 0,
error: {
kind: "device",
retryable: false,
field: null,
resource: "device",
operation: null,
detail: "injected fault",
},
},
2,
)
: ended("recognition", "failed");
const test = await scripted(
[
{
method: "recognition.start",
result: { operation: "recognition" },
after: [after],
},
],
false,
open,
"fixture",
);
await test.start();
await vi.waitFor(() =>
expect(test.events.some((event) => event.type === "error")).toBe(true),
);
expect(test.events.filter((event) => event.type === "error")).toEqual([
expect.objectContaining({ recoverable: false }),
]);
},
);
it("uses real child transport for recognition acceptance/ack and consumed-only playback", async () => {
const test = await logical("greeting");
await test.speech.setHotwords(["Nugu"]);
await test.start();
await vi.waitFor(() =>
expect(test.events.some((event) => event.type === "final")).toBe(true),
);
const stable = test.events.find((event) => event.type === "final");
if (stable?.type !== "final") throw new Error("Missing final");
expect(test.events[0]).toMatchObject({ type: "ready" });
expect(
(await test.requests()).some(
(request) => request.method === "recognition.ack",
),
).toBe(false);
expect(test.events.some((event) => event.type === "playback")).toBe(false);
await test.speech.consumeFinal(stable.generation, stable.requestId);
await test.speech.speak({
assistantTurnId: "private-assistant",
userTurnId: "private-user",
text: "A reply.",
});
await test.speech.complete("private-assistant");
await vi.waitFor(() =>
expect(test.events.some((event) => event.type === "boundary")).toBe(true),
);
const playback = test.events.filter((event) => event.type === "playback");
expect(playback.at(-1)).toMatchObject({
playedText: "A reply.",
cumulativeMs: 1000,
assistantTurnId: "private-assistant",
});
expect(test.events.indexOf(playback.at(-1)!)).toBeLessThan(
test.events.findIndex((event) => event.type === "boundary"),
);
const requests = await test.requests();
expect(
requests.find((request) => request.method === "resource.open").params,
).toMatchObject({
endpoint_ms: 2000,
aec: true,
preview: true,
interrupt_on_speech: true,
});
expect(
requests.find((request) => request.method === "recognition.ack").params,
).toEqual({ operation: expect.any(String), segment: "segment" });
expect(
requests.find((request) => request.method === "recognition.set_hotwords")
.params.terms,
).toEqual(["Nugu"]);
expect(JSON.stringify(requests)).not.toMatch(
/private-assistant|private-user|testAudioEndpoint|audioAddonPath/,
);
});
it("captures batched resource/recognition identities, revision one, stale previews and stable finals exactly once", async () => {
const test = await scripted([
{
method: "recognition.start",
result: { operation: "recognition" },
after: [
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 1,
text: "first",
}),
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 1,
text: "duplicate",
}),
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 3,
text: "new",
}),
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 2,
text: "old",
}),
notice("recognition.preview_cleared", "recognition", {
segment: "segment",
through_revision: 3,
}),
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 3,
text: "cleared",
}),
final(),
final(),
notice("recognition.preview", "recognition", {
segment: "segment",
revision: 4,
text: "after final",
}),
],
},
{ method: "synthesis.start", result: { operation: "synthesis" } },
{
method: "synthesis.append",
result: { accepted_index: 0 },
after: [
notice("synthesis.phrase", "synthesis", {
index: 0,
text: "Candidate only.",
}),
],
},
{
method: "synthesis.finish",
result: { accepted: true },
after: [
notice("playback.progress", "synthesis", {
consumed_frames: 24000,
through_frame: 24000,
played_text: "",
}),
notice("playback.boundary", "synthesis", { through_frame: 24000 }),
ended("synthesis"),
],
},
]);
await test.start();
expect(test.events[0]).toMatchObject({ type: "ready" });
expect(
test.events
.filter((event) => event.type === "preview")
.map((event) => event.text),
).toEqual(["first", "new"]);
expect(test.events.filter((event) => event.type === "final")).toHaveLength(1);
await test.speech.speak({
assistantTurnId: "assistant",
userTurnId: "user",
text: "Candidate only.",
});
expect(test.events.some((event) => event.type === "playback")).toBe(false);
await test.speech.complete("assistant");
expect(test.events.filter((event) => event.type === "playback")).toEqual([
{
type: "playback",
assistantTurnId: "assistant",
userTurnId: "user",
playedText: "",
cumulativeMs: 1000,
},
]);
});
it("barge-in cancels pending synthesis without losing stable recognition awaiting acceptance", async () => {
const gate = await releaseGate();
const test = await scripted([
{
method: "recognition.start",
result: { operation: "recognition" },
after: [final()],
},
{
method: "synthesis.start",
result: { operation: "synthesis" },
releaseFile: { path: gate.path },
},
{
method: "operation.cancel",
params: { operation: "synthesis" },
result: { outcome: "cancelled", stop_frame: 0 },
before: [ended("synthesis", "cancelled")],
},
{
method: "recognition.ack",
params: { operation: "recognition", segment: "segment" },
result: { acknowledged: true },
},
]);
await test.start();
const speaking = test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Never append.",
});
await vi.waitFor(async () =>
expect(
(await test.requests()).some(
(request) => request.method === "synthesis.start",
),
).toBe(true),
);
await test.speech.interrupt();
await gate.release();
await speaking;
const stable = test.events.find((event) => event.type === "final");
if (stable?.type !== "final") throw new Error("Missing final");
await test.speech.consumeFinal(stable.generation, stable.requestId);
const requests = await test.requests();
expect(
requests
.filter((request) => request.method === "operation.cancel")
.map((request) => request.params.operation),
).toEqual(["synthesis"]);
expect(
requests.some((request) => request.method === "synthesis.append"),
).toBe(false);
});
it("mute preserves final consumed accounting, discards retained recognition, and readies fresh recognition before batched unmute input", async () => {
const test = await scripted([
{
method: "recognition.start",
result: { operation: "recognition" },
after: [final()],
},
{ method: "synthesis.start", result: { operation: "synthesis" } },
{ method: "synthesis.append", result: { accepted_index: 0 } },
{
method: "resource.set_muted",
params: { resource: "device", epoch: 1, muted: true },
result: { epoch: 2, muted: true },
before: [
notice("playback.progress", "synthesis", {
consumed_frames: 12000,
through_frame: 12000,
played_text: "",
}),
ended("synthesis", "cancelled"),
ended("recognition", "cancelled"),
],
},
{
method: "resource.set_muted",
params: { resource: "device", epoch: 2, muted: false },
result: { epoch: 3, muted: false },
},
{
method: "recognition.start",
params: { resource: "device", epoch: 3 },
result: { operation: "fresh" },
after: [
final("stale", "recognition", 1),
final("wrong-operation", "recognition", 3),
final("fresh-segment", "fresh", 3),
],
},
{
method: "recognition.ack",
params: { operation: "fresh", segment: "fresh-segment" },
result: { acknowledged: true },
},
]);
await test.start();
const old = test.events.find((event) => event.type === "final");
await test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Partly consumed.",
});
await test.speech.setMuted(true);
if (old?.type !== "final") throw new Error("Missing old final");
await test.speech.consumeFinal(old.generation, old.requestId);
expect(test.events.find((event) => event.type === "playback")).toMatchObject({
playedText: "",
cumulativeMs: 500,
});
await test.speech.setMuted(false);
const finals = test.events.filter((event) => event.type === "final");
expect(finals).toHaveLength(2);
const fresh = finals[1]!;
expect(fresh.generation).toBe(3);
expect(test.events[test.events.indexOf(fresh) - 1]).toMatchObject({
type: "ready",
muted: false,
});
await test.speech.consumeFinal(fresh.generation, fresh.requestId);
expect(
(await test.requests()).filter(
(request) => request.method === "recognition.ack",
),
).toHaveLength(1);
});
it("does not let a delayed old acknowledgement erase a fresh final reusing its segment identity", async () => {
const gate = await releaseGate();
const test = await scripted([
{
method: "recognition.start",
result: { operation: "recognition" },
after: [final()],
},
{
method: "recognition.ack",
params: { operation: "recognition", segment: "segment" },
result: { acknowledged: true },
releaseFile: { path: gate.path },
},
{
method: "resource.set_muted",
result: { epoch: 2, muted: true },
before: [ended("recognition", "cancelled")],
},
{ method: "resource.set_muted", result: { epoch: 3, muted: false } },
{
method: "recognition.start",
result: { operation: "fresh" },
after: [final("segment", "fresh", 3)],
},
{
method: "recognition.ack",
params: { operation: "fresh", segment: "segment" },
result: { acknowledged: true },
},
]);
await test.start();
const old = test.events.find((event) => event.type === "final");
if (old?.type !== "final") throw new Error("Missing old final");
const acknowledging = test.speech.consumeFinal(old.generation, old.requestId);
await vi.waitFor(async () =>
expect(
(await test.requests()).some(
(request) => request.method === "recognition.ack",
),
).toBe(true),
);
await test.speech.setMuted(true);
await gate.release();
await acknowledging;
await test.speech.setMuted(false);
const fresh = test.events.filter((event) => event.type === "final").at(-1)!;
await test.speech.consumeFinal(fresh.generation, fresh.requestId);
expect(
(await test.requests())
.filter((request) => request.method === "recognition.ack")
.map((request) => request.params.operation),
).toEqual(["recognition", "fresh"]);
});
it("off fences an in-flight loading cue and cancels only its accepted opaque operation", async () => {
const gate = await releaseGate();
const test = await scripted(
[
{
method: "sound.play",
params: {
resource: "device",
epoch: 1,
sound: "opaque",
gain: 0.3,
repeat_ms: 2000,
},
result: { operation: "cue" },
releaseFile: { path: gate.path },
after: open.after!,
},
{
method: "operation.cancel",
params: { operation: "cue" },
result: { outcome: "cancelled", stop_frame: 0 },
before: [ended("cue", "cancelled")],
},
{ method: "recognition.start", result: { operation: "recognition" } },
],
true,
{ ...open, after: [] },
);
const starting = test.start();
await vi.waitFor(async () =>
expect(
(await test.requests()).some(
(request) => request.method === "sound.play",
),
).toBe(true),
);
const turningOff = test.speech.setCueState("off");
await gate.release();
await turningOff;
await starting;
expect(
(await test.requests())
.filter((request) => request.method === "operation.cancel")
.map((request) => request.params.operation),
).toEqual(["cue"]);
});
it("contains cue cancellation errors and coalesces rapid state changes without replacing speech", async () => {
const test = await scripted([
{ method: "recognition.start", result: { operation: "recognition" } },
{ method: "sound.play", result: { operation: "cue" } },
{
method: "operation.cancel",
params: { operation: "cue" },
error: { code: -32005, message: "Cue unavailable" },
},
{ method: "synthesis.start", result: { operation: "speech" } },
{ method: "synthesis.append", result: { accepted_index: 0 } },
]);
await test.start();
await test.speech.setCueState("loading");
await expect(test.speech.setCueState("off")).resolves.toBeUndefined();
const changes = Array.from({ length: 128 }, () =>
test.speech.setCueState("loading"),
);
changes.push(test.speech.setCueState("off"));
await Promise.all(changes);
await test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Still usable.",
});
expect(test.events.some((event) => event.type === "warning")).toBe(true);
expect(test.events.some((event) => event.type === "error")).toBe(false);
expect(
(await test.requests()).filter(
(request) => request.method === "sound.play",
),
).toHaveLength(1);
});
it("loops catalog-matched loading/listening/thinking/working cues without cancelling speech", async () => {
const test = await logical();
await test.start();
for (const state of ["listening", "thinking", "working"] as const)
await test.speech.setCueState(state);
await test.speech.setCueState("off");
await test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Pending speech.",
});
await test.speech.setCueState("listening");
const requests = await test.requests();
const sounds = requests.filter((request) => request.method === "sound.play");
expect(sounds.map((request) => request.params.sound)).toEqual([
"done",
"bell",
"ready",
"error",
]);
expect(
sounds.every(
(request) =>
request.params.repeat_ms >= 1800 && request.params.repeat_ms <= 2400,
),
).toBe(true);
const speech = requests.find(
(request) => request.method === "synthesis.append",
).params.operation;
expect(
requests
.filter((request) => request.method === "operation.cancel")
.some((request) => request.params.operation === speech),
).toBe(false);
});
it("restarts a closed prepared client, rechecks stable profile and assets, and never replays output or installs", async () => {
const test = await logical();
const previous = test.backend.client;
const instance = previous.serviceInfo?.instance;
await previous.close();
// A setup preview can precede installation; installed is readiness, not identity.
test.backend.profile = {
...test.backend.profile,
installed: false,
capabilities: Object.fromEntries(
Object.entries(test.backend.profile.capabilities).reverse(),
) as typeof capabilities,
};
await test.start();
expect(test.backend.client).not.toBe(previous);
expect(test.backend.client.serviceInfo?.instance).not.toBe(instance);
const methods = (await test.requests()).map((request) => request.method);
expect(methods.filter((method) => method === "assets.plan")).toHaveLength(1);
expect(methods).not.toContain("assets.install");
expect(methods).not.toContain("synthesis.start");
});
it("closes a failed profile selection without opening devices", async () => {
const test = await logical();
test.backend.profile = { ...test.backend.profile, label: "Changed profile" };
await expect(test.start()).rejects.toThrow("incompatible");
await expect(test.backend.client.start()).rejects.toThrow("closed");
expect(
(await test.requests()).map((request) => request.method),
).not.toContain("resource.open");
});
it("rejects recovery requiring assets instead of downloading or replaying", async () => {
const test = await logical("needs-setup");
await test.backend.client.close();
await expect(test.start()).rejects.toThrow("explicit setup");
const methods = (await test.requests()).map((request) => request.method);
expect(methods).not.toContain("resource.open");
expect(methods).not.toContain("assets.install");
});
it("retires a synthesis terminal delivered in its start response batch without resurrecting that turn", async () => {
const test = await scripted([
{ method: "recognition.start", result: { operation: "recognition" } },
{
method: "synthesis.start",
result: { operation: "failed" },
after: [ended("failed", "failed")],
},
{ method: "synthesis.start", result: { operation: "next" } },
{
method: "synthesis.append",
params: { operation: "next", index: 0, text: "Next." },
result: { accepted_index: 0 },
},
]);
await test.start();
await test.speech.speak({
assistantTurnId: "failed-reply",
userTurnId: "user",
text: "Never append.",
});
await test.speech.complete("failed-reply");
await test.speech.speak({
assistantTurnId: "next-reply",
userTurnId: "user",
text: "Next.",
});
expect(
(await test.requests())
.filter((request) => request.method === "synthesis.append")
.map((request) => request.params.operation),
).toEqual(["next"]);
});
it("serializes concurrent chunks and waits for a finished stream to drain before opening the next", async () => {
const test = await logical();
await test.speech.setCueState("off");
await test.start();
await Promise.all([
test.speech.speak({
assistantTurnId: "first",
userTurnId: "user1",
text: "One. ",
}),
test.speech.speak({
assistantTurnId: "first",
userTurnId: "user1",
text: "Two.",
}),
test.speech.complete("first"),
test.speech.speak({
assistantTurnId: "second",
userTurnId: "user2",
text: "Next.",
}),
test.speech.complete("second"),
]);
await vi.waitFor(() =>
expect(
test.events.filter((event) => event.type === "boundary"),
).toHaveLength(2),
);
const appends = (await test.requests()).filter(
(request) => request.method === "synthesis.append",
);
expect(
appends.map((request) => [request.params.index, request.params.text]),
).toEqual([
[0, "One. "],
[1, "Two."],
[0, "Next."],
]);
expect(
test.events
.filter((event) => event.type === "playback")
.map((event) => event.playedText),
).toEqual(["One. Two.", "One. Two.", "Next.", "Next."]);
});
it("separates adjacent prose fragments, preserves explicit whitespace, and starts new turns unchanged", async () => {
const test = await logical();
await test.speech.setCueState("off");
await test.start();
await test.speech.speak({
assistantTurnId: "first",
userTurnId: "u1",
text: "One.",
});
await test.speech.speak({
assistantTurnId: "first",
userTurnId: "u1",
text: "Two.",
});
await test.speech.speak({
assistantTurnId: "first",
userTurnId: "u1",
text: " Three.",
});
await test.speech.complete("first");
await vi.waitFor(() =>
expect(
test.events.filter((event) => event.type === "boundary"),
).toHaveLength(1),
);
await test.speech.speak({
assistantTurnId: "second",
userTurnId: "u2",
text: "Next.",
});
const appends = (await test.requests()).filter(
(request) => request.method === "synthesis.append",
);
expect(appends.map((request) => request.params.text)).toEqual([
"One.",
" Two.",
" Three.",
"Next.",
]);
});
it("counts inserted prose separation against the cumulative scalar limit and cancels overflow", async () => {
const test = await logical();
await test.speech.setCueState("off");
await test.start();
await test.speech.speak({
assistantTurnId: "large",
userTurnId: "user",
text: "x".repeat(8000),
});
await expect(
test.speech.speak({
assistantTurnId: "large",
userTurnId: "user",
text: "Next.",
}),
).rejects.toThrow("8000");
const requests = await test.requests();
expect(
requests.filter((request) => request.method === "synthesis.append"),
).toHaveLength(1);
expect(
requests.some((request) => request.method === "operation.cancel"),
).toBe(true);
});
it("rejects oversized streams and bounds queued output, then interrupts every queued command without replay", async () => {
const test = await logical();
await test.speech.setCueState("off");
await test.start();
await expect(
test.speech.speak({
assistantTurnId: "large",
userTurnId: "user",
text: "x".repeat(8001),
}),
).rejects.toThrow("8000");
const work = Array.from({ length: 64 }, () =>
test.speech.speak({
assistantTurnId: "queued",
userTurnId: "user",
text: "Must not replay.",
}),
);
await expect(
test.speech.speak({
assistantTurnId: "overflow",
userTurnId: "user",
text: "Overflow.",
}),
).rejects.toThrow("queue is full");
await test.speech.interrupt();
await Promise.all(work);
expect(
(await test.requests()).some(
(request) => request.method === "synthesis.append",
),
).toBe(false);
});
it("bounds unaccepted finals without losing the retained prefix or synthesizing text", async () => {
const test = await scripted([
{
method: "recognition.start",
result: { operation: "recognition" },
after: Array.from({ length: 33 }, (_, index) => final(`segment${index}`)),
},
]);
await test.start();
expect(test.events.filter((event) => event.type === "final")).toHaveLength(
32,
);
expect(test.events.at(-1)).toMatchObject({
type: "error",
recoverable: false,
});
expect(test.events.some((event) => event.type === "playback")).toBe(false);
});
it("defers unmuted recognition until delayed resource-open completion", async () => {
const gate = await releaseGate();
const test = await scripted(
[
{ method: "resource.set_muted", result: { epoch: 2, muted: true } },
{ method: "resource.set_muted", result: { epoch: 3, muted: false } },
{
method: "recognition.start",
params: { resource: "device", epoch: 3 },
result: { operation: "recognition" },
after: [final("fresh", "recognition", 3)],
},
],
false,
{
...open,
releaseFile: { path: gate.path, afterResult: true },
after: open.after!.map((event) => ({
...event,
params: { ...event.params, epoch: 3 },
})),
},
);
const starting = test.start();
await vi.waitFor(async () =>
expect(
(await test.requests()).some((req) => req.method === "resource.open"),
).toBe(true),
);
await test.speech.setMuted(true);
const unmuting = test.speech.setMuted(false);
await vi.waitFor(async () =>
expect(
(await test.requests()).some(
(req) =>
req.method === "resource.set_muted" && req.params.muted === false,
),
).toBe(true),
);
expect(
(await test.requests()).some((req) => req.method === "recognition.start"),
).toBe(false);
await gate.release();
await unmuting;
await starting;
expect(test.events.filter((event) => event.type === "final")).toHaveLength(1);
expect(test.events.at(-2)).toMatchObject({ type: "ready", muted: false });
});
it("closes an already-running prepared child on history initialization failure or stop before handoff", async () => {
for (const failHistory of [true, false]) {
const test = await logical();
const beginNative = vi.fn(() => {
if (failHistory) throw new Error("history failed");
});
const call = new NativeVoiceCall({
beginNative,
endNative: () => {},
nativeSnapshot: () => ({ error: null }),
} as never);
const starting = call.start({} as never, {
backend: test.backend,
modelSetting: "auto",
historyDirectory: await directory(),
});
if (failHistory) await expect(starting).rejects.toThrow("history failed");
else {
await call.stop();
await starting;
}
await expect(test.backend.client.start()).rejects.toThrow("closed");
expect(
(await test.requests()).some((req) => req.method === "resource.open"),
).toBe(false);
expect(
(await test.requests()).some((req) => req.method === "service.shutdown"),
).toBe(true);
}
});
it("retains final certified consumed history through shared close while fencing recognition and UI", async () => {
const progress = (text: string, frames: number) =>
notice("playback.progress", "speech", {
consumed_frames: frames,
through_frame: frames,
played_text: text,
});
const test = await scripted([
{ method: "recognition.start", result: { operation: "recognition" } },
{ method: "synthesis.start", result: { operation: "speech" } },
{
method: "synthesis.append",
result: { accepted_index: 0 },
after: [progress("Hello.", 12000)],
},
{
method: "synthesis.append",
params: { operation: "speech", index: 1, text: " More." },
result: { accepted_index: 1 },
},
{
method: "synthesis.append",
params: { operation: "speech", index: 2, text: " Tail." },
result: { accepted_index: 2 },
},
{ method: "recognition.set_hotwords", result: { accepted: true } },
{ method: "synthesis.finish", result: { accepted: true } },
{
method: "resource.close",
result: { closed: true },
before: [
progress("Hello. More.", 24000),
final("must-not-replay"),
ended("speech", "cancelled", 1, 24000),
ended("recognition", "cancelled"),
notice("resource.closed", null, {
capture_frame: 16000,
render_frame: 24000,
}),
],
},
]);
const root = await directory();
const editor = vi.fn(() => {
throw new Error("editor accessed");
});
const ctx = {
cwd: root,
isIdle: () => false,
abort: async () => {},
ui: { setEditorText: editor, getEditorText: editor },
sessionManager: {
getSessionId: () => "main",
getSessionName: () => "",
getBranch: () => [],
buildSessionProjection: () => buildSessionProjection([]),
},
};
const control = new VoiceController({
events: createEventBus(),
sendUserMessage: vi.fn(),
} as never);
cleanups.push(async () => control.dispose());
const child = {
refreshContext: () => {},
setActiveToolsByName: () => {},
compact: () => {},
prompt: async () => {
for (const delta of [
'{"heard":"hello","reply":"Hello.',
" More.",
' Tail.","hotwords":[]}',
]) {
Reflect.get(control, "consumeStructuredDelta").call(
control,
Reflect.get(control, "activeTurn"),
delta,
);
}
},
};
Reflect.set(control, "ensureChild", async () => child);
const call = new NativeVoiceCall(control, () => test.speech);
cleanups.push(() => call.stop());
await call.start(ctx as never, {
backend: test.backend,
historyDirectory: root,
modelSetting: "auto",
});
control.receiveNative("hello");
await vi.waitFor(async () =>
expect(
(await test.requests()).some((req) => req.method === "synthesis.finish"),
).toBe(true),
);
expect(
control.nativeHistory().find((entry) => entry.role === "assistant"),
).toMatchObject({ playedText: "Hello." });
expect(
(await test.requests())
.filter((req) => req.method === "synthesis.append")
.map((req) => req.params.text),
).toEqual(["Hello.", " More.", " Tail."]);
const stopping = call.stop();
expect(call.stop()).toBe(stopping);
await stopping;
expect(
control.nativeHistory().find((entry) => entry.role === "assistant"),
).toMatchObject({ playedText: "Hello. More.", playback: "partial" });
expect(
control.nativeHistory().filter((entry) => entry.role === "user"),
).toHaveLength(1);
expect(editor).not.toHaveBeenCalled();
expect(
(await test.requests()).filter((req) => req.method === "resource.close"),
).toHaveLength(1);
});
it("fences pending output and cue work during a shared healthy close", async () => {
const synthesisGate = await releaseGate();
const closeGate = await releaseGate();
const test = await scripted([
{ method: "recognition.start", result: { operation: "recognition" } },
{
method: "synthesis.start",
result: { operation: "speech" },
releaseFile: { path: synthesisGate.path },
},
{
method: "resource.close",
result: { closed: true },
releaseFile: { path: closeGate.path },
before: [
ended("speech", "cancelled", 1, 0),
ended("recognition", "cancelled"),
notice("resource.closed", null, { capture_frame: 0, render_frame: 0 }),
],
},
]);
await test.start();
const speaking = test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Never append.",
});
const completing = test.speech.complete("reply");
await vi.waitFor(async () =>
expect(
(await test.requests()).some((req) => req.method === "synthesis.start"),
).toBe(true),
);
const closing = test.speech.close();
expect(test.speech.close()).toBe(closing);
await vi.waitFor(async () =>
expect(
(await test.requests()).some((req) => req.method === "resource.close"),
).toBe(true),
);
await synthesisGate.release();
await closeGate.release();
await Promise.all([
closing,
speaking,
completing,
test.speech.setCueState("loading"),
]);
const methods = (await test.requests()).map((req) => req.method);
expect(methods).not.toContain("synthesis.append");
expect(methods).not.toContain("synthesis.finish");
expect(methods).not.toContain("sound.play");
expect(methods).not.toContain("operation.cancel");
expect(test.events.filter((event) => event.type === "error")).toHaveLength(0);
});
it("reconciles desired cue after synthesis terminal without competing with active speech", async () => {
const test = await scripted([
{ method: "recognition.start", result: { operation: "recognition" } },
{ method: "sound.play", result: { operation: "cue" } },
{
method: "synthesis.start",
result: { operation: "speech" },
before: [ended("cue", "cancelled")],
},
{ method: "synthesis.append", result: { accepted_index: 0 } },
{
method: "synthesis.finish",
result: { accepted: true },
after: [ended("speech")],
},
{
method: "sound.play",
result: { operation: "resumed" },
after: [
notice("audio.level", null, {
input: 0,
output: 0,
waveform: [],
playing: false,
busy: true,
}),
],
},
]);
await test.start();
await test.speech.setCueState("loading");
await test.speech.speak({
assistantTurnId: "reply",
userTurnId: "user",
text: "Words.",
});
await test.speech.setCueState("loading");
expect(
(await test.requests()).filter((req) => req.method === "sound.play"),
).toHaveLength(1);
await test.speech.complete("reply");
await vi.waitFor(() =>
expect(
test.events.some(
(event) => event.type === "audio-state" && event.cueOnly && !event.busy,
),
).toBe(true),
);
expect(
(await test.requests()).filter((req) => req.method === "sound.play"),
).toHaveLength(2);
});