repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/voice/src/speech-backend.ts
Rawimport type { ServiceInfo, SpeakEvent } from "./speak-client.ts";
import { SpeakClient } from "./speak-client.ts";
import type {
NativeSpeechEvent,
NativeSpeechOptions,
PreparedSpeech,
SpeechRequest,
} from "./speech-events.ts";
import { supportsSpeechProfile } from "./speech-events.ts";
import { CUE_DIGESTS, type CueState } from "./voice-sounds.ts";
const OPEN_TIMEOUT_MS = 180_000;
const CONTROL_TIMEOUT_MS = 5_000;
const MAX_FINALS = 32;
const MAX_STREAMS = 1;
const MAX_SEGMENTS = 4096;
const MAX_SCALARS = 8_000;
const MAX_OUTPUT_WORK = 64;
type Turn = {
assistantTurnId: string;
userTurnId: string;
operation: string;
index: number;
finished: boolean;
chars: number;
endsWithWhitespace: boolean;
};
type Segment = {
local: number;
revision: number;
ordinal: number;
final: boolean;
};
/** Maps generic RPC identities to call-local UI and conversation identities. */
export class NativeSpeechProcess {
private client: SpeakClient | undefined;
private info: ServiceInfo | undefined;
private onEvent: ((event: NativeSpeechEvent) => void) | undefined;
private unsubscribe: (() => void) | undefined;
private unfail: (() => void) | undefined;
private resource: string | undefined;
private epoch = 0;
private recognition: string | undefined;
private renderRate = 0;
private muted = false;
private closed = false;
private opened = false;
private recognitionPending = false;
private captureActive = false;
private hotwords: readonly string[] = [];
private nextCaption = 0;
private nextOrdinal = 0;
private segments = new Map<string, Segment>();
private finals = new Map<string, { generation: number; requestId: number }>();
private nextFinal = 0;
private turns = new Map<string, Turn>();
private byAssistant = new Map<string, Turn>();
private cueOperation: string | undefined;
private cueState: CueState = "loading";
private cueRevision = 0;
private cueWork = Promise.resolve();
private cueRunning = false;
private outputRevision = 0;
private outputWork = Promise.resolve();
private pendingOutput = 0;
private muteWork = Promise.resolve();
private pendingMute = 0;
private closing: Promise<void> | undefined;
async start(options: NativeSpeechOptions): Promise<void> {
if (this.client || this.closed)
throw new Error("Native speech process already used");
try {
await this.initialize(options);
} catch (error) {
await this.close();
throw error;
}
}
private async initialize(options: NativeSpeechOptions): Promise<void> {
this.onEvent = options.onEvent;
let client = options.backend.client;
let recovered = false;
this.client = client;
try {
await client.start();
} catch {
await client.close().catch(() => undefined);
if (this.closed) throw new Error("Speech backend is closed");
client = new SpeakClient(options.backend.command);
this.client = client;
options.backend.client = client;
recovered = true;
await client.start();
}
if (this.closed) throw new Error("Speech backend is closed");
if (recovered) {
const plan = await client.request<{ plan: { ready: boolean } }>(
"assets.plan",
{ profile: options.backend.profile.id },
);
if (!plan.plan.ready)
throw new Error("Speech assets need explicit setup before recovery");
}
this.info = await client.request<ServiceInfo>("service.info");
if (this.closed) throw new Error("Speech backend is closed");
this.unsubscribe = client.onEvent((event) => this.event(event));
this.unfail = client.onFailure((error) =>
this.emit({
type: "error",
error: error.message,
recoverable: this.info?.io_mode !== "fixture",
}),
);
const profile = this.info.profiles.find(
(item) => item.id === options.backend.profile.id,
);
if (
!profile?.installed ||
!sameProfile(profile, options.backend.profile) ||
!supportsSpeechProfile(profile, options.backend.preview)
)
throw new Error("Selected speech profile is unavailable or incompatible");
const openingMuted = this.muted;
const opened = await client.request<{
resource: string;
epoch: number;
operation: string;
}>(
"resource.open",
{
profile: profile.id,
aec: true,
preview: options.backend.preview,
endpoint_ms: 2000,
interrupt_on_speech: true,
muted: openingMuted,
},
{
timeoutMs: CONTROL_TIMEOUT_MS,
onResult: (result) => {
this.resource = result.resource;
this.epoch = result.epoch;
if (this.muted !== openingMuted)
void this.setMuted(this.muted).catch((error) =>
this.emit({
type: "error",
error: message(error),
recoverable: false,
}),
);
void this.setCueState(this.cueState);
},
},
);
const outcome = await client.waitForOperation(opened.operation, {
timeoutMs: OPEN_TIMEOUT_MS,
});
if (outcome.outcome !== "completed")
throw new Error(`Speech resource failed to open: ${outcome.reason}`);
if (this.closed) throw new Error("Speech backend is closed");
if (!this.renderRate) throw new Error("Speech resource has no render rate");
this.opened = true;
await this.muteWork;
if (!this.muted && !this.recognition) await this.startRecognition();
else if (this.muted) this.emit({ type: "ready", muted: true });
}
speak(request: SpeechRequest): Promise<void> {
return this.queueOutput(() => this.speakNow(request));
}
private async speakNow(request: SpeechRequest): Promise<void> {
if (request.text.length > MAX_SCALARS * 2)
throw new Error("Speech stream exceeds 8000 scalars");
if (!request.text.trim()) return;
const requestScalars = [...request.text].length;
if (requestScalars > MAX_SCALARS)
throw new Error("Speech stream exceeds 8000 scalars");
const client = this.readyClient();
if (this.muted) return;
const revision = this.outputRevision;
let turn = this.byAssistant.get(request.assistantTurnId);
if (!turn) {
for (const previous of this.turns.values()) {
if (previous.finished)
await client.waitForOperation(previous.operation, {
timeoutMs: OPEN_TIMEOUT_MS,
});
}
if (revision !== this.outputRevision || this.muted || this.closed) return;
if (this.turns.size >= MAX_STREAMS)
throw new Error("Speech stream queue is full");
await client.request<{ operation: string }>(
"synthesis.start",
this.fence(),
{
onResult: (result) => {
turn = {
assistantTurnId: request.assistantTurnId,
userTurnId: request.userTurnId,
operation: result.operation,
index: 0,
finished: false,
chars: 0,
endsWithWhitespace: false,
};
this.turns.set(turn.operation, turn);
this.byAssistant.set(turn.assistantTurnId, turn);
},
},
);
}
if (!turn) throw new Error("Speech stream was not accepted");
if (revision !== this.outputRevision || this.muted || this.closed) {
if (this.closed) return;
await this.cancel(turn.operation);
return;
}
if (!this.turns.has(turn.operation)) return;
if (turn.finished) throw new Error("Speech turn is already complete");
const separator =
turn.chars > 0 && !turn.endsWithWhitespace && !/^\s/u.test(request.text)
? " "
: "";
if (turn.chars + requestScalars + separator.length > MAX_SCALARS) {
await this.cancel(turn.operation);
throw new Error("Speech stream exceeds 8000 scalars");
}
const text = separator + request.text;
try {
await client.request("synthesis.append", {
operation: turn.operation,
index: turn.index,
text,
});
} catch (error) {
await this.cancel(turn.operation).catch(() => undefined);
throw error;
}
turn.index++;
turn.chars += requestScalars + separator.length;
turn.endsWithWhitespace = /\s$/u.test(request.text);
}
complete(assistantTurnId: string): Promise<void> {
return this.queueOutput(() => this.completeNow(assistantTurnId));
}
private async completeNow(assistantTurnId: string): Promise<void> {
const turn = this.byAssistant.get(assistantTurnId);
if (!turn || turn.finished) return;
turn.finished = true;
try {
await this.readyClient().request("synthesis.finish", {
operation: turn.operation,
});
} catch (error) {
await this.cancel(turn.operation).catch(() => undefined);
throw error;
}
}
async setHotwords(terms: readonly string[]): Promise<void> {
if (terms.length > 20 || terms.some((term) => [...term].length > 256))
throw new Error("Hotwords exceed limit");
this.hotwords = [...terms];
if (this.recognition)
await this.readyClient().request("recognition.set_hotwords", {
operation: this.recognition,
terms: this.hotwords,
});
}
async consumeFinal(generation: number, requestId: number): Promise<void> {
const entry = [...this.finals.entries()].find(
([, value]) =>
value.generation === generation && value.requestId === requestId,
);
if (!entry || !this.recognition) return;
const [segment, retained] = entry;
await this.readyClient().request("recognition.ack", {
operation: this.recognition,
segment,
});
if (this.finals.get(segment) === retained) this.finals.delete(segment);
}
async interrupt(): Promise<void> {
++this.outputRevision;
await Promise.all([
this.setCueState("off"),
...[...this.turns.keys()].map((operation) => this.cancel(operation)),
]);
}
setMuted(muted: boolean): Promise<void> {
if (this.pendingMute >= MAX_OUTPUT_WORK) {
void this.close();
return Promise.reject(new Error("Speech mute queue is full"));
}
this.pendingMute++;
this.muted = muted;
++this.outputRevision;
if (muted) void this.setCueState("off");
const work = this.muteWork
.catch(() => undefined)
.then(async () => {
if (!this.resource || this.closed) return;
await this.readyClient().request<{ epoch: number }>(
"resource.set_muted",
{ ...this.fence(), muted },
{
onResult: (result) => {
if (result.epoch === this.epoch) return;
this.epoch = result.epoch;
this.captureActive = false;
this.recognitionPending = false;
this.recognition = undefined;
this.segments.clear();
this.finals.clear();
this.turns.clear();
this.byAssistant.clear();
this.cueOperation = undefined;
},
},
);
if (
this.opened &&
!muted &&
!this.muted &&
!this.recognition &&
!this.closed
)
await this.startRecognition();
})
.finally(() => {
this.pendingMute--;
});
this.muteWork = work;
return work;
}
setCueState(state: CueState): Promise<void> {
this.cueState = state;
++this.cueRevision;
if (this.cueRunning) return this.cueWork;
this.cueRunning = true;
this.cueWork = Promise.resolve().then(async () => {
let applied = -1;
try {
while (!this.closed && applied !== this.cueRevision) {
applied = this.cueRevision;
await this.updateCue(this.cueState, applied);
}
} finally {
this.cueRunning = false;
}
});
return this.cueWork;
}
private async updateCue(state: CueState, revision: number): Promise<void> {
if (revision !== this.cueRevision) return;
const previous = this.cueOperation;
this.cueOperation = undefined;
if (previous) {
try {
await this.cancel(previous);
} catch (error) {
if (!this.closed)
this.emit({
type: "warning",
message: `Sound cue unavailable: ${message(error)}`,
});
return;
}
}
if (
revision !== this.cueRevision ||
state === "off" ||
this.muted ||
this.closed ||
!this.resource ||
this.turns.size > 0 ||
!this.client
)
return;
const sound = this.info?.sounds.find(
(item) => item.installed && item.sha256 === CUE_DIGESTS[state],
);
if (!sound) {
this.emit({
type: "warning",
message: `Sound cue unavailable: ${state}`,
});
return;
}
try {
await this.client.request<{ operation: string }>(
"sound.play",
{
...this.fence(),
sound: sound.id,
gain: 0.3,
repeat_ms:
state === "working" ? 1800 + Math.floor(Math.random() * 600) : 2000,
},
{
onResult: (result) => {
this.cueOperation = result.operation;
},
},
);
if (revision !== this.cueRevision || this.muted || this.closed) {
const operation = this.cueOperation;
this.cueOperation = undefined;
if (operation && !this.closed) await this.cancel(operation);
}
} catch (error) {
if (revision === this.cueRevision && !this.closed)
this.emit({
type: "warning",
message: `Sound cue unavailable: ${message(error)}`,
});
}
}
close(): Promise<void> {
this.closing ??= this.closeOwned();
return this.closing;
}
private async closeOwned(): Promise<void> {
if (this.closed) return;
this.closed = true;
++this.outputRevision;
++this.cueRevision;
const client = this.client;
if (!client) return;
if (this.resource)
await client
.request("resource.close", this.fence(), {
timeoutMs: CONTROL_TIMEOUT_MS,
})
.catch(() => undefined);
await client.close().catch(() => undefined);
this.unsubscribe?.();
this.unfail?.();
this.recognition = undefined;
this.segments.clear();
this.finals.clear();
this.turns.clear();
this.byAssistant.clear();
}
private async startRecognition(): Promise<void> {
const client = this.readyClient();
const result = await client.request<{ operation: string }>(
"recognition.start",
this.fence(),
{
onResult: (result) => {
this.recognition = result.operation;
if (!this.muted && !this.closed)
this.emit({ type: "ready", muted: false });
},
},
);
if (this.hotwords.length)
await client.request("recognition.set_hotwords", {
operation: result.operation,
terms: this.hotwords,
});
}
private event(event: SpeakEvent): void {
// During healthy close only certified playback accounting may cross the fence.
if (
this.closed &&
event.method !== "playback.progress" &&
event.method !== "operation.ended"
)
return;
if (event.params.resource && event.params.resource !== this.resource)
return;
const data = event.params.data;
if (
event.method !== "resource.health" &&
event.params.epoch !== null &&
event.params.epoch !== this.epoch
)
return;
if (
event.method.startsWith("recognition.") &&
(this.muted || event.params.operation !== this.recognition)
)
return;
if (event.method === "resource.ready") {
this.renderRate = Number(data["render_rate"]);
return;
}
if (event.method === "resource.health") {
if ((event.params.epoch ?? 0) <= this.epoch) return;
this.epoch = event.params.epoch ?? this.epoch;
this.captureActive = false;
this.recognitionPending = false;
this.recognition = undefined;
this.segments.clear();
this.finals.clear();
this.turns.clear();
this.byAssistant.clear();
++this.outputRevision;
++this.cueRevision;
this.cueOperation = undefined;
this.emit({
type: "error",
error: "Speech resource unavailable",
recoverable: this.info?.io_mode !== "fixture",
});
return;
}
if (event.method === "service.warning") {
this.emit({ type: "warning", message: String(data["message"]) });
return;
}
if (event.method === "recognition.activity") {
this.captureActive = Boolean(data["active"]);
const segment =
typeof data["segment"] === "string"
? this.segment(data["segment"])
: undefined;
this.emit({
type: "speech",
active: Boolean(data["active"]),
...(segment ? { captionId: segment.local } : {}),
});
return;
}
if (event.method === "recognition.pending") {
this.recognitionPending = Boolean(data["pending"]);
this.emit({ type: "input-state", pending: Boolean(data["pending"]) });
return;
}
if (event.method === "recognition.preview") {
const segment = this.segment(String(data["segment"]));
const revision = Number(data["revision"]);
if (segment.final || revision <= segment.revision) return;
segment.revision = revision;
this.emit({
type: "preview",
text: String(data["text"]),
captionId: segment.local,
revision,
});
return;
}
if (event.method === "recognition.preview_cleared") {
const segment = this.segment(String(data["segment"]));
const through = Number(data["through_revision"]);
if (segment.final || through < segment.revision) return;
segment.revision = through;
this.emit({
type: "caption-empty",
captionId: segment.local,
previewRevision: Number(data["through_revision"]),
});
return;
}
if (event.method === "recognition.empty") {
const segment = this.segment(String(data["segment"]));
if (segment.final) return;
segment.final = true;
this.emit({ type: "empty" });
return;
}
if (event.method === "recognition.final") {
const id = String(data["segment"]),
segment = this.segment(id);
if (segment.final) return;
if (this.finals.size >= MAX_FINALS) {
this.emit({
type: "error",
error: "Speech final retention overflow",
recoverable: false,
});
return;
}
segment.final = true;
const requestId = ++this.nextFinal;
this.finals.set(id, { generation: this.epoch, requestId });
this.emit({
type: "final",
text: String(data["text"]),
generation: this.epoch,
requestId,
ordinal: segment.ordinal,
captionId: segment.local,
previewRevision: segment.revision,
});
return;
}
if (event.method === "audio.level") {
const synthesisBusy = this.turns.size > 0;
const cueOnly =
this.cueOperation !== undefined &&
this.opened &&
!synthesisBusy &&
!this.captureActive &&
!this.recognitionPending;
this.emit({
type: "audio-state",
inputLevel: Number(data["input"]),
outputLevel: Number(data["output"]),
waveform: Array.isArray(data["waveform"])
? (data["waveform"] as number[])
: [],
playing: Boolean(data["playing"]),
busy: (Boolean(data["busy"]) && !cueOnly) || synthesisBusy,
cueOnly,
});
return;
}
if (event.method === "operation.ended") {
if (event.params.operation === this.cueOperation)
this.cueOperation = undefined;
if (event.params.operation === this.recognition) {
this.captureActive = false;
this.recognitionPending = false;
this.recognition = undefined;
this.segments.clear();
this.finals.clear();
if (!this.muted && !this.closed)
this.emit({
type: "error",
error: "Speech recognition ended",
recoverable: this.info?.io_mode !== "fixture",
});
}
}
const turn = event.params.operation
? this.turns.get(event.params.operation)
: undefined;
if (!turn) return;
if (event.method === "synthesis.phrase")
this.emit({
type: "phrase",
assistantTurnId: turn.assistantTurnId,
userTurnId: turn.userTurnId,
text: String(data["text"]),
});
if (event.method === "playback.progress")
this.emit({
type: "playback",
assistantTurnId: turn.assistantTurnId,
userTurnId: turn.userTurnId,
playedText: String(data["played_text"]),
cumulativeMs: Math.floor(
(Number(data["consumed_frames"]) * 1000) / this.renderRate,
),
});
if (event.method === "playback.boundary")
this.emit({ type: "boundary", assistantTurnId: turn.assistantTurnId });
if (event.method === "operation.ended") {
this.turns.delete(turn.operation);
this.byAssistant.delete(turn.assistantTurnId);
if (!this.closed) void this.setCueState(this.cueState);
if (!this.closed && data["outcome"] === "failed")
this.emit({
type: "error",
error: `Speech synthesis failed: ${String(data["reason"])}`,
recoverable: false,
});
}
}
private segment(id: string): Segment {
let value = this.segments.get(id);
if (!value) {
if (this.segments.size >= MAX_SEGMENTS) {
this.emit({
type: "error",
error: "Speech segment retention overflow",
recoverable: false,
});
throw new Error("Speech segment retention overflow");
}
value = {
local: ++this.nextCaption,
revision: -1,
ordinal: ++this.nextOrdinal,
final: false,
};
this.segments.set(id, value);
}
return value;
}
private queueOutput(work: () => Promise<void>): Promise<void> {
if (this.pendingOutput >= MAX_OUTPUT_WORK)
return Promise.reject(new Error("Speech command queue is full"));
const revision = this.outputRevision;
this.pendingOutput++;
const pending = this.outputWork
.catch(() => undefined)
.then(async () => {
if (revision !== this.outputRevision || this.muted || this.closed)
return;
await work();
})
.finally(() => {
this.pendingOutput--;
});
this.outputWork = pending;
return pending;
}
private async cancel(operation: string): Promise<void> {
await this.client?.request(
"operation.cancel",
{ operation },
{ timeoutMs: CONTROL_TIMEOUT_MS },
);
}
private fence(): { resource: string; epoch: number } {
if (!this.resource) throw new Error("Speech resource is unavailable");
return { resource: this.resource, epoch: this.epoch };
}
private readyClient(): SpeakClient {
if (!this.client || this.closed || !this.resource)
throw new Error("Speech backend is unavailable");
return this.client;
}
private emit(event: NativeSpeechEvent): void {
if (this.closed && event.type !== "playback") return;
try {
this.onEvent?.(event);
} catch {
/* UI failures cannot corrupt RPC state. */
}
}
}
function sameProfile(
actual: PreparedSpeech["profile"],
expected: PreparedSpeech["profile"],
): boolean {
return (
actual.id === expected.id &&
actual.label === expected.label &&
actual.description === expected.description &&
(
Object.keys(expected.capabilities) as Array<
keyof typeof expected.capabilities
>
).every((key) => actual.capabilities[key] === expected.capabilities[key])
);
}
function message(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}