import type { ServiceInfo, SpeakEvent } from "./speak-client.ts"; import { SpeakClient } from "./speak-client.ts"; import type { NativeSpeechEvent, NativeSpeechOptions, PreparedSpeech, SpeechRequest, } from "./speech-events.ts"; import { supportsSpeechProfile } from "./speech-events.ts"; import { CUE_DIGESTS, type CueState } from "./voice-sounds.ts"; const OPEN_TIMEOUT_MS = 180_000; const CONTROL_TIMEOUT_MS = 5_000; const MAX_FINALS = 32; const MAX_STREAMS = 1; const MAX_SEGMENTS = 4096; const MAX_SCALARS = 8_000; const MAX_OUTPUT_WORK = 64; type Turn = { assistantTurnId: string; userTurnId: string; operation: string; index: number; finished: boolean; chars: number; endsWithWhitespace: boolean; }; type Segment = { local: number; revision: number; ordinal: number; final: boolean; }; /** Maps generic RPC identities to call-local UI and conversation identities. */ export class NativeSpeechProcess { private client: SpeakClient | undefined; private info: ServiceInfo | undefined; private onEvent: ((event: NativeSpeechEvent) => void) | undefined; private unsubscribe: (() => void) | undefined; private unfail: (() => void) | undefined; private resource: string | undefined; private epoch = 0; private recognition: string | undefined; private renderRate = 0; private muted = false; private closed = false; private opened = false; private recognitionPending = false; private captureActive = false; private hotwords: readonly string[] = []; private nextCaption = 0; private nextOrdinal = 0; private segments = new Map(); private finals = new Map(); private nextFinal = 0; private turns = new Map(); private byAssistant = new Map(); private cueOperation: string | undefined; private cueState: CueState = "loading"; private cueRevision = 0; private cueWork = Promise.resolve(); private cueRunning = false; private outputRevision = 0; private outputWork = Promise.resolve(); private pendingOutput = 0; private muteWork = Promise.resolve(); private pendingMute = 0; private closing: Promise | undefined; async start(options: NativeSpeechOptions): Promise { if (this.client || this.closed) throw new Error("Native speech process already used"); try { await this.initialize(options); } catch (error) { await this.close(); throw error; } } private async initialize(options: NativeSpeechOptions): Promise { this.onEvent = options.onEvent; let client = options.backend.client; let recovered = false; this.client = client; try { await client.start(); } catch { await client.close().catch(() => undefined); if (this.closed) throw new Error("Speech backend is closed"); client = new SpeakClient(options.backend.command); this.client = client; options.backend.client = client; recovered = true; await client.start(); } if (this.closed) throw new Error("Speech backend is closed"); if (recovered) { const plan = await client.request<{ plan: { ready: boolean } }>( "assets.plan", { profile: options.backend.profile.id }, ); if (!plan.plan.ready) throw new Error("Speech assets need explicit setup before recovery"); } this.info = await client.request("service.info"); if (this.closed) throw new Error("Speech backend is closed"); this.unsubscribe = client.onEvent((event) => this.event(event)); this.unfail = client.onFailure((error) => this.emit({ type: "error", error: error.message, recoverable: this.info?.io_mode !== "fixture", }), ); const profile = this.info.profiles.find( (item) => item.id === options.backend.profile.id, ); if ( !profile?.installed || !sameProfile(profile, options.backend.profile) || !supportsSpeechProfile(profile, options.backend.preview) ) throw new Error("Selected speech profile is unavailable or incompatible"); const openingMuted = this.muted; const opened = await client.request<{ resource: string; epoch: number; operation: string; }>( "resource.open", { profile: profile.id, aec: true, preview: options.backend.preview, endpoint_ms: 2000, interrupt_on_speech: true, muted: openingMuted, }, { timeoutMs: CONTROL_TIMEOUT_MS, onResult: (result) => { this.resource = result.resource; this.epoch = result.epoch; if (this.muted !== openingMuted) void this.setMuted(this.muted).catch((error) => this.emit({ type: "error", error: message(error), recoverable: false, }), ); void this.setCueState(this.cueState); }, }, ); const outcome = await client.waitForOperation(opened.operation, { timeoutMs: OPEN_TIMEOUT_MS, }); if (outcome.outcome !== "completed") throw new Error(`Speech resource failed to open: ${outcome.reason}`); if (this.closed) throw new Error("Speech backend is closed"); if (!this.renderRate) throw new Error("Speech resource has no render rate"); this.opened = true; await this.muteWork; if (!this.muted && !this.recognition) await this.startRecognition(); else if (this.muted) this.emit({ type: "ready", muted: true }); } speak(request: SpeechRequest): Promise { return this.queueOutput(() => this.speakNow(request)); } private async speakNow(request: SpeechRequest): Promise { if (request.text.length > MAX_SCALARS * 2) throw new Error("Speech stream exceeds 8000 scalars"); if (!request.text.trim()) return; const requestScalars = [...request.text].length; if (requestScalars > MAX_SCALARS) throw new Error("Speech stream exceeds 8000 scalars"); const client = this.readyClient(); if (this.muted) return; const revision = this.outputRevision; let turn = this.byAssistant.get(request.assistantTurnId); if (!turn) { for (const previous of this.turns.values()) { if (previous.finished) await client.waitForOperation(previous.operation, { timeoutMs: OPEN_TIMEOUT_MS, }); } if (revision !== this.outputRevision || this.muted || this.closed) return; if (this.turns.size >= MAX_STREAMS) throw new Error("Speech stream queue is full"); await client.request<{ operation: string }>( "synthesis.start", this.fence(), { onResult: (result) => { turn = { assistantTurnId: request.assistantTurnId, userTurnId: request.userTurnId, operation: result.operation, index: 0, finished: false, chars: 0, endsWithWhitespace: false, }; this.turns.set(turn.operation, turn); this.byAssistant.set(turn.assistantTurnId, turn); }, }, ); } if (!turn) throw new Error("Speech stream was not accepted"); if (revision !== this.outputRevision || this.muted || this.closed) { if (this.closed) return; await this.cancel(turn.operation); return; } if (!this.turns.has(turn.operation)) return; if (turn.finished) throw new Error("Speech turn is already complete"); const separator = turn.chars > 0 && !turn.endsWithWhitespace && !/^\s/u.test(request.text) ? " " : ""; if (turn.chars + requestScalars + separator.length > MAX_SCALARS) { await this.cancel(turn.operation); throw new Error("Speech stream exceeds 8000 scalars"); } const text = separator + request.text; try { await client.request("synthesis.append", { operation: turn.operation, index: turn.index, text, }); } catch (error) { await this.cancel(turn.operation).catch(() => undefined); throw error; } turn.index++; turn.chars += requestScalars + separator.length; turn.endsWithWhitespace = /\s$/u.test(request.text); } complete(assistantTurnId: string): Promise { return this.queueOutput(() => this.completeNow(assistantTurnId)); } private async completeNow(assistantTurnId: string): Promise { const turn = this.byAssistant.get(assistantTurnId); if (!turn || turn.finished) return; turn.finished = true; try { await this.readyClient().request("synthesis.finish", { operation: turn.operation, }); } catch (error) { await this.cancel(turn.operation).catch(() => undefined); throw error; } } async setHotwords(terms: readonly string[]): Promise { if (terms.length > 20 || terms.some((term) => [...term].length > 256)) throw new Error("Hotwords exceed limit"); this.hotwords = [...terms]; if (this.recognition) await this.readyClient().request("recognition.set_hotwords", { operation: this.recognition, terms: this.hotwords, }); } async consumeFinal(generation: number, requestId: number): Promise { const entry = [...this.finals.entries()].find( ([, value]) => value.generation === generation && value.requestId === requestId, ); if (!entry || !this.recognition) return; const [segment, retained] = entry; await this.readyClient().request("recognition.ack", { operation: this.recognition, segment, }); if (this.finals.get(segment) === retained) this.finals.delete(segment); } async interrupt(): Promise { ++this.outputRevision; await Promise.all([ this.setCueState("off"), ...[...this.turns.keys()].map((operation) => this.cancel(operation)), ]); } setMuted(muted: boolean): Promise { if (this.pendingMute >= MAX_OUTPUT_WORK) { void this.close(); return Promise.reject(new Error("Speech mute queue is full")); } this.pendingMute++; this.muted = muted; ++this.outputRevision; if (muted) void this.setCueState("off"); const work = this.muteWork .catch(() => undefined) .then(async () => { if (!this.resource || this.closed) return; await this.readyClient().request<{ epoch: number }>( "resource.set_muted", { ...this.fence(), muted }, { onResult: (result) => { if (result.epoch === this.epoch) return; this.epoch = result.epoch; this.captureActive = false; this.recognitionPending = false; this.recognition = undefined; this.segments.clear(); this.finals.clear(); this.turns.clear(); this.byAssistant.clear(); this.cueOperation = undefined; }, }, ); if ( this.opened && !muted && !this.muted && !this.recognition && !this.closed ) await this.startRecognition(); }) .finally(() => { this.pendingMute--; }); this.muteWork = work; return work; } setCueState(state: CueState): Promise { this.cueState = state; ++this.cueRevision; if (this.cueRunning) return this.cueWork; this.cueRunning = true; this.cueWork = Promise.resolve().then(async () => { let applied = -1; try { while (!this.closed && applied !== this.cueRevision) { applied = this.cueRevision; await this.updateCue(this.cueState, applied); } } finally { this.cueRunning = false; } }); return this.cueWork; } private async updateCue(state: CueState, revision: number): Promise { if (revision !== this.cueRevision) return; const previous = this.cueOperation; this.cueOperation = undefined; if (previous) { try { await this.cancel(previous); } catch (error) { if (!this.closed) this.emit({ type: "warning", message: `Sound cue unavailable: ${message(error)}`, }); return; } } if ( revision !== this.cueRevision || state === "off" || this.muted || this.closed || !this.resource || this.turns.size > 0 || !this.client ) return; const sound = this.info?.sounds.find( (item) => item.installed && item.sha256 === CUE_DIGESTS[state], ); if (!sound) { this.emit({ type: "warning", message: `Sound cue unavailable: ${state}`, }); return; } try { await this.client.request<{ operation: string }>( "sound.play", { ...this.fence(), sound: sound.id, gain: 0.3, repeat_ms: state === "working" ? 1800 + Math.floor(Math.random() * 600) : 2000, }, { onResult: (result) => { this.cueOperation = result.operation; }, }, ); if (revision !== this.cueRevision || this.muted || this.closed) { const operation = this.cueOperation; this.cueOperation = undefined; if (operation && !this.closed) await this.cancel(operation); } } catch (error) { if (revision === this.cueRevision && !this.closed) this.emit({ type: "warning", message: `Sound cue unavailable: ${message(error)}`, }); } } close(): Promise { this.closing ??= this.closeOwned(); return this.closing; } private async closeOwned(): Promise { if (this.closed) return; this.closed = true; ++this.outputRevision; ++this.cueRevision; const client = this.client; if (!client) return; if (this.resource) await client .request("resource.close", this.fence(), { timeoutMs: CONTROL_TIMEOUT_MS, }) .catch(() => undefined); await client.close().catch(() => undefined); this.unsubscribe?.(); this.unfail?.(); this.recognition = undefined; this.segments.clear(); this.finals.clear(); this.turns.clear(); this.byAssistant.clear(); } private async startRecognition(): Promise { const client = this.readyClient(); const result = await client.request<{ operation: string }>( "recognition.start", this.fence(), { onResult: (result) => { this.recognition = result.operation; if (!this.muted && !this.closed) this.emit({ type: "ready", muted: false }); }, }, ); if (this.hotwords.length) await client.request("recognition.set_hotwords", { operation: result.operation, terms: this.hotwords, }); } private event(event: SpeakEvent): void { // During healthy close only certified playback accounting may cross the fence. if ( this.closed && event.method !== "playback.progress" && event.method !== "operation.ended" ) return; if (event.params.resource && event.params.resource !== this.resource) return; const data = event.params.data; if ( event.method !== "resource.health" && event.params.epoch !== null && event.params.epoch !== this.epoch ) return; if ( event.method.startsWith("recognition.") && (this.muted || event.params.operation !== this.recognition) ) return; if (event.method === "resource.ready") { this.renderRate = Number(data["render_rate"]); return; } if (event.method === "resource.health") { if ((event.params.epoch ?? 0) <= this.epoch) return; this.epoch = event.params.epoch ?? this.epoch; this.captureActive = false; this.recognitionPending = false; this.recognition = undefined; this.segments.clear(); this.finals.clear(); this.turns.clear(); this.byAssistant.clear(); ++this.outputRevision; ++this.cueRevision; this.cueOperation = undefined; this.emit({ type: "error", error: "Speech resource unavailable", recoverable: this.info?.io_mode !== "fixture", }); return; } if (event.method === "service.warning") { this.emit({ type: "warning", message: String(data["message"]) }); return; } if (event.method === "recognition.activity") { this.captureActive = Boolean(data["active"]); const segment = typeof data["segment"] === "string" ? this.segment(data["segment"]) : undefined; this.emit({ type: "speech", active: Boolean(data["active"]), ...(segment ? { captionId: segment.local } : {}), }); return; } if (event.method === "recognition.pending") { this.recognitionPending = Boolean(data["pending"]); this.emit({ type: "input-state", pending: Boolean(data["pending"]) }); return; } if (event.method === "recognition.preview") { const segment = this.segment(String(data["segment"])); const revision = Number(data["revision"]); if (segment.final || revision <= segment.revision) return; segment.revision = revision; this.emit({ type: "preview", text: String(data["text"]), captionId: segment.local, revision, }); return; } if (event.method === "recognition.preview_cleared") { const segment = this.segment(String(data["segment"])); const through = Number(data["through_revision"]); if (segment.final || through < segment.revision) return; segment.revision = through; this.emit({ type: "caption-empty", captionId: segment.local, previewRevision: Number(data["through_revision"]), }); return; } if (event.method === "recognition.empty") { const segment = this.segment(String(data["segment"])); if (segment.final) return; segment.final = true; this.emit({ type: "empty" }); return; } if (event.method === "recognition.final") { const id = String(data["segment"]), segment = this.segment(id); if (segment.final) return; if (this.finals.size >= MAX_FINALS) { this.emit({ type: "error", error: "Speech final retention overflow", recoverable: false, }); return; } segment.final = true; const requestId = ++this.nextFinal; this.finals.set(id, { generation: this.epoch, requestId }); this.emit({ type: "final", text: String(data["text"]), generation: this.epoch, requestId, ordinal: segment.ordinal, captionId: segment.local, previewRevision: segment.revision, }); return; } if (event.method === "audio.level") { const synthesisBusy = this.turns.size > 0; const cueOnly = this.cueOperation !== undefined && this.opened && !synthesisBusy && !this.captureActive && !this.recognitionPending; this.emit({ type: "audio-state", inputLevel: Number(data["input"]), outputLevel: Number(data["output"]), waveform: Array.isArray(data["waveform"]) ? (data["waveform"] as number[]) : [], playing: Boolean(data["playing"]), busy: (Boolean(data["busy"]) && !cueOnly) || synthesisBusy, cueOnly, }); return; } if (event.method === "operation.ended") { if (event.params.operation === this.cueOperation) this.cueOperation = undefined; if (event.params.operation === this.recognition) { this.captureActive = false; this.recognitionPending = false; this.recognition = undefined; this.segments.clear(); this.finals.clear(); if (!this.muted && !this.closed) this.emit({ type: "error", error: "Speech recognition ended", recoverable: this.info?.io_mode !== "fixture", }); } } const turn = event.params.operation ? this.turns.get(event.params.operation) : undefined; if (!turn) return; if (event.method === "synthesis.phrase") this.emit({ type: "phrase", assistantTurnId: turn.assistantTurnId, userTurnId: turn.userTurnId, text: String(data["text"]), }); if (event.method === "playback.progress") this.emit({ type: "playback", assistantTurnId: turn.assistantTurnId, userTurnId: turn.userTurnId, playedText: String(data["played_text"]), cumulativeMs: Math.floor( (Number(data["consumed_frames"]) * 1000) / this.renderRate, ), }); if (event.method === "playback.boundary") this.emit({ type: "boundary", assistantTurnId: turn.assistantTurnId }); if (event.method === "operation.ended") { this.turns.delete(turn.operation); this.byAssistant.delete(turn.assistantTurnId); if (!this.closed) void this.setCueState(this.cueState); if (!this.closed && data["outcome"] === "failed") this.emit({ type: "error", error: `Speech synthesis failed: ${String(data["reason"])}`, recoverable: false, }); } } private segment(id: string): Segment { let value = this.segments.get(id); if (!value) { if (this.segments.size >= MAX_SEGMENTS) { this.emit({ type: "error", error: "Speech segment retention overflow", recoverable: false, }); throw new Error("Speech segment retention overflow"); } value = { local: ++this.nextCaption, revision: -1, ordinal: ++this.nextOrdinal, final: false, }; this.segments.set(id, value); } return value; } private queueOutput(work: () => Promise): Promise { if (this.pendingOutput >= MAX_OUTPUT_WORK) return Promise.reject(new Error("Speech command queue is full")); const revision = this.outputRevision; this.pendingOutput++; const pending = this.outputWork .catch(() => undefined) .then(async () => { if (revision !== this.outputRevision || this.muted || this.closed) return; await work(); }) .finally(() => { this.pendingOutput--; }); this.outputWork = pending; return pending; } private async cancel(operation: string): Promise { await this.client?.request( "operation.cancel", { operation }, { timeoutMs: CONTROL_TIMEOUT_MS }, ); } private fence(): { resource: string; epoch: number } { if (!this.resource) throw new Error("Speech resource is unavailable"); return { resource: this.resource, epoch: this.epoch }; } private readyClient(): SpeakClient { if (!this.client || this.closed || !this.resource) throw new Error("Speech backend is unavailable"); return this.client; } private emit(event: NativeSpeechEvent): void { if (this.closed && event.type !== "playback") return; try { this.onEvent?.(event); } catch { /* UI failures cannot corrupt RPC state. */ } } } function sameProfile( actual: PreparedSpeech["profile"], expected: PreparedSpeech["profile"], ): boolean { return ( actual.id === expected.id && actual.label === expected.label && actual.description === expected.description && ( Object.keys(expected.capabilities) as Array< keyof typeof expected.capabilities > ).every((key) => actual.capabilities[key] === expected.capabilities[key]) ); } function message(error: unknown): string { return error instanceof Error ? error.message : String(error); }