import { join } from "node:path"; import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import type { VoiceController, VoiceModelSetting } from "./controller.ts"; import { voiceDataDirectory } from "./paths.ts"; import { NativeSpeechProcess } from "./speech-backend.ts"; import type { NativeSpeechEvent, PreparedSpeech } from "./speech-events.ts"; import { MAX_NATIVE_INPUT_CHARS, TURN_HARD_CAP_MS, turnPolicy, } from "./turn-policy.ts"; import { showNativeVoiceHistory } from "./voice-history-view.ts"; import type { CueState } from "./voice-sounds.ts"; import { createNativeVoiceWidget, type NativeVoiceWidget, type NativeVoiceWidgetState, } from "./voice-widget.ts"; export type NativeVoiceCallOptions = { backend: PreparedSpeech; modelSetting: VoiceModelSetting; historyDirectory: string; modality?: NonNullable< Parameters[1]["modality"] >; classifierModelSetting?: VoiceModelSetting; }; type SpeechFactory = () => NativeSpeechProcess; const MAX_RECOVERIES = 2; const RECOVERY_DEADLINE_MS = 15_000; const MAX_SERIAL_WORK = 64; /** Epoch-fenced bridge between controller and supervised local audio child. */ export class NativeVoiceCall { private process: NativeSpeechProcess | undefined; private draining: NativeSpeechProcess | undefined; private stopping: Promise | undefined; private cueOnly = false; private unsubscribe: (() => void) | undefined; private epoch = 0; private serial = Promise.resolve(); private serialCount = 0; private operation = 0; private options: NativeVoiceCallOptions | undefined; private context: ExtensionContext | undefined; private muted = false; private inputPending = false; private speechBusy = false; private progressSpeechId: string | undefined; private captionId = 0; private previewFloor = 0; private cueState: CueState | undefined; private cueWarningShown = false; private recovering = false; private recoveries = 0; private finalTimer: ReturnType | undefined; private classifyAbort: AbortController | undefined; private finalDeadline: number | undefined; private speechRevision = 0; private finals: Array<{ text: string; generation: number; requestId: number; ordinal: number; }> = []; private finalKeys = new Set(); private state: NativeVoiceWidgetState = { status: "unavailable", activity: "unavailable", error: null, partial: "", spoken: "", waveform: [], inputLevel: 0, outputLevel: 0, playing: false, muted: false, }; constructor( private readonly controller: VoiceController, private readonly makeProcess: SpeechFactory = () => new NativeSpeechProcess(), ) {} get active(): boolean { return this.context !== undefined && this.state.status !== "unavailable"; } get isMuted(): boolean { return this.muted; } async start( ctx: ExtensionContext, options: NativeVoiceCallOptions, ): Promise { const epoch = this.epoch + 1; try { await this.stop(); } catch (error) { await options.backend.client.close().catch(() => undefined); throw error; } if (epoch !== this.epoch) { await options.backend.client.close().catch(() => undefined); return; } this.context = ctx; this.options = options; this.muted = false; this.cueState = undefined; this.inputPending = false; this.speechBusy = false; this.progressSpeechId = undefined; this.cueWarningShown = false; this.recoveries = 0; this.resetSerial(); this.state = { ...this.state, status: "loading", activity: "ready", error: null, voiceError: null, partial: "", spoken: "", waveform: [], inputLevel: 0, outputLevel: 0, playing: false, muted: false, }; try { this.controller.beginNative(ctx, options); this.subscribe(epoch); await this.open(epoch); } catch (error) { if (epoch === this.epoch) this.unavailable(message(error)); await options.backend.client.close().catch(() => undefined); throw error; } } private widget: NativeVoiceWidget | undefined; createWidget( tui: Parameters[0], theme: Parameters[1], ): NativeVoiceWidget { this.widget?.dispose(); this.widget = createNativeVoiceWidget(tui, theme, () => this.state); return this.widget; } scrollWidget(...args: Parameters): void { if (this.context) this.widget?.scroll(...args); } stop(): Promise { ++this.epoch; if (this.stopping) return this.stopping; const stopping = this.stopOwned(); this.stopping = stopping; void stopping .finally(() => { if (this.stopping === stopping) this.stopping = undefined; }) .catch(() => undefined); return stopping; } private async stopOwned(): Promise { const process = this.process; this.process = undefined; this.unsubscribe?.(); this.unsubscribe = undefined; this.clearFinals(); this.resetSerial(); this.recovering = false; this.cueState = undefined; this.context = undefined; this.state = { ...this.state, status: "unavailable", activity: "unavailable", voiceError: null, partial: "", spoken: "", waveform: [], inputLevel: 0, outputLevel: 0, playing: false, muted: false, }; try { this.widget?.invalidate(); } catch { /* A disposed UI must not obstruct audio shutdown. */ } this.draining = process; try { await process?.close().catch(() => undefined); } finally { this.draining = undefined; } try { this.controller.endNative(); this.state = { ...this.state, voiceError: this.controller.nativeSnapshot().error, }; } finally { this.widget?.invalidate(); } } async mute(muted: boolean): Promise { if (!this.context) throw new Error("Start native voice before changing mute."); const epoch = this.epoch; const opening = this.state.status === "loading" || this.recovering; if (muted || opening) this.muted = muted; if (muted) this.inputPending = false; this.clearFinals(); if (muted) { this.resetSerial(); this.controller.onHumanSpeech(false); this.controller.interruptNative(); } this.state = { ...this.state, muted: this.muted, humanSpeaking: this.controller.nativeSnapshot().humanSpeaking, status: opening ? this.state.status : this.muted ? "muted" : "ready", }; this.widget?.invalidate(); const process = this.process; if (muted) void this.syncCueState(); if (opening) { void process?.setMuted(muted).catch((error) => { if (epoch === this.epoch && process === this.process) this.unavailable(message(error)); }); void this.syncCueState(); return; } if (!process) throw new Error("Audio is unavailable; resume native voice explicitly."); try { await process.setMuted(muted); if (epoch !== this.epoch || process !== this.process) return; this.muted = muted; this.state = { ...this.state, muted, status: muted ? "muted" : "ready" }; this.widget?.invalidate(); void this.syncCueState(); } catch (error) { if (epoch === this.epoch) this.unavailable(message(error)); throw error; } } async resume(): Promise { if (!this.options || !this.context) throw new Error("Native voice has not been configured."); if (this.recovering) throw new Error( "Native audio is already recovering; wait or stop the call.", ); if (this.process && !this.recovering) { await this.mute(false); return; } const epoch = ++this.epoch; this.clearFinals(); this.resetSerial(); this.muted = false; this.recoveries = 0; this.controller.beginNative(this.context, this.options); this.subscribe(epoch); await this.recover(epoch, true); } async showHistory( ctx: ExtensionContext, historyDirectory?: string, ): Promise { const epoch = this.epoch; const entries = await this.controller.loadNativeHistory({ cwd: ctx.cwd, historyDirectory: historyDirectory ?? this.options?.historyDirectory ?? join(voiceDataDirectory(), "history"), mainSessionId: ctx.sessionManager.getSessionId(), }); if (epoch !== this.epoch) return; await showNativeVoiceHistory(ctx, entries); } private subscribe(epoch: number): void { this.unsubscribe?.(); this.unsubscribe = this.controller.subscribe((event) => { if (epoch !== this.epoch) return; if (event.type === "status-interrupt") { if (this.progressSpeechId === event.assistantTurnId) { this.progressSpeechId = undefined; void this.process ?.interrupt() .catch((error) => this.recordError(error)); } return; } if (this.muted && event.type !== "native-state") return; if (event.type === "assistant-speech") { const revision = this.speechRevision; if (event.kind === "status") this.progressSpeechId = event.assistantTurnId; this.enqueueFor( epoch, () => revision !== this.speechRevision || (event.kind === "status" && !this.canSendProgress(event.assistantTurnId)) ? Promise.resolve() : (this.process?.speak({ assistantTurnId: event.assistantTurnId, userTurnId: event.userTurnId, text: event.text, ...(event.kind ? { kind: event.kind } : {}), }) ?? Promise.resolve()), event.kind === "status" ? event.assistantTurnId : undefined, ); } if (event.type === "assistant-complete") { const revision = this.speechRevision; this.enqueueFor( epoch, () => revision !== this.speechRevision || (event.kind === "status" && !this.canSendProgress(event.assistantTurnId)) ? Promise.resolve() : (this.process?.complete(event.assistantTurnId) ?? Promise.resolve()), event.kind === "status" ? event.assistantTurnId : undefined, ); } if (event.type === "hotwords") this.enqueueFor( epoch, () => this.process?.setHotwords(event.terms) ?? Promise.resolve(), ); if (event.type === "native-state") this.syncControllerState(); }); } private async open(epoch: number): Promise { const options = this.options; if (!options || epoch !== this.epoch) return; const process = this.makeProcess(); this.cueState = undefined; this.inputPending = false; this.speechBusy = false; this.progressSpeechId = undefined; this.captionId = 0; this.previewFloor = 0; this.process = process; const starting = process.start({ ...options, onEvent: (event) => this.onEvent(epoch, process, event), }); if (this.muted) { void process.setMuted(true).catch((error) => { if (epoch === this.epoch && process === this.process) this.unavailable(message(error)); }); } await starting; if (epoch !== this.epoch || this.process !== process) { await process.close().catch(() => undefined); return; } this.recovering = false; this.state = { ...this.state, status: this.muted ? "muted" : "ready", error: null, }; this.syncControllerState(); void this.syncCueState(); } private onEvent( epoch: number, process: NativeSpeechProcess, event: NativeSpeechEvent, ): void { if (process === this.draining && event.type === "playback") { this.controller.markNativePlayback( event.assistantTurnId, event.playedText, ); this.controller.setPlaybackDuration(event.userTurnId, event.cumulativeMs); return; } if (epoch !== this.epoch || process !== this.process) return; if (event.type === "ready") { this.muted = event.muted ?? this.muted; this.state = { ...this.state, muted: this.muted }; this.recovering = false; this.state = { ...this.state, status: this.muted ? "muted" : "ready", error: null, }; this.syncControllerState(); return; } if (event.type === "warning") { this.notifyCueWarning(event.message); return; } if (event.type === "phrase") { this.state = { ...this.state, spoken: event.text }; this.widget?.invalidate(); return; } if (event.type === "error") { if (!event.recoverable || this.state.status === "loading" || this.muted) { this.unavailable(event.error); return; } this.state = { ...this.state, error: event.error }; this.widget?.invalidate(); this.runBackground(epoch, this.recover(epoch)); return; } if (event.type === "input-state") { this.inputPending = event.pending; if (event.pending) this.controller.cancelNativeProgress(); return; } if (event.type === "audio-state") { this.speechBusy = event.busy ?? event.playing; this.cueOnly = event.cueOnly ?? false; this.state = { ...this.state, waveform: event.waveform, inputLevel: event.inputLevel, outputLevel: event.outputLevel, playing: event.playing, }; this.widget?.invalidate(); this.maybeNarrateProgress(); return; } if (event.type === "caption-empty") { if (this.muted || this.recovering || event.captionId < this.captionId) return; this.captionId = event.captionId; this.previewFloor = Math.max( this.previewFloor, event.previewRevision + 1, ); this.state = { ...this.state, partial: "", provisional: this.controller.nativeSnapshot().humanSpeaking, }; this.widget?.invalidate(); return; } if (event.type === "preview") { if ( this.muted || this.recovering || this.state.status !== "ready" || event.captionId < this.captionId || (event.captionId === this.captionId && event.revision < this.previewFloor) ) return; if (event.captionId > this.captionId) this.previewFloor = 0; this.captionId = event.captionId; this.previewFloor = Math.max(this.previewFloor, event.revision); this.state = { ...this.state, partial: event.text, provisional: true }; this.widget?.invalidate(); return; } if (event.type === "speech") { if (event.active && !this.muted) { const newCaption = event.captionId !== undefined && event.captionId > this.captionId; if (newCaption && event.captionId !== undefined) { this.captionId = event.captionId; this.previewFloor = 0; } this.state = { ...this.state, provisional: true, partial: !newCaption && this.state.provisional ? this.state.partial : "", }; this.widget?.invalidate(); } this.controller.onHumanSpeech(!this.muted && event.active); this.syncControllerState(); if (event.active) { this.speechRevision++; this.clearFinalTimer(); this.finalDeadline = undefined; if (!this.muted) this.runBackground(epoch, this.interrupt()); } else if (this.finals.length) { this.scheduleFinals(epoch, process); } return; } if (event.type === "empty") { if (!this.controller.nativeSnapshot().humanSpeaking) { this.state = { ...this.state, partial: "", provisional: false }; this.widget?.invalidate(); } if (!this.muted && !this.recovering) this.controller.onEmptySpeechBoundary(); return; } if (event.type === "final") { this.controller.cancelNativeProgress(); this.queueFinal(epoch, process, event); return; } if (event.type === "playback") { this.controller.markNativePlayback( event.assistantTurnId, event.playedText, ); this.controller.setPlaybackDuration(event.userTurnId, event.cumulativeMs); return; } if (event.type === "boundary") { if (this.progressSpeechId === event.assistantTurnId) this.progressSpeechId = undefined; this.controller.turnBoundary(event.assistantTurnId); } } private queueFinal( epoch: number, process: NativeSpeechProcess, event: Extract, ): void { // Native mute/recovery resets ASR; acknowledging an old generation would be stale. if (this.muted || this.recovering) return; const key = `${event.generation}:${event.requestId}`; if (this.finalKeys.has(key)) return; if (event.text.length > MAX_NATIVE_INPUT_CHARS) { this.unavailable("Speech final exceeds 100,000-character sanity bound."); return; } const pendingChars = this.finals.reduce( (sum, final) => sum + final.text.length + 1, 0, ); if ( this.finals.length >= 8 || pendingChars + event.text.length > MAX_NATIVE_INPUT_CHARS ) { if (this.controller.nativeSnapshot().humanSpeaking) { this.unavailable( "Speech-final input bound exceeded while speaking; resume explicitly.", ); return; } this.submitFinals(epoch, process); } if (this.finalKeys.size >= 32) { this.unavailable("Speech-final queue overflow; resume explicitly."); return; } this.finalKeys.add(key); if (!event.text.trim()) { this.consumeFinal(epoch, process, event, key); return; } this.finals.push({ text: event.text, generation: event.generation, requestId: event.requestId, ordinal: event.ordinal ?? event.captionId ?? event.requestId, }); this.finals.sort((a, b) => a.ordinal - b.ordinal); if (event.captionId === undefined || event.captionId >= this.captionId) { this.captionId = event.captionId ?? this.captionId; if (event.captionId !== undefined) this.previewFloor = Math.max( this.previewFloor, (event.previewRevision ?? Number.MAX_SAFE_INTEGER - 1) + 1, ); this.state = { ...this.state, partial: event.text, provisional: false, captionRevision: (this.state.captionRevision ?? 0) + (this.controller.nativeSnapshot().humanSpeaking ? 0 : 1), }; this.widget?.invalidate(); } this.scheduleFinals(epoch, process); } private scheduleFinals(epoch: number, process: NativeSpeechProcess): void { this.clearFinalTimer(); if (this.controller.nativeSnapshot().humanSpeaking) return; const abort = new AbortController(); this.finalDeadline ??= Date.now() + TURN_HARD_CAP_MS; const deadline = this.finalDeadline; this.classifyAbort = abort; const joined = this.finals .map((final) => final.text) .join(" ") .slice(-4_000); const submit = async () => { let waitMs = 1_500; try { waitMs = turnPolicy( await this.controller.classifyTurn({ text: joined, signal: abort.signal, }), ).waitMs; } catch { // Classifier failure retains bounded fallback endpointing. } if ( abort.signal.aborted || epoch !== this.epoch || process !== this.process || this.muted || this.recovering ) return; if (this.finalTimer) clearTimeout(this.finalTimer); this.finalTimer = setTimeout( () => this.submitFinals(epoch, process), Math.min(waitMs, Math.max(0, deadline - Date.now())), ); }; void submit(); this.finalTimer = setTimeout( () => this.submitFinals(epoch, process), Math.max(0, deadline - Date.now()), ); } private submitFinals(epoch: number, process: NativeSpeechProcess): void { if (this.controller.nativeSnapshot().humanSpeaking) return; if (this.finalTimer) clearTimeout(this.finalTimer); this.finalTimer = undefined; this.finalDeadline = undefined; this.classifyAbort?.abort(); this.classifyAbort = undefined; if ( epoch !== this.epoch || process !== this.process || this.muted || this.recovering ) return; const finals = this.finals.splice(0, 8); const first = finals[0]; if (!first) return; const revision = this.speechRevision; this.enqueueFor(epoch, async () => { if (process !== this.process || this.muted || this.recovering) return; if ( revision !== this.speechRevision || this.controller.nativeSnapshot().humanSpeaking || this.finals.some((pending) => pending.ordinal < first.ordinal) ) { // A queued delivery cannot turn a mid-thought pause into a committed turn. const chars = [...finals, ...this.finals].reduce( (sum, final) => sum + final.text.length + 1, -1, ); if ( finals.length + this.finals.length > 8 || chars > MAX_NATIVE_INPUT_CHARS ) { this.unavailable( "Speech-final input bound exceeded while speaking; resume explicitly.", ); return; } this.finals.unshift(...finals); this.finals.sort((a, b) => a.ordinal - b.ordinal); if (!this.controller.nativeSnapshot().humanSpeaking) this.scheduleFinals(epoch, process); return; } this.controller.receiveNative( finals.map((final) => final.text).join(" "), ); for (const final of finals) { if (process !== this.process || this.muted || this.recovering) return; await process.consumeFinal(final.generation, final.requestId); this.finalKeys.delete(`${final.generation}:${final.requestId}`); } }); } private consumeFinal( epoch: number, process: NativeSpeechProcess, event: Extract, key: string, ): void { this.enqueueFor(epoch, async () => { if (process !== this.process) return; await process.consumeFinal(event.generation, event.requestId); this.finalKeys.delete(key); }); } private async interrupt(): Promise { // Barge-in cancels output, not recognition already waiting for submission. this.controller.interruptNative(true); await this.process?.interrupt().catch((error) => this.recordError(error)); } private async recover(epoch: number, explicit = false): Promise { if (this.recovering || epoch !== this.epoch || this.muted) return; if (!explicit && ++this.recoveries > MAX_RECOVERIES) { this.unavailable("Audio recovery exhausted. Resume explicitly."); return; } this.recovering = true; this.clearFinals(); this.resetSerial(); this.controller.onHumanSpeech(false); this.controller.interruptNative(); this.state = { ...this.state, status: "reconnecting", humanSpeaking: false, partial: "", playing: false, }; this.widget?.invalidate(); const old = this.process; this.process = undefined; void this.setCueState(old, "off", true); await old?.close().catch(() => undefined); if (epoch !== this.epoch) return; let timer: ReturnType | undefined; let attempted: NativeSpeechProcess | undefined; try { const opening = this.open(epoch); attempted = this.currentProcess(); await Promise.race([ opening, new Promise((_, reject) => { timer = setTimeout( () => reject(new Error("Audio recovery timed out")), RECOVERY_DEADLINE_MS, ); }), ]); } catch (error) { if (this.process === attempted) this.process = undefined; await attempted?.close().catch(() => undefined); if (epoch !== this.epoch) return; this.recovering = false; this.state = { ...this.state, error: message(error) }; this.runBackground(epoch, this.recover(epoch)); } finally { if (timer) clearTimeout(timer); } } private unavailable(error: string): void { const process = this.process; this.process = undefined; void this.setCueState(process, "off", true); this.recovering = false; this.clearFinals(); this.resetSerial(); this.state = { ...this.state, status: "unavailable", activity: "unavailable", error, partial: "", playing: false, }; try { this.controller.endNative(); } catch (cause) { this.state = { ...this.state, error: `${error}; ${message(cause)}` }; } void process?.close().catch(() => undefined); try { this.widget?.invalidate(); } catch { /* A disposed UI must not obstruct audio shutdown. */ } } private runBackground(epoch: number, work: Promise): void { void work.catch((cause) => { if (epoch === this.epoch) this.unavailable(message(cause)); }); } private canSendProgress(id: string): boolean { return ( this.controller.isNativeProgressCurrent(id) && !this.inputPending && !this.finals.length && !this.finalKeys.size && !this.muted && !this.recovering && this.state.status === "ready" ); } private isProgressIdle(): boolean { return Boolean( this.process && this.context && this.state.status === "ready" && !this.muted && !this.recovering && !this.inputPending && !this.controller.nativeSnapshot().humanSpeaking && !this.speechBusy && (!this.state.playing || this.cueOnly) && !this.finals.length && !this.finalKeys.size && this.serialCount === 0, ); } private maybeNarrateProgress(): void { if (this.isProgressIdle()) this.controller.narrateExecutorProgress(() => this.isProgressIdle()); } private syncControllerState(): void { const { status, error, humanSpeaking } = this.controller.nativeSnapshot(); if ( this.state.activity !== status || this.state.voiceError !== error || this.state.humanSpeaking !== humanSpeaking ) { this.state = { ...this.state, activity: status, voiceError: error, humanSpeaking, }; this.widget?.invalidate(); } void this.syncCueState(); } private async syncCueState(): Promise { const process = this.process; if (!process) return; const snapshot = this.controller.nativeSnapshot(); const state: CueState = this.muted || this.recovering || this.state.status === "unavailable" || snapshot.humanSpeaking ? "off" : this.state.status === "loading" ? "loading" : (snapshot.cueState ?? "listening"); // Worker owns loading until it reports ready; parent only enforces priority off. if (state === "loading" && this.cueState !== "off") return; await this.setCueState(process, state); } private async setCueState( process: NativeSpeechProcess | undefined, state: CueState, force = false, ): Promise { if (!process || (!force && process !== this.process)) return; if (!force && this.cueState === state) return; this.cueState = state; const epoch = this.epoch; try { await process.setCueState(state); } catch (error) { if (!force && epoch === this.epoch && process === this.process) this.notifyCueWarning(`Sound cue unavailable: ${message(error)}`); } } private notifyCueWarning(warning: string): void { if (this.cueWarningShown) return; this.cueWarningShown = true; this.context?.ui?.notify(warning.slice(0, 1_000), "warning"); } private clearFinalTimer(): void { if (this.finalTimer) clearTimeout(this.finalTimer); this.finalTimer = undefined; this.classifyAbort?.abort(); this.classifyAbort = undefined; } private clearFinals(): void { this.clearFinalTimer(); this.finalDeadline = undefined; this.finals.length = 0; this.finalKeys.clear(); } private resetSerial(): void { this.operation++; this.serial = Promise.resolve(); this.serialCount = 0; } private enqueueFor( epoch: number, work: () => Promise, statusTurnId?: string, ): void { if (this.serialCount >= MAX_SERIAL_WORK) { this.recordError(new Error("Native audio command queue overflow")); return; } const operation = this.operation; this.serialCount++; this.serial = this.serial .then(async () => { if (epoch !== this.epoch || operation !== this.operation) return; await work(); }) .catch((error) => { if (epoch === this.epoch && operation === this.operation) { this.recordError(error); if ( statusTurnId && this.controller.isNativeProgressCurrent(statusTurnId) ) this.controller.cancelNativeProgress(); } }) .finally(() => { if (epoch === this.epoch && operation === this.operation) this.serialCount = Math.max(0, this.serialCount - 1); }); } private currentProcess(): NativeSpeechProcess | undefined { return this.process; } private recordError(error: unknown): void { this.state = { ...this.state, error: message(error) }; this.widget?.invalidate(); } } function message(error: unknown): string { return error instanceof Error ? error.message : String(error); }