Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/voice/src/native-call.ts

Raw
import { join } from "node:path";
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
import type { VoiceController, VoiceModelSetting } from "./controller.ts";
import { voiceDataDirectory } from "./paths.ts";
import { NativeSpeechProcess } from "./speech-backend.ts";
import type { NativeSpeechEvent, PreparedSpeech } from "./speech-events.ts";
import {
	MAX_NATIVE_INPUT_CHARS,
	TURN_HARD_CAP_MS,
	turnPolicy,
} from "./turn-policy.ts";
import { showNativeVoiceHistory } from "./voice-history-view.ts";
import type { CueState } from "./voice-sounds.ts";
import {
	createNativeVoiceWidget,
	type NativeVoiceWidget,
	type NativeVoiceWidgetState,
} from "./voice-widget.ts";

export type NativeVoiceCallOptions = {
	backend: PreparedSpeech;
	modelSetting: VoiceModelSetting;
	historyDirectory: string;
	modality?: NonNullable<
		Parameters<VoiceController["beginNative"]>[1]["modality"]
	>;
	classifierModelSetting?: VoiceModelSetting;
};

type SpeechFactory = () => NativeSpeechProcess;
const MAX_RECOVERIES = 2;
const RECOVERY_DEADLINE_MS = 15_000;
const MAX_SERIAL_WORK = 64;

/** Epoch-fenced bridge between controller and supervised local audio child. */
export class NativeVoiceCall {
	private process: NativeSpeechProcess | undefined;
	private draining: NativeSpeechProcess | undefined;
	private stopping: Promise<void> | undefined;
	private cueOnly = false;
	private unsubscribe: (() => void) | undefined;
	private epoch = 0;
	private serial = Promise.resolve();
	private serialCount = 0;
	private operation = 0;
	private options: NativeVoiceCallOptions | undefined;
	private context: ExtensionContext | undefined;
	private muted = false;
	private inputPending = false;
	private speechBusy = false;
	private progressSpeechId: string | undefined;
	private captionId = 0;
	private previewFloor = 0;
	private cueState: CueState | undefined;
	private cueWarningShown = false;
	private recovering = false;
	private recoveries = 0;
	private finalTimer: ReturnType<typeof setTimeout> | undefined;
	private classifyAbort: AbortController | undefined;
	private finalDeadline: number | undefined;
	private speechRevision = 0;
	private finals: Array<{
		text: string;
		generation: number;
		requestId: number;
		ordinal: number;
	}> = [];
	private finalKeys = new Set<string>();
	private state: NativeVoiceWidgetState = {
		status: "unavailable",
		activity: "unavailable",
		error: null,
		partial: "",
		spoken: "",
		waveform: [],
		inputLevel: 0,
		outputLevel: 0,
		playing: false,
		muted: false,
	};

	constructor(
		private readonly controller: VoiceController,
		private readonly makeProcess: SpeechFactory = () =>
			new NativeSpeechProcess(),
	) {}

	get active(): boolean {
		return this.context !== undefined && this.state.status !== "unavailable";
	}

	get isMuted(): boolean {
		return this.muted;
	}

	async start(
		ctx: ExtensionContext,
		options: NativeVoiceCallOptions,
	): Promise<void> {
		const epoch = this.epoch + 1;
		try {
			await this.stop();
		} catch (error) {
			await options.backend.client.close().catch(() => undefined);
			throw error;
		}
		if (epoch !== this.epoch) {
			await options.backend.client.close().catch(() => undefined);
			return;
		}
		this.context = ctx;
		this.options = options;
		this.muted = false;
		this.cueState = undefined;
		this.inputPending = false;
		this.speechBusy = false;
		this.progressSpeechId = undefined;
		this.cueWarningShown = false;
		this.recoveries = 0;
		this.resetSerial();
		this.state = {
			...this.state,
			status: "loading",
			activity: "ready",
			error: null,
			voiceError: null,
			partial: "",
			spoken: "",
			waveform: [],
			inputLevel: 0,
			outputLevel: 0,
			playing: false,
			muted: false,
		};
		try {
			this.controller.beginNative(ctx, options);
			this.subscribe(epoch);
			await this.open(epoch);
		} catch (error) {
			if (epoch === this.epoch) this.unavailable(message(error));
			await options.backend.client.close().catch(() => undefined);
			throw error;
		}
	}

	private widget: NativeVoiceWidget | undefined;

	createWidget(
		tui: Parameters<typeof createNativeVoiceWidget>[0],
		theme: Parameters<typeof createNativeVoiceWidget>[1],
	): NativeVoiceWidget {
		this.widget?.dispose();
		this.widget = createNativeVoiceWidget(tui, theme, () => this.state);
		return this.widget;
	}

	scrollWidget(...args: Parameters<NativeVoiceWidget["scroll"]>): void {
		if (this.context) this.widget?.scroll(...args);
	}

	stop(): Promise<void> {
		++this.epoch;
		if (this.stopping) return this.stopping;
		const stopping = this.stopOwned();
		this.stopping = stopping;
		void stopping
			.finally(() => {
				if (this.stopping === stopping) this.stopping = undefined;
			})
			.catch(() => undefined);
		return stopping;
	}

	private async stopOwned(): Promise<void> {
		const process = this.process;
		this.process = undefined;
		this.unsubscribe?.();
		this.unsubscribe = undefined;
		this.clearFinals();
		this.resetSerial();
		this.recovering = false;
		this.cueState = undefined;
		this.context = undefined;
		this.state = {
			...this.state,
			status: "unavailable",
			activity: "unavailable",
			voiceError: null,
			partial: "",
			spoken: "",
			waveform: [],
			inputLevel: 0,
			outputLevel: 0,
			playing: false,
			muted: false,
		};
		try {
			this.widget?.invalidate();
		} catch {
			/* A disposed UI must not obstruct audio shutdown. */
		}
		this.draining = process;
		try {
			await process?.close().catch(() => undefined);
		} finally {
			this.draining = undefined;
		}
		try {
			this.controller.endNative();
			this.state = {
				...this.state,
				voiceError: this.controller.nativeSnapshot().error,
			};
		} finally {
			this.widget?.invalidate();
		}
	}

	async mute(muted: boolean): Promise<void> {
		if (!this.context)
			throw new Error("Start native voice before changing mute.");
		const epoch = this.epoch;
		const opening = this.state.status === "loading" || this.recovering;
		if (muted || opening) this.muted = muted;
		if (muted) this.inputPending = false;
		this.clearFinals();
		if (muted) {
			this.resetSerial();
			this.controller.onHumanSpeech(false);
			this.controller.interruptNative();
		}
		this.state = {
			...this.state,
			muted: this.muted,
			humanSpeaking: this.controller.nativeSnapshot().humanSpeaking,
			status: opening ? this.state.status : this.muted ? "muted" : "ready",
		};
		this.widget?.invalidate();
		const process = this.process;
		if (muted) void this.syncCueState();
		if (opening) {
			void process?.setMuted(muted).catch((error) => {
				if (epoch === this.epoch && process === this.process)
					this.unavailable(message(error));
			});
			void this.syncCueState();
			return;
		}
		if (!process)
			throw new Error("Audio is unavailable; resume native voice explicitly.");
		try {
			await process.setMuted(muted);
			if (epoch !== this.epoch || process !== this.process) return;
			this.muted = muted;
			this.state = { ...this.state, muted, status: muted ? "muted" : "ready" };
			this.widget?.invalidate();
			void this.syncCueState();
		} catch (error) {
			if (epoch === this.epoch) this.unavailable(message(error));
			throw error;
		}
	}

	async resume(): Promise<void> {
		if (!this.options || !this.context)
			throw new Error("Native voice has not been configured.");
		if (this.recovering)
			throw new Error(
				"Native audio is already recovering; wait or stop the call.",
			);
		if (this.process && !this.recovering) {
			await this.mute(false);
			return;
		}
		const epoch = ++this.epoch;
		this.clearFinals();
		this.resetSerial();
		this.muted = false;
		this.recoveries = 0;
		this.controller.beginNative(this.context, this.options);
		this.subscribe(epoch);
		await this.recover(epoch, true);
	}

	async showHistory(
		ctx: ExtensionContext,
		historyDirectory?: string,
	): Promise<void> {
		const epoch = this.epoch;
		const entries = await this.controller.loadNativeHistory({
			cwd: ctx.cwd,
			historyDirectory:
				historyDirectory ??
				this.options?.historyDirectory ??
				join(voiceDataDirectory(), "history"),
			mainSessionId: ctx.sessionManager.getSessionId(),
		});
		if (epoch !== this.epoch) return;
		await showNativeVoiceHistory(ctx, entries);
	}

	private subscribe(epoch: number): void {
		this.unsubscribe?.();
		this.unsubscribe = this.controller.subscribe((event) => {
			if (epoch !== this.epoch) return;
			if (event.type === "status-interrupt") {
				if (this.progressSpeechId === event.assistantTurnId) {
					this.progressSpeechId = undefined;
					void this.process
						?.interrupt()
						.catch((error) => this.recordError(error));
				}
				return;
			}
			if (this.muted && event.type !== "native-state") return;
			if (event.type === "assistant-speech") {
				const revision = this.speechRevision;
				if (event.kind === "status")
					this.progressSpeechId = event.assistantTurnId;
				this.enqueueFor(
					epoch,
					() =>
						revision !== this.speechRevision ||
						(event.kind === "status" &&
							!this.canSendProgress(event.assistantTurnId))
							? Promise.resolve()
							: (this.process?.speak({
									assistantTurnId: event.assistantTurnId,
									userTurnId: event.userTurnId,
									text: event.text,
									...(event.kind ? { kind: event.kind } : {}),
								}) ?? Promise.resolve()),
					event.kind === "status" ? event.assistantTurnId : undefined,
				);
			}
			if (event.type === "assistant-complete") {
				const revision = this.speechRevision;
				this.enqueueFor(
					epoch,
					() =>
						revision !== this.speechRevision ||
						(event.kind === "status" &&
							!this.canSendProgress(event.assistantTurnId))
							? Promise.resolve()
							: (this.process?.complete(event.assistantTurnId) ??
								Promise.resolve()),
					event.kind === "status" ? event.assistantTurnId : undefined,
				);
			}
			if (event.type === "hotwords")
				this.enqueueFor(
					epoch,
					() => this.process?.setHotwords(event.terms) ?? Promise.resolve(),
				);
			if (event.type === "native-state") this.syncControllerState();
		});
	}

	private async open(epoch: number): Promise<void> {
		const options = this.options;
		if (!options || epoch !== this.epoch) return;
		const process = this.makeProcess();
		this.cueState = undefined;
		this.inputPending = false;
		this.speechBusy = false;
		this.progressSpeechId = undefined;
		this.captionId = 0;
		this.previewFloor = 0;
		this.process = process;
		const starting = process.start({
			...options,
			onEvent: (event) => this.onEvent(epoch, process, event),
		});
		if (this.muted) {
			void process.setMuted(true).catch((error) => {
				if (epoch === this.epoch && process === this.process)
					this.unavailable(message(error));
			});
		}
		await starting;
		if (epoch !== this.epoch || this.process !== process) {
			await process.close().catch(() => undefined);
			return;
		}
		this.recovering = false;
		this.state = {
			...this.state,
			status: this.muted ? "muted" : "ready",
			error: null,
		};
		this.syncControllerState();
		void this.syncCueState();
	}

	private onEvent(
		epoch: number,
		process: NativeSpeechProcess,
		event: NativeSpeechEvent,
	): void {
		if (process === this.draining && event.type === "playback") {
			this.controller.markNativePlayback(
				event.assistantTurnId,
				event.playedText,
			);
			this.controller.setPlaybackDuration(event.userTurnId, event.cumulativeMs);
			return;
		}
		if (epoch !== this.epoch || process !== this.process) return;
		if (event.type === "ready") {
			this.muted = event.muted ?? this.muted;
			this.state = { ...this.state, muted: this.muted };
			this.recovering = false;
			this.state = {
				...this.state,
				status: this.muted ? "muted" : "ready",
				error: null,
			};
			this.syncControllerState();
			return;
		}
		if (event.type === "warning") {
			this.notifyCueWarning(event.message);
			return;
		}
		if (event.type === "phrase") {
			this.state = { ...this.state, spoken: event.text };
			this.widget?.invalidate();
			return;
		}
		if (event.type === "error") {
			if (!event.recoverable || this.state.status === "loading" || this.muted) {
				this.unavailable(event.error);
				return;
			}
			this.state = { ...this.state, error: event.error };
			this.widget?.invalidate();
			this.runBackground(epoch, this.recover(epoch));
			return;
		}
		if (event.type === "input-state") {
			this.inputPending = event.pending;
			if (event.pending) this.controller.cancelNativeProgress();
			return;
		}
		if (event.type === "audio-state") {
			this.speechBusy = event.busy ?? event.playing;
			this.cueOnly = event.cueOnly ?? false;
			this.state = {
				...this.state,
				waveform: event.waveform,
				inputLevel: event.inputLevel,
				outputLevel: event.outputLevel,
				playing: event.playing,
			};
			this.widget?.invalidate();
			this.maybeNarrateProgress();
			return;
		}
		if (event.type === "caption-empty") {
			if (this.muted || this.recovering || event.captionId < this.captionId)
				return;
			this.captionId = event.captionId;
			this.previewFloor = Math.max(
				this.previewFloor,
				event.previewRevision + 1,
			);
			this.state = {
				...this.state,
				partial: "",
				provisional: this.controller.nativeSnapshot().humanSpeaking,
			};
			this.widget?.invalidate();
			return;
		}
		if (event.type === "preview") {
			if (
				this.muted ||
				this.recovering ||
				this.state.status !== "ready" ||
				event.captionId < this.captionId ||
				(event.captionId === this.captionId &&
					event.revision < this.previewFloor)
			)
				return;
			if (event.captionId > this.captionId) this.previewFloor = 0;
			this.captionId = event.captionId;
			this.previewFloor = Math.max(this.previewFloor, event.revision);
			this.state = { ...this.state, partial: event.text, provisional: true };
			this.widget?.invalidate();
			return;
		}
		if (event.type === "speech") {
			if (event.active && !this.muted) {
				const newCaption =
					event.captionId !== undefined && event.captionId > this.captionId;
				if (newCaption && event.captionId !== undefined) {
					this.captionId = event.captionId;
					this.previewFloor = 0;
				}
				this.state = {
					...this.state,
					provisional: true,
					partial:
						!newCaption && this.state.provisional ? this.state.partial : "",
				};
				this.widget?.invalidate();
			}
			this.controller.onHumanSpeech(!this.muted && event.active);
			this.syncControllerState();
			if (event.active) {
				this.speechRevision++;
				this.clearFinalTimer();
				this.finalDeadline = undefined;
				if (!this.muted) this.runBackground(epoch, this.interrupt());
			} else if (this.finals.length) {
				this.scheduleFinals(epoch, process);
			}
			return;
		}
		if (event.type === "empty") {
			if (!this.controller.nativeSnapshot().humanSpeaking) {
				this.state = { ...this.state, partial: "", provisional: false };
				this.widget?.invalidate();
			}
			if (!this.muted && !this.recovering)
				this.controller.onEmptySpeechBoundary();
			return;
		}
		if (event.type === "final") {
			this.controller.cancelNativeProgress();
			this.queueFinal(epoch, process, event);
			return;
		}
		if (event.type === "playback") {
			this.controller.markNativePlayback(
				event.assistantTurnId,
				event.playedText,
			);
			this.controller.setPlaybackDuration(event.userTurnId, event.cumulativeMs);
			return;
		}
		if (event.type === "boundary") {
			if (this.progressSpeechId === event.assistantTurnId)
				this.progressSpeechId = undefined;
			this.controller.turnBoundary(event.assistantTurnId);
		}
	}

	private queueFinal(
		epoch: number,
		process: NativeSpeechProcess,
		event: Extract<NativeSpeechEvent, { type: "final" }>,
	): void {
		// Native mute/recovery resets ASR; acknowledging an old generation would be stale.
		if (this.muted || this.recovering) return;
		const key = `${event.generation}:${event.requestId}`;
		if (this.finalKeys.has(key)) return;
		if (event.text.length > MAX_NATIVE_INPUT_CHARS) {
			this.unavailable("Speech final exceeds 100,000-character sanity bound.");
			return;
		}
		const pendingChars = this.finals.reduce(
			(sum, final) => sum + final.text.length + 1,
			0,
		);
		if (
			this.finals.length >= 8 ||
			pendingChars + event.text.length > MAX_NATIVE_INPUT_CHARS
		) {
			if (this.controller.nativeSnapshot().humanSpeaking) {
				this.unavailable(
					"Speech-final input bound exceeded while speaking; resume explicitly.",
				);
				return;
			}
			this.submitFinals(epoch, process);
		}
		if (this.finalKeys.size >= 32) {
			this.unavailable("Speech-final queue overflow; resume explicitly.");
			return;
		}
		this.finalKeys.add(key);
		if (!event.text.trim()) {
			this.consumeFinal(epoch, process, event, key);
			return;
		}
		this.finals.push({
			text: event.text,
			generation: event.generation,
			requestId: event.requestId,
			ordinal: event.ordinal ?? event.captionId ?? event.requestId,
		});
		this.finals.sort((a, b) => a.ordinal - b.ordinal);
		if (event.captionId === undefined || event.captionId >= this.captionId) {
			this.captionId = event.captionId ?? this.captionId;
			if (event.captionId !== undefined)
				this.previewFloor = Math.max(
					this.previewFloor,
					(event.previewRevision ?? Number.MAX_SAFE_INTEGER - 1) + 1,
				);
			this.state = {
				...this.state,
				partial: event.text,
				provisional: false,
				captionRevision:
					(this.state.captionRevision ?? 0) +
					(this.controller.nativeSnapshot().humanSpeaking ? 0 : 1),
			};
			this.widget?.invalidate();
		}
		this.scheduleFinals(epoch, process);
	}

	private scheduleFinals(epoch: number, process: NativeSpeechProcess): void {
		this.clearFinalTimer();
		if (this.controller.nativeSnapshot().humanSpeaking) return;
		const abort = new AbortController();
		this.finalDeadline ??= Date.now() + TURN_HARD_CAP_MS;
		const deadline = this.finalDeadline;
		this.classifyAbort = abort;
		const joined = this.finals
			.map((final) => final.text)
			.join(" ")
			.slice(-4_000);
		const submit = async () => {
			let waitMs = 1_500;
			try {
				waitMs = turnPolicy(
					await this.controller.classifyTurn({
						text: joined,
						signal: abort.signal,
					}),
				).waitMs;
			} catch {
				// Classifier failure retains bounded fallback endpointing.
			}
			if (
				abort.signal.aborted ||
				epoch !== this.epoch ||
				process !== this.process ||
				this.muted ||
				this.recovering
			)
				return;
			if (this.finalTimer) clearTimeout(this.finalTimer);
			this.finalTimer = setTimeout(
				() => this.submitFinals(epoch, process),
				Math.min(waitMs, Math.max(0, deadline - Date.now())),
			);
		};
		void submit();
		this.finalTimer = setTimeout(
			() => this.submitFinals(epoch, process),
			Math.max(0, deadline - Date.now()),
		);
	}

	private submitFinals(epoch: number, process: NativeSpeechProcess): void {
		if (this.controller.nativeSnapshot().humanSpeaking) return;
		if (this.finalTimer) clearTimeout(this.finalTimer);
		this.finalTimer = undefined;
		this.finalDeadline = undefined;
		this.classifyAbort?.abort();
		this.classifyAbort = undefined;
		if (
			epoch !== this.epoch ||
			process !== this.process ||
			this.muted ||
			this.recovering
		)
			return;
		const finals = this.finals.splice(0, 8);
		const first = finals[0];
		if (!first) return;
		const revision = this.speechRevision;
		this.enqueueFor(epoch, async () => {
			if (process !== this.process || this.muted || this.recovering) return;
			if (
				revision !== this.speechRevision ||
				this.controller.nativeSnapshot().humanSpeaking ||
				this.finals.some((pending) => pending.ordinal < first.ordinal)
			) {
				// A queued delivery cannot turn a mid-thought pause into a committed turn.
				const chars = [...finals, ...this.finals].reduce(
					(sum, final) => sum + final.text.length + 1,
					-1,
				);
				if (
					finals.length + this.finals.length > 8 ||
					chars > MAX_NATIVE_INPUT_CHARS
				) {
					this.unavailable(
						"Speech-final input bound exceeded while speaking; resume explicitly.",
					);
					return;
				}
				this.finals.unshift(...finals);
				this.finals.sort((a, b) => a.ordinal - b.ordinal);
				if (!this.controller.nativeSnapshot().humanSpeaking)
					this.scheduleFinals(epoch, process);
				return;
			}
			this.controller.receiveNative(
				finals.map((final) => final.text).join(" "),
			);
			for (const final of finals) {
				if (process !== this.process || this.muted || this.recovering) return;
				await process.consumeFinal(final.generation, final.requestId);
				this.finalKeys.delete(`${final.generation}:${final.requestId}`);
			}
		});
	}

	private consumeFinal(
		epoch: number,
		process: NativeSpeechProcess,
		event: Extract<NativeSpeechEvent, { type: "final" }>,
		key: string,
	): void {
		this.enqueueFor(epoch, async () => {
			if (process !== this.process) return;
			await process.consumeFinal(event.generation, event.requestId);
			this.finalKeys.delete(key);
		});
	}

	private async interrupt(): Promise<void> {
		// Barge-in cancels output, not recognition already waiting for submission.
		this.controller.interruptNative(true);
		await this.process?.interrupt().catch((error) => this.recordError(error));
	}

	private async recover(epoch: number, explicit = false): Promise<void> {
		if (this.recovering || epoch !== this.epoch || this.muted) return;
		if (!explicit && ++this.recoveries > MAX_RECOVERIES) {
			this.unavailable("Audio recovery exhausted. Resume explicitly.");
			return;
		}
		this.recovering = true;
		this.clearFinals();
		this.resetSerial();
		this.controller.onHumanSpeech(false);
		this.controller.interruptNative();
		this.state = {
			...this.state,
			status: "reconnecting",
			humanSpeaking: false,
			partial: "",
			playing: false,
		};
		this.widget?.invalidate();
		const old = this.process;
		this.process = undefined;
		void this.setCueState(old, "off", true);
		await old?.close().catch(() => undefined);
		if (epoch !== this.epoch) return;
		let timer: ReturnType<typeof setTimeout> | undefined;
		let attempted: NativeSpeechProcess | undefined;
		try {
			const opening = this.open(epoch);
			attempted = this.currentProcess();
			await Promise.race([
				opening,
				new Promise<never>((_, reject) => {
					timer = setTimeout(
						() => reject(new Error("Audio recovery timed out")),
						RECOVERY_DEADLINE_MS,
					);
				}),
			]);
		} catch (error) {
			if (this.process === attempted) this.process = undefined;
			await attempted?.close().catch(() => undefined);
			if (epoch !== this.epoch) return;
			this.recovering = false;
			this.state = { ...this.state, error: message(error) };
			this.runBackground(epoch, this.recover(epoch));
		} finally {
			if (timer) clearTimeout(timer);
		}
	}

	private unavailable(error: string): void {
		const process = this.process;
		this.process = undefined;
		void this.setCueState(process, "off", true);
		this.recovering = false;
		this.clearFinals();
		this.resetSerial();
		this.state = {
			...this.state,
			status: "unavailable",
			activity: "unavailable",
			error,
			partial: "",
			playing: false,
		};
		try {
			this.controller.endNative();
		} catch (cause) {
			this.state = { ...this.state, error: `${error}; ${message(cause)}` };
		}
		void process?.close().catch(() => undefined);
		try {
			this.widget?.invalidate();
		} catch {
			/* A disposed UI must not obstruct audio shutdown. */
		}
	}

	private runBackground(epoch: number, work: Promise<void>): void {
		void work.catch((cause) => {
			if (epoch === this.epoch) this.unavailable(message(cause));
		});
	}

	private canSendProgress(id: string): boolean {
		return (
			this.controller.isNativeProgressCurrent(id) &&
			!this.inputPending &&
			!this.finals.length &&
			!this.finalKeys.size &&
			!this.muted &&
			!this.recovering &&
			this.state.status === "ready"
		);
	}

	private isProgressIdle(): boolean {
		return Boolean(
			this.process &&
				this.context &&
				this.state.status === "ready" &&
				!this.muted &&
				!this.recovering &&
				!this.inputPending &&
				!this.controller.nativeSnapshot().humanSpeaking &&
				!this.speechBusy &&
				(!this.state.playing || this.cueOnly) &&
				!this.finals.length &&
				!this.finalKeys.size &&
				this.serialCount === 0,
		);
	}

	private maybeNarrateProgress(): void {
		if (this.isProgressIdle())
			this.controller.narrateExecutorProgress(() => this.isProgressIdle());
	}

	private syncControllerState(): void {
		const { status, error, humanSpeaking } = this.controller.nativeSnapshot();
		if (
			this.state.activity !== status ||
			this.state.voiceError !== error ||
			this.state.humanSpeaking !== humanSpeaking
		) {
			this.state = {
				...this.state,
				activity: status,
				voiceError: error,
				humanSpeaking,
			};
			this.widget?.invalidate();
		}
		void this.syncCueState();
	}

	private async syncCueState(): Promise<void> {
		const process = this.process;
		if (!process) return;
		const snapshot = this.controller.nativeSnapshot();
		const state: CueState =
			this.muted ||
			this.recovering ||
			this.state.status === "unavailable" ||
			snapshot.humanSpeaking
				? "off"
				: this.state.status === "loading"
					? "loading"
					: (snapshot.cueState ?? "listening");
		// Worker owns loading until it reports ready; parent only enforces priority off.
		if (state === "loading" && this.cueState !== "off") return;
		await this.setCueState(process, state);
	}

	private async setCueState(
		process: NativeSpeechProcess | undefined,
		state: CueState,
		force = false,
	): Promise<void> {
		if (!process || (!force && process !== this.process)) return;
		if (!force && this.cueState === state) return;
		this.cueState = state;
		const epoch = this.epoch;
		try {
			await process.setCueState(state);
		} catch (error) {
			if (!force && epoch === this.epoch && process === this.process)
				this.notifyCueWarning(`Sound cue unavailable: ${message(error)}`);
		}
	}

	private notifyCueWarning(warning: string): void {
		if (this.cueWarningShown) return;
		this.cueWarningShown = true;
		this.context?.ui?.notify(warning.slice(0, 1_000), "warning");
	}

	private clearFinalTimer(): void {
		if (this.finalTimer) clearTimeout(this.finalTimer);
		this.finalTimer = undefined;
		this.classifyAbort?.abort();
		this.classifyAbort = undefined;
	}

	private clearFinals(): void {
		this.clearFinalTimer();
		this.finalDeadline = undefined;
		this.finals.length = 0;
		this.finalKeys.clear();
	}

	private resetSerial(): void {
		this.operation++;
		this.serial = Promise.resolve();
		this.serialCount = 0;
	}

	private enqueueFor(
		epoch: number,
		work: () => Promise<void>,
		statusTurnId?: string,
	): void {
		if (this.serialCount >= MAX_SERIAL_WORK) {
			this.recordError(new Error("Native audio command queue overflow"));
			return;
		}
		const operation = this.operation;
		this.serialCount++;
		this.serial = this.serial
			.then(async () => {
				if (epoch !== this.epoch || operation !== this.operation) return;
				await work();
			})
			.catch((error) => {
				if (epoch === this.epoch && operation === this.operation) {
					this.recordError(error);
					if (
						statusTurnId &&
						this.controller.isNativeProgressCurrent(statusTurnId)
					)
						this.controller.cancelNativeProgress();
				}
			})
			.finally(() => {
				if (epoch === this.epoch && operation === this.operation)
					this.serialCount = Math.max(0, this.serialCount - 1);
			});
	}

	private currentProcess(): NativeSpeechProcess | undefined {
		return this.process;
	}

	private recordError(error: unknown): void {
		this.state = { ...this.state, error: message(error) };
		this.widget?.invalidate();
	}
}

function message(error: unknown): string {
	return error instanceof Error ? error.message : String(error);
}