Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/voice/src/speech-backend.ts

Raw
import type { ServiceInfo, SpeakEvent } from "./speak-client.ts";
import { SpeakClient } from "./speak-client.ts";
import type {
	NativeSpeechEvent,
	NativeSpeechOptions,
	PreparedSpeech,
	SpeechRequest,
} from "./speech-events.ts";
import { supportsSpeechProfile } from "./speech-events.ts";
import { CUE_DIGESTS, type CueState } from "./voice-sounds.ts";

const OPEN_TIMEOUT_MS = 180_000;
const CONTROL_TIMEOUT_MS = 5_000;
const MAX_FINALS = 32;
const MAX_STREAMS = 1;
const MAX_SEGMENTS = 4096;
const MAX_SCALARS = 8_000;
const MAX_OUTPUT_WORK = 64;

type Turn = {
	assistantTurnId: string;
	userTurnId: string;
	operation: string;
	index: number;
	finished: boolean;
	chars: number;
	endsWithWhitespace: boolean;
};
type Segment = {
	local: number;
	revision: number;
	ordinal: number;
	final: boolean;
};

/** Maps generic RPC identities to call-local UI and conversation identities. */
export class NativeSpeechProcess {
	private client: SpeakClient | undefined;
	private info: ServiceInfo | undefined;
	private onEvent: ((event: NativeSpeechEvent) => void) | undefined;
	private unsubscribe: (() => void) | undefined;
	private unfail: (() => void) | undefined;
	private resource: string | undefined;
	private epoch = 0;
	private recognition: string | undefined;
	private renderRate = 0;
	private muted = false;
	private closed = false;
	private opened = false;
	private recognitionPending = false;
	private captureActive = false;
	private hotwords: readonly string[] = [];
	private nextCaption = 0;
	private nextOrdinal = 0;
	private segments = new Map<string, Segment>();
	private finals = new Map<string, { generation: number; requestId: number }>();
	private nextFinal = 0;
	private turns = new Map<string, Turn>();
	private byAssistant = new Map<string, Turn>();
	private cueOperation: string | undefined;
	private cueState: CueState = "loading";
	private cueRevision = 0;
	private cueWork = Promise.resolve();
	private cueRunning = false;
	private outputRevision = 0;
	private outputWork = Promise.resolve();
	private pendingOutput = 0;
	private muteWork = Promise.resolve();
	private pendingMute = 0;
	private closing: Promise<void> | undefined;

	async start(options: NativeSpeechOptions): Promise<void> {
		if (this.client || this.closed)
			throw new Error("Native speech process already used");
		try {
			await this.initialize(options);
		} catch (error) {
			await this.close();
			throw error;
		}
	}

	private async initialize(options: NativeSpeechOptions): Promise<void> {
		this.onEvent = options.onEvent;
		let client = options.backend.client;
		let recovered = false;
		this.client = client;
		try {
			await client.start();
		} catch {
			await client.close().catch(() => undefined);
			if (this.closed) throw new Error("Speech backend is closed");
			client = new SpeakClient(options.backend.command);
			this.client = client;
			options.backend.client = client;
			recovered = true;
			await client.start();
		}
		if (this.closed) throw new Error("Speech backend is closed");
		if (recovered) {
			const plan = await client.request<{ plan: { ready: boolean } }>(
				"assets.plan",
				{ profile: options.backend.profile.id },
			);
			if (!plan.plan.ready)
				throw new Error("Speech assets need explicit setup before recovery");
		}
		this.info = await client.request<ServiceInfo>("service.info");
		if (this.closed) throw new Error("Speech backend is closed");
		this.unsubscribe = client.onEvent((event) => this.event(event));
		this.unfail = client.onFailure((error) =>
			this.emit({
				type: "error",
				error: error.message,
				recoverable: this.info?.io_mode !== "fixture",
			}),
		);
		const profile = this.info.profiles.find(
			(item) => item.id === options.backend.profile.id,
		);
		if (
			!profile?.installed ||
			!sameProfile(profile, options.backend.profile) ||
			!supportsSpeechProfile(profile, options.backend.preview)
		)
			throw new Error("Selected speech profile is unavailable or incompatible");
		const openingMuted = this.muted;
		const opened = await client.request<{
			resource: string;
			epoch: number;
			operation: string;
		}>(
			"resource.open",
			{
				profile: profile.id,
				aec: true,
				preview: options.backend.preview,
				endpoint_ms: 2000,
				interrupt_on_speech: true,
				muted: openingMuted,
			},
			{
				timeoutMs: CONTROL_TIMEOUT_MS,
				onResult: (result) => {
					this.resource = result.resource;
					this.epoch = result.epoch;
					if (this.muted !== openingMuted)
						void this.setMuted(this.muted).catch((error) =>
							this.emit({
								type: "error",
								error: message(error),
								recoverable: false,
							}),
						);
					void this.setCueState(this.cueState);
				},
			},
		);
		const outcome = await client.waitForOperation(opened.operation, {
			timeoutMs: OPEN_TIMEOUT_MS,
		});
		if (outcome.outcome !== "completed")
			throw new Error(`Speech resource failed to open: ${outcome.reason}`);
		if (this.closed) throw new Error("Speech backend is closed");
		if (!this.renderRate) throw new Error("Speech resource has no render rate");
		this.opened = true;
		await this.muteWork;
		if (!this.muted && !this.recognition) await this.startRecognition();
		else if (this.muted) this.emit({ type: "ready", muted: true });
	}

	speak(request: SpeechRequest): Promise<void> {
		return this.queueOutput(() => this.speakNow(request));
	}

	private async speakNow(request: SpeechRequest): Promise<void> {
		if (request.text.length > MAX_SCALARS * 2)
			throw new Error("Speech stream exceeds 8000 scalars");
		if (!request.text.trim()) return;
		const requestScalars = [...request.text].length;
		if (requestScalars > MAX_SCALARS)
			throw new Error("Speech stream exceeds 8000 scalars");
		const client = this.readyClient();
		if (this.muted) return;
		const revision = this.outputRevision;
		let turn = this.byAssistant.get(request.assistantTurnId);
		if (!turn) {
			for (const previous of this.turns.values()) {
				if (previous.finished)
					await client.waitForOperation(previous.operation, {
						timeoutMs: OPEN_TIMEOUT_MS,
					});
			}
			if (revision !== this.outputRevision || this.muted || this.closed) return;
			if (this.turns.size >= MAX_STREAMS)
				throw new Error("Speech stream queue is full");
			await client.request<{ operation: string }>(
				"synthesis.start",
				this.fence(),
				{
					onResult: (result) => {
						turn = {
							assistantTurnId: request.assistantTurnId,
							userTurnId: request.userTurnId,
							operation: result.operation,
							index: 0,
							finished: false,
							chars: 0,
							endsWithWhitespace: false,
						};
						this.turns.set(turn.operation, turn);
						this.byAssistant.set(turn.assistantTurnId, turn);
					},
				},
			);
		}
		if (!turn) throw new Error("Speech stream was not accepted");
		if (revision !== this.outputRevision || this.muted || this.closed) {
			if (this.closed) return;
			await this.cancel(turn.operation);
			return;
		}
		if (!this.turns.has(turn.operation)) return;
		if (turn.finished) throw new Error("Speech turn is already complete");
		const separator =
			turn.chars > 0 && !turn.endsWithWhitespace && !/^\s/u.test(request.text)
				? " "
				: "";
		if (turn.chars + requestScalars + separator.length > MAX_SCALARS) {
			await this.cancel(turn.operation);
			throw new Error("Speech stream exceeds 8000 scalars");
		}
		const text = separator + request.text;
		try {
			await client.request("synthesis.append", {
				operation: turn.operation,
				index: turn.index,
				text,
			});
		} catch (error) {
			await this.cancel(turn.operation).catch(() => undefined);
			throw error;
		}
		turn.index++;
		turn.chars += requestScalars + separator.length;
		turn.endsWithWhitespace = /\s$/u.test(request.text);
	}

	complete(assistantTurnId: string): Promise<void> {
		return this.queueOutput(() => this.completeNow(assistantTurnId));
	}

	private async completeNow(assistantTurnId: string): Promise<void> {
		const turn = this.byAssistant.get(assistantTurnId);
		if (!turn || turn.finished) return;
		turn.finished = true;
		try {
			await this.readyClient().request("synthesis.finish", {
				operation: turn.operation,
			});
		} catch (error) {
			await this.cancel(turn.operation).catch(() => undefined);
			throw error;
		}
	}

	async setHotwords(terms: readonly string[]): Promise<void> {
		if (terms.length > 20 || terms.some((term) => [...term].length > 256))
			throw new Error("Hotwords exceed limit");
		this.hotwords = [...terms];
		if (this.recognition)
			await this.readyClient().request("recognition.set_hotwords", {
				operation: this.recognition,
				terms: this.hotwords,
			});
	}

	async consumeFinal(generation: number, requestId: number): Promise<void> {
		const entry = [...this.finals.entries()].find(
			([, value]) =>
				value.generation === generation && value.requestId === requestId,
		);
		if (!entry || !this.recognition) return;
		const [segment, retained] = entry;
		await this.readyClient().request("recognition.ack", {
			operation: this.recognition,
			segment,
		});
		if (this.finals.get(segment) === retained) this.finals.delete(segment);
	}

	async interrupt(): Promise<void> {
		++this.outputRevision;
		await Promise.all([
			this.setCueState("off"),
			...[...this.turns.keys()].map((operation) => this.cancel(operation)),
		]);
	}

	setMuted(muted: boolean): Promise<void> {
		if (this.pendingMute >= MAX_OUTPUT_WORK) {
			void this.close();
			return Promise.reject(new Error("Speech mute queue is full"));
		}
		this.pendingMute++;
		this.muted = muted;
		++this.outputRevision;
		if (muted) void this.setCueState("off");
		const work = this.muteWork
			.catch(() => undefined)
			.then(async () => {
				if (!this.resource || this.closed) return;
				await this.readyClient().request<{ epoch: number }>(
					"resource.set_muted",
					{ ...this.fence(), muted },
					{
						onResult: (result) => {
							if (result.epoch === this.epoch) return;
							this.epoch = result.epoch;
							this.captureActive = false;
							this.recognitionPending = false;
							this.recognition = undefined;
							this.segments.clear();
							this.finals.clear();
							this.turns.clear();
							this.byAssistant.clear();
							this.cueOperation = undefined;
						},
					},
				);
				if (
					this.opened &&
					!muted &&
					!this.muted &&
					!this.recognition &&
					!this.closed
				)
					await this.startRecognition();
			})
			.finally(() => {
				this.pendingMute--;
			});
		this.muteWork = work;
		return work;
	}

	setCueState(state: CueState): Promise<void> {
		this.cueState = state;
		++this.cueRevision;
		if (this.cueRunning) return this.cueWork;
		this.cueRunning = true;
		this.cueWork = Promise.resolve().then(async () => {
			let applied = -1;
			try {
				while (!this.closed && applied !== this.cueRevision) {
					applied = this.cueRevision;
					await this.updateCue(this.cueState, applied);
				}
			} finally {
				this.cueRunning = false;
			}
		});
		return this.cueWork;
	}

	private async updateCue(state: CueState, revision: number): Promise<void> {
		if (revision !== this.cueRevision) return;
		const previous = this.cueOperation;
		this.cueOperation = undefined;
		if (previous) {
			try {
				await this.cancel(previous);
			} catch (error) {
				if (!this.closed)
					this.emit({
						type: "warning",
						message: `Sound cue unavailable: ${message(error)}`,
					});
				return;
			}
		}
		if (
			revision !== this.cueRevision ||
			state === "off" ||
			this.muted ||
			this.closed ||
			!this.resource ||
			this.turns.size > 0 ||
			!this.client
		)
			return;
		const sound = this.info?.sounds.find(
			(item) => item.installed && item.sha256 === CUE_DIGESTS[state],
		);
		if (!sound) {
			this.emit({
				type: "warning",
				message: `Sound cue unavailable: ${state}`,
			});
			return;
		}
		try {
			await this.client.request<{ operation: string }>(
				"sound.play",
				{
					...this.fence(),
					sound: sound.id,
					gain: 0.3,
					repeat_ms:
						state === "working" ? 1800 + Math.floor(Math.random() * 600) : 2000,
				},
				{
					onResult: (result) => {
						this.cueOperation = result.operation;
					},
				},
			);
			if (revision !== this.cueRevision || this.muted || this.closed) {
				const operation = this.cueOperation;
				this.cueOperation = undefined;
				if (operation && !this.closed) await this.cancel(operation);
			}
		} catch (error) {
			if (revision === this.cueRevision && !this.closed)
				this.emit({
					type: "warning",
					message: `Sound cue unavailable: ${message(error)}`,
				});
		}
	}

	close(): Promise<void> {
		this.closing ??= this.closeOwned();
		return this.closing;
	}

	private async closeOwned(): Promise<void> {
		if (this.closed) return;
		this.closed = true;
		++this.outputRevision;
		++this.cueRevision;
		const client = this.client;
		if (!client) return;
		if (this.resource)
			await client
				.request("resource.close", this.fence(), {
					timeoutMs: CONTROL_TIMEOUT_MS,
				})
				.catch(() => undefined);
		await client.close().catch(() => undefined);
		this.unsubscribe?.();
		this.unfail?.();
		this.recognition = undefined;
		this.segments.clear();
		this.finals.clear();
		this.turns.clear();
		this.byAssistant.clear();
	}

	private async startRecognition(): Promise<void> {
		const client = this.readyClient();
		const result = await client.request<{ operation: string }>(
			"recognition.start",
			this.fence(),
			{
				onResult: (result) => {
					this.recognition = result.operation;
					if (!this.muted && !this.closed)
						this.emit({ type: "ready", muted: false });
				},
			},
		);
		if (this.hotwords.length)
			await client.request("recognition.set_hotwords", {
				operation: result.operation,
				terms: this.hotwords,
			});
	}

	private event(event: SpeakEvent): void {
		// During healthy close only certified playback accounting may cross the fence.
		if (
			this.closed &&
			event.method !== "playback.progress" &&
			event.method !== "operation.ended"
		)
			return;
		if (event.params.resource && event.params.resource !== this.resource)
			return;
		const data = event.params.data;
		if (
			event.method !== "resource.health" &&
			event.params.epoch !== null &&
			event.params.epoch !== this.epoch
		)
			return;
		if (
			event.method.startsWith("recognition.") &&
			(this.muted || event.params.operation !== this.recognition)
		)
			return;
		if (event.method === "resource.ready") {
			this.renderRate = Number(data["render_rate"]);
			return;
		}
		if (event.method === "resource.health") {
			if ((event.params.epoch ?? 0) <= this.epoch) return;
			this.epoch = event.params.epoch ?? this.epoch;
			this.captureActive = false;
			this.recognitionPending = false;
			this.recognition = undefined;
			this.segments.clear();
			this.finals.clear();
			this.turns.clear();
			this.byAssistant.clear();
			++this.outputRevision;
			++this.cueRevision;
			this.cueOperation = undefined;
			this.emit({
				type: "error",
				error: "Speech resource unavailable",
				recoverable: this.info?.io_mode !== "fixture",
			});
			return;
		}
		if (event.method === "service.warning") {
			this.emit({ type: "warning", message: String(data["message"]) });
			return;
		}
		if (event.method === "recognition.activity") {
			this.captureActive = Boolean(data["active"]);
			const segment =
				typeof data["segment"] === "string"
					? this.segment(data["segment"])
					: undefined;
			this.emit({
				type: "speech",
				active: Boolean(data["active"]),
				...(segment ? { captionId: segment.local } : {}),
			});
			return;
		}
		if (event.method === "recognition.pending") {
			this.recognitionPending = Boolean(data["pending"]);
			this.emit({ type: "input-state", pending: Boolean(data["pending"]) });
			return;
		}
		if (event.method === "recognition.preview") {
			const segment = this.segment(String(data["segment"]));
			const revision = Number(data["revision"]);
			if (segment.final || revision <= segment.revision) return;
			segment.revision = revision;
			this.emit({
				type: "preview",
				text: String(data["text"]),
				captionId: segment.local,
				revision,
			});
			return;
		}
		if (event.method === "recognition.preview_cleared") {
			const segment = this.segment(String(data["segment"]));
			const through = Number(data["through_revision"]);
			if (segment.final || through < segment.revision) return;
			segment.revision = through;
			this.emit({
				type: "caption-empty",
				captionId: segment.local,
				previewRevision: Number(data["through_revision"]),
			});
			return;
		}
		if (event.method === "recognition.empty") {
			const segment = this.segment(String(data["segment"]));
			if (segment.final) return;
			segment.final = true;
			this.emit({ type: "empty" });
			return;
		}
		if (event.method === "recognition.final") {
			const id = String(data["segment"]),
				segment = this.segment(id);
			if (segment.final) return;
			if (this.finals.size >= MAX_FINALS) {
				this.emit({
					type: "error",
					error: "Speech final retention overflow",
					recoverable: false,
				});
				return;
			}
			segment.final = true;
			const requestId = ++this.nextFinal;
			this.finals.set(id, { generation: this.epoch, requestId });
			this.emit({
				type: "final",
				text: String(data["text"]),
				generation: this.epoch,
				requestId,
				ordinal: segment.ordinal,
				captionId: segment.local,
				previewRevision: segment.revision,
			});
			return;
		}
		if (event.method === "audio.level") {
			const synthesisBusy = this.turns.size > 0;
			const cueOnly =
				this.cueOperation !== undefined &&
				this.opened &&
				!synthesisBusy &&
				!this.captureActive &&
				!this.recognitionPending;
			this.emit({
				type: "audio-state",
				inputLevel: Number(data["input"]),
				outputLevel: Number(data["output"]),
				waveform: Array.isArray(data["waveform"])
					? (data["waveform"] as number[])
					: [],
				playing: Boolean(data["playing"]),
				busy: (Boolean(data["busy"]) && !cueOnly) || synthesisBusy,
				cueOnly,
			});
			return;
		}
		if (event.method === "operation.ended") {
			if (event.params.operation === this.cueOperation)
				this.cueOperation = undefined;
			if (event.params.operation === this.recognition) {
				this.captureActive = false;
				this.recognitionPending = false;
				this.recognition = undefined;
				this.segments.clear();
				this.finals.clear();
				if (!this.muted && !this.closed)
					this.emit({
						type: "error",
						error: "Speech recognition ended",
						recoverable: this.info?.io_mode !== "fixture",
					});
			}
		}
		const turn = event.params.operation
			? this.turns.get(event.params.operation)
			: undefined;
		if (!turn) return;
		if (event.method === "synthesis.phrase")
			this.emit({
				type: "phrase",
				assistantTurnId: turn.assistantTurnId,
				userTurnId: turn.userTurnId,
				text: String(data["text"]),
			});
		if (event.method === "playback.progress")
			this.emit({
				type: "playback",
				assistantTurnId: turn.assistantTurnId,
				userTurnId: turn.userTurnId,
				playedText: String(data["played_text"]),
				cumulativeMs: Math.floor(
					(Number(data["consumed_frames"]) * 1000) / this.renderRate,
				),
			});
		if (event.method === "playback.boundary")
			this.emit({ type: "boundary", assistantTurnId: turn.assistantTurnId });
		if (event.method === "operation.ended") {
			this.turns.delete(turn.operation);
			this.byAssistant.delete(turn.assistantTurnId);
			if (!this.closed) void this.setCueState(this.cueState);
			if (!this.closed && data["outcome"] === "failed")
				this.emit({
					type: "error",
					error: `Speech synthesis failed: ${String(data["reason"])}`,
					recoverable: false,
				});
		}
	}

	private segment(id: string): Segment {
		let value = this.segments.get(id);
		if (!value) {
			if (this.segments.size >= MAX_SEGMENTS) {
				this.emit({
					type: "error",
					error: "Speech segment retention overflow",
					recoverable: false,
				});
				throw new Error("Speech segment retention overflow");
			}
			value = {
				local: ++this.nextCaption,
				revision: -1,
				ordinal: ++this.nextOrdinal,
				final: false,
			};
			this.segments.set(id, value);
		}
		return value;
	}
	private queueOutput(work: () => Promise<void>): Promise<void> {
		if (this.pendingOutput >= MAX_OUTPUT_WORK)
			return Promise.reject(new Error("Speech command queue is full"));
		const revision = this.outputRevision;
		this.pendingOutput++;
		const pending = this.outputWork
			.catch(() => undefined)
			.then(async () => {
				if (revision !== this.outputRevision || this.muted || this.closed)
					return;
				await work();
			})
			.finally(() => {
				this.pendingOutput--;
			});
		this.outputWork = pending;
		return pending;
	}

	private async cancel(operation: string): Promise<void> {
		await this.client?.request(
			"operation.cancel",
			{ operation },
			{ timeoutMs: CONTROL_TIMEOUT_MS },
		);
	}
	private fence(): { resource: string; epoch: number } {
		if (!this.resource) throw new Error("Speech resource is unavailable");
		return { resource: this.resource, epoch: this.epoch };
	}
	private readyClient(): SpeakClient {
		if (!this.client || this.closed || !this.resource)
			throw new Error("Speech backend is unavailable");
		return this.client;
	}
	private emit(event: NativeSpeechEvent): void {
		if (this.closed && event.type !== "playback") return;
		try {
			this.onEvent?.(event);
		} catch {
			/* UI failures cannot corrupt RPC state. */
		}
	}
}

function sameProfile(
	actual: PreparedSpeech["profile"],
	expected: PreparedSpeech["profile"],
): boolean {
	return (
		actual.id === expected.id &&
		actual.label === expected.label &&
		actual.description === expected.description &&
		(
			Object.keys(expected.capabilities) as Array<
				keyof typeof expected.capabilities
			>
		).every((key) => actual.capabilities[key] === expected.capabilities[key])
	);
}
function message(error: unknown): string {
	return error instanceof Error ? error.message : String(error);
}