Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/voice/index.ts

Raw
import { isAbsolute, join } from "node:path";
import type {
	ExtensionAPI,
	ExtensionContext,
} from "@earendil-works/pi-coding-agent";
import { VoiceController, type VoiceModelSetting } from "./src/controller.ts";
import { dbg } from "./src/debug.ts";
import { NativeVoiceCall } from "./src/native-call.ts";
import { runNativeSetup } from "./src/native-setup-ui.ts";
import { voiceDataDirectory } from "./src/paths.ts";
import {
	defineIntegerSetting,
	parseBooleanSetting,
	resolveSetting,
	type SettingDeclaration,
} from "./src/pi-ext-settings.ts";

const COMMAND_SETTING: SettingDeclaration<readonly string[]> = {
	key: "voice.command",
	label: "Speech service command",
	description: "Executable and arguments for the external speech service.",
	env: "PI_VOICE_COMMAND",
	default: ["speak_to_me"],
	parse: (raw, source) => {
		let value = raw;
		if (typeof raw === "string" && source === "env") {
			try {
				value = JSON.parse(raw);
			} catch {
				throw new Error(
					"expected a nonempty JSON array of nonempty command strings",
				);
			}
		}
		if (
			Array.isArray(value) &&
			value.length > 0 &&
			value.every(
				(part) =>
					typeof part === "string" && part.length > 0 && !part.includes("\0"),
			)
		)
			return value;
		throw new Error(
			"expected a nonempty JSON array of nonempty command strings",
		);
	},
};
const PROFILE_SETTING: SettingDeclaration<string | undefined> = {
	key: "voice.profile",
	label: "Speech service profile",
	description: "Optional opaque profile ID supplied by the speech service.",
	env: "PI_VOICE_PROFILE",
	default: undefined,
	parse: (raw, source) => {
		if (raw === undefined && source === "default") return undefined;
		if (typeof raw === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(raw))
			return raw;
		throw new Error("expected an opaque speech service profile ID");
	},
};
const PREVIEW_SETTING: SettingDeclaration<boolean> = {
	key: "voice.preview",
	label: "Live transcription preview",
	description:
		"Show provisional transcription while the speech service recognizes input.",
	env: "PI_VOICE_PREVIEW",
	default: true,
	parse: (raw, source) => {
		const value = parseBooleanSetting(raw, source);
		if (value === undefined) throw new Error("expected a boolean");
		return value;
	},
};
const MAX_SENTENCES_SETTING: SettingDeclaration<number> = defineIntegerSetting({
	key: "voice.maxSentences",
	label: "Reply sentence limit",
	description: "Maximum sentences per spoken reply.",
	env: "PI_VOICE_MAX_SENTENCES",
	default: 6,
	range: { min: 1, max: 20 },
});
type VoiceShortcut = Parameters<ExtensionAPI["registerShortcut"]>[0];

const TOGGLE_SHORTCUT_SETTING: SettingDeclaration<VoiceShortcut | null> = {
	key: "voice.toggleShortcut",
	label: "Voice start/stop shortcut",
	description:
		"Key combination for starting or stopping native voice; empty disables it.",
	env: "PI_VOICE_TOGGLE_SHORTCUT",
	default: "ctrl+alt+v",
	parse: parseVoiceShortcut,
};
const MUTE_SHORTCUT_SETTING: SettingDeclaration<VoiceShortcut | null> = {
	key: "voice.muteShortcut",
	label: "Voice mute shortcut",
	description:
		"Key combination for toggling native voice mute; empty disables it.",
	env: "PI_VOICE_MUTE_SHORTCUT",
	default: "ctrl+alt+b",
	parse: parseVoiceShortcut,
};
const MAX_SPEECH_SECONDS_SETTING: SettingDeclaration<number> =
	defineIntegerSetting({
		key: "voice.maxSpeechSeconds",
		label: "Reply duration limit",
		description: "Maximum seconds of generated speech per reply.",
		env: "PI_VOICE_MAX_SPEECH_SECONDS",
		default: 40,
		range: { min: 1, max: 120 },
	});
const CLASSIFIER_MODEL_SETTING: SettingDeclaration<VoiceModelSetting> = {
	key: "voice.classifierModel",
	label: "Classifier model",
	description: "Use auto or an authenticated provider/model identifier.",
	env: "PI_VOICE_CLASSIFIER_MODEL",
	default: "auto",
	parse: (raw) => {
		if (
			typeof raw === "string" &&
			raw.length <= 256 &&
			!raw.includes("\0") &&
			!/\s/u.test(raw) &&
			(raw === "auto" ||
				/^[a-z0-9][a-z0-9._-]*\/[a-z0-9@~][a-z0-9._:@~-]*(?:\/[a-z0-9@~][a-z0-9._:@~-]*)*$/iu.test(
					raw,
				))
		)
			return raw as VoiceModelSetting;
		throw new Error('expected "auto" or provider/model (≤256 characters)');
	},
};
const MODEL_SETTING: SettingDeclaration<VoiceModelSetting> = {
	key: "voice.model",
	label: "Conversation model",
	description: "Use auto or an authenticated provider/model identifier.",
	env: "PI_VOICE_MODEL",
	default: "auto",
	parse: (raw) => {
		if (typeof raw !== "string")
			throw new Error('expected "auto" or provider/model');
		const value = raw.trim();
		if (
			value === "auto" ||
			/^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._:-]*$/iu.test(value)
		)
			return value as VoiceModelSetting;
		throw new Error('expected "auto" or provider/model');
	},
};

type VoiceCommand =
	| "start"
	| "help"
	| "setup"
	| "history"
	| "mute"
	| "unmute"
	| "stop"
	| "resume";

export function parseVoiceCommandArgs(args: string): {
	command: VoiceCommand;
	root?: string;
} {
	if (args.length > 8192) throw new Error("Voice command is too long.");
	const input = args.trim();
	const token = input.split(/\s/u, 1)[0] ?? "";
	const command =
		!token || token === "--root" || token === "native" ? "start" : token;
	if (
		![
			"start",
			"help",
			"setup",
			"history",
			"mute",
			"unmute",
			"stop",
			"resume",
		].includes(command)
	)
		throw new Error(
			"usage: /voice [start|setup|history|mute|unmute|stop|resume|help] [--root ABSOLUTE_PATH]",
		);
	const remainder =
		token === "--root" ? input : input.slice(token.length).trim();
	if (!remainder) return { command: command as VoiceCommand };
	const match = /^--root\s+(.+)$/u.exec(remainder);
	if (!match)
		throw new Error(
			"usage: /voice [start|setup|history] [--root ABSOLUTE_PATH]",
		);
	let root = match[1]?.trim() ?? "";
	if (
		(root.startsWith('"') && root.endsWith('"')) ||
		(root.startsWith("'") && root.endsWith("'"))
	)
		root = root.slice(1, -1);
	if (!root || root.length > 4096 || root.includes("\0") || !isAbsolute(root))
		throw new Error("--root must be an absolute path without NUL");
	if (command !== "start" && command !== "setup" && command !== "history")
		throw new Error("--root is only valid with start, setup or history");
	return { command: command as VoiceCommand, root };
}

export default function voice(pi: ExtensionAPI): void {
	const controller = new VoiceController(pi);
	const nativeCall = new NativeVoiceCall(controller);
	let nativeStart: AbortController | undefined;
	let nativeEpoch = 0;
	const shortcutSettings = [
		resolveSetting(
			pi,
			{ cwd: process.cwd(), isProjectTrusted: () => false },
			TOGGLE_SHORTCUT_SETTING,
		),
		resolveSetting(
			pi,
			{ cwd: process.cwd(), isProjectTrusted: () => false },
			MUTE_SHORTCUT_SETTING,
		),
	];
	const shortcutErrors: string[] = [];
	const stopNative = async (ctx?: ExtensionContext): Promise<void> => {
		const epoch = ++nativeEpoch;
		nativeStart?.abort();
		nativeStart = undefined;
		try {
			await nativeCall.stop();
		} finally {
			if (epoch === nativeEpoch && ctx?.hasUI)
				ctx.ui.setWidget("voice", undefined);
		}
	};
	pi.on("session_start", async (_event, ctx) => {
		for (const setting of shortcutSettings)
			if (!setting.ok)
				ctx.ui.notify(`${setting.error}; shortcut disabled`, "warning");
		for (const error of shortcutErrors)
			ctx.ui.notify(`${error}; shortcut disabled`, "warning");
		await stopNative(ctx);
		controller.start(ctx);
		dbg?.("session.start");
	});
	pi.on("session_before_switch", async (_event, ctx) => {
		await stopNative(ctx);
		controller.onExecutorSessionSwitch();
	});
	pi.on("session_before_fork", async (_event, ctx) => {
		await stopNative(ctx);
	});
	pi.on("session_before_tree", async (_event, ctx) => {
		await stopNative(ctx);
		controller.onExecutorSessionSwitch();
	});
	pi.on("session_tree", async (_event, ctx) => {
		await stopNative(ctx);
		controller.start(ctx);
	});
	pi.on("agent_start", () => controller.onExecutorStart());
	pi.on("message_start", (event) =>
		controller.onExecutorMessageStart(event.message),
	);
	pi.on("message_update", () => controller.onExecutorMessageUpdate());
	pi.on("message_end", (event) =>
		controller.onExecutorMessageEnd(event.message),
	);
	pi.on("agent_end", (event) => controller.onExecutorAgentEnd(event.messages));
	pi.on("tool_execution_start", (event) =>
		controller.onExecutorTool(event.toolName, event.toolCallId, event.args),
	);
	pi.on("tool_execution_end", (event) =>
		controller.onExecutorToolEnd(event.toolCallId, event.isError, event.result),
	);
	pi.on("agent_settled", (event) =>
		controller.onExecutorSettled(event.aborted),
	);
	pi.on("ui_prompt_start", (event) =>
		controller.onExecutorPromptStart(event.title),
	);
	pi.on("ui_prompt_end", () => controller.onExecutorPromptEnd());
	pi.on("session_shutdown", async (_event, ctx) => {
		dbg?.("session.shutdown");
		await stopNative(ctx);
		controller.dispose();
	});
	let muteShortcutEpoch: number | undefined;
	const registeredShortcuts = new Set<string>();
	for (const [index, setting] of shortcutSettings.entries()) {
		if (!setting.ok || setting.value === null) continue;
		const shortcut = setting.value;
		if (registeredShortcuts.has(shortcut)) {
			shortcutErrors.push(`Duplicate voice shortcut ${shortcut}`);
			continue;
		}
		registeredShortcuts.add(shortcut);
		pi.registerShortcut(shortcut, {
			description:
				index === 0 ? "Toggle native voice" : "Toggle native voice mute",
			handler: async (ctx) => {
				const epoch = nativeEpoch;
				if (index === 1) {
					if (muteShortcutEpoch === epoch) return;
					muteShortcutEpoch = epoch;
				}
				try {
					if (index === 0) {
						if (nativeStart || nativeCall.active) await stopNative(ctx);
						else await handleNative({ command: "start" }, ctx);
					} else {
						await nativeCall.mute(!nativeCall.isMuted);
						if (epoch !== nativeEpoch) return;
						ctx.ui.notify(
							nativeCall.isMuted
								? "Native voice muted."
								: "Native voice unmuted.",
							"info",
						);
					}
				} catch (cause) {
					if (index === 0 || epoch === nativeEpoch)
						ctx.ui.notify(
							`Native voice shortcut failed${detail(cause)}`,
							"error",
						);
				} finally {
					if (index === 1 && muteShortcutEpoch === epoch)
						muteShortcutEpoch = undefined;
				}
			},
		});
	}
	for (const [shortcut, channel, lines] of [
		["alt+pageUp", "spoken", -3],
		["alt+pageDown", "spoken", 3],
		["alt+shift+pageUp", "heard", -3],
		["alt+shift+pageDown", "heard", 3],
	] as const) {
		pi.registerShortcut(shortcut, {
			description: `Scroll voice ${channel === "heard" ? "microphone text" : "captions"} ${lines < 0 ? "up" : "down"}`,
			handler: () => nativeCall.scrollWidget(channel, lines),
		});
	}
	pi.registerCommand("voice", {
		description:
			"Start Voice through speech service or manage setup, history and call controls",
		getArgumentCompletions: (prefix) =>
			["start", "setup", "history", "mute", "unmute", "stop", "resume", "help"]
				.filter((value) => value.startsWith(prefix))
				.map((value) => ({ value, label: value })),
		handler: async (args, ctx) => {
			let command: ReturnType<typeof parseVoiceCommandArgs>;
			try {
				command = parseVoiceCommandArgs(args);
			} catch (cause) {
				ctx.ui.notify(
					cause instanceof Error ? cause.message : "Invalid voice command.",
					"warning",
				);
				return;
			}
			try {
				await handleNative(command, ctx);
			} catch (cause) {
				ctx.ui.notify(`Native voice failed${detail(cause)}`, "error");
			}
		},
	});
	async function handleNative(
		command: ReturnType<typeof parseVoiceCommandArgs>,
		ctx: ExtensionContext,
	): Promise<void> {
		if (command.command === "help") {
			ctx.ui.notify(
				"Voice: /voice starts a call through configured speech service. Controls: start · setup · history · mute · unmute · stop · resume. --root stores Voice history only; service assets are service-owned.",
				"info",
			);
			return;
		}
		if (command.command === "history") {
			await nativeCall.showHistory(
				ctx,
				command.root ? join(command.root, "history") : undefined,
			);
			return;
		}
		if (command.command === "mute") {
			await nativeCall.mute(true);
			return;
		}
		if (command.command === "unmute") {
			await nativeCall.mute(false);
			return;
		}
		if (command.command === "stop") {
			await stopNative(ctx);
			return;
		}
		if (command.command === "resume") {
			++nativeEpoch;
			nativeStart?.abort();
			nativeStart = undefined;
			try {
				await nativeCall.resume();
			} catch (cause) {
				if (
					cause instanceof Error &&
					cause.message === "Native voice has not been configured."
				) {
					await handleNative({ command: "start" }, ctx);
					return;
				}
				throw cause;
			}
			return;
		}
		if (!ctx.hasUI) {
			ctx.ui.notify(
				"Native voice setup requires an interactive Pi UI.",
				"warning",
			);
			return;
		}
		const settings = settingsFor(pi, ctx);
		if (!settings) return;
		const epoch = ++nativeEpoch;
		nativeStart?.abort();
		const start = new AbortController();
		nativeStart = start;
		const root = command.root ?? voiceDataDirectory();
		let mounted = false;
		let live = false;
		let prepared: Awaited<ReturnType<typeof runNativeSetup>>;
		let startInvoked = false;
		try {
			await nativeCall.stop();
			if (epoch !== nativeEpoch || start.signal.aborted) return;
			const toggleShortcut = shortcutSettings[0];
			const setup = await runNativeSetup(ctx, {
				command: settings.command,
				...(settings.profile === undefined
					? {}
					: { profile: settings.profile }),
				preview: settings.preview,
				signal: start.signal,
				...(toggleShortcut?.ok && toggleShortcut.value
					? { cancelShortcut: toggleShortcut.value }
					: {}),
			});
			if (!setup) return;
			prepared = setup;
			if (start.signal.aborted || epoch !== nativeEpoch) return;
			if (command.command === "setup") {
				ctx.ui.notify("Voice setup is ready.", "info");
				return;
			}
			ctx.ui.setWidget(
				"voice",
				(tui, theme) => nativeCall.createWidget(tui, theme),
				{ placement: "aboveEditor" },
			);
			mounted = true;
			startInvoked = true;
			await nativeCall.start(ctx, {
				backend: setup,
				modelSetting: settings.model,
				historyDirectory: join(root, "history"),
				classifierModelSetting: settings.classifierModel,
				modality: {
					maxSentences: settings.maxSentences,
					maxPlaybackMs: settings.maxSpeechSeconds * 1_000,
				},
			});
			live = true;
		} catch (cause) {
			if (!start.signal.aborted && epoch === nativeEpoch)
				ctx.ui.notify(`Native voice setup failed${detail(cause)}`, "error");
		} finally {
			if (!live) {
				start.abort();
				if (prepared) {
					if (startInvoked) await nativeCall.stop().catch(() => undefined);
					else await prepared.client.close().catch(() => undefined);
				}
			}
			if (epoch === nativeEpoch) {
				if (nativeStart === start) nativeStart = undefined;
				if (!live || !mounted) ctx.ui.setWidget("voice", undefined);
			}
		}
	}
}

function settingsFor(
	pi: ExtensionAPI,
	ctx: ExtensionContext,
):
	| {
			command: readonly string[];
			profile: string | undefined;
			preview: boolean;
			model: VoiceModelSetting;
			classifierModel: VoiceModelSetting;
			maxSentences: number;
			maxSpeechSeconds: number;
	  }
	| undefined {
	const command = resolveSetting(pi, ctx, COMMAND_SETTING);
	const profile = resolveSetting(pi, ctx, PROFILE_SETTING);
	const preview = resolveSetting(pi, ctx, PREVIEW_SETTING);
	const model = resolveSetting(pi, ctx, MODEL_SETTING);
	const classifierModel = resolveSetting(pi, ctx, CLASSIFIER_MODEL_SETTING);
	const maxSentences = resolveSetting(pi, ctx, MAX_SENTENCES_SETTING);
	const maxSpeechSeconds = resolveSetting(pi, ctx, MAX_SPEECH_SECONDS_SETTING);
	const settings = [
		command,
		profile,
		preview,
		model,
		classifierModel,
		maxSentences,
		maxSpeechSeconds,
	];
	const invalid = settings.find((result) => !result.ok);
	if (invalid) {
		ctx.ui.notify(`${invalid.error}; voice did not open`, "warning");
		return;
	}
	if (
		!command.ok ||
		!profile.ok ||
		!preview.ok ||
		!model.ok ||
		!classifierModel.ok ||
		!maxSentences.ok ||
		!maxSpeechSeconds.ok
	)
		return;
	return {
		command: command.value,
		profile: profile.value,
		preview: preview.value,
		model: model.value,
		classifierModel: classifierModel.value,
		maxSentences: maxSentences.value,
		maxSpeechSeconds: maxSpeechSeconds.value,
	};
}
function parseVoiceShortcut(raw: unknown): VoiceShortcut | null {
	if (raw === "") return null;
	if (
		typeof raw !== "string" ||
		raw.length > 64 ||
		!/^(?:(?:ctrl|alt|shift|super)\+)*(?:[a-z0-9]|f(?:[1-9]|1[0-2]))$/iu.test(
			raw,
		)
	)
		throw new Error(
			"expected key combination, e.g. ctrl+alt+v, or empty to disable",
		);
	const parts = raw.toLowerCase().split("+");
	const key = parts.pop();
	const order = ["ctrl", "alt", "shift", "super"];
	const modifiers = parts.sort((a, b) => order.indexOf(a) - order.indexOf(b));
	if (new Set(modifiers).size !== modifiers.length)
		throw new Error("duplicate shortcut modifier");
	return [...modifiers, key].join("+") as VoiceShortcut;
}

function detail(cause: unknown): string {
	return cause instanceof Error && cause.message
		? `: ${cause.message.slice(0, 200)}`
		: "";
}

export const __test: {
	COMMAND_SETTING: typeof COMMAND_SETTING;
	PROFILE_SETTING: typeof PROFILE_SETTING;
	PREVIEW_SETTING: typeof PREVIEW_SETTING;
	MODEL_SETTING: typeof MODEL_SETTING;
	CLASSIFIER_MODEL_SETTING: typeof CLASSIFIER_MODEL_SETTING;
	MAX_SENTENCES_SETTING: typeof MAX_SENTENCES_SETTING;
	MAX_SPEECH_SECONDS_SETTING: typeof MAX_SPEECH_SECONDS_SETTING;
	TOGGLE_SHORTCUT_SETTING: typeof TOGGLE_SHORTCUT_SETTING;
	MUTE_SHORTCUT_SETTING: typeof MUTE_SHORTCUT_SETTING;
} = {
	COMMAND_SETTING,
	PROFILE_SETTING,
	PREVIEW_SETTING,
	MODEL_SETTING,
	CLASSIFIER_MODEL_SETTING,
	MAX_SENTENCES_SETTING,
	MAX_SPEECH_SECONDS_SETTING,
	TOGGLE_SHORTCUT_SETTING,
	MUTE_SHORTCUT_SETTING,
};