import { isAbsolute, join } from "node:path"; import type { ExtensionAPI, ExtensionContext, } from "@earendil-works/pi-coding-agent"; import { VoiceController, type VoiceModelSetting } from "./src/controller.ts"; import { dbg } from "./src/debug.ts"; import { NativeVoiceCall } from "./src/native-call.ts"; import { runNativeSetup } from "./src/native-setup-ui.ts"; import { voiceDataDirectory } from "./src/paths.ts"; import { defineIntegerSetting, parseBooleanSetting, resolveSetting, type SettingDeclaration, } from "./src/pi-ext-settings.ts"; const COMMAND_SETTING: SettingDeclaration = { key: "voice.command", label: "Speech service command", description: "Executable and arguments for the external speech service.", env: "PI_VOICE_COMMAND", default: ["speak_to_me"], parse: (raw, source) => { let value = raw; if (typeof raw === "string" && source === "env") { try { value = JSON.parse(raw); } catch { throw new Error( "expected a nonempty JSON array of nonempty command strings", ); } } if ( Array.isArray(value) && value.length > 0 && value.every( (part) => typeof part === "string" && part.length > 0 && !part.includes("\0"), ) ) return value; throw new Error( "expected a nonempty JSON array of nonempty command strings", ); }, }; const PROFILE_SETTING: SettingDeclaration = { key: "voice.profile", label: "Speech service profile", description: "Optional opaque profile ID supplied by the speech service.", env: "PI_VOICE_PROFILE", default: undefined, parse: (raw, source) => { if (raw === undefined && source === "default") return undefined; if (typeof raw === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(raw)) return raw; throw new Error("expected an opaque speech service profile ID"); }, }; const PREVIEW_SETTING: SettingDeclaration = { key: "voice.preview", label: "Live transcription preview", description: "Show provisional transcription while the speech service recognizes input.", env: "PI_VOICE_PREVIEW", default: true, parse: (raw, source) => { const value = parseBooleanSetting(raw, source); if (value === undefined) throw new Error("expected a boolean"); return value; }, }; const MAX_SENTENCES_SETTING: SettingDeclaration = defineIntegerSetting({ key: "voice.maxSentences", label: "Reply sentence limit", description: "Maximum sentences per spoken reply.", env: "PI_VOICE_MAX_SENTENCES", default: 6, range: { min: 1, max: 20 }, }); type VoiceShortcut = Parameters[0]; const TOGGLE_SHORTCUT_SETTING: SettingDeclaration = { key: "voice.toggleShortcut", label: "Voice start/stop shortcut", description: "Key combination for starting or stopping native voice; empty disables it.", env: "PI_VOICE_TOGGLE_SHORTCUT", default: "ctrl+alt+v", parse: parseVoiceShortcut, }; const MUTE_SHORTCUT_SETTING: SettingDeclaration = { key: "voice.muteShortcut", label: "Voice mute shortcut", description: "Key combination for toggling native voice mute; empty disables it.", env: "PI_VOICE_MUTE_SHORTCUT", default: "ctrl+alt+b", parse: parseVoiceShortcut, }; const MAX_SPEECH_SECONDS_SETTING: SettingDeclaration = defineIntegerSetting({ key: "voice.maxSpeechSeconds", label: "Reply duration limit", description: "Maximum seconds of generated speech per reply.", env: "PI_VOICE_MAX_SPEECH_SECONDS", default: 40, range: { min: 1, max: 120 }, }); const CLASSIFIER_MODEL_SETTING: SettingDeclaration = { key: "voice.classifierModel", label: "Classifier model", description: "Use auto or an authenticated provider/model identifier.", env: "PI_VOICE_CLASSIFIER_MODEL", default: "auto", parse: (raw) => { if ( typeof raw === "string" && raw.length <= 256 && !raw.includes("\0") && !/\s/u.test(raw) && (raw === "auto" || /^[a-z0-9][a-z0-9._-]*\/[a-z0-9@~][a-z0-9._:@~-]*(?:\/[a-z0-9@~][a-z0-9._:@~-]*)*$/iu.test( raw, )) ) return raw as VoiceModelSetting; throw new Error('expected "auto" or provider/model (≤256 characters)'); }, }; const MODEL_SETTING: SettingDeclaration = { key: "voice.model", label: "Conversation model", description: "Use auto or an authenticated provider/model identifier.", env: "PI_VOICE_MODEL", default: "auto", parse: (raw) => { if (typeof raw !== "string") throw new Error('expected "auto" or provider/model'); const value = raw.trim(); if ( value === "auto" || /^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._:-]*$/iu.test(value) ) return value as VoiceModelSetting; throw new Error('expected "auto" or provider/model'); }, }; type VoiceCommand = | "start" | "help" | "setup" | "history" | "mute" | "unmute" | "stop" | "resume"; export function parseVoiceCommandArgs(args: string): { command: VoiceCommand; root?: string; } { if (args.length > 8192) throw new Error("Voice command is too long."); const input = args.trim(); const token = input.split(/\s/u, 1)[0] ?? ""; const command = !token || token === "--root" || token === "native" ? "start" : token; if ( ![ "start", "help", "setup", "history", "mute", "unmute", "stop", "resume", ].includes(command) ) throw new Error( "usage: /voice [start|setup|history|mute|unmute|stop|resume|help] [--root ABSOLUTE_PATH]", ); const remainder = token === "--root" ? input : input.slice(token.length).trim(); if (!remainder) return { command: command as VoiceCommand }; const match = /^--root\s+(.+)$/u.exec(remainder); if (!match) throw new Error( "usage: /voice [start|setup|history] [--root ABSOLUTE_PATH]", ); let root = match[1]?.trim() ?? ""; if ( (root.startsWith('"') && root.endsWith('"')) || (root.startsWith("'") && root.endsWith("'")) ) root = root.slice(1, -1); if (!root || root.length > 4096 || root.includes("\0") || !isAbsolute(root)) throw new Error("--root must be an absolute path without NUL"); if (command !== "start" && command !== "setup" && command !== "history") throw new Error("--root is only valid with start, setup or history"); return { command: command as VoiceCommand, root }; } export default function voice(pi: ExtensionAPI): void { const controller = new VoiceController(pi); const nativeCall = new NativeVoiceCall(controller); let nativeStart: AbortController | undefined; let nativeEpoch = 0; const shortcutSettings = [ resolveSetting( pi, { cwd: process.cwd(), isProjectTrusted: () => false }, TOGGLE_SHORTCUT_SETTING, ), resolveSetting( pi, { cwd: process.cwd(), isProjectTrusted: () => false }, MUTE_SHORTCUT_SETTING, ), ]; const shortcutErrors: string[] = []; const stopNative = async (ctx?: ExtensionContext): Promise => { const epoch = ++nativeEpoch; nativeStart?.abort(); nativeStart = undefined; try { await nativeCall.stop(); } finally { if (epoch === nativeEpoch && ctx?.hasUI) ctx.ui.setWidget("voice", undefined); } }; pi.on("session_start", async (_event, ctx) => { for (const setting of shortcutSettings) if (!setting.ok) ctx.ui.notify(`${setting.error}; shortcut disabled`, "warning"); for (const error of shortcutErrors) ctx.ui.notify(`${error}; shortcut disabled`, "warning"); await stopNative(ctx); controller.start(ctx); dbg?.("session.start"); }); pi.on("session_before_switch", async (_event, ctx) => { await stopNative(ctx); controller.onExecutorSessionSwitch(); }); pi.on("session_before_fork", async (_event, ctx) => { await stopNative(ctx); }); pi.on("session_before_tree", async (_event, ctx) => { await stopNative(ctx); controller.onExecutorSessionSwitch(); }); pi.on("session_tree", async (_event, ctx) => { await stopNative(ctx); controller.start(ctx); }); pi.on("agent_start", () => controller.onExecutorStart()); pi.on("message_start", (event) => controller.onExecutorMessageStart(event.message), ); pi.on("message_update", () => controller.onExecutorMessageUpdate()); pi.on("message_end", (event) => controller.onExecutorMessageEnd(event.message), ); pi.on("agent_end", (event) => controller.onExecutorAgentEnd(event.messages)); pi.on("tool_execution_start", (event) => controller.onExecutorTool(event.toolName, event.toolCallId, event.args), ); pi.on("tool_execution_end", (event) => controller.onExecutorToolEnd(event.toolCallId, event.isError, event.result), ); pi.on("agent_settled", (event) => controller.onExecutorSettled(event.aborted), ); pi.on("ui_prompt_start", (event) => controller.onExecutorPromptStart(event.title), ); pi.on("ui_prompt_end", () => controller.onExecutorPromptEnd()); pi.on("session_shutdown", async (_event, ctx) => { dbg?.("session.shutdown"); await stopNative(ctx); controller.dispose(); }); let muteShortcutEpoch: number | undefined; const registeredShortcuts = new Set(); for (const [index, setting] of shortcutSettings.entries()) { if (!setting.ok || setting.value === null) continue; const shortcut = setting.value; if (registeredShortcuts.has(shortcut)) { shortcutErrors.push(`Duplicate voice shortcut ${shortcut}`); continue; } registeredShortcuts.add(shortcut); pi.registerShortcut(shortcut, { description: index === 0 ? "Toggle native voice" : "Toggle native voice mute", handler: async (ctx) => { const epoch = nativeEpoch; if (index === 1) { if (muteShortcutEpoch === epoch) return; muteShortcutEpoch = epoch; } try { if (index === 0) { if (nativeStart || nativeCall.active) await stopNative(ctx); else await handleNative({ command: "start" }, ctx); } else { await nativeCall.mute(!nativeCall.isMuted); if (epoch !== nativeEpoch) return; ctx.ui.notify( nativeCall.isMuted ? "Native voice muted." : "Native voice unmuted.", "info", ); } } catch (cause) { if (index === 0 || epoch === nativeEpoch) ctx.ui.notify( `Native voice shortcut failed${detail(cause)}`, "error", ); } finally { if (index === 1 && muteShortcutEpoch === epoch) muteShortcutEpoch = undefined; } }, }); } for (const [shortcut, channel, lines] of [ ["alt+pageUp", "spoken", -3], ["alt+pageDown", "spoken", 3], ["alt+shift+pageUp", "heard", -3], ["alt+shift+pageDown", "heard", 3], ] as const) { pi.registerShortcut(shortcut, { description: `Scroll voice ${channel === "heard" ? "microphone text" : "captions"} ${lines < 0 ? "up" : "down"}`, handler: () => nativeCall.scrollWidget(channel, lines), }); } pi.registerCommand("voice", { description: "Start Voice through speech service or manage setup, history and call controls", getArgumentCompletions: (prefix) => ["start", "setup", "history", "mute", "unmute", "stop", "resume", "help"] .filter((value) => value.startsWith(prefix)) .map((value) => ({ value, label: value })), handler: async (args, ctx) => { let command: ReturnType; try { command = parseVoiceCommandArgs(args); } catch (cause) { ctx.ui.notify( cause instanceof Error ? cause.message : "Invalid voice command.", "warning", ); return; } try { await handleNative(command, ctx); } catch (cause) { ctx.ui.notify(`Native voice failed${detail(cause)}`, "error"); } }, }); async function handleNative( command: ReturnType, ctx: ExtensionContext, ): Promise { if (command.command === "help") { ctx.ui.notify( "Voice: /voice starts a call through configured speech service. Controls: start · setup · history · mute · unmute · stop · resume. --root stores Voice history only; service assets are service-owned.", "info", ); return; } if (command.command === "history") { await nativeCall.showHistory( ctx, command.root ? join(command.root, "history") : undefined, ); return; } if (command.command === "mute") { await nativeCall.mute(true); return; } if (command.command === "unmute") { await nativeCall.mute(false); return; } if (command.command === "stop") { await stopNative(ctx); return; } if (command.command === "resume") { ++nativeEpoch; nativeStart?.abort(); nativeStart = undefined; try { await nativeCall.resume(); } catch (cause) { if ( cause instanceof Error && cause.message === "Native voice has not been configured." ) { await handleNative({ command: "start" }, ctx); return; } throw cause; } return; } if (!ctx.hasUI) { ctx.ui.notify( "Native voice setup requires an interactive Pi UI.", "warning", ); return; } const settings = settingsFor(pi, ctx); if (!settings) return; const epoch = ++nativeEpoch; nativeStart?.abort(); const start = new AbortController(); nativeStart = start; const root = command.root ?? voiceDataDirectory(); let mounted = false; let live = false; let prepared: Awaited>; let startInvoked = false; try { await nativeCall.stop(); if (epoch !== nativeEpoch || start.signal.aborted) return; const toggleShortcut = shortcutSettings[0]; const setup = await runNativeSetup(ctx, { command: settings.command, ...(settings.profile === undefined ? {} : { profile: settings.profile }), preview: settings.preview, signal: start.signal, ...(toggleShortcut?.ok && toggleShortcut.value ? { cancelShortcut: toggleShortcut.value } : {}), }); if (!setup) return; prepared = setup; if (start.signal.aborted || epoch !== nativeEpoch) return; if (command.command === "setup") { ctx.ui.notify("Voice setup is ready.", "info"); return; } ctx.ui.setWidget( "voice", (tui, theme) => nativeCall.createWidget(tui, theme), { placement: "aboveEditor" }, ); mounted = true; startInvoked = true; await nativeCall.start(ctx, { backend: setup, modelSetting: settings.model, historyDirectory: join(root, "history"), classifierModelSetting: settings.classifierModel, modality: { maxSentences: settings.maxSentences, maxPlaybackMs: settings.maxSpeechSeconds * 1_000, }, }); live = true; } catch (cause) { if (!start.signal.aborted && epoch === nativeEpoch) ctx.ui.notify(`Native voice setup failed${detail(cause)}`, "error"); } finally { if (!live) { start.abort(); if (prepared) { if (startInvoked) await nativeCall.stop().catch(() => undefined); else await prepared.client.close().catch(() => undefined); } } if (epoch === nativeEpoch) { if (nativeStart === start) nativeStart = undefined; if (!live || !mounted) ctx.ui.setWidget("voice", undefined); } } } } function settingsFor( pi: ExtensionAPI, ctx: ExtensionContext, ): | { command: readonly string[]; profile: string | undefined; preview: boolean; model: VoiceModelSetting; classifierModel: VoiceModelSetting; maxSentences: number; maxSpeechSeconds: number; } | undefined { const command = resolveSetting(pi, ctx, COMMAND_SETTING); const profile = resolveSetting(pi, ctx, PROFILE_SETTING); const preview = resolveSetting(pi, ctx, PREVIEW_SETTING); const model = resolveSetting(pi, ctx, MODEL_SETTING); const classifierModel = resolveSetting(pi, ctx, CLASSIFIER_MODEL_SETTING); const maxSentences = resolveSetting(pi, ctx, MAX_SENTENCES_SETTING); const maxSpeechSeconds = resolveSetting(pi, ctx, MAX_SPEECH_SECONDS_SETTING); const settings = [ command, profile, preview, model, classifierModel, maxSentences, maxSpeechSeconds, ]; const invalid = settings.find((result) => !result.ok); if (invalid) { ctx.ui.notify(`${invalid.error}; voice did not open`, "warning"); return; } if ( !command.ok || !profile.ok || !preview.ok || !model.ok || !classifierModel.ok || !maxSentences.ok || !maxSpeechSeconds.ok ) return; return { command: command.value, profile: profile.value, preview: preview.value, model: model.value, classifierModel: classifierModel.value, maxSentences: maxSentences.value, maxSpeechSeconds: maxSpeechSeconds.value, }; } function parseVoiceShortcut(raw: unknown): VoiceShortcut | null { if (raw === "") return null; if ( typeof raw !== "string" || raw.length > 64 || !/^(?:(?:ctrl|alt|shift|super)\+)*(?:[a-z0-9]|f(?:[1-9]|1[0-2]))$/iu.test( raw, ) ) throw new Error( "expected key combination, e.g. ctrl+alt+v, or empty to disable", ); const parts = raw.toLowerCase().split("+"); const key = parts.pop(); const order = ["ctrl", "alt", "shift", "super"]; const modifiers = parts.sort((a, b) => order.indexOf(a) - order.indexOf(b)); if (new Set(modifiers).size !== modifiers.length) throw new Error("duplicate shortcut modifier"); return [...modifiers, key].join("+") as VoiceShortcut; } function detail(cause: unknown): string { return cause instanceof Error && cause.message ? `: ${cause.message.slice(0, 200)}` : ""; } export const __test: { COMMAND_SETTING: typeof COMMAND_SETTING; PROFILE_SETTING: typeof PROFILE_SETTING; PREVIEW_SETTING: typeof PREVIEW_SETTING; MODEL_SETTING: typeof MODEL_SETTING; CLASSIFIER_MODEL_SETTING: typeof CLASSIFIER_MODEL_SETTING; MAX_SENTENCES_SETTING: typeof MAX_SENTENCES_SETTING; MAX_SPEECH_SECONDS_SETTING: typeof MAX_SPEECH_SECONDS_SETTING; TOGGLE_SHORTCUT_SETTING: typeof TOGGLE_SHORTCUT_SETTING; MUTE_SHORTCUT_SETTING: typeof MUTE_SHORTCUT_SETTING; } = { COMMAND_SETTING, PROFILE_SETTING, PREVIEW_SETTING, MODEL_SETTING, CLASSIFIER_MODEL_SETTING, MAX_SENTENCES_SETTING, MAX_SPEECH_SECONDS_SETTING, TOGGLE_SHORTCUT_SETTING, MUTE_SHORTCUT_SETTING, };