repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/voice/index.ts
Rawimport { isAbsolute, join } from "node:path";
import type {
ExtensionAPI,
ExtensionContext,
} from "@earendil-works/pi-coding-agent";
import { VoiceController, type VoiceModelSetting } from "./src/controller.ts";
import { dbg } from "./src/debug.ts";
import { NativeVoiceCall } from "./src/native-call.ts";
import { runNativeSetup } from "./src/native-setup-ui.ts";
import { voiceDataDirectory } from "./src/paths.ts";
import {
defineIntegerSetting,
parseBooleanSetting,
resolveSetting,
type SettingDeclaration,
} from "./src/pi-ext-settings.ts";
const COMMAND_SETTING: SettingDeclaration<readonly string[]> = {
key: "voice.command",
label: "Speech service command",
description: "Executable and arguments for the external speech service.",
env: "PI_VOICE_COMMAND",
default: ["speak_to_me"],
parse: (raw, source) => {
let value = raw;
if (typeof raw === "string" && source === "env") {
try {
value = JSON.parse(raw);
} catch {
throw new Error(
"expected a nonempty JSON array of nonempty command strings",
);
}
}
if (
Array.isArray(value) &&
value.length > 0 &&
value.every(
(part) =>
typeof part === "string" && part.length > 0 && !part.includes("\0"),
)
)
return value;
throw new Error(
"expected a nonempty JSON array of nonempty command strings",
);
},
};
const PROFILE_SETTING: SettingDeclaration<string | undefined> = {
key: "voice.profile",
label: "Speech service profile",
description: "Optional opaque profile ID supplied by the speech service.",
env: "PI_VOICE_PROFILE",
default: undefined,
parse: (raw, source) => {
if (raw === undefined && source === "default") return undefined;
if (typeof raw === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(raw))
return raw;
throw new Error("expected an opaque speech service profile ID");
},
};
const PREVIEW_SETTING: SettingDeclaration<boolean> = {
key: "voice.preview",
label: "Live transcription preview",
description:
"Show provisional transcription while the speech service recognizes input.",
env: "PI_VOICE_PREVIEW",
default: true,
parse: (raw, source) => {
const value = parseBooleanSetting(raw, source);
if (value === undefined) throw new Error("expected a boolean");
return value;
},
};
const MAX_SENTENCES_SETTING: SettingDeclaration<number> = defineIntegerSetting({
key: "voice.maxSentences",
label: "Reply sentence limit",
description: "Maximum sentences per spoken reply.",
env: "PI_VOICE_MAX_SENTENCES",
default: 6,
range: { min: 1, max: 20 },
});
type VoiceShortcut = Parameters<ExtensionAPI["registerShortcut"]>[0];
const TOGGLE_SHORTCUT_SETTING: SettingDeclaration<VoiceShortcut | null> = {
key: "voice.toggleShortcut",
label: "Voice start/stop shortcut",
description:
"Key combination for starting or stopping native voice; empty disables it.",
env: "PI_VOICE_TOGGLE_SHORTCUT",
default: "ctrl+alt+v",
parse: parseVoiceShortcut,
};
const MUTE_SHORTCUT_SETTING: SettingDeclaration<VoiceShortcut | null> = {
key: "voice.muteShortcut",
label: "Voice mute shortcut",
description:
"Key combination for toggling native voice mute; empty disables it.",
env: "PI_VOICE_MUTE_SHORTCUT",
default: "ctrl+alt+b",
parse: parseVoiceShortcut,
};
const MAX_SPEECH_SECONDS_SETTING: SettingDeclaration<number> =
defineIntegerSetting({
key: "voice.maxSpeechSeconds",
label: "Reply duration limit",
description: "Maximum seconds of generated speech per reply.",
env: "PI_VOICE_MAX_SPEECH_SECONDS",
default: 40,
range: { min: 1, max: 120 },
});
const CLASSIFIER_MODEL_SETTING: SettingDeclaration<VoiceModelSetting> = {
key: "voice.classifierModel",
label: "Classifier model",
description: "Use auto or an authenticated provider/model identifier.",
env: "PI_VOICE_CLASSIFIER_MODEL",
default: "auto",
parse: (raw) => {
if (
typeof raw === "string" &&
raw.length <= 256 &&
!raw.includes("\0") &&
!/\s/u.test(raw) &&
(raw === "auto" ||
/^[a-z0-9][a-z0-9._-]*\/[a-z0-9@~][a-z0-9._:@~-]*(?:\/[a-z0-9@~][a-z0-9._:@~-]*)*$/iu.test(
raw,
))
)
return raw as VoiceModelSetting;
throw new Error('expected "auto" or provider/model (≤256 characters)');
},
};
const MODEL_SETTING: SettingDeclaration<VoiceModelSetting> = {
key: "voice.model",
label: "Conversation model",
description: "Use auto or an authenticated provider/model identifier.",
env: "PI_VOICE_MODEL",
default: "auto",
parse: (raw) => {
if (typeof raw !== "string")
throw new Error('expected "auto" or provider/model');
const value = raw.trim();
if (
value === "auto" ||
/^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._:-]*$/iu.test(value)
)
return value as VoiceModelSetting;
throw new Error('expected "auto" or provider/model');
},
};
type VoiceCommand =
| "start"
| "help"
| "setup"
| "history"
| "mute"
| "unmute"
| "stop"
| "resume";
export function parseVoiceCommandArgs(args: string): {
command: VoiceCommand;
root?: string;
} {
if (args.length > 8192) throw new Error("Voice command is too long.");
const input = args.trim();
const token = input.split(/\s/u, 1)[0] ?? "";
const command =
!token || token === "--root" || token === "native" ? "start" : token;
if (
![
"start",
"help",
"setup",
"history",
"mute",
"unmute",
"stop",
"resume",
].includes(command)
)
throw new Error(
"usage: /voice [start|setup|history|mute|unmute|stop|resume|help] [--root ABSOLUTE_PATH]",
);
const remainder =
token === "--root" ? input : input.slice(token.length).trim();
if (!remainder) return { command: command as VoiceCommand };
const match = /^--root\s+(.+)$/u.exec(remainder);
if (!match)
throw new Error(
"usage: /voice [start|setup|history] [--root ABSOLUTE_PATH]",
);
let root = match[1]?.trim() ?? "";
if (
(root.startsWith('"') && root.endsWith('"')) ||
(root.startsWith("'") && root.endsWith("'"))
)
root = root.slice(1, -1);
if (!root || root.length > 4096 || root.includes("\0") || !isAbsolute(root))
throw new Error("--root must be an absolute path without NUL");
if (command !== "start" && command !== "setup" && command !== "history")
throw new Error("--root is only valid with start, setup or history");
return { command: command as VoiceCommand, root };
}
export default function voice(pi: ExtensionAPI): void {
const controller = new VoiceController(pi);
const nativeCall = new NativeVoiceCall(controller);
let nativeStart: AbortController | undefined;
let nativeEpoch = 0;
const shortcutSettings = [
resolveSetting(
pi,
{ cwd: process.cwd(), isProjectTrusted: () => false },
TOGGLE_SHORTCUT_SETTING,
),
resolveSetting(
pi,
{ cwd: process.cwd(), isProjectTrusted: () => false },
MUTE_SHORTCUT_SETTING,
),
];
const shortcutErrors: string[] = [];
const stopNative = async (ctx?: ExtensionContext): Promise<void> => {
const epoch = ++nativeEpoch;
nativeStart?.abort();
nativeStart = undefined;
try {
await nativeCall.stop();
} finally {
if (epoch === nativeEpoch && ctx?.hasUI)
ctx.ui.setWidget("voice", undefined);
}
};
pi.on("session_start", async (_event, ctx) => {
for (const setting of shortcutSettings)
if (!setting.ok)
ctx.ui.notify(`${setting.error}; shortcut disabled`, "warning");
for (const error of shortcutErrors)
ctx.ui.notify(`${error}; shortcut disabled`, "warning");
await stopNative(ctx);
controller.start(ctx);
dbg?.("session.start");
});
pi.on("session_before_switch", async (_event, ctx) => {
await stopNative(ctx);
controller.onExecutorSessionSwitch();
});
pi.on("session_before_fork", async (_event, ctx) => {
await stopNative(ctx);
});
pi.on("session_before_tree", async (_event, ctx) => {
await stopNative(ctx);
controller.onExecutorSessionSwitch();
});
pi.on("session_tree", async (_event, ctx) => {
await stopNative(ctx);
controller.start(ctx);
});
pi.on("agent_start", () => controller.onExecutorStart());
pi.on("message_start", (event) =>
controller.onExecutorMessageStart(event.message),
);
pi.on("message_update", () => controller.onExecutorMessageUpdate());
pi.on("message_end", (event) =>
controller.onExecutorMessageEnd(event.message),
);
pi.on("agent_end", (event) => controller.onExecutorAgentEnd(event.messages));
pi.on("tool_execution_start", (event) =>
controller.onExecutorTool(event.toolName, event.toolCallId, event.args),
);
pi.on("tool_execution_end", (event) =>
controller.onExecutorToolEnd(event.toolCallId, event.isError, event.result),
);
pi.on("agent_settled", (event) =>
controller.onExecutorSettled(event.aborted),
);
pi.on("ui_prompt_start", (event) =>
controller.onExecutorPromptStart(event.title),
);
pi.on("ui_prompt_end", () => controller.onExecutorPromptEnd());
pi.on("session_shutdown", async (_event, ctx) => {
dbg?.("session.shutdown");
await stopNative(ctx);
controller.dispose();
});
let muteShortcutEpoch: number | undefined;
const registeredShortcuts = new Set<string>();
for (const [index, setting] of shortcutSettings.entries()) {
if (!setting.ok || setting.value === null) continue;
const shortcut = setting.value;
if (registeredShortcuts.has(shortcut)) {
shortcutErrors.push(`Duplicate voice shortcut ${shortcut}`);
continue;
}
registeredShortcuts.add(shortcut);
pi.registerShortcut(shortcut, {
description:
index === 0 ? "Toggle native voice" : "Toggle native voice mute",
handler: async (ctx) => {
const epoch = nativeEpoch;
if (index === 1) {
if (muteShortcutEpoch === epoch) return;
muteShortcutEpoch = epoch;
}
try {
if (index === 0) {
if (nativeStart || nativeCall.active) await stopNative(ctx);
else await handleNative({ command: "start" }, ctx);
} else {
await nativeCall.mute(!nativeCall.isMuted);
if (epoch !== nativeEpoch) return;
ctx.ui.notify(
nativeCall.isMuted
? "Native voice muted."
: "Native voice unmuted.",
"info",
);
}
} catch (cause) {
if (index === 0 || epoch === nativeEpoch)
ctx.ui.notify(
`Native voice shortcut failed${detail(cause)}`,
"error",
);
} finally {
if (index === 1 && muteShortcutEpoch === epoch)
muteShortcutEpoch = undefined;
}
},
});
}
for (const [shortcut, channel, lines] of [
["alt+pageUp", "spoken", -3],
["alt+pageDown", "spoken", 3],
["alt+shift+pageUp", "heard", -3],
["alt+shift+pageDown", "heard", 3],
] as const) {
pi.registerShortcut(shortcut, {
description: `Scroll voice ${channel === "heard" ? "microphone text" : "captions"} ${lines < 0 ? "up" : "down"}`,
handler: () => nativeCall.scrollWidget(channel, lines),
});
}
pi.registerCommand("voice", {
description:
"Start Voice through speech service or manage setup, history and call controls",
getArgumentCompletions: (prefix) =>
["start", "setup", "history", "mute", "unmute", "stop", "resume", "help"]
.filter((value) => value.startsWith(prefix))
.map((value) => ({ value, label: value })),
handler: async (args, ctx) => {
let command: ReturnType<typeof parseVoiceCommandArgs>;
try {
command = parseVoiceCommandArgs(args);
} catch (cause) {
ctx.ui.notify(
cause instanceof Error ? cause.message : "Invalid voice command.",
"warning",
);
return;
}
try {
await handleNative(command, ctx);
} catch (cause) {
ctx.ui.notify(`Native voice failed${detail(cause)}`, "error");
}
},
});
async function handleNative(
command: ReturnType<typeof parseVoiceCommandArgs>,
ctx: ExtensionContext,
): Promise<void> {
if (command.command === "help") {
ctx.ui.notify(
"Voice: /voice starts a call through configured speech service. Controls: start · setup · history · mute · unmute · stop · resume. --root stores Voice history only; service assets are service-owned.",
"info",
);
return;
}
if (command.command === "history") {
await nativeCall.showHistory(
ctx,
command.root ? join(command.root, "history") : undefined,
);
return;
}
if (command.command === "mute") {
await nativeCall.mute(true);
return;
}
if (command.command === "unmute") {
await nativeCall.mute(false);
return;
}
if (command.command === "stop") {
await stopNative(ctx);
return;
}
if (command.command === "resume") {
++nativeEpoch;
nativeStart?.abort();
nativeStart = undefined;
try {
await nativeCall.resume();
} catch (cause) {
if (
cause instanceof Error &&
cause.message === "Native voice has not been configured."
) {
await handleNative({ command: "start" }, ctx);
return;
}
throw cause;
}
return;
}
if (!ctx.hasUI) {
ctx.ui.notify(
"Native voice setup requires an interactive Pi UI.",
"warning",
);
return;
}
const settings = settingsFor(pi, ctx);
if (!settings) return;
const epoch = ++nativeEpoch;
nativeStart?.abort();
const start = new AbortController();
nativeStart = start;
const root = command.root ?? voiceDataDirectory();
let mounted = false;
let live = false;
let prepared: Awaited<ReturnType<typeof runNativeSetup>>;
let startInvoked = false;
try {
await nativeCall.stop();
if (epoch !== nativeEpoch || start.signal.aborted) return;
const toggleShortcut = shortcutSettings[0];
const setup = await runNativeSetup(ctx, {
command: settings.command,
...(settings.profile === undefined
? {}
: { profile: settings.profile }),
preview: settings.preview,
signal: start.signal,
...(toggleShortcut?.ok && toggleShortcut.value
? { cancelShortcut: toggleShortcut.value }
: {}),
});
if (!setup) return;
prepared = setup;
if (start.signal.aborted || epoch !== nativeEpoch) return;
if (command.command === "setup") {
ctx.ui.notify("Voice setup is ready.", "info");
return;
}
ctx.ui.setWidget(
"voice",
(tui, theme) => nativeCall.createWidget(tui, theme),
{ placement: "aboveEditor" },
);
mounted = true;
startInvoked = true;
await nativeCall.start(ctx, {
backend: setup,
modelSetting: settings.model,
historyDirectory: join(root, "history"),
classifierModelSetting: settings.classifierModel,
modality: {
maxSentences: settings.maxSentences,
maxPlaybackMs: settings.maxSpeechSeconds * 1_000,
},
});
live = true;
} catch (cause) {
if (!start.signal.aborted && epoch === nativeEpoch)
ctx.ui.notify(`Native voice setup failed${detail(cause)}`, "error");
} finally {
if (!live) {
start.abort();
if (prepared) {
if (startInvoked) await nativeCall.stop().catch(() => undefined);
else await prepared.client.close().catch(() => undefined);
}
}
if (epoch === nativeEpoch) {
if (nativeStart === start) nativeStart = undefined;
if (!live || !mounted) ctx.ui.setWidget("voice", undefined);
}
}
}
}
function settingsFor(
pi: ExtensionAPI,
ctx: ExtensionContext,
):
| {
command: readonly string[];
profile: string | undefined;
preview: boolean;
model: VoiceModelSetting;
classifierModel: VoiceModelSetting;
maxSentences: number;
maxSpeechSeconds: number;
}
| undefined {
const command = resolveSetting(pi, ctx, COMMAND_SETTING);
const profile = resolveSetting(pi, ctx, PROFILE_SETTING);
const preview = resolveSetting(pi, ctx, PREVIEW_SETTING);
const model = resolveSetting(pi, ctx, MODEL_SETTING);
const classifierModel = resolveSetting(pi, ctx, CLASSIFIER_MODEL_SETTING);
const maxSentences = resolveSetting(pi, ctx, MAX_SENTENCES_SETTING);
const maxSpeechSeconds = resolveSetting(pi, ctx, MAX_SPEECH_SECONDS_SETTING);
const settings = [
command,
profile,
preview,
model,
classifierModel,
maxSentences,
maxSpeechSeconds,
];
const invalid = settings.find((result) => !result.ok);
if (invalid) {
ctx.ui.notify(`${invalid.error}; voice did not open`, "warning");
return;
}
if (
!command.ok ||
!profile.ok ||
!preview.ok ||
!model.ok ||
!classifierModel.ok ||
!maxSentences.ok ||
!maxSpeechSeconds.ok
)
return;
return {
command: command.value,
profile: profile.value,
preview: preview.value,
model: model.value,
classifierModel: classifierModel.value,
maxSentences: maxSentences.value,
maxSpeechSeconds: maxSpeechSeconds.value,
};
}
function parseVoiceShortcut(raw: unknown): VoiceShortcut | null {
if (raw === "") return null;
if (
typeof raw !== "string" ||
raw.length > 64 ||
!/^(?:(?:ctrl|alt|shift|super)\+)*(?:[a-z0-9]|f(?:[1-9]|1[0-2]))$/iu.test(
raw,
)
)
throw new Error(
"expected key combination, e.g. ctrl+alt+v, or empty to disable",
);
const parts = raw.toLowerCase().split("+");
const key = parts.pop();
const order = ["ctrl", "alt", "shift", "super"];
const modifiers = parts.sort((a, b) => order.indexOf(a) - order.indexOf(b));
if (new Set(modifiers).size !== modifiers.length)
throw new Error("duplicate shortcut modifier");
return [...modifiers, key].join("+") as VoiceShortcut;
}
function detail(cause: unknown): string {
return cause instanceof Error && cause.message
? `: ${cause.message.slice(0, 200)}`
: "";
}
export const __test: {
COMMAND_SETTING: typeof COMMAND_SETTING;
PROFILE_SETTING: typeof PROFILE_SETTING;
PREVIEW_SETTING: typeof PREVIEW_SETTING;
MODEL_SETTING: typeof MODEL_SETTING;
CLASSIFIER_MODEL_SETTING: typeof CLASSIFIER_MODEL_SETTING;
MAX_SENTENCES_SETTING: typeof MAX_SENTENCES_SETTING;
MAX_SPEECH_SECONDS_SETTING: typeof MAX_SPEECH_SECONDS_SETTING;
TOGGLE_SHORTCUT_SETTING: typeof TOGGLE_SHORTCUT_SETTING;
MUTE_SHORTCUT_SETTING: typeof MUTE_SHORTCUT_SETTING;
} = {
COMMAND_SETTING,
PROFILE_SETTING,
PREVIEW_SETTING,
MODEL_SETTING,
CLASSIFIER_MODEL_SETTING,
MAX_SENTENCES_SETTING,
MAX_SPEECH_SECONDS_SETTING,
TOGGLE_SHORTCUT_SETTING,
MUTE_SHORTCUT_SETTING,
};