Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/minimax/src/tools.ts

Raw
import type { JsonObject } from "@earendil-works/pi-ai";
import type {
	AgentToolResult,
	ExtensionAPI,
} from "@earendil-works/pi-coding-agent";
import { type TSchema, Type } from "typebox";
import { credentials, region } from "./access.ts";
import { ApiError, request } from "./api.ts";
import { imageSource } from "./artifacts.ts";
import { dbg } from "./debug.ts";
import { estimatedUsage, PRICE_SOURCE } from "./images.ts";
import { PREVIEW_KEY } from "./preview.ts";
import { observeQuota, quotaLine } from "./quota.ts";
import { renderCall, renderResult } from "./render.ts";

export type MediaDetails = Record<string, unknown>;
const VLM_USD = 0.01;
/** Loose by design: details vary per action, and codemode only needs an object. */
export const outputSchema: TSchema = Type.Record(Type.String(), Type.Unknown());

/** Concise model text; full machine data stays in details, mirrored for scripts. */
export function result(
	text: string,
	details: MediaDetails,
): AgentToolResult<MediaDetails> {
	// Render-only preview bytes never reach scripts or the model.
	const { [PREVIEW_KEY]: _preview, ...structured } = details;
	return {
		content: [{ type: "text", text }],
		details,
		structuredContent: structured as JsonObject,
	};
}
export function failure(
	error: unknown,
	extra: MediaDetails = {},
	note?: string,
): AgentToolResult<MediaDetails> {
	const message =
		error instanceof Error ? error.message : "MiniMax operation failed.";
	return {
		...result(note ? `${message} ${note}` : message, {
			state: error instanceof ApiError ? error.kind : "failed",
			errorKind:
				error instanceof ApiError ? error.kind : "local-or-incompatible",
			providerCharge: "unknown",
			creditsConsumed: "unknown",
			...extra,
			error: message,
		}),
		isError: true,
	};
}
/** One short trailing meta line for model text: estimate plus quota when observed. */
export function meta(...parts: (string | undefined)[]): string {
	return parts.filter((part): part is string => !!part).join(" · ");
}
export function estimate(usd: number): string {
	return `est $${Number(usd.toPrecision(3))}`;
}

export function registerUnderstanding(
	pi: ExtensionAPI,
	lifetime: () => AbortSignal,
): void {
	pi.registerTool({
		name: "minimax_understand_image",
		label: "MiniMax image understanding",
		description:
			"Explicit M Plan image understanding via coding_plan/vlm, distinct from ordinary chat vision. JPEG/PNG/WebP, absolute local path or public HTTPS source ≤8 MiB; prompt ≤7000 chars. Uses the shared M Plan allowance; no approval dialog.",
		parameters: Type.Object(
			{
				prompt: Type.String({ minLength: 1, maxLength: 7000 }),
				source: Type.String({ minLength: 1, maxLength: 8192 }),
			},
			{ additionalProperties: false },
		),
		outputSchema,
		renderCall: renderCall("vision"),
		renderResult: renderResult("vision"),
		async execute(_id, params, signal, update, ctx) {
			try {
				const owned = AbortSignal.any([
					lifetime(),
					...(signal ? [signal] : []),
				]);
				region(pi, ctx);
				if (!params.prompt.trim())
					throw new Error("Understanding prompt must be nonempty.");
				const key = await credentials(ctx, owned);
				const image = await imageSource(params.source, owned);
				update?.({
					content: [{ type: "text", text: "Understanding image…" }],
					details: {},
				});
				const response = await request(
					key,
					"/v1/coding_plan/vlm",
					{ prompt: params.prompt, image_url: image },
					owned,
				);
				const answer = response["content"];
				if (typeof answer !== "string" || !answer.trim())
					throw new Error("MiniMax understanding response missing text.");
				if (answer.length > 64000)
					throw new Error(
						"MiniMax understanding response exceeds 64000 characters.",
					);
				dbg?.("request.finish", { operation: "understand", success: true });
				const quota = await observeQuota(pi, key, owned);
				return {
					...result(
						`${answer}\n\n${meta("MiniMax vlm", estimate(VLM_USD), quotaLine(quota))}`,
						{
							operation: "image-understanding",
							answer,
							route: "/v1/coding_plan/vlm",
							access: "observed-this-operation",
							accounting: {
								kind: "catalog-estimate",
								usd: VLM_USD,
								unit: "request",
								rate: VLM_USD,
								source: PRICE_SOURCE,
								providerCharge: "unknown",
								creditsConsumed: "unknown",
								tokens: "not reported",
							},
							...(quota ? { quota } : {}),
						},
					),
					usage: estimatedUsage(VLM_USD),
				};
			} catch (error) {
				return failure(error);
			}
		},
	});
}