import type { JsonObject } from "@earendil-works/pi-ai"; import type { AgentToolResult, ExtensionAPI, } from "@earendil-works/pi-coding-agent"; import { type TSchema, Type } from "typebox"; import { credentials, region } from "./access.ts"; import { ApiError, request } from "./api.ts"; import { imageSource } from "./artifacts.ts"; import { dbg } from "./debug.ts"; import { estimatedUsage, PRICE_SOURCE } from "./images.ts"; import { PREVIEW_KEY } from "./preview.ts"; import { observeQuota, quotaLine } from "./quota.ts"; import { renderCall, renderResult } from "./render.ts"; export type MediaDetails = Record; const VLM_USD = 0.01; /** Loose by design: details vary per action, and codemode only needs an object. */ export const outputSchema: TSchema = Type.Record(Type.String(), Type.Unknown()); /** Concise model text; full machine data stays in details, mirrored for scripts. */ export function result( text: string, details: MediaDetails, ): AgentToolResult { // Render-only preview bytes never reach scripts or the model. const { [PREVIEW_KEY]: _preview, ...structured } = details; return { content: [{ type: "text", text }], details, structuredContent: structured as JsonObject, }; } export function failure( error: unknown, extra: MediaDetails = {}, note?: string, ): AgentToolResult { const message = error instanceof Error ? error.message : "MiniMax operation failed."; return { ...result(note ? `${message} ${note}` : message, { state: error instanceof ApiError ? error.kind : "failed", errorKind: error instanceof ApiError ? error.kind : "local-or-incompatible", providerCharge: "unknown", creditsConsumed: "unknown", ...extra, error: message, }), isError: true, }; } /** One short trailing meta line for model text: estimate plus quota when observed. */ export function meta(...parts: (string | undefined)[]): string { return parts.filter((part): part is string => !!part).join(" · "); } export function estimate(usd: number): string { return `est $${Number(usd.toPrecision(3))}`; } export function registerUnderstanding( pi: ExtensionAPI, lifetime: () => AbortSignal, ): void { pi.registerTool({ name: "minimax_understand_image", label: "MiniMax image understanding", description: "Explicit M Plan image understanding via coding_plan/vlm, distinct from ordinary chat vision. JPEG/PNG/WebP, absolute local path or public HTTPS source ≤8 MiB; prompt ≤7000 chars. Uses the shared M Plan allowance; no approval dialog.", parameters: Type.Object( { prompt: Type.String({ minLength: 1, maxLength: 7000 }), source: Type.String({ minLength: 1, maxLength: 8192 }), }, { additionalProperties: false }, ), outputSchema, renderCall: renderCall("vision"), renderResult: renderResult("vision"), async execute(_id, params, signal, update, ctx) { try { const owned = AbortSignal.any([ lifetime(), ...(signal ? [signal] : []), ]); region(pi, ctx); if (!params.prompt.trim()) throw new Error("Understanding prompt must be nonempty."); const key = await credentials(ctx, owned); const image = await imageSource(params.source, owned); update?.({ content: [{ type: "text", text: "Understanding image…" }], details: {}, }); const response = await request( key, "/v1/coding_plan/vlm", { prompt: params.prompt, image_url: image }, owned, ); const answer = response["content"]; if (typeof answer !== "string" || !answer.trim()) throw new Error("MiniMax understanding response missing text."); if (answer.length > 64000) throw new Error( "MiniMax understanding response exceeds 64000 characters.", ); dbg?.("request.finish", { operation: "understand", success: true }); const quota = await observeQuota(pi, key, owned); return { ...result( `${answer}\n\n${meta("MiniMax vlm", estimate(VLM_USD), quotaLine(quota))}`, { operation: "image-understanding", answer, route: "/v1/coding_plan/vlm", access: "observed-this-operation", accounting: { kind: "catalog-estimate", usd: VLM_USD, unit: "request", rate: VLM_USD, source: PRICE_SOURCE, providerCharge: "unknown", creditsConsumed: "unknown", tokens: "not reported", }, ...(quota ? { quota } : {}), }, ), usage: estimatedUsage(VLM_USD), }; } catch (error) { return failure(error); } }, }); }