repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

Add ui visual tools

047f3e47c9a1a8bba9eed546a325b008293b73e3

Oliver Krylow <o.krylow@isp-insoft.de> ·

unsigned

Browse filesCompare with HEAD

Changed files (8)

extensions/ui/__tests__/harness.test.ts

Modified
24 24 const extension = t.session.extensionRunner.extensions[0];
26 26 expect([...extension.commands.keys()]).toEqual(["ui"]);
27 - expect([...extension.tools.keys()]).toHaveLength(0);
27 + expect([...extension.tools.keys()]).toEqual([
28 + "ui_card",
29 + "ui_table",
30 + "ui_diagram",
31 + "ui_form",
32 + "ui_mockup",
33 + "ui_html",
34 + "ui_update",
35 + "ui_wait_action",
36 + ]);
28 37 expect(t.events.uiCallsFor("setWidget")).toHaveLength(0);

extensions/ui/__tests__/visual-tools.test.ts

Added
new file mode 100644
1 +import { describe, expect, it } from "vitest";
2 +import { createVisualTools } from "../visual-tools";
3 +import { WidgetStore } from "../widget-state";
5 +describe("ui visual tools", () => {
6 + it("creates the expected visual tool family", () => {
7 + const tools = createVisualTools({
8 + store: new WidgetStore({ now: () => 1 }),
9 + });
10 + expect(tools.map((tool) => tool.name)).toEqual([
11 + "ui_card",
12 + "ui_table",
13 + "ui_diagram",
14 + "ui_form",
15 + "ui_mockup",
16 + "ui_html",
17 + "ui_update",
18 + "ui_wait_action",
19 + ]);
20 + });
22 + it("mounts raw HTML widgets with fallback text", async () => {
23 + const events: unknown[] = [];
24 + const [, , , , , htmlTool] = createVisualTools({
25 + store: new WidgetStore({ now: () => 2 }),
26 + publish: (event) => events.push(event),
27 + });
29 + const result = await htmlTool.execute(
30 + "call-1",
31 + { title: "Demo", fallback: "fallback", html: "<button>go</button>" },
32 + undefined,
33 + undefined,
34 + {} as never,
35 + );
37 + expect(result.content).toEqual([{ type: "text", text: "fallback" }]);
38 + expect(result.details).toMatchObject({
39 + widgetId: "widget-call-1",
40 + type: "widget_mount",
41 + title: "Demo",
42 + });
43 + expect(events).toHaveLength(1);
44 + });
46 + it("updates existing widgets", async () => {
47 + const store = new WidgetStore({ now: () => 3 });
48 + store.mount({
49 + toolCallId: "call-2",
50 + title: "Demo",
51 + fallback: "fallback",
52 + html: "<p>old</p>",
53 + });
54 + const tools = createVisualTools({ store });
55 + const updateTool = tools.find((tool) => tool.name === "ui_update");
56 + expect(updateTool).toBeDefined();
58 + const result = await updateTool?.execute(
59 + "call-update",
60 + { widgetId: "widget-call-2", html: "<p>new</p>" },
61 + undefined,
62 + undefined,
63 + {} as never,
64 + );
66 + expect(result?.content).toEqual([
67 + { type: "text", text: "Updated widget-call-2" },
68 + ]);
69 + expect(store.get("widget-call-2")?.html).toBe("<p>new</p>");
70 + });
71 +});

extensions/ui/__tests__/widget-state.test.ts

Modified
30 30 fallback: "fallback",
31 31 html: "<button>go</button>",
33 - const update = store.update(mount.widgetId, { html: "<button>stop</button>" });
33 + const update = store.update(mount.widgetId, {
34 + html: "<button>stop</button>",
35 + });
34 36 const action = store.action(mount.widgetId, {
35 37 toolCallId: "call-2",
36 38 label: "Approve",
39 41 const error = store.error(mount.widgetId, "boom");
41 - expect(update).toMatchObject({ type: "widget_update", widgetId: mount.widgetId });
43 + expect(update).toMatchObject({
44 + type: "widget_update",
45 + widgetId: mount.widgetId,
46 + });
42 47 expect(action).toMatchObject({
43 48 type: "widget_action",
44 49 widgetId: mount.widgetId,

extensions/ui/index.ts

Modified
8 8 import { humanError, redactToken, type UiError, uiError } from "./errors";
9 9 import { createUiRuntime, type UiRuntime } from "./runtime";
10 10 import type { UiSeedSessionManager } from "./session-seed";
11 +import { createVisualTools } from "./visual-tools";
12 13 const COMMANDS = ["off", "status", "open", "qr"];
13 14 const WIDGET_KEY = "ui";
15 16 export default function uiExtension(pi: ExtensionAPI): void {
17 + for (const tool of createVisualTools()) pi.registerTool(tool);
16 18 let runtime: UiRuntime | undefined;
17 19 let terminalUnsubscribe: (() => void) | undefined;

extensions/ui/package.json

Modified
13 13 "peerDependencies": {
14 14 "@earendil-works/pi-coding-agent": "*",
15 - "@earendil-works/pi-tui": "*"
15 + "@earendil-works/pi-tui": "*",
16 + "typebox": "*"
17 18 "license": "MIT",
18 19 "description": "Browser UI runtime shell for Pi.",

extensions/ui/visual-tools.ts

Added
new file mode 100644
1 +import {
2 + defineTool,
3 + type ToolDefinition,
4 +} from "@earendil-works/pi-coding-agent";
5 +import { Type } from "typebox";
6 +import type { UiWidgetEvent } from "./protocol";
7 +import { WidgetStore } from "./widget-state";
9 +export type VisualToolsOptions = {
10 + store?: WidgetStore;
11 + publish?: (event: UiWidgetEvent) => void;
12 +};
14 +const mountParams = Type.Object({
15 + title: Type.String(),
16 + fallback: Type.String(),
17 + html: Type.String(),
18 +});
20 +const updateParams = Type.Object({
21 + widgetId: Type.String(),
22 + html: Type.String(),
23 +});
25 +const waitParams = Type.Object({
26 + widgetId: Type.String(),
27 + prompt: Type.Optional(Type.String()),
28 +});
30 +export function createVisualTools(
31 + options: VisualToolsOptions = {},
32 +): ToolDefinition[] {
33 + const store = options.store ?? new WidgetStore();
34 + return [
35 + mountTool("ui_card", "Card", store, options.publish),
36 + mountTool("ui_table", "Table", store, options.publish),
37 + mountTool("ui_diagram", "Diagram", store, options.publish),
38 + mountTool("ui_form", "Form", store, options.publish),
39 + mountTool("ui_mockup", "Mockup", store, options.publish),
40 + mountTool("ui_html", "HTML", store, options.publish),
41 + defineTool({
42 + name: "ui_update",
43 + label: "UI Update",
44 + description: "Update an existing trusted UI widget by id.",
45 + parameters: updateParams,
46 + async execute(_toolCallId, params) {
47 + const event = store.update(params.widgetId, { html: params.html });
48 + options.publish?.(event);
49 + return {
50 + content: [{ type: "text", text: `Updated ${params.widgetId}` }],
51 + details: event,
52 + };
53 + },
54 + }),
55 + defineTool({
56 + name: "ui_wait_action",
57 + label: "UI Wait Action",
58 + description:
59 + "Declare that the agent is waiting for a visible action from a trusted UI widget.",
60 + parameters: waitParams,
61 + async execute(_toolCallId, params) {
62 + return {
63 + content: [
64 + {
65 + type: "text",
66 + text: `Waiting for action from ${params.widgetId}${params.prompt ? `: ${params.prompt}` : ""}`,
67 + },
68 + ],
69 + details: params,
70 + };
71 + },
72 + }),
73 + ];
76 +function mountTool(
77 + name: string,
78 + label: string,
79 + store: WidgetStore,
80 + publish: ((event: UiWidgetEvent) => void) | undefined,
81 +): ToolDefinition {
82 + return defineTool({
83 + name,
84 + label: `UI ${label}`,
85 + description: `Render a trusted ${label.toLowerCase()} widget in the browser UI stream.`,
86 + parameters: mountParams,
87 + async execute(toolCallId, params) {
88 + const event = store.mount({
89 + toolCallId,
90 + title: params.title,
91 + fallback: params.fallback,
92 + html: params.html,
93 + });
94 + publish?.(event);
95 + return {
96 + content: [{ type: "text", text: params.fallback }],
97 + details: event,
98 + };
99 + },
100 + });

extensions/ui/widget-state.ts

Modified
104 104 return this.#widgets.get(widgetId);
107 - #audit(record: WidgetRecord, event: Omit<UiWidgetEvent, "id" | "at">): UiWidgetEvent {
107 + #audit(
108 + record: WidgetRecord,
109 + event: Omit<UiWidgetEvent, "id" | "at">,
110 + ): UiWidgetEvent {
108 111 const full = { id: 0, at: this.#now(), ...event };
109 112 record.audit.push(full);
110 113 return full;

package-lock.json

Modified
483 483 "peerDependencies": {
484 484 "@earendil-works/pi-coding-agent": "*",
485 - "@earendil-works/pi-tui": "*"
485 + "@earendil-works/pi-tui": "*",
486 + "typebox": "*"
488 489 "extensions/ultra": {