repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

Render ui visual widgets

d26bc574afb1116733eed155cb68f27aa56d31e0

Oliver Krylow <o.krylow@isp-insoft.de> ·

unsigned

Browse filesCompare with HEAD

Changed files (5)

extensions/ui/__tests__/visual-block.test.ts

Added
new file mode 100644
1 +import { describe, expect, it } from "vitest";
2 +import {
3 + renderVisualBlockShell,
4 + visualBlockSummary,
5 +} from "../web/components/visual-block.js";
7 +describe("ui visual block rendering", () => {
8 + it("renders widget fallback and ids", () => {
9 + expect(
10 + visualBlockSummary({
11 + type: "widget_mount",
12 + widgetId: "widget-1",
13 + title: "Demo",
14 + fallback: "Fallback text",
15 + status: "mounted",
16 + }),
17 + ).toBe("Demo · widget-1 · mounted · Fallback text");
18 + });
20 + it("renders widget errors with a fix hint", () => {
21 + expect(
22 + visualBlockSummary({
23 + type: "widget_error",
24 + widgetId: "widget-1",
25 + message: "boom",
26 + status: "error",
27 + }),
28 + ).toBe("widget-1 · error · boom · agent: fix widget HTML/JS");
29 + });
31 + it("renders bounded iframe shell", () => {
32 + const html = renderVisualBlockShell({
33 + widgetId: "widget-1",
34 + title: "Demo",
35 + fallback: "Fallback",
36 + srcdoc: "<p>Hi</p>",
37 + });
38 + expect(html).toContain("ui-visual-block");
39 + expect(html).toContain("widget-1");
40 + expect(html).toContain("iframe");
41 + expect(html).toContain("Fallback");
42 + });
43 +});

extensions/ui/runtime.ts

Modified
198 198 return;
200 + if (message.type === "widget_action") {
201 + publish({
202 + type: "widget_action",
203 + widgetId: message.widgetId,
204 + label: message.label,
205 + payload: message.payload,
206 + });
207 + return;
209 + if (message.type === "widget_prompt") {
210 + const text = `[widget ${message.widgetId}] ${message.text}`;
211 + publish({
212 + type: "widget_action",
213 + widgetId: message.widgetId,
214 + label: "prompt",
215 + payload: { text: message.text },
216 + });
217 + await sendPrompt(rpc, text);
218 + return;
220 + if (message.type === "widget_notify") {
221 + publish({
222 + type: "ui_notify",
223 + requestId: message.widgetId,
224 + message: message.message,
225 + });
226 + return;
228 + if (message.type === "widget_error") {
229 + publish({
230 + type: "widget_error",
231 + widgetId: message.widgetId,
232 + message: message.message,
233 + });
234 + return;
200 236 const parsed = parseComposerInput(message.text);
201 237 if (parsed.kind === "bash") {
202 238 publish({ type: "message", role: "user", text: message.text });
222 258 | { value: string }
223 259 | { confirmed: boolean }
224 260 | { cancelled: true }
225 - ));
261 + ))
262 + | {
263 + type: "widget_action";
264 + widgetId: string;
265 + label: string;
266 + payload?: unknown;
268 + | { type: "widget_prompt"; widgetId: string; text: string }
269 + | { type: "widget_notify"; widgetId: string; message: string }
270 + | { type: "widget_error"; widgetId: string; message: string };
227 272 function isBrowserMessage(value: unknown): value is BrowserMessage {
228 273 if (!value || typeof value !== "object") return false;
231 276 return typeof (value as { text?: unknown }).text === "string";
232 277 if (type === "ui_response")
233 278 return typeof (value as { id?: unknown }).id === "string";
279 + if (type === "widget_action")
280 + return (
281 + typeof (value as { widgetId?: unknown }).widgetId === "string" &&
282 + typeof (value as { label?: unknown }).label === "string"
283 + );
284 + if (type === "widget_prompt")
285 + return (
286 + typeof (value as { widgetId?: unknown }).widgetId === "string" &&
287 + typeof (value as { text?: unknown }).text === "string"
288 + );
289 + if (type === "widget_notify" || type === "widget_error")
290 + return (
291 + typeof (value as { widgetId?: unknown }).widgetId === "string" &&
292 + typeof (value as { message?: unknown }).message === "string"
293 + );
234 294 return (
235 295 type === "abort" ||
236 296 type === "cycle_model" ||

extensions/ui/web/app.css.ts

Modified
12 12 #ui-stream { min-width: 0; padding: 12px; overflow: auto; }
13 13 .ui-event { border: 2px solid color-mix(in srgb, CanvasText 50%, transparent); padding: 8px; margin: 0 0 8px; white-space: pre-wrap; }
14 14 .ui-event-user { border-width: 4px; }
15 +.ui-visual-block-row { overflow: auto; contain: layout paint; }
16 +.ui-visual-block-row header { font-weight: 800; margin-bottom: 8px; }
17 +.ui-visual-fallback { color: color-mix(in srgb, CanvasText 70%, Canvas); margin-bottom: 8px; }
18 +.ui-visual-block-row iframe { display: block; width: 100%; height: min(60vh, 600px); max-height: 600px; border: 2px solid CanvasText; background: Canvas; }
19 +.ui-visual-error-hint { margin-top: 8px; font-weight: 800; }
15 20 #ui-composer { display: grid; grid-template-columns: 1fr auto; gap: 8px; border-top: 3px solid CanvasText; padding: 8px; }
16 21 #ui-input-wrap { position: relative; min-width: 0; }
17 22 #ui-input { width: 100%; min-height: 42px; resize: vertical; font: inherit; background: Canvas; color: CanvasText; border: 2px solid CanvasText; padding: 8px; }

extensions/ui/web/app.js.ts

Modified
1 1 import { renderComposerComponentJs } from "./components/composer.js";
2 2 import { renderDialogsComponentJs } from "./components/dialogs.js";
3 +import { renderVisualBlockComponentJs } from "./components/visual-block.js";
4 5 export function renderAppJs(): string {
5 6 return `const status = document.querySelector('#ui-status');
17 18 ${renderComposerComponentJs()}
18 19 ${renderDialogsComponentJs()}
20 +${renderVisualBlockComponentJs()}
20 22 function authHeaders() {
21 23 const token = new URL(location.href).searchParams.get('token');
56 58 row.dataset.eventId = String(event.id);
57 59 row.textContent = event.text || event.message || event.state || JSON.stringify(event);
58 60 if (event.type === 'ui_request') renderUiRequest(row, event);
61 + if (event.type && event.type.startsWith('widget_')) renderVisualBlock(row, event);
59 62 stream.append(row);
61 64 stream.scrollTop = stream.scrollHeight;
100 +window.addEventListener('message', event => {
101 + const data = event.data;
102 + if (!data || typeof data !== 'object' || !String(data.type || '').startsWith('widget_')) return;
103 + socket?.send(JSON.stringify(data));
104 + renderEvents([{ id: Number.MAX_SAFE_INTEGER - 2, at: Date.now(), type: 'widget_action', widgetId: data.widgetId, label: data.label || data.type, payload: data.payload || data.text || data.message }]);
105 +});
97 107 form.addEventListener('submit', event => {
98 108 event.preventDefault();
99 109 const text = input.value.trim();

extensions/ui/web/components/visual-block.js.ts

Added
new file mode 100644
1 +import type { UiWidgetEvent } from "../../protocol";
2 +import { renderWidgetBridgeScript } from "./widget-bridge.js";
4 +export type VisualBlockShellInput = {
5 + widgetId: string;
6 + title: string;
7 + fallback: string;
8 + srcdoc: string;
9 +};
11 +export function visualBlockSummary(event: Partial<UiWidgetEvent>): string {
12 + if (event.type === "widget_error") {
13 + return [event.widgetId, "error", event.message, "agent: fix widget HTML/JS"]
14 + .filter(Boolean)
15 + .join(" · ");
16 + }
17 + return [event.title, event.widgetId, event.status, event.fallback]
18 + .filter(Boolean)
19 + .join(" · ");
22 +export function renderVisualBlockShell(input: VisualBlockShellInput): string {
23 + return `<section class="ui-visual-block" data-widget-id="${escapeAttr(input.widgetId)}">
24 +<header>${escapeHtml(input.title)} <code>${escapeHtml(input.widgetId)}</code></header>
25 +<div class="ui-visual-fallback">${escapeHtml(input.fallback)}</div>
26 +<iframe title="${escapeAttr(input.title)}" srcdoc="${escapeAttr(renderWidgetBridgeScript(input.widgetId) + input.srcdoc)}"></iframe>
27 +</section>`;
30 +export function renderVisualBlockComponentJs(): string {
31 + return `function renderVisualBlock(row, event) {
32 + row.classList.add('ui-visual-block-row');
33 + row.textContent = '';
34 + const header = document.createElement('header');
35 + header.textContent = [event.title, event.widgetId, event.status, event.label, event.message].filter(Boolean).join(' · ');
36 + row.append(header);
37 + if (event.fallback) {
38 + const fallback = document.createElement('div');
39 + fallback.className = 'ui-visual-fallback';
40 + fallback.textContent = event.fallback;
41 + row.append(fallback);
42 + }
43 + if (event.type === 'widget_mount' || event.type === 'widget_update') {
44 + const iframe = document.createElement('iframe');
45 + iframe.title = event.title || event.widgetId;
46 + iframe.srcdoc = widgetBridgeScript(event.widgetId) + (event.html || '');
47 + row.append(iframe);
48 + }
49 + if (event.type === 'widget_error') {
50 + const hint = document.createElement('div');
51 + hint.className = 'ui-visual-error-hint';
52 + hint.textContent = 'agent: fix widget HTML/JS';
53 + row.append(hint);
54 + }
56 +function widgetBridgeScript(widgetId) {
57 + return ${JSON.stringify(renderWidgetBridgeScript("__WIDGET_ID__"))}.replaceAll('__WIDGET_ID__', widgetId);
59 +`;
62 +function escapeAttr(value: string): string {
63 + return escapeHtml(value).replaceAll('"', "&quot;");
66 +function escapeHtml(value: string): string {
67 + return value
68 + .replaceAll("&", "&amp;")
69 + .replaceAll("<", "&lt;")
70 + .replaceAll(">", "&gt;");