repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

feat(ultra): provider-scoped model tier overrides

e096e5259b9dd5e1c92b0d1f4ed89dcebd44b6f6

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Add ultra.providerModelTiers: per-provider field overrides folded over
ultra.modelTiers when the session model's provider matches.
Rubric rendering, tier resolution, and journal routing use the folded
tiers; overrides may reference only tiers defined in the merged base.

Changed files (5)

extensions/ultra/README.md

Modified
149 149 | `ultra.subagentSkills` | `"all"` | Load host skills into sub-agents, or not. |
150 150 | `ultra.subagentSessionRetention` | `"all"` | Retain parent-linked child sessions; `"none"` is stateless. |
151 151 | `ultra.modelTiers` | built-in tiers | Named model, thinking, and orchestration guidance tiers. |
152 +| `ultra.providerModelTiers` | `{}` | Per-active-provider tier overrides folded over `ultra.modelTiers`. |
152 153 | `ultra.actionSummarizerModel` | `null` | Fast model for concise tool-action phrases; off when null. |
154 155 `ultra.actionSummarizerModel` accepts a concrete `provider/model` ref.
182 183 Changing effective model or thinking routes starts a fresh checkpoint journal instead of resuming children with stale routes.
183 184 Changes only to the tier's orchestrator-facing selection instructions do not invalidate execution checkpoints.
186 +`ultra.providerModelTiers` maps provider names to per-tier field overrides that Ultra folds over `ultra.modelTiers` whenever the session model's provider matches.
187 +Provider names are matched exactly against the provider part of a `provider/model` reference.
188 +Overrides may set any of the three tier fields; tiers without an entry keep the base values, and providers without a block keep the base tiers.
189 +Provider blocks may only override tiers that exist in the merged base `ultra.modelTiers`; an unknown tier name is a settings error.
190 +The orchestrator rubric and all tier resolution use the folded tiers, so switching the session model's provider re-routes affected tiers on the next turn.
192 +For example, keep tier-neutral defaults and give each provider its own models:
194 +```json
196 + "ultra": {
197 + "providerModelTiers": {
198 + "openai-codex": {
199 + "tiny": { "model": "openai-codex/gpt-5.6-luna" },
200 + "medium": { "model": "openai-codex/gpt-5.6-terra" },
201 + "large": { "model": "openai-codex/gpt-6-sol" },
202 + "huge": { "model": "openai-codex/gpt-6-astra" }
203 + },
204 + "zai": {
205 + "tiny": { "model": "zai/glm-4.7-flash", "thinkingLevel": "low" },
206 + "medium": { "model": "zai/glm-4.7" },
207 + "large": { "model": "zai/glm-4.7", "thinkingLevel": "max" }
212 +```
185 214 For example, override one built-in tier and add a custom one:
187 216 ```json

extensions/ultra/__tests__/settings.test.ts

Modified
1 1 import { describe, expect, it } from "vitest";
2 2 import {
3 3 DEFAULT_MODEL_TIERS,
4 + effectiveModelTiers,
4 5 loadUltraSettings,
5 6 mergeUltraSettings,
6 7 modelTierPrompt,
19 20 subagentSkills: "all",
20 21 subagentSessionRetention: "all",
21 22 modelTiers: DEFAULT_MODEL_TIERS,
23 + providerModelTiers: {},
22 24 actionSummarizerModel: null,
313 +describe("providerModelTiers setting", () => {
314 + const base = loadUltraSettings({});
316 + it("defaults to no provider overrides and extracts typed provider blocks", () => {
317 + expect(ULTRA_DEFAULTS.providerModelTiers).toEqual({});
318 + expect(
319 + pickUltra({
320 + ultra: {
321 + providerModelTiers: {
322 + zai: { small: { model: "zai/glm-4.7", thinkingLevel: "high" } },
323 + },
324 + },
325 + }),
326 + ).toEqual({
327 + providerModelTiers: {
328 + zai: { small: { model: "zai/glm-4.7", thinkingLevel: "high" } },
329 + },
330 + });
331 + expect(effectiveModelTiers(base, "openai-codex")).toBe(base.modelTiers);
332 + expect(effectiveModelTiers(base, undefined)).toBe(base.modelTiers);
333 + });
335 + it("folds matching provider overrides per field and renders them", () => {
336 + const settings = loadUltraSettings({
337 + global: {
338 + ultra: {
339 + providerModelTiers: {
340 + zai: {
341 + tiny: { model: "zai/glm-4.7-flash" },
342 + large: {
343 + model: "zai/glm-4.7",
344 + instructions: "Provider flagship only.",
345 + },
346 + },
347 + },
348 + },
349 + },
350 + project: {
351 + ultra: {
352 + providerModelTiers: {
353 + zai: { large: { model: "zai/glm-4.6" } },
354 + "klaus-custom": { medium: { model: "klaus/fable" } },
355 + },
356 + },
357 + },
358 + });
359 + const zai = effectiveModelTiers(settings, "zai");
360 + expect(zai.large).toEqual({
361 + model: "zai/glm-4.6",
362 + thinkingLevel: DEFAULT_MODEL_TIERS.large.thinkingLevel,
363 + instructions: "Provider flagship only.",
364 + });
365 + expect(zai.tiny.model).toBe("zai/glm-4.7-flash");
366 + expect(zai.small).toEqual(DEFAULT_MODEL_TIERS.small);
367 + expect(effectiveModelTiers(settings, "klaus-custom").medium.model).toBe(
368 + "klaus/fable",
369 + );
370 + expect(effectiveModelTiers(settings, "unknown-provider")).toBe(
371 + settings.modelTiers,
372 + );
373 + expect(settings.modelTiers.large.model).toBeNull();
374 + expect(modelTierPrompt(zai)).toContain(
375 + "- large: zai/glm-4.6; high; Provider flagship only.",
376 + );
377 + });
379 + it.each([
380 + [7, "ultra.providerModelTiers must be an object."],
381 + [{ "bad provider": {} }, "invalid provider name"],
382 + [{ zai: 7 }, "ultra.providerModelTiers.zai must be an object."],
384 + { zai: { small: { model: "small" } } },
385 + "ultra.providerModelTiers.zai.small.model must be provider/model",
386 + ],
388 + { zai: { missing_tier: { model: "zai/glm-4.7" } } },
389 + "ultra.providerModelTiers.zai.missing_tier: unknown tier.",
390 + ],
391 + ] as const)(
392 + "rejects malformed provider tier configuration %j",
393 + (providerModelTiers, error) => {
394 + expect(() =>
395 + loadUltraSettings({
396 + global: { ultra: { providerModelTiers } },
397 + }),
398 + ).toThrow(error);
399 + },
400 + );
401 +});
311 403 describe("pickUltra", () => {
312 404 it("extracts only correctly-typed known keys from the `ultra` block", () => {
313 405 expect(
396 488 small: { ...DEFAULT_MODEL_TIERS.small, model: "openai/gpt-5-mini" },
397 489 large: { ...DEFAULT_MODEL_TIERS.large, model: "openai/gpt-5" },
491 + providerModelTiers: {},
399 492 actionSummarizerModel: null,
442 535 small: { ...DEFAULT_MODEL_TIERS.small, model: "openai/gpt-5-mini" },
443 536 large: { ...DEFAULT_MODEL_TIERS.large, model: "openai/gpt-5" },
538 + providerModelTiers: {},
445 539 actionSummarizerModel: "openai/gpt-5-mini",

extensions/ultra/implementation.ts

Modified
39 39 type StepDrivers,
40 40 type StepRunner,
41 41 } from "./runner.ts";
42 -import { loadUltraSettings, type UltraSettings } from "./settings.ts";
42 +import {
43 + effectiveModelTiers,
44 + loadUltraSettings,
45 + type UltraSettings,
46 +} from "./settings.ts";
43 47 import {
44 48 type DiscoveredWorkflow,
45 49 DYNAMIC_EXTENSION_TOOL_NAMES,
113 117 const manager = SettingsManager.create(ctx.cwd, undefined, {
114 118 projectTrusted: ctx.isProjectTrusted(),
116 - return loadUltraSettings({
120 + const cfg = loadUltraSettings({
117 121 global: manager.getGlobalSettings(),
118 122 project: manager.getProjectSettings(),
124 + return {
125 + ...cfg,
126 + modelTiers: effectiveModelTiers(cfg, ctx.model?.provider),
127 + };
122 130 // Workflow files are tiny and human-authored during live sessions. Rescan at

extensions/ultra/index.ts

Modified
12 12 import { type AutocompleteItem, Markdown } from "@earendil-works/pi-tui";
13 13 import type { WorkflowResult } from "./engine.ts";
14 14 import { formatUltraResultMarkdown } from "./result.ts";
15 -import { loadUltraSettings, modelTierPrompt } from "./settings.ts";
15 +import {
16 + effectiveModelTiers,
17 + loadUltraSettings,
18 + modelTierPrompt,
19 +} from "./settings.ts";
17 21 const ULTRA_AUTHORING_SKILL_PATH = fileURLToPath(
18 22 new URL("./prompts/ultra-authoring/SKILL.md", import.meta.url),
88 92 if (!settings.enabled) return;
89 93 event.systemPromptOptions.sections.ultra_model_tier_policy =
90 - modelTierPrompt(settings.modelTiers);
94 + modelTierPrompt(effectiveModelTiers(settings, ctx.model?.provider));
93 97 pi.registerCommand("ultra", {

extensions/ultra/settings.ts

Modified
25 25 export type ModelTiers = Record<string, ModelTier>;
26 -type UltraOverrides = Partial<Omit<UltraSettings, "modelTiers">> & {
26 +/** Provider name → tier name → per-field overrides for when that provider is active. */
27 +export type ProviderModelTiers = Record<
28 + string,
29 + Record<string, Partial<ModelTier>>
30 +>;
31 +type UltraOverrides = Partial<
32 + Omit<UltraSettings, "modelTiers" | "providerModelTiers">
33 +> & {
27 34 modelTiers?: Record<string, Partial<ModelTier>>;
35 + providerModelTiers?: Record<string, Record<string, Partial<ModelTier>>>;
30 38 export const DEFAULT_MODEL_TIERS: ModelTiers = {
77 85 subagentSessionRetention: "all" | "none";
78 86 /** Settings-defined model, thinking, and selection policy used by step.model. */
79 87 modelTiers: ModelTiers;
88 + /** Per-provider tier overrides folded over `modelTiers` when the session model's
89 + * provider matches; providers without a block keep the base tiers. */
90 + providerModelTiers: ProviderModelTiers;
80 91 /** Fast model used to turn raw tool args into board action phrases; null disables it. */
81 92 actionSummarizerModel: string | null;
91 102 subagentSkills: "all",
92 103 subagentSessionRetention: "all",
93 104 modelTiers: DEFAULT_MODEL_TIERS,
105 + providerModelTiers: {},
94 106 actionSummarizerModel: null,
98 110 return v !== null && typeof v === "object" && !Array.isArray(v);
101 -function pickModelTiers(raw: unknown): UltraOverrides["modelTiers"] {
113 +function pickModelTiers(
114 + raw: unknown,
115 + prefix = "ultra.modelTiers",
116 +): UltraOverrides["modelTiers"] {
102 117 if (raw === undefined) return undefined;
103 - if (!isPlainObject(raw))
104 - throw new Error("ultra.modelTiers must be an object.");
118 + if (!isPlainObject(raw)) throw new Error(`${prefix} must be an object.`);
105 119 const out: NonNullable<UltraOverrides["modelTiers"]> = {};
106 120 for (const [name, value] of Object.entries(raw)) {
107 - const key = `ultra.modelTiers.${name}`;
121 + const key = `${prefix}.${name}`;
108 122 if (!/^[a-z][a-z0-9_-]*$/.test(name) || name in Object.prototype)
109 123 throw new Error(`${key}: invalid tier name.`);
110 124 if (!isPlainObject(value)) throw new Error(`${key} must be an object.`);
146 160 return out;
163 +function pickProviderModelTiers(
164 + raw: unknown,
165 +): UltraOverrides["providerModelTiers"] {
166 + if (raw === undefined) return undefined;
167 + if (!isPlainObject(raw))
168 + throw new Error("ultra.providerModelTiers must be an object.");
169 + const out: NonNullable<UltraOverrides["providerModelTiers"]> = {};
170 + for (const [provider, tiers] of Object.entries(raw)) {
171 + if (!/^[^\s/:]+$/.test(provider) || provider in Object.prototype)
172 + throw new Error(
173 + `ultra.providerModelTiers.${provider}: invalid provider name.`,
174 + );
175 + out[provider] =
176 + pickModelTiers(tiers, `ultra.providerModelTiers.${provider}`) ?? {};
178 + return out;
149 181 /** The sole tier rubric, rendered from the effective settings for each agent turn. */
150 182 export function modelTierPrompt(tiers: ModelTiers): string {
151 183 return [
201 233 out.subagentSessionRetention = ultra.subagentSessionRetention;
202 234 const modelTiers = pickModelTiers(ultra.modelTiers);
203 235 if (modelTiers) out.modelTiers = modelTiers;
236 + const providerModelTiers = pickProviderModelTiers(ultra.providerModelTiers);
237 + if (providerModelTiers) out.providerModelTiers = providerModelTiers;
204 238 if (ultra.actionSummarizerModel === null) out.actionSummarizerModel = null;
205 239 else if (
206 240 typeof ultra.actionSummarizerModel === "string" &&
223 257 for (const [name, tier] of Object.entries(pickModelTiers(source) ?? {}))
224 258 modelTiers[name] = { ...modelTiers[name], ...tier };
260 + const providerModelTiers: ProviderModelTiers = {};
261 + for (const partial of partials) {
262 + for (const [provider, tiers] of Object.entries(
263 + partial.providerModelTiers ?? {},
264 + )) {
265 + providerModelTiers[provider] ??= {};
266 + const current = providerModelTiers[provider];
267 + for (const [name, tier] of Object.entries(
268 + pickModelTiers(tiers, `ultra.providerModelTiers.${provider}`) ?? {},
269 + )) {
270 + const base = current[name] ?? {};
271 + current[name] = { ...base, ...tier };
275 + for (const [provider, tiers] of Object.entries(providerModelTiers))
276 + for (const name of Object.keys(tiers))
277 + if (!Object.hasOwn(modelTiers, name))
278 + throw new Error(
279 + `ultra.providerModelTiers.${provider}.${name}: unknown tier.`,
280 + );
226 281 for (const [name, tier] of Object.entries(modelTiers)) {
228 283 tier.model === undefined ||
257 312 ? p.subagentSessionRetention
258 313 : into.subagentSessionRetention,
259 314 modelTiers,
315 + providerModelTiers,
260 316 actionSummarizerModel:
261 317 p.actionSummarizerModel === null ||
262 318 typeof p.actionSummarizerModel === "string"
263 319 ? p.actionSummarizerModel
264 320 : into.actionSummarizerModel,
266 - { ...ULTRA_DEFAULTS, modelTiers },
322 + { ...ULTRA_DEFAULTS, modelTiers, providerModelTiers },
326 +/** Fold provider-scoped overrides over the base tiers; unknown providers keep the base tiers. */
327 +export function effectiveModelTiers(
328 + settings: UltraSettings,
329 + provider: string | undefined,
330 +): ModelTiers {
331 + const overrides =
332 + provider === undefined ? undefined : settings.providerModelTiers[provider];
333 + if (!overrides) return settings.modelTiers;
334 + const out: ModelTiers = {};
335 + for (const [name, tier] of Object.entries(settings.modelTiers))
336 + out[name] = { ...tier, ...overrides[name] };
337 + return out;
270 340 export interface SettingsSources {
271 341 global?: unknown;
272 342 project?: unknown;