ultimate-pi 0.18.0 → 0.19.0

package/.pi/extensions/pi-model-router-harness.ts

@@ -1,42 +0,0 @@
-/**
- * Vendored [pi-model-router](https://github.com/yeliu84/pi-model-router), gated behind
- * a project-local `.pi/model-router.json` from `/harness-setup` so the extension
- * (and built-in fallback tiers) never load before harness bootstrap.
- */
-
-import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import vendorModelRouter from "../../vendor/pi-model-router/extensions/index.js";
-
-function isHarnessRouterReady(cwd: string): boolean {
-	const path = join(cwd, ".pi", "model-router.json");
-	if (!existsSync(path)) {
-		return false;
-	}
-	try {
-		const data: unknown = JSON.parse(readFileSync(path, "utf8"));
-		if (typeof data !== "object" || data === null) {
-			return false;
-		}
-		const profiles = (data as { profiles?: unknown }).profiles;
-		return (
-			typeof profiles === "object" &&
-			profiles !== null &&
-			Object.keys(profiles).length > 0
-		);
-	} catch {
-		return false;
-	}
-}
-
-export default function piModelRouterHarness(pi: ExtensionAPI) {
-	const cwd = process.cwd();
-	if (!isHarnessRouterReady(cwd)) {
-		console.warn(
-			"[ultimate-pi] Model router disabled until `.pi/model-router.json` exists (generate via /harness-setup Step 3.5).",
-		);
-		return;
-	}
-	vendorModelRouter(pi);
-}

package/.pi/harness/evolution/meta-optimizer.mjs

@@ -1,99 +0,0 @@
-#!/usr/bin/env node
-/**
- * meta-optimizer — read harness JSONL index and emit tuning proposals (no LLM).
- */
-
-import { readFile, readdir } from "node:fs/promises";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
-const RUNS = join(ROOT, ".pi", "harness", "runs");
-const OUT = join(ROOT, ".pi", "harness", "router", "proposals", "meta-optimizer-proposal.json");
-
-async function loadIndexLines() {
-	const indexPath = join(RUNS, "index.jsonl");
-	try {
-		const raw = await readFile(indexPath, "utf-8");
-		return raw
-			.trim()
-			.split("\n")
-			.filter(Boolean)
-			.map((line) => JSON.parse(line));
-	} catch {
-		return [];
-	}
-}
-
-async function countEventTypes(runId) {
-	const eventsPath = join(RUNS, runId, "events.jsonl");
-	try {
-		const raw = await readFile(eventsPath, "utf-8");
-		const counts = {};
-		for (const line of raw.trim().split("\n").filter(Boolean)) {
-			const row = JSON.parse(line);
-			const t = row.type ?? "unknown";
-			counts[t] = (counts[t] ?? 0) + 1;
-		}
-		return counts;
-	} catch {
-		return {};
-	}
-}
-
-async function main() {
-	const index = await loadIndexLines();
-	const recent = index.slice(-20);
-	let totalToolSpans = 0;
-	let runs = 0;
-	const policyHints = [];
-
-	for (const row of recent) {
-		const runId = row.run_id;
-		if (!runId) continue;
-		runs += 1;
-		try {
-			const trace = JSON.parse(
-				await readFile(join(RUNS, runId, "trace.json"), "utf-8"),
-			);
-			totalToolSpans += trace.tool_span_count ?? trace.tool_spans?.length ?? 0;
-		} catch {
-			/* skip */
-		}
-		const events = await countEventTypes(runId);
-		if ((events.tool_result ?? 0) > 50) {
-			policyHints.push({
-				run_id: runId,
-				hint: "High tool volume; consider stricter phase caps or router to smaller model for plan phase.",
-			});
-		}
-	}
-
-	const avgTools = runs > 0 ? totalToolSpans / runs : 0;
-	const proposal = {
-		schema_version: "1.0.0",
-		generated_at: new Date().toISOString(),
-		source: "meta-optimizer",
-		sample_runs: runs,
-		avg_tool_spans_per_run: avgTools,
-		router_hints: policyHints,
-		recommendation:
-			avgTools > 30
-				? "Consider lowering execute-phase tool budget or enabling HARNESS_BUDGET_HARD_STOP."
-				: "No automatic router change; metrics within nominal band.",
-	};
-
-	await import("node:fs/promises").then(({ mkdir, writeFile }) =>
-		mkdir(dirname(OUT), { recursive: true }).then(() =>
-			writeFile(OUT, `${JSON.stringify(proposal, null, 2)}\n`),
-		),
-	);
-
-	console.log(JSON.stringify(proposal, null, 2));
-	console.log(`\nWrote ${OUT}`);
-}
-
-main().catch((err) => {
-	console.error(err);
-	process.exit(1);
-});

package/.pi/harness/specs/router-tuning-proposal.schema.json

@@ -1,114 +0,0 @@
-{
-	"$schema": "https://json-schema.org/draft/2020-12/schema",
-	"$id": "https://ultimate-pi.local/.pi/harness/specs/router-tuning-proposal.schema.json",
-	"title": "RouterTuningProposal",
-	"description": "Evidence-gated, approval-required proposal for model-router updates.",
-	"type": "object",
-	"additionalProperties": false,
-	"required": [
-		"schema_version",
-		"proposal_id",
-		"created_at",
-		"router_path",
-		"base_router_sha256",
-		"candidate_router_sha256",
-		"evidence",
-		"status",
-		"approval",
-		"candidate_router"
-	],
-	"properties": {
-		"schema_version": {
-			"type": "string",
-			"const": "1.0.0"
-		},
-		"proposal_id": {
-			"type": "string",
-			"minLength": 1
-		},
-		"created_at": {
-			"type": "string",
-			"minLength": 1
-		},
-		"router_path": {
-			"type": "string",
-			"const": ".pi/model-router.json"
-		},
-		"base_router_sha256": {
-			"type": "string",
-			"pattern": "^[a-f0-9]{64}$"
-		},
-		"candidate_router_sha256": {
-			"type": "string",
-			"pattern": "^[a-f0-9]{64}$"
-		},
-		"status": {
-			"type": "string",
-			"enum": ["proposed", "approved_applied", "rejected"]
-		},
-		"evidence": {
-			"type": "object",
-			"additionalProperties": false,
-			"required": [
-				"sample_count",
-				"min_sample_count",
-				"success_rate_delta",
-				"cost_per_task_delta",
-				"regression_guard_passed",
-				"trace_refs"
-			],
-			"properties": {
-				"sample_count": {
-					"type": "integer",
-					"minimum": 1
-				},
-				"min_sample_count": {
-					"type": "integer",
-					"minimum": 1
-				},
-				"success_rate_delta": {
-					"type": "number"
-				},
-				"cost_per_task_delta": {
-					"type": "number"
-				},
-				"regression_guard_passed": {
-					"type": "boolean"
-				},
-				"trace_refs": {
-					"type": "array",
-					"items": {
-						"type": "string",
-						"minLength": 1
-					}
-				},
-				"notes": {
-					"type": "string"
-				}
-			}
-		},
-		"approval": {
-			"type": "object",
-			"additionalProperties": false,
-			"required": ["required", "approved_by", "approved_at", "justification"],
-			"properties": {
-				"required": {
-					"type": "boolean",
-					"const": true
-				},
-				"approved_by": {
-					"type": ["string", "null"]
-				},
-				"approved_at": {
-					"type": ["string", "null"]
-				},
-				"justification": {
-					"type": ["string", "null"]
-				}
-			}
-		},
-		"candidate_router": {
-			"type": "object"
-		}
-	}
-}

package/.pi/model-router.example.json

@@ -1,36 +0,0 @@
-{
-	"defaultProfile": "auto",
-	"debug": false,
-	"classifierModel": "openai/gpt-5.4-nano",
-	"phaseBias": 0.5,
-	"maxSessionBudget": 1.0,
-	"largeContextThreshold": 100000,
-	"rules": [
-		{
-			"matches": ["deploy", "production", "release"],
-			"tier": "high",
-			"reason": "Safety check for production tasks"
-		},
-		{ "matches": "changelog", "tier": "low" }
-	],
-	"profiles": {
-		"auto": {
-			"high": {
-				"model": "openai/gpt-5.5",
-				"thinking": "high",
-				"fallbacks": ["openai/gpt-5.4-nano"]
-			},
-			"medium": { "model": "openai/gpt-5.5", "thinking": "medium" },
-			"low": { "model": "openai/gpt-5.5", "thinking": "low" }
-		},
-		"opencode-go": {
-			"high": {
-				"model": "opencode-go/qwen3.6-plus",
-				"thinking": "high",
-				"fallbacks": ["opencode-go/deepseek-v4-flash"]
-			},
-			"medium": { "model": "opencode-go/qwen3.6-plus", "thinking": "medium" },
-			"low": { "model": "opencode-go/qwen3.6-plus", "thinking": "low" }
-		}
-	}
-}

package/.pi/prompts/harness-critic.md

@@ -1,10 +0,0 @@
----
-description: "Deprecated alias — use /harness-review (includes adversary phase)."
-argument-hint: "[--run <run-id>] [--trace <trace-ref>] [--risk low|med|high]"
----
-
-# harness-critic
-
-**This command is deprecated.** Run **`/harness-review`** instead — Phase 4 runs `harness/adversary` after benchmark and policy verdict pass (skip with `--quick`).
-
-If you must continue this turn only: forward to `/harness-review` with the same `$ARGUMENTS` (omit `--quick` if you need adversary). Do not spawn adversary in isolation unless the user explicitly requested adversary-only review.

package/.pi/prompts/harness-eval.md

@@ -1,10 +0,0 @@
----
-description: "Deprecated alias — use /harness-review (post-run master orchestrator)."
-argument-hint: "[--run <run-id>] [--quick] [--trace <trace-ref>]"
----
-
-# harness-eval
-
-**This command is deprecated.** Run **`/harness-review`** instead — it orchestrates deterministic gates, benchmark eval, policy verdict, and adversary review in one flow (ADR 0039).
-
-If you must continue this turn only: forward all work to `/harness-review` with the same arguments (`$ARGUMENTS`). Do not spawn a separate benchmark-only pass unless the user explicitly asked for benchmark-only diagnostics.

package/.pi/prompts/harness-router-tune.md

@@ -1,52 +0,0 @@
----
-description: Propose model-router updates from eval evidence; apply only with explicit approval.
-argument-hint: "--evidence <evidence.json> --candidate <candidate-router.json> [--proposal <out.json>]"
----
-
-# harness-router-tune
-
-Orchestrator — scripts + `harness/meta-optimizer` spawn. **Never** write `.pi/model-router.json` directly.
-
-## Step 0 — Parse arguments
-
-- required: `--evidence <evidence.json>`, `--candidate <candidate-router.json>`
-- optional: `--proposal <out.json>`
-
-If missing required args:
-
-`Usage: /harness-router-tune --evidence <path> --candidate <path> [--proposal <out.json>]`
-
-## Orchestration (required)
-
-1. Parent validates evidence paths exist.
-2. Optionally spawn:
-
-```
-subagent({ agentScope: "both", agent: "harness/meta-optimizer", task: "mode: tune, evidence paths…" })
-```
-
-3. Parent runs proposal script:
-
-```bash
-node .pi/harness/router/propose-router-tuning.mjs \
-  --evidence <evidence.json> \
-  --candidate <candidate-router.json> \
-  --proposal-out .pi/harness/router/proposals/<id>.json
-```
-
-4. `ask_user` approve / reject / edit (harness-decisions).
-5. Apply only after approval:
-
-```bash
-node .pi/harness/router/apply-router-proposal.mjs \
-  --proposal .pi/harness/router/proposals/<id>.json \
-  --approve-by "<human>" \
-  --justification "<reason>" \
-  --write
-```
-
-## Completion
-
-- `tuning_status`: `proposed`, `human_required`, or `rejected`
-- Evidence gate summary
-- Confirm `.pi/model-router.json` was not mutated without apply script

package/.pi/scripts/harness-generate-model-router.mjs

@@ -1,327 +0,0 @@
-#!/usr/bin/env node
-/**
- * Generate `.pi/model-router.json` from Pi's authenticated providers (auth.json + env),
- * not from raw env-var heuristics alone.
- *
- * Uses @earendil-works/pi-coding-agent ModelRegistry.getAvailable() — same source as /login.
- *
- * Usage: node harness-generate-model-router.mjs [--force] [--dry-run]
- *   --force    overwrite existing .pi/model-router.json
- *   --dry-run  print JSON to stdout, do not write
- *
- * Requires @earendil-works/pi-coding-agent (peer of ultimate-pi; bundled with pi).
- */
-
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { createRequire } from "node:module";
-import { dirname, join } from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
-
-const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
-const UP_PKG = join(SCRIPT_DIR, "..", "..");
-const OUT_PATH = join(process.cwd(), ".pi", "model-router.json");
-
-const PROVIDER_PRIORITY = [
-	"openai",
-	"opencode-go",
-	"anthropic",
-	"google",
-	"openrouter",
-	"groq",
-	"mistral",
-	"amazon",
-];
-
-/** Substring hints per tier (first match in available ids wins). */
-const TIER_HINTS = {
-	high: [
-		"gpt-5.5-pro",
-		"deepseek-v4-pro",
-		"gpt-5.4-pro",
-		"claude-opus",
-		"sonnet-4",
-		"gemini-2.5-pro",
-		"pro",
-	],
-	medium: [
-		"gpt-5.5",
-		"qwen3.6-plus",
-		"kimi-k2.6",
-		"gpt-5.4",
-		"claude-sonnet",
-		"gemini-flash",
-		"plus",
-	],
-	low: [
-		"deepseek-v4-flash",
-		"gpt-5.4-nano",
-		"haiku",
-		"flash-lite",
-		"flash",
-		"mini",
-	],
-};
-
-function fail(msg) {
-	console.error(`harness-generate-model-router: ${msg}`);
-	process.exit(1);
-}
-
-async function loadPiCodingAgent() {
-	const scopes = ["@earendil-works", "@mariozechner"];
-	const agentRoots = scopes.flatMap((scope) => [
-		join(UP_PKG, "node_modules", scope, "pi-coding-agent"),
-		join(UP_PKG, ".pi", "npm", "node_modules", scope, "pi-coding-agent"),
-	]);
-	for (const root of agentRoots) {
-		const entry = join(root, "dist", "index.js");
-		if (existsSync(entry)) {
-			return import(pathToFileURL(entry).href);
-		}
-	}
-	for (const spec of ["@earendil-works/pi-coding-agent", "@mariozechner/pi-coding-agent"]) {
-		for (const base of [UP_PKG, process.cwd()]) {
-			try {
-				const req = createRequire(join(base, "package.json"));
-				return req(spec);
-			} catch {
-				/* try next */
-			}
-		}
-	}
-	fail(
-		"@earendil-works/pi-coding-agent not found (install pi or npm i in ultimate-pi). Peer: @earendil-works/pi-coding-agent",
-	);
-}
-
-function canonicalRef(provider, modelId) {
-	return `${provider}/${modelId}`;
-}
-
-function pickTierModel(models, tier) {
-	const hints = TIER_HINTS[tier];
-	for (const hint of hints) {
-		const exact = models.find((m) => m.id === hint);
-		if (exact) return canonicalRef(exact.provider, exact.id);
-	}
-	for (const hint of hints) {
-		const match = models.find((m) => m.id.includes(hint));
-		if (match) return canonicalRef(match.provider, match.id);
-	}
-	if (models.length === 0) return null;
-	if (tier === "high") {
-		const reasoning = models.find((m) => m.reasoning);
-		if (reasoning) return canonicalRef(reasoning.provider, reasoning.id);
-	}
-	if (tier === "low") {
-		return canonicalRef(models[models.length - 1].provider, models[models.length - 1].id);
-	}
-	return canonicalRef(models[0].provider, models[0].id);
-}
-
-function modelsForProvider(available, provider) {
-	return available.filter((m) => m.provider === provider);
-}
-
-function choosePrimaryProvider(available) {
-	const byProvider = new Map();
-	for (const m of available) {
-		if (!byProvider.has(m.provider)) byProvider.set(m.provider, []);
-		byProvider.get(m.provider).push(m);
-	}
-	for (const p of PROVIDER_PRIORITY) {
-		if (byProvider.has(p)) return { provider: p, models: byProvider.get(p) };
-	}
-	const first = [...byProvider.keys()].sort()[0];
-	return { provider: first, models: byProvider.get(first) ?? [] };
-}
-
-function buildFallbacks(available, primaryProvider, highModel) {
-	const fallbacks = [];
-	for (const p of ["anthropic", "google", "openai", "opencode-go"]) {
-		if (p === primaryProvider) continue;
-		const alt = available.filter((m) => m.provider === p);
-		if (alt.length === 0) continue;
-		const ref = pickTierModel(alt, "medium");
-		if (ref && ref !== highModel) fallbacks.push(ref);
-	}
-	return fallbacks.slice(0, 3);
-}
-
-/** Session-locked router: one model SKU per profile; tiers vary thinking only. */
-function buildRoutedProfile(available, provider) {
-	const models = modelsForProvider(available, provider);
-	if (models.length === 0) return null;
-	const sku =
-		pickTierModel(models, "medium") ??
-		pickTierModel(models, "high") ??
-		pickTierModel(models, "low");
-	if (!sku) return null;
-	const fallbacks = buildFallbacks(available, provider, sku);
-	const high = { model: sku, thinking: "high" };
-	if (fallbacks.length) high.fallbacks = fallbacks;
-	return {
-		high,
-		medium: { model: sku, thinking: "medium" },
-		low: { model: sku, thinking: "low" },
-	};
-}
-
-function addCheapDeepProfiles(profiles, available, provider) {
-	const models = modelsForProvider(available, provider);
-	if (models.length === 0) return;
-	const sku =
-		pickTierModel(models, "medium") ??
-		pickTierModel(models, "high") ??
-		pickTierModel(models, "low");
-	if (!sku) return;
-	const fallbacks = buildFallbacks(available, provider, sku);
-	const deepHigh = { model: sku, thinking: "xhigh" };
-	if (fallbacks.length) deepHigh.fallbacks = fallbacks;
-	profiles.cheap = {
-		high: { model: sku, thinking: "low" },
-		medium: { model: sku, thinking: "off" },
-		low: { model: sku, thinking: "off" },
-	};
-	profiles.deep = {
-		high: deepHigh,
-		medium: { model: sku, thinking: "medium" },
-		low: { model: sku, thinking: "low" },
-	};
-}
-
-function resolveClassifierModel(available) {
-	const openaiModels = modelsForProvider(available, "openai");
-	if (openaiModels.length > 0) {
-		return (
-			pickTierModel(openaiModels, "low") ??
-			canonicalRef(openaiModels[openaiModels.length - 1].provider, openaiModels[openaiModels.length - 1].id)
-		);
-	}
-	const { models } = choosePrimaryProvider(available);
-	return pickTierModel(models, "medium");
-}
-
-/** OpenAI-backed default profile name exposed as `router/auto`. */
-const OPENAI_PROFILE_NAME = "auto";
-
-function routerProfileName(provider) {
-	return provider === "openai" ? OPENAI_PROFILE_NAME : provider;
-}
-
-function resolveDefaultProfile(profiles) {
-	if (profiles[OPENAI_PROFILE_NAME]) return OPENAI_PROFILE_NAME;
-	if (profiles["opencode-go"]) return "opencode-go";
-	return (
-		Object.keys(profiles).find((name) => name !== "cheap" && name !== "deep") ??
-		OPENAI_PROFILE_NAME
-	);
-}
-
-async function main() {
-	const force = process.argv.includes("--force");
-	const dryRun = process.argv.includes("--dry-run");
-
-	if (existsSync(OUT_PATH) && !force) {
-		console.log(
-			"✓ .pi/model-router.json already exists — preserving (use --force to regenerate)",
-		);
-		process.exit(0);
-	}
-
-	const { AuthStorage, ModelRegistry } = await loadPiCodingAgent();
-	const authStorage = AuthStorage.create();
-	const modelRegistry = ModelRegistry.create(authStorage);
-	const available = await modelRegistry.getAvailable();
-
-	if (available.length === 0) {
-		console.log(
-			"✗ No authenticated Pi providers — skip model-router.json",
-		);
-		console.log(
-			"  Log in inside pi: /login (or set API keys in ~/.pi/agent/auth.json)",
-		);
-		const providers = authStorage.list();
-		if (providers.length > 0) {
-			console.log(
-				`  Stored providers in auth.json (may need refresh): ${providers.join(", ")}`,
-			);
-		}
-		process.exit(0);
-	}
-
-	const profiles = {};
-	for (const provider of ["openai", "opencode-go"]) {
-		const profile = buildRoutedProfile(available, provider);
-		if (profile) profiles[routerProfileName(provider)] = profile;
-	}
-
-	if (Object.keys(profiles).length === 0) {
-		const { provider: primaryProvider, models: primaryModels } =
-			choosePrimaryProvider(available);
-		const profile = buildRoutedProfile(available, primaryProvider);
-		if (!profile) {
-			fail("could not assign tier models from available registry");
-		}
-		profiles[primaryProvider] = profile;
-	}
-
-	const cheapDeepSource = profiles["opencode-go"]
-		? "opencode-go"
-		: resolveDefaultProfile(profiles);
-	addCheapDeepProfiles(profiles, available, cheapDeepSource);
-
-	const defaultProfile = resolveDefaultProfile(profiles);
-	const classifierModel = resolveClassifierModel(available);
-	if (!classifierModel) {
-		fail("could not assign classifier model from available registry");
-	}
-
-	const config = {
-		defaultProfile,
-		debug: false,
-		classifierModel,
-		phaseBias: 0.5,
-		maxSessionBudget: 1.0,
-		largeContextThreshold: 100000,
-		rules: [
-			{
-				matches: ["deploy", "production", "release"],
-				tier: "high",
-				reason: "Safety check for production tasks",
-			},
-			{ matches: "changelog", tier: "low" },
-		],
-		profiles,
-	};
-
-	const json = `${JSON.stringify(config, null, 2)}\n`;
-	const providerSet = [...new Set(available.map((m) => m.provider))].sort();
-	const autoProfile = profiles[OPENAI_PROFILE_NAME];
-	const opencodeProfile = profiles["opencode-go"];
-
-	if (dryRun) {
-		process.stdout.write(json);
-		process.exit(0);
-	}
-
-	mkdirSync(dirname(OUT_PATH), { recursive: true });
-	writeFileSync(OUT_PATH, json, "utf8");
-
-	console.log("✓ Generated .pi/model-router.json from Pi authenticated providers:");
-	console.log(`  Default profile: ${defaultProfile}`);
-	console.log(`  Classifier: ${classifierModel}`);
-	console.log(`  Authenticated providers: ${providerSet.join(", ")}`);
-	console.log(`  Available models: ${available.length}`);
-	if (autoProfile) {
-		console.log(`  auto (openai) high: ${autoProfile.high.model}`);
-	}
-	if (opencodeProfile) {
-		console.log(`  opencode-go high: ${opencodeProfile.high.model}`);
-	}
-}
-
-main().catch((err) => {
-	console.error(err);
-	process.exit(1);
-});