pi-harness-runtime 0.10.43 → 1.1.1

package/packages/provider-selector/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@pi-harness/provider-selector",
+  "version": "0.10.30",
+  "description": "Multi-Provider Selector — route requests by cost, quality, and latency (RFC-0012)",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "bun-types": "^1.0.0"
+  }
+}

package/packages/provider-selector/test/selector.test.ts

@@ -0,0 +1,192 @@
+/**
+ * Provider Selector Tests (RFC-0012)
+ */
+
+import { describe, it, expect } from "bun:test";
+import {
+	selectProvider,
+	rank,
+	compareCost,
+	compareLatency,
+	filterByCapability,
+	filterByRegion,
+} from "../src/index.js";
+import type { Provider } from "../src/index.js";
+
+const makeProvider = (overrides?: Partial<Provider>): Provider => ({
+	id: "p1",
+	name: "Provider 1",
+	baseURL: "https://api.example.com",
+	capabilities: ["text"],
+	region: "us",
+	inputCostPer1M: 1.0,
+	outputCostPer1M: 3.0,
+	latencyMs: 500,
+	qualityScore: 80,
+	maxContextTokens: 128000,
+	available: true,
+	...overrides,
+});
+
+const PROVIDERS: Provider[] = [
+	makeProvider({
+		id: "cheap",
+		name: "Cheap",
+		inputCostPer1M: 0.1,
+		latencyMs: 200,
+		qualityScore: 60,
+	}),
+	makeProvider({
+		id: "fast",
+		name: "Fast",
+		inputCostPer1M: 2.0,
+		latencyMs: 50,
+		qualityScore: 90,
+	}),
+	makeProvider({
+		id: "quality",
+		name: "Quality",
+		inputCostPer1M: 5.0,
+		latencyMs: 800,
+		qualityScore: 98,
+	}),
+];
+
+describe("selectProvider", () => {
+	it("selects a provider", () => {
+		const result = selectProvider(PROVIDERS);
+		expect(result).not.toBeNull();
+		expect(result!.provider.available).toBe(true);
+	});
+
+	it("returns null for empty list", () => {
+		expect(selectProvider([])).toBeNull();
+	});
+
+	it("returns null when none available", () => {
+		const unavailable = PROVIDERS.map((p) => ({ ...p, available: false }));
+		expect(selectProvider(unavailable)).toBeNull();
+	});
+
+	it("prefers cheapest for cost criteria", () => {
+		const result = selectProvider(PROVIDERS, "cost");
+		expect(result!.provider.id).toBe("cheap");
+	});
+
+	it("prefers fastest for latency criteria", () => {
+		const result = selectProvider(PROVIDERS, "latency");
+		expect(result!.provider.id).toBe("fast");
+	});
+
+	it("prefers high quality for quality criteria", () => {
+		const result = selectProvider(PROVIDERS, "quality");
+		expect(result!.score).toBeGreaterThan(0.8);
+	});
+
+	it("includes alternatives", () => {
+		const result = selectProvider(PROVIDERS);
+		expect(result!.alternatives.length).toBeGreaterThan(0);
+	});
+
+	it("includes reason", () => {
+		const result = selectProvider(PROVIDERS);
+		expect(result!.reason).toContain("score:");
+	});
+});
+
+describe("rank", () => {
+	it("returns sorted providers", () => {
+		const ranked = rank(PROVIDERS);
+		expect(ranked.length).toBe(3);
+		expect(ranked[0].available).toBe(true);
+	});
+
+	it("uses criteria for ranking", () => {
+		const byCost = rank(PROVIDERS, "cost");
+		const byLatency = rank(PROVIDERS, "latency");
+		expect(byCost[0].id).not.toBe(byLatency[0].id);
+	});
+
+	it("filters unavailable", () => {
+		const mixed = [
+			...PROVIDERS,
+			{ ...PROVIDERS[0], id: "down", available: false },
+		];
+		expect(rank(mixed).length).toBe(3);
+	});
+});
+
+describe("compareCost", () => {
+	it("negative when a is cheaper", () => {
+		const cheap = makeProvider({
+			id: "c",
+			inputCostPer1M: 0.5,
+			outputCostPer1M: 1.0,
+		});
+		const expensive = makeProvider({
+			id: "e",
+			inputCostPer1M: 5.0,
+			outputCostPer1M: 10.0,
+		});
+		expect(compareCost(cheap, expensive)).toBeLessThan(0);
+	});
+
+	it("positive when a is more expensive", () => {
+		const cheap = makeProvider({
+			id: "c",
+			inputCostPer1M: 0.5,
+			outputCostPer1M: 1.0,
+		});
+		const expensive = makeProvider({
+			id: "e",
+			inputCostPer1M: 5.0,
+			outputCostPer1M: 10.0,
+		});
+		expect(compareCost(expensive, cheap)).toBeGreaterThan(0);
+	});
+});
+
+describe("compareLatency", () => {
+	it("negative when a is faster", () => {
+		const fast = makeProvider({ id: "f", latencyMs: 50 });
+		const slow = makeProvider({ id: "s", latencyMs: 500 });
+		expect(compareLatency(fast, slow)).toBeLessThan(0);
+	});
+});
+
+describe("filterByCapability", () => {
+	it("filters to matching capability", () => {
+		const mixed = [
+			makeProvider({ id: "t", capabilities: ["text"] }),
+			makeProvider({ id: "tv", capabilities: ["text", "vision"] }),
+		];
+		expect(filterByCapability(mixed, "vision")).toHaveLength(1);
+		expect(filterByCapability(mixed, "vision")[0].id).toBe("tv");
+	});
+});
+
+describe("filterByRegion", () => {
+	it("filters by region", () => {
+		const mixed = [
+			makeProvider({ id: "us1", region: "us" }),
+			makeProvider({ id: "eu1", region: "eu" }),
+		];
+		expect(filterByRegion(mixed, "us")).toHaveLength(1);
+	});
+
+	it("global returns all", () => {
+		const mixed = [
+			makeProvider({ region: "us" }),
+			makeProvider({ region: "eu" }),
+		];
+		expect(filterByRegion(mixed, "global")).toHaveLength(2);
+	});
+
+	it("global regions always match", () => {
+		const mixed = [
+			makeProvider({ id: "g", region: "global" }),
+			makeProvider({ id: "us", region: "us" }),
+		];
+		expect(filterByRegion(mixed, "eu")).toHaveLength(1);
+	});
+});

package/packages/providers/README.md

@@ -0,0 +1,3 @@
+# packages/providers
+
+Provider adapter interfaces and provider-specific implementations will live here.

package/packages/providers/adapters.ts

@@ -0,0 +1,362 @@
+/**
+ * Provider Adapter — RFC-0002
+ *
+ * Normalizes provider-specific behavior for:
+ * - model invocation
+ * - error detection
+ * - quota detection
+ * - reset time discovery
+ * - retry policy
+ * - model capability metadata
+ */
+
+import type {
+	ProviderConfig,
+	ProviderCapability,
+	ProviderRequest,
+	ProviderResponse,
+} from "../../packages/types/src/runtime-types.js";
+
+export interface AdapterConfig {
+	provider: ProviderConfig;
+	quotaSignalExtractor?: (error: unknown) => QuotaSignal | null;
+}
+
+export interface QuotaSignal {
+	exhausted: boolean;
+	resetsAt?: string;
+	retryAfterMs?: number;
+}
+
+export interface AdapterResult {
+	response: ProviderResponse;
+	quotaSignal?: QuotaSignal;
+	retryable: boolean;
+}
+
+export interface ProviderAdapter {
+	readonly id: string;
+	readonly name: string;
+
+	invoke(request: ProviderRequest): Promise<AdapterResult>;
+
+	parseError(error: unknown): {
+		quotaExceeded: boolean;
+		rateLimited: boolean;
+		timeout: boolean;
+		serverError: boolean;
+		clientError: boolean;
+		quotaSignal?: QuotaSignal;
+	};
+
+	getCapabilities(): ProviderCapability[];
+
+	supportsModel(model: string): boolean;
+
+	getDefaultModel(): string;
+
+	getMaxTokens(model?: string): number;
+}
+
+/**
+ * Base adapter with common functionality
+ */
+export abstract class BaseProviderAdapter implements ProviderAdapter {
+	abstract readonly id: string;
+	abstract readonly name: string;
+
+	constructor(protected config: ProviderConfig) {}
+
+	abstract invoke(request: ProviderRequest): Promise<AdapterResult>;
+
+	abstract parseError(error: unknown): {
+		quotaExceeded: boolean;
+		rateLimited: boolean;
+		timeout: boolean;
+		serverError: boolean;
+		clientError: boolean;
+		quotaSignal?: QuotaSignal;
+	};
+
+	getCapabilities(): ProviderCapability[] {
+		return this.config.capabilities;
+	}
+
+	supportsModel(model: string): boolean {
+		return this.config.models.includes(model);
+	}
+
+	getDefaultModel(): string {
+		return this.config.models[0] ?? "default";
+	}
+
+	getMaxTokens(model?: string): number {
+		// Default token limits per model family
+		const limits: Record<string, number> = {
+			"minimax/MiniMax-M3": 32768,
+			"minimax/MiniMax-Text-01": 1000000,
+			"anthropic/claude-3-5-sonnet": 200000,
+			"openai/gpt-4o": 128000,
+			"openai/gpt-4-turbo": 128000,
+		};
+		return limits[model ?? this.getDefaultModel()] ?? 4096;
+	}
+
+	/**
+	 * Parse usage from response
+	 */
+	protected parseUsage(response: unknown): ProviderResponse["usage"] {
+		const r = response as Record<string, unknown>;
+		return {
+			input: (r.input_tokens as number) ?? (r.prompt_tokens as number) ?? 0,
+			output:
+				(r.output_tokens as number) ?? (r.completion_tokens as number) ?? 0,
+			cacheRead: (r.cache_read_tokens as number) ?? 0,
+			cacheWrite: (r.cache_write_tokens as number) ?? 0,
+			cost: (r.cost as number) ?? 0,
+		};
+	}
+}
+
+/**
+ * MiniMax adapter
+ */
+export class MiniMaxAdapter extends BaseProviderAdapter {
+	readonly id = "minimax";
+	readonly name = "MiniMax";
+
+	async invoke(request: ProviderRequest): Promise<AdapterResult> {
+		// In practice, this would call the MiniMax API
+		// For now, return a mock response
+		return {
+			response: {
+				content: "Mock response",
+				usage: { input: 100, output: 200, cost: 0.001 },
+				model: request.model,
+				finishReason: "stop",
+			},
+			retryable: false,
+		};
+	}
+
+	parseError(error: unknown): {
+		quotaExceeded: boolean;
+		rateLimited: boolean;
+		timeout: boolean;
+		serverError: boolean;
+		clientError: boolean;
+		quotaSignal?: QuotaSignal;
+	} {
+		const e = error as Record<string, unknown>;
+		const msg = String(e.message ?? e.error ?? "").toLowerCase();
+
+		return {
+			quotaExceeded: msg.includes("2056") || msg.includes("quota"),
+			rateLimited: msg.includes("rate limit") || msg.includes("429"),
+			timeout: msg.includes("timeout") || msg.includes("timed out"),
+			serverError:
+				msg.includes("500") || msg.includes("502") || msg.includes("503"),
+			clientError:
+				msg.includes("400") || msg.includes("401") || msg.includes("403"),
+			quotaSignal:
+				msg.includes("quota") || msg.includes("2056")
+					? { exhausted: true, resetsAt: undefined }
+					: undefined,
+		};
+	}
+}
+
+/**
+ * OpenAI adapter
+ */
+export class OpenAIAdapter extends BaseProviderAdapter {
+	readonly id = "openai";
+	readonly name = "OpenAI";
+
+	async invoke(request: ProviderRequest): Promise<AdapterResult> {
+		return {
+			response: {
+				content: "Mock response",
+				usage: { input: 100, output: 200, cost: 0.002 },
+				model: request.model,
+				finishReason: "stop",
+			},
+			retryable: false,
+		};
+	}
+
+	parseError(error: unknown): {
+		quotaExceeded: boolean;
+		rateLimited: boolean;
+		timeout: boolean;
+		serverError: boolean;
+		clientError: boolean;
+		quotaSignal?: QuotaSignal;
+	} {
+		const e = error as Record<string, unknown>;
+		const msg = String(e.message ?? e.error ?? "").toLowerCase();
+		const code = String(e.code ?? "");
+
+		return {
+			quotaExceeded:
+				code === "insufficient_quota" || code === "context_length_exceeded",
+			rateLimited: code === "rate_limit_exceeded" || msg.includes("429"),
+			timeout: msg.includes("timeout"),
+			serverError: code.startsWith("5"),
+			clientError: code.startsWith("4"),
+			quotaSignal:
+				code === "insufficient_quota"
+					? { exhausted: true, resetsAt: undefined }
+					: undefined,
+		};
+	}
+}
+
+/**
+ * Claude adapter — Anthropic's Claude models
+ */
+export class ClaudeAdapter extends BaseProviderAdapter {
+	readonly id = "anthropic";
+	readonly name = "Anthropic Claude";
+
+	async invoke(request: ProviderRequest): Promise<AdapterResult> {
+		// In practice, this would call the Claude API via Anthropic SDK
+		// For now, return a mock response
+		return {
+			response: {
+				content: "Mock response",
+				usage: { input: 100, output: 200, cost: 0.003 },
+				model: request.model,
+				finishReason: "stop",
+			},
+			retryable: false,
+		};
+	}
+
+	parseError(error: unknown): {
+		quotaExceeded: boolean;
+		rateLimited: boolean;
+		timeout: boolean;
+		serverError: boolean;
+		clientError: boolean;
+		quotaSignal?: QuotaSignal;
+	} {
+		const e = error as Record<string, unknown>;
+		const msg = String(e.message ?? e.error ?? "").toLowerCase();
+		const type = String(e.type ?? "");
+		const errorCode = String(e.code ?? "");
+
+		// Anthropic-specific error handling
+		const isRateLimited =
+			type.includes("rate_limit_error") ||
+			msg.includes("rate limit") ||
+			errorCode.includes("429");
+
+		const isOverloaded =
+			type.includes("overloaded_error") || msg.includes("overloaded");
+
+		return {
+			quotaExceeded:
+				type.includes("quota_error") ||
+				type.includes("insufficient_quota") ||
+				msg.includes("quota exceeded"),
+			rateLimited: isRateLimited || isOverloaded,
+			timeout: type.includes("timeout") || msg.includes("timeout"),
+			serverError: type.includes("api_error") || isOverloaded,
+			clientError:
+				type.includes("invalid_request_error") ||
+				type.includes("authentication_error") ||
+				type.includes("permission_error"),
+			quotaSignal: type.includes("quota_error")
+				? { exhausted: true, resetsAt: undefined }
+				: undefined,
+		};
+	}
+
+	getDefaultModel(): string {
+		return "anthropic/claude-3-5-sonnet-20240620";
+	}
+
+	getMaxTokens(model?: string): number {
+		const limits: Record<string, number> = {
+			"anthropic/claude-3-5-sonnet-20240620": 200000,
+			"anthropic/claude-3-5-haiku-20240620": 200000,
+			"anthropic/claude-3-opus-20240229": 200000,
+			"anthropic/claude-3-sonnet-20240229": 200000,
+			"anthropic/claude-3-haiku-20240307": 200000,
+			"anthropic/claude-2.1": 200000,
+			"anthropic/claude-2": 100000,
+			"anthropic/claude-instant": 100000,
+		};
+		return limits[model ?? this.getDefaultModel()] ?? 200000;
+	}
+}
+
+/**
+ * Adapter registry
+ */
+export class AdapterRegistry {
+	private adapters: Map<string, ProviderAdapter> = new Map();
+
+	register(adapter: ProviderAdapter): void {
+		this.adapters.set(adapter.id, adapter);
+	}
+
+	get(id: string): ProviderAdapter | undefined {
+		return this.adapters.get(id);
+	}
+
+	list(): ProviderAdapter[] {
+		return Array.from(this.adapters.values());
+	}
+
+	/**
+	 * Create default registry with standard adapters
+	 */
+	static createDefault(): AdapterRegistry {
+		const registry = new AdapterRegistry();
+
+		registry.register(
+			new MiniMaxAdapter({
+				id: "minimax",
+				name: "MiniMax",
+				models: ["minimax/MiniMax-M3", "minimax/MiniMax-Text-01"],
+				capabilities: ["code", "review", "plan", "test"],
+				rateLimits: {},
+			}),
+		);
+
+		registry.register(
+			new OpenAIAdapter({
+				id: "openai",
+				name: "OpenAI",
+				models: ["openai/gpt-4o", "openai/gpt-4-turbo"],
+				capabilities: ["code", "review", "plan", "test", "refactor"],
+				rateLimits: {},
+			}),
+		);
+
+		registry.register(
+			new ClaudeAdapter({
+				id: "anthropic",
+				name: "Anthropic Claude",
+				models: [
+					"anthropic/claude-3-5-sonnet-20240620",
+					"anthropic/claude-3-5-haiku-20240620",
+					"anthropic/claude-3-opus-20240229",
+				],
+				capabilities: [
+					"code",
+					"review",
+					"plan",
+					"test",
+					"refactor",
+					"analysis",
+				],
+				rateLimits: {},
+			}),
+		);
+
+		return registry;
+	}
+}