pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/capability-registry/src/query.ts

@@ -0,0 +1,99 @@
+/**
+ * Capability Query Functions (RFC-0051)
+ */
+
+import type {
+	Capability,
+	CapabilityQuery,
+	CapabilityProfile,
+	ModelWithCapability,
+} from "./types.js";
+
+const LATENCY_ORDER: Record<string, number> = {
+	fast: 1,
+	medium: 2,
+	slow: 3,
+};
+
+/**
+ * Query capabilities from a registry store
+ */
+export function queryCapabilities(
+	getAllModelsWithCapability: (
+		capability: Capability,
+	) => Array<{
+		providerId: string;
+		modelId: string;
+		profile: CapabilityProfile;
+	}>,
+	capability: Capability,
+	requirements?: CapabilityQuery,
+): ModelWithCapability[] {
+	const all = getAllModelsWithCapability(capability);
+
+	return all
+		.filter((m) => {
+			if (
+				requirements?.minScore !== undefined &&
+				m.profile.score < requirements.minScore
+			) {
+				return false;
+			}
+			if (
+				requirements?.maxLatency !== undefined &&
+				LATENCY_ORDER[m.profile.latency] >
+					LATENCY_ORDER[requirements.maxLatency]
+			) {
+				return false;
+			}
+			if (
+				requirements?.requiresContextWindow !== undefined &&
+				m.profile.contextWindow < requirements.requiresContextWindow
+			) {
+				return false;
+			}
+			if (
+				requirements?.requiresMaxOutput !== undefined &&
+				m.profile.maxOutputTokens < requirements.requiresMaxOutput
+			) {
+				return false;
+			}
+			return true;
+		})
+		.sort((a, b) => b.profile.score - a.profile.score);
+}
+
+/**
+ * Find the best model for a capability
+ */
+export function findBestModel(
+	candidates: ModelWithCapability[],
+): ModelWithCapability | undefined {
+	if (candidates.length === 0) return undefined;
+	return candidates[0];
+}
+
+/**
+ * Filter by latency preference
+ */
+export function filterByLatency(
+	candidates: ModelWithCapability[],
+	preference: "fast" | "balanced" | "quality",
+): ModelWithCapability[] {
+	switch (preference) {
+		case "fast":
+			return candidates.filter((c) => c.profile.latency === "fast");
+		case "quality":
+			return candidates.sort((a, b) => b.profile.score - a.profile.score);
+		case "balanced":
+		default:
+			// Return all candidates sorted by a balanced score
+			return candidates.sort((a, b) => {
+				const latencyScoreA = 4 - LATENCY_ORDER[a.profile.latency];
+				const latencyScoreB = 4 - LATENCY_ORDER[b.profile.latency];
+				const balancedA = a.profile.score * 0.7 + latencyScoreA * 30;
+				const balancedB = b.profile.score * 0.7 + latencyScoreB * 30;
+				return balancedB - balancedA;
+			});
+	}
+}

package/packages/capability-registry/src/registry.ts

@@ -0,0 +1,222 @@
+/**
+ * Capability Registry Implementation (RFC-0051)
+ */
+
+import type {
+	Capability,
+	CapabilityProfile,
+	CapabilityQuery,
+	CapabilityRegistry,
+	CapabilityRegistryEvent,
+	ModelWithCapability,
+} from "./types.js";
+import { DEFAULT_CAPABILITIES } from "./defaults.js";
+import { queryCapabilities } from "./query.js";
+
+type EventHandler = (event: CapabilityRegistryEvent) => void;
+
+/**
+ * In-memory capability registry with event emission
+ */
+export class InMemoryCapabilityRegistry implements CapabilityRegistry {
+	private capabilities: Map<string, Map<string, CapabilityProfile[]>> =
+		new Map();
+	private capabilityIndex: Map<
+		Capability,
+		Array<{ providerId: string; modelId: string; profile: CapabilityProfile }>
+	> = new Map();
+	private eventHandlers: Set<EventHandler> = new Set();
+
+	constructor(loadDefaults = true) {
+		if (loadDefaults) {
+			this.loadDefaults();
+		}
+	}
+
+	/**
+	 * Load default capability profiles
+	 */
+	private loadDefaults(): void {
+		for (const [providerId, models] of Object.entries(DEFAULT_CAPABILITIES)) {
+			for (const [modelId, profiles] of Object.entries(models)) {
+				this.register(providerId, modelId, profiles);
+			}
+		}
+	}
+
+	/**
+	 * Register capabilities for a model
+	 */
+	register(
+		providerId: string,
+		modelId: string,
+		capabilities: CapabilityProfile[],
+	): void {
+		const key = this.makeKey(providerId, modelId);
+
+		// Store capabilities
+		if (!this.capabilities.has(key)) {
+			this.capabilities.set(key, new Map());
+		}
+		const modelCaps = this.capabilities.get(key)!;
+		modelCaps.set(providerId, capabilities);
+
+		// Also store in flat provider/model map
+		if (!this.capabilities.has("by_model")) {
+			this.capabilities.set("by_model", new Map());
+		}
+		this.capabilities.get("by_model")!.set(key, capabilities);
+
+		// Update capability index
+		for (const profile of capabilities) {
+			if (!this.capabilityIndex.has(profile.capability)) {
+				this.capabilityIndex.set(profile.capability, []);
+			}
+			this.capabilityIndex.get(profile.capability)!.push({
+				providerId,
+				modelId,
+				profile,
+			});
+		}
+
+		this.emit({
+			type: "capability.registered",
+			providerId,
+			modelId,
+			count: capabilities.length,
+		});
+	}
+
+	/**
+	 * Unregister a model's capabilities
+	 */
+	unregister(providerId: string, modelId: string): void {
+		const key = this.makeKey(providerId, modelId);
+		const existing = this.capabilities.get("by_model")?.get(key);
+
+		if (existing) {
+			// Remove from capability index
+			for (const profile of existing) {
+				const index = this.capabilityIndex.get(profile.capability);
+				if (index) {
+					const idx = index.findIndex(
+						(e) => e.providerId === providerId && e.modelId === modelId,
+					);
+					if (idx >= 0) {
+						index.splice(idx, 1);
+					}
+				}
+			}
+
+			// Remove from storage
+			this.capabilities.get("by_model")?.delete(key);
+		}
+
+		this.emit({
+			type: "capability.unregistered",
+			providerId,
+			modelId,
+		});
+	}
+
+	/**
+	 * Get capabilities for a specific model
+	 */
+	getCapabilities(providerId: string, modelId: string): CapabilityProfile[] {
+		const key = this.makeKey(providerId, modelId);
+		return this.capabilities.get("by_model")?.get(key) ?? [];
+	}
+
+	/**
+	 * Query models by capability
+	 */
+	query(
+		capability: Capability,
+		requirements?: CapabilityQuery,
+	): ModelWithCapability[] {
+		const results = queryCapabilities(
+			(cap) => this.capabilityIndex.get(cap) ?? [],
+			capability,
+			requirements,
+		);
+
+		this.emit({
+			type: "capability.queried",
+			capability,
+			results: results.length,
+		});
+
+		return results;
+	}
+
+	/**
+	 * List all registered providers
+	 */
+	listProviders(): string[] {
+		const providers = new Set<string>();
+		for (const key of this.capabilities.get("by_model")?.keys() ?? []) {
+			const [providerId] = this.parseKey(key);
+			providers.add(providerId);
+		}
+		return Array.from(providers);
+	}
+
+	/**
+	 * List all models for a provider
+	 */
+	listModels(providerId: string): string[] {
+		const models: string[] = [];
+		for (const key of this.capabilities.get("by_model")?.keys() ?? []) {
+			const [p, modelId] = this.parseKey(key);
+			if (p === providerId) {
+				models.push(modelId);
+			}
+		}
+		return models;
+	}
+
+	/**
+	 * Subscribe to events
+	 */
+	onEvent(handler: EventHandler): () => void {
+		this.eventHandlers.add(handler);
+		return () => this.eventHandlers.delete(handler);
+	}
+
+	/**
+	 * Emit an event to all handlers
+	 */
+	private emit(event: CapabilityRegistryEvent): void {
+		for (const handler of this.eventHandlers) {
+			try {
+				handler(event);
+			} catch {
+				// Ignore handler errors
+			}
+		}
+	}
+
+	/**
+	 * Create a unique key for provider/model pair
+	 */
+	private makeKey(providerId: string, modelId: string): string {
+		return `${providerId}::${modelId}`;
+	}
+
+	/**
+	 * Parse a key into provider/model
+	 */
+	private parseKey(key: string): [string, string] {
+		const [providerId, modelId] = key.split("::");
+		return [providerId, modelId];
+	}
+}
+
+/**
+ * Create a new capability registry
+ */
+export function createCapabilityRegistry(
+	loadDefaults = true,
+): InMemoryCapabilityRegistry {
+	return new InMemoryCapabilityRegistry(loadDefaults);
+}

package/packages/capability-registry/src/types.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Capability Registry Types (RFC-0051)
+ */
+export type Capability = "code_generation" | "code_review" | "planning" | "test_generation" | "e2e_testing" | "refactoring" | "analysis" | "debugging" | "documentation" | "vision" | "function_calling" | "json_mode" | "streaming";
+export type Latency = "fast" | "medium" | "slow";
+export interface CapabilityProfile {
+    capability: Capability;
+    score: number;
+    latency: Latency;
+    contextWindow: number;
+    maxOutputTokens: number;
+}
+export interface CapabilityQuery {
+    minScore?: number;
+    maxLatency?: Latency;
+    requiresContextWindow?: number;
+    requiresMaxOutput?: number;
+}
+export interface ModelWithCapability {
+    providerId: string;
+    modelId: string;
+    profile: CapabilityProfile;
+}
+export interface CapabilityRegistry {
+    register(providerId: string, modelId: string, capabilities: CapabilityProfile[]): void;
+    unregister(providerId: string, modelId: string): void;
+    getCapabilities(providerId: string, modelId: string): CapabilityProfile[];
+    query(capability: Capability, requirements?: CapabilityQuery): ModelWithCapability[];
+    listProviders(): string[];
+    listModels(providerId: string): string[];
+}
+export type CapabilityRegistryEvent = {
+    type: "capability.registered";
+    providerId: string;
+    modelId: string;
+    count: number;
+} | {
+    type: "capability.unregistered";
+    providerId: string;
+    modelId: string;
+} | {
+    type: "capability.queried";
+    capability: Capability;
+    results: number;
+};
+export declare const LATENCY_RANK: Record<Latency, number>;
+//# sourceMappingURL=types.d.ts.map

package/packages/capability-registry/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,MAAM,UAAU,GACnB,iBAAiB,GACjB,aAAa,GACb,UAAU,GACV,iBAAiB,GACjB,aAAa,GACb,aAAa,GACb,UAAU,GACV,WAAW,GACX,eAAe,GACf,QAAQ,GACR,kBAAkB,GAClB,WAAW,GACX,WAAW,CAAC;AAEf,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IACjC,UAAU,EAAE,UAAU,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;CACxB;AAID,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;CAC3B;AAID,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CACP,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,iBAAiB,EAAE,GAC/B,IAAI,CAAC;IACR,UAAU,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAAC;IAC1E,KAAK,CACJ,UAAU,EAAE,UAAU,EACtB,YAAY,CAAC,EAAE,eAAe,GAC5B,mBAAmB,EAAE,CAAC;IACzB,aAAa,IAAI,MAAM,EAAE,CAAC;IAC1B,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACzC;AAID,MAAM,MAAM,uBAAuB,GAChC;IACA,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACb,GACD;IAAE,IAAI,EAAE,yBAAyB,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,UAAU,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAI3E,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAIhD,CAAC"}

package/packages/capability-registry/src/types.js

@@ -0,0 +1,10 @@
+/**
+ * Capability Registry Types (RFC-0051)
+ */
+// ─── Latency Ranking ─────────────────────────────────────────────────
+export const LATENCY_RANK = {
+    fast: 1,
+    medium: 2,
+    slow: 3,
+};
+//# sourceMappingURL=types.js.map

package/packages/capability-registry/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA0EH,wEAAwE;AAExE,MAAM,CAAC,MAAM,YAAY,GAA4B;IACpD,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;CACP,CAAC"}

package/packages/capability-registry/src/types.ts

@@ -0,0 +1,83 @@
+/**
+ * Capability Registry Types (RFC-0051)
+ */
+
+// ─── Capability Types ─────────────────────────────────────────────────
+
+export type Capability =
+	| "code_generation"
+	| "code_review"
+	| "planning"
+	| "test_generation"
+	| "e2e_testing"
+	| "refactoring"
+	| "analysis"
+	| "debugging"
+	| "documentation"
+	| "vision"
+	| "function_calling"
+	| "json_mode"
+	| "streaming";
+
+export type Latency = "fast" | "medium" | "slow";
+
+export interface CapabilityProfile {
+	capability: Capability;
+	score: number; // 0-100, quality rating
+	latency: Latency;
+	contextWindow: number;
+	maxOutputTokens: number;
+}
+
+// ─── Query Types ──────────────────────────────────────────────────────
+
+export interface CapabilityQuery {
+	minScore?: number;
+	maxLatency?: Latency;
+	requiresContextWindow?: number;
+	requiresMaxOutput?: number;
+}
+
+export interface ModelWithCapability {
+	providerId: string;
+	modelId: string;
+	profile: CapabilityProfile;
+}
+
+// ─── Registry Types ───────────────────────────────────────────────────
+
+export interface CapabilityRegistry {
+	register(
+		providerId: string,
+		modelId: string,
+		capabilities: CapabilityProfile[],
+	): void;
+	unregister(providerId: string, modelId: string): void;
+	getCapabilities(providerId: string, modelId: string): CapabilityProfile[];
+	query(
+		capability: Capability,
+		requirements?: CapabilityQuery,
+	): ModelWithCapability[];
+	listProviders(): string[];
+	listModels(providerId: string): string[];
+}
+
+// ─── Event Types ─────────────────────────────────────────────────────
+
+export type CapabilityRegistryEvent =
+	| {
+			type: "capability.registered";
+			providerId: string;
+			modelId: string;
+			count: number;
+	  }
+	| { type: "capability.unregistered"; providerId: string; modelId: string }
+	| { type: "capability.queried"; capability: Capability; results: number };
+
+// ─── Latency Ranking ─────────────────────────────────────────────────
+
+export const LATENCY_RANK: Record<Latency, number> = {
+	fast: 1,
+	medium: 2,
+	slow: 3,
+};

package/packages/capability-registry/test/registry.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Capability Registry Tests (RFC-0051)
+ */
+
+import { describe, it, expect, beforeEach } from "bun:test";
+import {
+	createCapabilityRegistry,
+	type InMemoryCapabilityRegistry,
+	DEFAULT_CAPABILITIES,
+	queryCapabilities,
+	findBestModel,
+	filterByLatency,
+	LATENCY_RANK,
+} from "../src/index.js";
+import type { CapabilityProfile, Capability } from "../src/types.js";
+
+describe("CapabilityRegistry", () => {
+	let registry: InMemoryCapabilityRegistry;
+
+	beforeEach(() => {
+		registry = createCapabilityRegistry(true);
+	});
+
+	describe("initialization", () => {
+		it("should load default capabilities", () => {
+			const providers = registry.listProviders();
+			expect(providers.length).toBeGreaterThan(0);
+		});
+
+		it("should have OpenAI providers", () => {
+			const providers = registry.listProviders();
+			expect(providers).toContain("openai");
+		});
+
+		it("should have Anthropic providers", () => {
+			const providers = registry.listProviders();
+			expect(providers).toContain("anthropic");
+		});
+	});
+
+	describe("register and unregister", () => {
+		it("should register new capabilities", () => {
+			const newCaps: CapabilityProfile[] = [
+				{
+					capability: "code_generation",
+					score: 90,
+					latency: "fast",
+					contextWindow: 100000,
+					maxOutputTokens: 4096,
+				},
+			];
+
+			registry.register("test-provider", "test-model", newCaps);
+
+			const caps = registry.getCapabilities("test-provider", "test-model");
+			expect(caps).toHaveLength(1);
+			expect(caps[0].capability).toBe("code_generation");
+			expect(caps[0].score).toBe(90);
+		});
+
+		it("should unregister capabilities", () => {
+			registry.unregister("openai", "gpt-4o");
+
+			const caps = registry.getCapabilities("openai", "gpt-4o");
+			expect(caps).toHaveLength(0);
+		});
+
+		it("should track events", () => {
+			const events: any[] = [];
+			registry.onEvent((e) => events.push(e));
+
+			const caps: CapabilityProfile[] = [
+				{
+					capability: "planning",
+					score: 85,
+					latency: "fast",
+					contextWindow: 50000,
+					maxOutputTokens: 2048,
+				},
+			];
+
+			registry.register("event-test", "model-v1", caps);
+
+			expect(events).toContainEqual({
+				type: "capability.registered",
+				providerId: "event-test",
+				modelId: "model-v1",
+				count: 1,
+			});
+		});
+	});
+
+	describe("query", () => {
+		it("should query by capability", () => {
+			const results = registry.query("code_generation");
+			expect(results.length).toBeGreaterThan(0);
+		});
+
+		it("should filter by minimum score", () => {
+			const results = registry.query("code_generation", { minScore: 95 });
+			expect(results.every((r) => r.profile.score >= 95)).toBe(true);
+		});
+
+		it("should filter by latency", () => {
+			const results = registry.query("code_generation", { maxLatency: "fast" });
+			expect(
+				results.every(
+					(r) => LATENCY_RANK[r.profile.latency] <= LATENCY_RANK["fast"],
+				),
+			).toBe(true);
+		});
+
+		it("should filter by context window", () => {
+			const results = registry.query("code_generation", {
+				requiresContextWindow: 200000,
+			});
+			expect(results.every((r) => r.profile.contextWindow >= 200000)).toBe(
+				true,
+			);
+		});
+
+		it("should sort by score descending", () => {
+			const results = registry.query("code_generation");
+			for (let i = 1; i < results.length; i++) {
+				expect(results[i - 1].profile.score).toBeGreaterThanOrEqual(
+					results[i].profile.score,
+				);
+			}
+		});
+
+		it("should emit query event", () => {
+			const events: any[] = [];
+			registry.onEvent((e) => events.push(e));
+
+			registry.query("code_generation");
+
+			expect(events).toContainEqual(
+				expect.objectContaining({
+					type: "capability.queried",
+					capability: "code_generation",
+				}),
+			);
+		});
+	});
+
+	describe("listProviders and listModels", () => {
+		it("should list providers", () => {
+			const providers = registry.listProviders();
+			expect(Array.isArray(providers)).toBe(true);
+			expect(providers.length).toBeGreaterThan(0);
+		});
+
+		it("should list models for a provider", () => {
+			const models = registry.listModels("openai");
+			expect(models).toContain("gpt-4o");
+		});
+
+		it("should return empty array for unknown provider", () => {
+			const models = registry.listModels("unknown-provider");
+			expect(models).toHaveLength(0);
+		});
+	});
+
+	describe("findBestModel", () => {
+		it("should find the highest scoring model", () => {
+			const candidates = registry.query("code_generation");
+			const best = findBestModel(candidates);
+
+			expect(best).toBeDefined();
+			const maxScore = Math.max(...candidates.map((c) => c.profile.score));
+			expect(best!.profile.score).toBe(maxScore);
+		});
+
+		it("should return undefined for empty array", () => {
+			const result = findBestModel([]);
+			expect(result).toBeUndefined();
+		});
+	});
+
+	describe("filterByLatency", () => {
+		it("should filter to fast latency", () => {
+			const candidates = registry.query("code_generation");
+			const fast = filterByLatency(candidates, "fast");
+
+			expect(fast.every((c) => c.profile.latency === "fast")).toBe(true);
+		});
+
+		it("should sort by quality when preference is quality", () => {
+			const candidates = registry.query("code_generation");
+			const sorted = filterByLatency(candidates, "quality");
+
+			expect(sorted[0].profile.score).toBeGreaterThanOrEqual(
+				sorted[1].profile.score,
+			);
+		});
+	});
+});
+
+describe("LATENCY_RANK", () => {
+	it("should have correct ordering", () => {
+		expect(LATENCY_RANK["fast"]).toBeLessThan(LATENCY_RANK["medium"]);
+		expect(LATENCY_RANK["medium"]).toBeLessThan(LATENCY_RANK["slow"]);
+	});
+});

package/packages/capability-registry/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "types": [
+      "node"
+    ],
+    "lib": [
+      "ES2022",
+      "DOM"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "test"
+  ]
+}