pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/cost-optimizer/test/cost-optimizer.test.ts

@@ -0,0 +1,438 @@
+/**
+ * Cost Optimizer Tests (RFC-0055)
+ */
+
+import { describe, it, expect, beforeEach } from "bun:test";
+import {
+	createCostOptimizer,
+	type InMemoryCostOptimizer,
+	DEFAULT_CONFIG,
+	createOptimizerConfig,
+	generateCostId,
+	createCostEntry,
+	calculateSummary,
+	calculateBudgetStatus,
+	exceedsBudget,
+	calculateModelCost,
+	calculateQualityImpact,
+	findCheaperAlternatives,
+	optimizeModelSelection,
+	forecastCosts,
+	quickForecast,
+} from "../src/index.js";
+import type {
+	CostEntry,
+	CostPeriod,
+	CostBudget,
+	TaskRequirements,
+} from "../src/index.js";
+
+describe("CostOptimizer", () => {
+	let optimizer: InMemoryCostOptimizer;
+
+	beforeEach(() => {
+		optimizer = createCostOptimizer();
+	});
+
+	describe("trackCost", () => {
+		it("should track a cost entry", () => {
+			const entry = optimizer.trackCost({
+				jobId: "job-1",
+				taskId: "task-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.005,
+				currency: "USD",
+			});
+
+			expect(entry.id).toBeDefined();
+			expect(entry.timestamp).toBeDefined();
+			expect(entry.cost).toBe(0.005);
+		});
+
+		it("should accumulate costs", () => {
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "openai",
+				modelId: "gpt-4o",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.003,
+				currency: "USD",
+			});
+
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "minimax",
+				modelId: "MiniMax-Text-01",
+				inputTokens: 2000,
+				outputTokens: 1000,
+				cost: 0.001,
+				currency: "USD",
+			});
+
+			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
+			expect(summary.total).toBe(0.004);
+		});
+
+		it("should emit tracked event", () => {
+			const events: any[] = [];
+			optimizer.onEvent((e) => events.push(e));
+
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.005,
+				currency: "USD",
+			});
+
+			expect(events).toContainEqual(
+				expect.objectContaining({
+					type: "cost.tracked",
+				}),
+			);
+		});
+
+		it("should emit budget warning when threshold exceeded", () => {
+			const events: any[] = [];
+			optimizer.onEvent((e) => events.push(e));
+
+			// Track enough to exceed 80% of $10 daily budget
+			for (let i = 0; i < 9; i++) {
+				optimizer.trackCost({
+					jobId: "job-1",
+					providerId: "anthropic",
+					modelId: "claude-sonnet-4",
+					inputTokens: 1000,
+					outputTokens: 500,
+					cost: 0.95,
+					currency: "USD",
+				});
+			}
+
+			const warnings = events.filter((e) => e.type === "cost.budget_warning");
+			expect(warnings.length).toBeGreaterThan(0);
+		});
+	});
+
+	describe("getSummary", () => {
+		it("should calculate summary by provider", () => {
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.01,
+				currency: "USD",
+			});
+
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "openai",
+				modelId: "gpt-4o",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.02,
+				currency: "USD",
+			});
+
+			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
+			expect(summary.total).toBe(0.03);
+			expect(summary.byProvider["anthropic"]).toBe(0.01);
+			expect(summary.byProvider["openai"]).toBe(0.02);
+		});
+
+		it("should calculate summary by job", () => {
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.01,
+				currency: "USD",
+			});
+
+			optimizer.trackCost({
+				jobId: "job-2",
+				providerId: "openai",
+				modelId: "gpt-4o",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.02,
+				currency: "USD",
+			});
+
+			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
+			expect(summary.byJob["job-1"]).toBe(0.01);
+			expect(summary.byJob["job-2"]).toBe(0.02);
+		});
+	});
+
+	describe("getBudgetStatus", () => {
+		it("should return budget status", () => {
+			const status = optimizer.getBudgetStatus();
+			expect(status.daily).toBeDefined();
+			expect(status.weekly).toBeDefined();
+			expect(status.monthly).toBeDefined();
+			expect(status.exhausted).toBe(false);
+		});
+
+		it("should track daily usage", () => {
+			optimizer.trackCost({
+				jobId: "job-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 5,
+				currency: "USD",
+			});
+
+			const status = optimizer.getBudgetStatus();
+			expect(status.daily.used).toBe(5);
+			expect(status.daily.remaining).toBe(5); // $10 budget - $5 used
+		});
+	});
+
+	describe("canAfford", () => {
+		it("should return true for affordable costs", () => {
+			expect(optimizer.canAfford(1)).toBe(true);
+		});
+
+		it("should return false when budget exhausted", () => {
+			// Exhaust daily budget
+			for (let i = 0; i < 10; i++) {
+				optimizer.trackCost({
+					jobId: "job-1",
+					providerId: "anthropic",
+					modelId: "claude-sonnet-4",
+					inputTokens: 1000,
+					outputTokens: 500,
+					cost: 1.05,
+					currency: "USD",
+				});
+			}
+
+			expect(optimizer.canAfford(1)).toBe(false);
+		});
+	});
+
+	describe("shouldSwitchToCheaper", () => {
+		it("should suggest cheaper alternatives", () => {
+			const recommendation = optimizer.shouldSwitchToCheaper(
+				{ id: "claude-sonnet-4", providerId: "anthropic" },
+				["code_generation"],
+			);
+
+			expect(recommendation).not.toBeNull();
+			expect(recommendation!.costSavings).toBeGreaterThan(0);
+		});
+
+		it("should respect quality loss limit", () => {
+			const recommendation = optimizer.shouldSwitchToCheaper(
+				{ id: "claude-sonnet-4", providerId: "anthropic" },
+				["code_generation"],
+				{ maxQualityLoss: 5 },
+			);
+
+			if (recommendation) {
+				expect(recommendation.qualityImpact).toBeLessThanOrEqual(5);
+			}
+		});
+	});
+
+	describe("forecastCosts", () => {
+		it("should forecast costs", () => {
+			const requirements: TaskRequirements[] = [
+				{
+					requiredCapabilities: ["code_generation"],
+					minContextWindow: 100000,
+					estimatedInputTokens: 5000,
+					estimatedOutputTokens: 2000,
+				},
+				{
+					requiredCapabilities: ["code_review"],
+					minContextWindow: 100000,
+					estimatedInputTokens: 3000,
+					estimatedOutputTokens: 1000,
+				},
+			];
+
+			const forecast = optimizer.forecastCosts(requirements);
+			expect(forecast.estimatedTotal).toBeGreaterThan(0);
+			expect(forecast.confidence).toBeGreaterThan(0);
+			expect(forecast.assumptions.length).toBeGreaterThan(0);
+		});
+	});
+
+	describe("optimizeModelSelection", () => {
+		it("should return optimized selections", () => {
+			const requirements: TaskRequirements = {
+				requiredCapabilities: ["code_generation"],
+				minContextWindow: 100000,
+				estimatedInputTokens: 5000,
+				estimatedOutputTokens: 2000,
+			};
+
+			const selections = optimizer.optimizeModelSelection(
+				"task-1",
+				requirements,
+			);
+			expect(selections.length).toBeGreaterThan(0);
+			expect(selections[0].model).toBeDefined();
+			expect(selections[0].estimatedCost).toBeGreaterThanOrEqual(0);
+		});
+
+		it("should filter by requirements", () => {
+			const requirements: TaskRequirements = {
+				requiredCapabilities: ["code_generation", "vision"],
+				minContextWindow: 500000,
+				estimatedInputTokens: 5000,
+				estimatedOutputTokens: 2000,
+			};
+
+			const selections = optimizer.optimizeModelSelection(
+				"task-1",
+				requirements,
+			);
+			expect(selections.length).toBeGreaterThan(0);
+			// All should have required capabilities
+			expect(selections.every((s) => s.model.id !== "MiniMax-Text-01")).toBe(
+				true,
+			);
+		});
+	});
+});
+
+describe("Utility Functions", () => {
+	describe("generateCostId", () => {
+		it("should generate unique IDs", () => {
+			const id1 = generateCostId();
+			const id2 = generateCostId();
+			expect(id1).not.toBe(id2);
+			expect(id1).toContain("cost_");
+		});
+	});
+
+	describe("createCostEntry", () => {
+		it("should create cost entry with generated id and timestamp", () => {
+			const entry = createCostEntry({
+				jobId: "job-1",
+				providerId: "anthropic",
+				modelId: "claude-sonnet-4",
+				inputTokens: 1000,
+				outputTokens: 500,
+				cost: 0.005,
+				currency: "USD",
+			});
+
+			expect(entry.id).toContain("cost_");
+			expect(entry.timestamp).toBeDefined();
+		});
+
+		it("should use provided id and timestamp", () => {
+			const entry = createCostEntry(
+				{
+					jobId: "job-1",
+					providerId: "anthropic",
+					modelId: "claude-sonnet-4",
+					inputTokens: 1000,
+					outputTokens: 500,
+					cost: 0.005,
+					currency: "USD",
+				},
+				"custom-id",
+				"2024-01-01T00:00:00Z",
+			);
+
+			expect(entry.id).toBe("custom-id");
+			expect(entry.timestamp).toBe("2024-01-01T00:00:00Z");
+		});
+	});
+
+	describe("calculateModelCost", () => {
+		it("should calculate model cost", () => {
+			const cost = calculateModelCost(
+				{
+					id: "test",
+					providerId: "test",
+					name: "Test",
+					contextWindow: 100000,
+					maxOutputTokens: 4096,
+					pricing: { inputPer1M: 1, outputPer1M: 2 },
+					capabilities: [],
+					latency: "fast",
+					qualityScore: 80,
+				},
+				1000000,
+				500000,
+			);
+
+			expect(cost).toBe(2); // $1 + $1
+		});
+	});
+
+	describe("quickForecast", () => {
+		it("should provide quick forecast", () => {
+			const cost = quickForecast(1000, 500);
+			expect(cost).toBeGreaterThan(0);
+		});
+	});
+
+	describe("calculateBudgetStatus", () => {
+		it("should calculate budget status", () => {
+			const budget: CostBudget = { daily: 10, weekly: 50, monthly: 200 };
+			const entries: CostEntry[] = [];
+			const status = calculateBudgetStatus(entries, budget);
+
+			expect(status.daily.budget).toBe(10);
+			expect(status.weekly.budget).toBe(50);
+			expect(status.monthly.budget).toBe(200);
+			expect(status.exhausted).toBe(false);
+		});
+	});
+
+	describe("exceedsBudget", () => {
+		it("should detect when cost exceeds budget", () => {
+			const budget: CostBudget = { daily: 10 };
+			const status = calculateBudgetStatus([], budget);
+
+			expect(exceedsBudget(5, budget, status)).toBe(false);
+			expect(exceedsBudget(15, budget, status)).toBe(true);
+		});
+	});
+});
+
+describe("Config", () => {
+	describe("DEFAULT_CONFIG", () => {
+		it("should have default values", () => {
+			expect(DEFAULT_CONFIG.defaultBudget.daily).toBe(10);
+			expect(DEFAULT_CONFIG.alertThreshold).toBe(0.8);
+		});
+	});
+
+	describe("createOptimizerConfig", () => {
+		it("should create custom config", () => {
+			const overrides = { defaultBudget: { daily: 20 } };
+			const config = createOptimizerConfig(overrides);
+
+			expect(config.defaultBudget.daily).toBe(20);
+			// Verify weekly is preserved from defaults
+			expect(config.defaultBudget.weekly).toBe(50);
+		});
+
+		it("should preserve defaults when not overridden", () => {
+			const config = createOptimizerConfig({});
+			expect(config.defaultBudget.daily).toBe(10);
+			expect(config.defaultBudget.weekly).toBe(50);
+			expect(config.defaultBudget.monthly).toBe(200);
+		});
+	});
+});

package/packages/cost-optimizer/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "types": [
+      "node"
+    ],
+    "lib": [
+      "ES2022",
+      "DOM"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "test"
+  ]
+}

package/packages/evaluation-engine/dist/engine.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Evaluation Engine (RFC-0057)
+ *
+ * Determines whether an agent-produced result is acceptable before advancing.
+ */
+import type { EvaluationRequest, EvaluationResult, EvaluationState, ScoringWeights } from "./types.js";
+export declare class EvaluationEngine {
+    private weights;
+    private state;
+    constructor(weights?: Partial<ScoringWeights>);
+    /**
+     * Evaluate a task against its acceptance criteria
+     */
+    evaluate(request: EvaluationRequest): EvaluationResult;
+    /**
+     * Get current evaluation state
+     */
+    getState(): EvaluationState;
+    /**
+     * Reset engine state
+     */
+    reset(): void;
+}
+export declare function createEvaluationEngine(weights?: Partial<ScoringWeights>): EvaluationEngine;
+//# sourceMappingURL=engine.d.ts.map

package/packages/evaluation-engine/dist/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACX,iBAAiB,EACjB,gBAAgB,EAKhB,eAAe,EAIf,cAAc,EACd,MAAM,YAAY,CAAC;AA2TpB,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,KAAK,CAA8B;gBAE/B,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAI7C;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB;IAyHtD;;OAEG;IACH,QAAQ,IAAI,eAAe;IAI3B;;OAEG;IACH,KAAK,IAAI,IAAI;CAGb;AAID,wBAAgB,sBAAsB,CACrC,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,gBAAgB,CAElB"}