pi-harness-runtime 0.10.7 → 0.10.11

package/packages/cost-optimizer/src/tracker.ts

@@ -1,122 +0,0 @@
-/**
- * Cost Tracking Functions (RFC-0055)
- */
-
-import type { CostEntry, CostPeriod, CostSummary } from "./types.js";
-
-let idCounter = 0;
-
-/**
- * Generate a unique ID for cost entries
- */
-export function generateCostId(): string {
-	return `cost_${Date.now()}_${++idCounter}`;
-}
-
-/**
- * Create a new cost entry
- */
-export function createCostEntry(
-	entry: Omit<CostEntry, "id" | "timestamp">,
-	id?: string,
-	timestamp?: string,
-): CostEntry {
-	return {
-		...entry,
-		id: id ?? generateCostId(),
-		timestamp: timestamp ?? new Date().toISOString(),
-	};
-}
-
-/**
- * Get the period boundaries for a given period type
- */
-export function getPeriodBoundaries(period: CostPeriod): {
-	start: Date;
-	end: Date;
-} {
-	const now = new Date();
-	const end = new Date(now);
-
-	switch (period.type) {
-		case "day": {
-			const start = new Date(now);
-			start.setHours(0, 0, 0, 0);
-			return { start, end };
-		}
-
-		case "week": {
-			const start = new Date(now);
-			start.setDate(now.getDate() - now.getDay());
-			start.setHours(0, 0, 0, 0);
-			return { start, end };
-		}
-
-		case "month": {
-			const start = new Date(now.getFullYear(), now.getMonth(), 1);
-			return { start, end };
-		}
-
-		case "custom": {
-			return {
-				start: new Date(period.start),
-				end: new Date(period.end),
-			};
-		}
-
-		default:
-			return { start: new Date(0), end: now };
-	}
-}
-
-/**
- * Check if a timestamp falls within a period
- */
-export function isInPeriod(timestamp: string, period: CostPeriod): boolean {
-	const { start, end } = getPeriodBoundaries(period);
-	const ts = new Date(timestamp);
-	return ts >= start && ts <= end;
-}
-
-/**
- * Calculate summary from cost entries
- */
-export function calculateSummary(
-	entries: CostEntry[],
-	period: CostPeriod,
-): CostSummary {
-	const filtered = entries.filter((e) => isInPeriod(e.timestamp, period));
-
-	const total = filtered.reduce((sum, e) => sum + e.cost, 0);
-	const currency = filtered[0]?.currency ?? "USD";
-
-	const byProvider: Record<string, number> = {};
-	const byModel: Record<string, number> = {};
-	const byJob: Record<string, number> = {};
-
-	for (const entry of filtered) {
-		byProvider[entry.providerId] =
-			(byProvider[entry.providerId] ?? 0) + entry.cost;
-		byModel[entry.modelId] = (byModel[entry.modelId] ?? 0) + entry.cost;
-		if (entry.jobId) {
-			byJob[entry.jobId] = (byJob[entry.jobId] ?? 0) + entry.cost;
-		}
-	}
-
-	return {
-		total: Math.round(total * 10000) / 10000, // higher precision for small amounts
-		currency,
-		byProvider,
-		byModel,
-		byJob,
-		period,
-	};
-}
-
-/**
- * Get total cost for a period
- */
-export function getTotalCost(entries: CostEntry[], period: CostPeriod): number {
-	const summary = calculateSummary(entries, period);
-	return summary.total;
-}

package/packages/cost-optimizer/src/types.ts

@@ -1,134 +0,0 @@
-/**
- * Cost Optimizer Types (RFC-0055)
- */
-
-// Re-export Currency type
-export type Currency = "USD" | "THB";
-
-// --- Cost Entry Types -------------------------------------------------
-
-export interface CostEntry {
-	id: string;
-	jobId: string;
-	taskId?: string;
-	providerId: string;
-	modelId: string;
-	inputTokens: number;
-	outputTokens: number;
-	cost: number;
-	currency: Currency;
-	timestamp: string;
-}
-
-// --- Budget Types -----------------------------------------------------
-
-export interface CostBudget {
-	daily?: number;
-	weekly?: number;
-	monthly?: number;
-	perJob?: number;
-	perTask?: number;
-}
-
-export interface BudgetStatus {
-	daily: { used: number; budget: number; remaining: number };
-	weekly: { used: number; budget: number; remaining: number };
-	monthly: { used: number; budget: number; remaining: number };
-	exhausted: boolean;
-	nextReset?: string;
-}
-
-export interface CostSummary {
-	total: number;
-	currency: Currency;
-	byProvider: Record<string, number>;
-	byModel: Record<string, number>;
-	byJob: Record<string, number>;
-	period: CostPeriod;
-}
-
-export interface CostPeriod {
-	start: string;
-	end: string;
-	type: "day" | "week" | "month" | "custom";
-}
-
-// --- Optimization Types ------------------------------------------------
-
-export interface CostOptimizerConfig {
-	defaultBudget: CostBudget;
-	currency: Currency;
-	alertThreshold: number;
-	autoSwitchToCheaper: boolean;
-	maxQualityLossPercent: number;
-}
-
-export interface SwitchOptions {
-	maxQualityLoss?: number;
-	maxLatencyIncrease?: "fast" | "medium" | "slow";
-}
-
-export interface ModelSwitchRecommendation {
-	currentModel: { id: string; providerId: string };
-	recommendedModel: { id: string; providerId: string };
-	costSavings: number;
-	qualityImpact: number;
-	reason: string;
-}
-
-export interface CostForecast {
-	estimatedTotal: number;
-	byTaskType: Record<string, number>;
-	confidence: number;
-	assumptions: string[];
-}
-
-export interface TaskRequirements {
-	requiredCapabilities: string[];
-	minContextWindow: number;
-	maxLatency?: "fast" | "medium" | "slow";
-	estimatedInputTokens: number;
-	estimatedOutputTokens: number;
-}
-
-export interface OptimizedSelection {
-	model: { id: string; providerId: string; name: string };
-	estimatedCost: number;
-	qualityScore: number;
-	reason: string;
-	tradeoffs: string[];
-}
-
-// --- Optimizer Interface ---------------------------------------------
-
-export interface CostOptimizer {
-	trackCost(entry: Omit<CostEntry, "id" | "timestamp">): CostEntry;
-	getSummary(period: CostPeriod): CostSummary;
-	getBudgetStatus(): BudgetStatus;
-	canAfford(cost: number): boolean;
-	shouldSwitchToCheaper(
-		currentModel: { id: string; providerId: string },
-		requiredCapabilities: string[],
-		options?: SwitchOptions,
-	): ModelSwitchRecommendation | null;
-	forecastCosts(jobRequirements: TaskRequirements[]): CostForecast;
-	optimizeModelSelection(
-		taskId: string,
-		requirements: TaskRequirements,
-	): OptimizedSelection[];
-}
-
-// --- Event Types -----------------------------------------------------
-
-export type CostOptimizerEvent =
-	| { type: "cost.tracked"; entryId: string; cost: number; total: number }
-	| { type: "cost.budget_warning"; threshold: number; percentage: number }
-	| { type: "cost.budget_exhausted"; period: string }
-	| {
-			type: "cost.switch_recommended";
-			from: string;
-			to: string;
-			savings: number;
-	  }
-	| { type: "cost.forecast"; estimated: number; confidence: number }
-	| { type: "cost.optimization_applied"; model: string; savings: number };

package/packages/cost-optimizer/test/cost-optimizer.test.ts

@@ -1,438 +0,0 @@
-/**
- * Cost Optimizer Tests (RFC-0055)
- */
-
-import { describe, it, expect, beforeEach } from "bun:test";
-import {
-	createCostOptimizer,
-	type InMemoryCostOptimizer,
-	DEFAULT_CONFIG,
-	createOptimizerConfig,
-	generateCostId,
-	createCostEntry,
-	calculateSummary,
-	calculateBudgetStatus,
-	exceedsBudget,
-	calculateModelCost,
-	calculateQualityImpact,
-	findCheaperAlternatives,
-	optimizeModelSelection,
-	forecastCosts,
-	quickForecast,
-} from "../src/index.js";
-import type {
-	CostEntry,
-	CostPeriod,
-	CostBudget,
-	TaskRequirements,
-} from "../src/index.js";
-
-describe("CostOptimizer", () => {
-	let optimizer: InMemoryCostOptimizer;
-
-	beforeEach(() => {
-		optimizer = createCostOptimizer();
-	});
-
-	describe("trackCost", () => {
-		it("should track a cost entry", () => {
-			const entry = optimizer.trackCost({
-				jobId: "job-1",
-				taskId: "task-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.005,
-				currency: "USD",
-			});
-
-			expect(entry.id).toBeDefined();
-			expect(entry.timestamp).toBeDefined();
-			expect(entry.cost).toBe(0.005);
-		});
-
-		it("should accumulate costs", () => {
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "openai",
-				modelId: "gpt-4o",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.003,
-				currency: "USD",
-			});
-
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "minimax",
-				modelId: "MiniMax-Text-01",
-				inputTokens: 2000,
-				outputTokens: 1000,
-				cost: 0.001,
-				currency: "USD",
-			});
-
-			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
-			expect(summary.total).toBe(0.004);
-		});
-
-		it("should emit tracked event", () => {
-			const events: any[] = [];
-			optimizer.onEvent((e) => events.push(e));
-
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.005,
-				currency: "USD",
-			});
-
-			expect(events).toContainEqual(
-				expect.objectContaining({
-					type: "cost.tracked",
-				}),
-			);
-		});
-
-		it("should emit budget warning when threshold exceeded", () => {
-			const events: any[] = [];
-			optimizer.onEvent((e) => events.push(e));
-
-			// Track enough to exceed 80% of $10 daily budget
-			for (let i = 0; i < 9; i++) {
-				optimizer.trackCost({
-					jobId: "job-1",
-					providerId: "anthropic",
-					modelId: "claude-sonnet-4",
-					inputTokens: 1000,
-					outputTokens: 500,
-					cost: 0.95,
-					currency: "USD",
-				});
-			}
-
-			const warnings = events.filter((e) => e.type === "cost.budget_warning");
-			expect(warnings.length).toBeGreaterThan(0);
-		});
-	});
-
-	describe("getSummary", () => {
-		it("should calculate summary by provider", () => {
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.01,
-				currency: "USD",
-			});
-
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "openai",
-				modelId: "gpt-4o",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.02,
-				currency: "USD",
-			});
-
-			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
-			expect(summary.total).toBe(0.03);
-			expect(summary.byProvider["anthropic"]).toBe(0.01);
-			expect(summary.byProvider["openai"]).toBe(0.02);
-		});
-
-		it("should calculate summary by job", () => {
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.01,
-				currency: "USD",
-			});
-
-			optimizer.trackCost({
-				jobId: "job-2",
-				providerId: "openai",
-				modelId: "gpt-4o",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.02,
-				currency: "USD",
-			});
-
-			const summary = optimizer.getSummary({ start: "", end: "", type: "day" });
-			expect(summary.byJob["job-1"]).toBe(0.01);
-			expect(summary.byJob["job-2"]).toBe(0.02);
-		});
-	});
-
-	describe("getBudgetStatus", () => {
-		it("should return budget status", () => {
-			const status = optimizer.getBudgetStatus();
-			expect(status.daily).toBeDefined();
-			expect(status.weekly).toBeDefined();
-			expect(status.monthly).toBeDefined();
-			expect(status.exhausted).toBe(false);
-		});
-
-		it("should track daily usage", () => {
-			optimizer.trackCost({
-				jobId: "job-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 5,
-				currency: "USD",
-			});
-
-			const status = optimizer.getBudgetStatus();
-			expect(status.daily.used).toBe(5);
-			expect(status.daily.remaining).toBe(5); // $10 budget - $5 used
-		});
-	});
-
-	describe("canAfford", () => {
-		it("should return true for affordable costs", () => {
-			expect(optimizer.canAfford(1)).toBe(true);
-		});
-
-		it("should return false when budget exhausted", () => {
-			// Exhaust daily budget
-			for (let i = 0; i < 10; i++) {
-				optimizer.trackCost({
-					jobId: "job-1",
-					providerId: "anthropic",
-					modelId: "claude-sonnet-4",
-					inputTokens: 1000,
-					outputTokens: 500,
-					cost: 1.05,
-					currency: "USD",
-				});
-			}
-
-			expect(optimizer.canAfford(1)).toBe(false);
-		});
-	});
-
-	describe("shouldSwitchToCheaper", () => {
-		it("should suggest cheaper alternatives", () => {
-			const recommendation = optimizer.shouldSwitchToCheaper(
-				{ id: "claude-sonnet-4", providerId: "anthropic" },
-				["code_generation"],
-			);
-
-			expect(recommendation).not.toBeNull();
-			expect(recommendation!.costSavings).toBeGreaterThan(0);
-		});
-
-		it("should respect quality loss limit", () => {
-			const recommendation = optimizer.shouldSwitchToCheaper(
-				{ id: "claude-sonnet-4", providerId: "anthropic" },
-				["code_generation"],
-				{ maxQualityLoss: 5 },
-			);
-
-			if (recommendation) {
-				expect(recommendation.qualityImpact).toBeLessThanOrEqual(5);
-			}
-		});
-	});
-
-	describe("forecastCosts", () => {
-		it("should forecast costs", () => {
-			const requirements: TaskRequirements[] = [
-				{
-					requiredCapabilities: ["code_generation"],
-					minContextWindow: 100000,
-					estimatedInputTokens: 5000,
-					estimatedOutputTokens: 2000,
-				},
-				{
-					requiredCapabilities: ["code_review"],
-					minContextWindow: 100000,
-					estimatedInputTokens: 3000,
-					estimatedOutputTokens: 1000,
-				},
-			];
-
-			const forecast = optimizer.forecastCosts(requirements);
-			expect(forecast.estimatedTotal).toBeGreaterThan(0);
-			expect(forecast.confidence).toBeGreaterThan(0);
-			expect(forecast.assumptions.length).toBeGreaterThan(0);
-		});
-	});
-
-	describe("optimizeModelSelection", () => {
-		it("should return optimized selections", () => {
-			const requirements: TaskRequirements = {
-				requiredCapabilities: ["code_generation"],
-				minContextWindow: 100000,
-				estimatedInputTokens: 5000,
-				estimatedOutputTokens: 2000,
-			};
-
-			const selections = optimizer.optimizeModelSelection(
-				"task-1",
-				requirements,
-			);
-			expect(selections.length).toBeGreaterThan(0);
-			expect(selections[0].model).toBeDefined();
-			expect(selections[0].estimatedCost).toBeGreaterThanOrEqual(0);
-		});
-
-		it("should filter by requirements", () => {
-			const requirements: TaskRequirements = {
-				requiredCapabilities: ["code_generation", "vision"],
-				minContextWindow: 500000,
-				estimatedInputTokens: 5000,
-				estimatedOutputTokens: 2000,
-			};
-
-			const selections = optimizer.optimizeModelSelection(
-				"task-1",
-				requirements,
-			);
-			expect(selections.length).toBeGreaterThan(0);
-			// All should have required capabilities
-			expect(selections.every((s) => s.model.id !== "MiniMax-Text-01")).toBe(
-				true,
-			);
-		});
-	});
-});
-
-describe("Utility Functions", () => {
-	describe("generateCostId", () => {
-		it("should generate unique IDs", () => {
-			const id1 = generateCostId();
-			const id2 = generateCostId();
-			expect(id1).not.toBe(id2);
-			expect(id1).toContain("cost_");
-		});
-	});
-
-	describe("createCostEntry", () => {
-		it("should create cost entry with generated id and timestamp", () => {
-			const entry = createCostEntry({
-				jobId: "job-1",
-				providerId: "anthropic",
-				modelId: "claude-sonnet-4",
-				inputTokens: 1000,
-				outputTokens: 500,
-				cost: 0.005,
-				currency: "USD",
-			});
-
-			expect(entry.id).toContain("cost_");
-			expect(entry.timestamp).toBeDefined();
-		});
-
-		it("should use provided id and timestamp", () => {
-			const entry = createCostEntry(
-				{
-					jobId: "job-1",
-					providerId: "anthropic",
-					modelId: "claude-sonnet-4",
-					inputTokens: 1000,
-					outputTokens: 500,
-					cost: 0.005,
-					currency: "USD",
-				},
-				"custom-id",
-				"2024-01-01T00:00:00Z",
-			);
-
-			expect(entry.id).toBe("custom-id");
-			expect(entry.timestamp).toBe("2024-01-01T00:00:00Z");
-		});
-	});
-
-	describe("calculateModelCost", () => {
-		it("should calculate model cost", () => {
-			const cost = calculateModelCost(
-				{
-					id: "test",
-					providerId: "test",
-					name: "Test",
-					contextWindow: 100000,
-					maxOutputTokens: 4096,
-					pricing: { inputPer1M: 1, outputPer1M: 2 },
-					capabilities: [],
-					latency: "fast",
-					qualityScore: 80,
-				},
-				1000000,
-				500000,
-			);
-
-			expect(cost).toBe(2); // $1 + $1
-		});
-	});
-
-	describe("quickForecast", () => {
-		it("should provide quick forecast", () => {
-			const cost = quickForecast(1000, 500);
-			expect(cost).toBeGreaterThan(0);
-		});
-	});
-
-	describe("calculateBudgetStatus", () => {
-		it("should calculate budget status", () => {
-			const budget: CostBudget = { daily: 10, weekly: 50, monthly: 200 };
-			const entries: CostEntry[] = [];
-			const status = calculateBudgetStatus(entries, budget);
-
-			expect(status.daily.budget).toBe(10);
-			expect(status.weekly.budget).toBe(50);
-			expect(status.monthly.budget).toBe(200);
-			expect(status.exhausted).toBe(false);
-		});
-	});
-
-	describe("exceedsBudget", () => {
-		it("should detect when cost exceeds budget", () => {
-			const budget: CostBudget = { daily: 10 };
-			const status = calculateBudgetStatus([], budget);
-
-			expect(exceedsBudget(5, budget, status)).toBe(false);
-			expect(exceedsBudget(15, budget, status)).toBe(true);
-		});
-	});
-});
-
-describe("Config", () => {
-	describe("DEFAULT_CONFIG", () => {
-		it("should have default values", () => {
-			expect(DEFAULT_CONFIG.defaultBudget.daily).toBe(10);
-			expect(DEFAULT_CONFIG.alertThreshold).toBe(0.8);
-		});
-	});
-
-	describe("createOptimizerConfig", () => {
-		it("should create custom config", () => {
-			const overrides = { defaultBudget: { daily: 20 } };
-			const config = createOptimizerConfig(overrides);
-
-			expect(config.defaultBudget.daily).toBe(20);
-			// Verify weekly is preserved from defaults
-			expect(config.defaultBudget.weekly).toBe(50);
-		});
-
-		it("should preserve defaults when not overridden", () => {
-			const config = createOptimizerConfig({});
-			expect(config.defaultBudget.daily).toBe(10);
-			expect(config.defaultBudget.weekly).toBe(50);
-			expect(config.defaultBudget.monthly).toBe(200);
-		});
-	});
-});

package/packages/cost-optimizer/tsconfig.json

@@ -1,22 +0,0 @@
-{
-  "extends": "../../tsconfig.base.json",
-  "compilerOptions": {
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "types": [
-      "node"
-    ],
-    "lib": [
-      "ES2022",
-      "DOM"
-    ]
-  },
-  "include": [
-    "src/**/*"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist",
-    "test"
-  ]
-}