pi-harness-runtime 0.10.11 → 0.10.12

package/packages/cost-optimizer/src/optimizer.ts

@@ -0,0 +1,277 @@
+/**
+ * Cost Optimizer Implementation (RFC-0055)
+ */
+
+import type {
+	CostEntry,
+	CostOptimizerConfig,
+	CostOptimizerEvent,
+	CostPeriod,
+	CostSummary,
+	CostOptimizer,
+	ModelSwitchRecommendation,
+	OptimizedSelection,
+	SwitchOptions,
+	TaskRequirements,
+} from "./types.js";
+import { DEFAULT_CONFIG } from "./config.js";
+import { createCostEntry, calculateSummary } from "./tracker.js";
+import { calculateBudgetStatus } from "./budget.js";
+import {
+	findCheaperAlternatives,
+	generateSwitchRecommendation,
+	optimizeModelSelection,
+} from "./selection.js";
+import { forecastCosts as calculateForecast } from "./forecast.js";
+
+type EventHandler = (event: CostOptimizerEvent) => void;
+
+interface ModelData {
+	id: string;
+	providerId: string;
+	name: string;
+	contextWindow: number;
+	maxOutputTokens: number;
+	pricing: { inputPer1M: number; outputPer1M: number };
+	capabilities: string[];
+	latency: "fast" | "medium" | "slow";
+	qualityScore: number;
+}
+
+const DEFAULT_MODELS: ModelData[] = [
+	{
+		id: "claude-sonnet-4",
+		providerId: "anthropic",
+		name: "Claude Sonnet 4",
+		contextWindow: 200000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 3, outputPer1M: 15 },
+		capabilities: ["code_generation", "code_review", "planning", "analysis"],
+		latency: "fast",
+		qualityScore: 90,
+	},
+	{
+		id: "gpt-4o-mini",
+		providerId: "openai",
+		name: "GPT-4o Mini",
+		contextWindow: 128000,
+		maxOutputTokens: 16384,
+		pricing: { inputPer1M: 0.15, outputPer1M: 0.6 },
+		capabilities: ["code_generation", "code_review", "function_calling"],
+		latency: "fast",
+		qualityScore: 82,
+	},
+	{
+		id: "MiniMax-Text-01",
+		providerId: "minimax",
+		name: "MiniMax Text 01",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.1, outputPer1M: 0.5 },
+		capabilities: ["code_generation", "function_calling"],
+		latency: "fast",
+		qualityScore: 75,
+	},
+	{
+		id: "gemini-1.5-flash",
+		providerId: "google",
+		name: "Gemini 1.5 Flash",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.035, outputPer1M: 0.14 },
+		capabilities: ["code_generation", "vision"],
+		latency: "fast",
+		qualityScore: 78,
+	},
+];
+
+/**
+ * In-memory Cost Optimizer
+ */
+export class InMemoryCostOptimizer implements CostOptimizer {
+	private config: CostOptimizerConfig;
+	private history: CostEntry[] = [];
+	private eventHandlers: Set<EventHandler> = new Set();
+
+	constructor(config?: Partial<CostOptimizerConfig>) {
+		this.config = { ...DEFAULT_CONFIG, ...config };
+	}
+
+	/**
+	 * Track a cost entry
+	 */
+	trackCost(entry: Omit<CostEntry, "id" | "timestamp">): CostEntry {
+		const costEntry = createCostEntry(entry);
+		this.history.push(costEntry);
+
+		// Get current total
+		const period: CostPeriod = { start: "", end: "", type: "day" };
+		const summary = this.getSummary(period);
+
+		this.emit({
+			type: "cost.tracked",
+			entryId: costEntry.id,
+			cost: costEntry.cost,
+			total: summary.total,
+		});
+
+		// Check budget warnings
+		const status = this.getBudgetStatus();
+		const dailyPct =
+			status.daily.budget > 0 ? status.daily.used / status.daily.budget : 0;
+
+		if (dailyPct >= this.config.alertThreshold) {
+			this.emit({
+				type: "cost.budget_warning",
+				threshold: this.config.alertThreshold,
+				percentage: Math.round(dailyPct * 100),
+			});
+		}
+
+		if (status.exhausted) {
+			this.emit({
+				type: "cost.budget_exhausted",
+				period: "daily",
+			});
+		}
+
+		return costEntry;
+	}
+
+	/**
+	 * Get cost summary for a period
+	 */
+	getSummary(period: CostPeriod): CostSummary {
+		return calculateSummary(this.history, period);
+	}
+
+	/**
+	 * Get current budget status
+	 */
+	getBudgetStatus() {
+		return calculateBudgetStatus(this.history, this.config.defaultBudget);
+	}
+
+	/**
+	 * Check if a cost can be afforded
+	 */
+	canAfford(cost: number): boolean {
+		const status = this.getBudgetStatus();
+
+		if (this.config.defaultBudget.daily !== undefined) {
+			if (cost > status.daily.remaining) return false;
+		}
+		if (this.config.defaultBudget.weekly !== undefined) {
+			if (cost > status.weekly.remaining) return false;
+		}
+		if (this.config.defaultBudget.monthly !== undefined) {
+			if (cost > status.monthly.remaining) return false;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Suggest cheaper alternatives
+	 */
+	shouldSwitchToCheaper(
+		currentModel: { id: string; providerId: string },
+		requiredCapabilities: string[],
+		options?: SwitchOptions,
+	): ModelSwitchRecommendation | null {
+		// Find current model
+		const current = DEFAULT_MODELS.find(
+			(m) =>
+				m.id === currentModel.id && m.providerId === currentModel.providerId,
+		);
+
+		if (!current) return null;
+
+		// Find cheaper alternatives
+		const alternatives = findCheaperAlternatives(
+			current,
+			requiredCapabilities,
+			options,
+		);
+
+		if (alternatives.length === 0) return null;
+
+		// Return the best alternative
+		const best = alternatives[0];
+		return generateSwitchRecommendation(
+			current,
+			best,
+			1000, // Default input tokens for comparison
+			500, // Default output tokens for comparison
+		);
+	}
+
+	/**
+	 * Forecast costs for job requirements
+	 */
+	forecastCosts(jobRequirements: TaskRequirements[]) {
+		const forecast = calculateForecast(jobRequirements);
+
+		this.emit({
+			type: "cost.forecast",
+			estimated: forecast.estimatedTotal,
+			confidence: forecast.confidence,
+		});
+
+		return forecast;
+	}
+
+	/**
+	 * Get optimized model selections
+	 */
+	optimizeModelSelection(
+		_taskId: string,
+		requirements: TaskRequirements,
+	): OptimizedSelection[] {
+		return optimizeModelSelection(requirements);
+	}
+
+	/**
+	 * Subscribe to events
+	 */
+	onEvent(handler: EventHandler): () => void {
+		this.eventHandlers.add(handler);
+		return () => this.eventHandlers.delete(handler);
+	}
+
+	/**
+	 * Emit an event
+	 */
+	private emit(event: CostOptimizerEvent): void {
+		for (const handler of this.eventHandlers) {
+			try {
+				handler(event);
+			} catch {
+				// Ignore handler errors
+			}
+		}
+	}
+
+	/**
+	 * Get cost history
+	 */
+	getHistory(): CostEntry[] {
+		return [...this.history];
+	}
+
+	/**
+	 * Clear history
+	 */
+	clearHistory(): void {
+		this.history = [];
+	}
+}
+
+/**
+ * Create a new cost optimizer
+ */
+export function createCostOptimizer(
+	config?: Partial<CostOptimizerConfig>,
+): InMemoryCostOptimizer {
+	return new InMemoryCostOptimizer(config);
+}

package/packages/cost-optimizer/src/selection.ts

@@ -0,0 +1,264 @@
+/**
+ * Model Selection Optimization Functions (RFC-0055)
+ */
+
+import type {
+	ModelSwitchRecommendation,
+	OptimizedSelection,
+	SwitchOptions,
+	TaskRequirements,
+} from "./types.js";
+
+// Default model data (would be injected from Model Registry)
+interface ModelData {
+	id: string;
+	providerId: string;
+	name: string;
+	contextWindow: number;
+	maxOutputTokens: number;
+	pricing: {
+		inputPer1M: number;
+		outputPer1M: number;
+	};
+	capabilities: string[];
+	latency: "fast" | "medium" | "slow";
+	qualityScore: number;
+}
+
+const DEFAULT_MODELS: ModelData[] = [
+	{
+		id: "claude-sonnet-4",
+		providerId: "anthropic",
+		name: "Claude Sonnet 4",
+		contextWindow: 200000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 3, outputPer1M: 15 },
+		capabilities: ["code_generation", "code_review", "planning", "analysis"],
+		latency: "fast",
+		qualityScore: 90,
+	},
+	{
+		id: "gpt-4o-mini",
+		providerId: "openai",
+		name: "GPT-4o Mini",
+		contextWindow: 128000,
+		maxOutputTokens: 16384,
+		pricing: { inputPer1M: 0.15, outputPer1M: 0.6 },
+		capabilities: ["code_generation", "code_review", "function_calling"],
+		latency: "fast",
+		qualityScore: 82,
+	},
+	{
+		id: "MiniMax-Text-01",
+		providerId: "minimax",
+		name: "MiniMax Text 01",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.1, outputPer1M: 0.5 },
+		capabilities: ["code_generation", "function_calling"],
+		latency: "fast",
+		qualityScore: 75,
+	},
+	{
+		id: "gemini-1.5-flash",
+		providerId: "google",
+		name: "Gemini 1.5 Flash",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.035, outputPer1M: 0.14 },
+		capabilities: ["code_generation", "vision"],
+		latency: "fast",
+		qualityScore: 78,
+	},
+	{
+		id: "gpt-4o",
+		providerId: "openai",
+		name: "GPT-4o",
+		contextWindow: 128000,
+		maxOutputTokens: 16384,
+		pricing: { inputPer1M: 5, outputPer1M: 15 },
+		capabilities: [
+			"code_generation",
+			"code_review",
+			"vision",
+			"function_calling",
+		],
+		latency: "medium",
+		qualityScore: 92,
+	},
+];
+
+/**
+ * Calculate cost for a model
+ */
+export function calculateModelCost(
+	model: ModelData,
+	inputTokens: number,
+	outputTokens: number,
+): number {
+	const inputCost = (inputTokens / 1_000_000) * model.pricing.inputPer1M;
+	const outputCost = (outputTokens / 1_000_000) * model.pricing.outputPer1M;
+	return Math.round((inputCost + outputCost) * 100) / 100;
+}
+
+/**
+ * Calculate quality impact between models
+ */
+export function calculateQualityImpact(
+	current: ModelData,
+	candidate: ModelData,
+): number {
+	return current.qualityScore - candidate.qualityScore;
+}
+
+/**
+ * Get quality score for a model
+ */
+export function getQualityScore(
+	model: ModelData,
+	requiredCapabilities: string[],
+): number {
+	// Base score
+	let score = model.qualityScore;
+
+	// Bonus for matching required capabilities
+	const matchedCaps = requiredCapabilities.filter((cap) =>
+		model.capabilities.includes(cap),
+	).length;
+
+	score += matchedCaps * 5;
+
+	return Math.min(score, 100);
+}
+
+/**
+ * Find cheaper alternatives for a model
+ */
+export function findCheaperAlternatives(
+	currentModel: ModelData,
+	requiredCapabilities: string[],
+	options?: SwitchOptions,
+): ModelData[] {
+	const maxQualityLoss = options?.maxQualityLoss ?? 20;
+
+	return DEFAULT_MODELS.filter((m) => {
+		// Must be different model
+		if (m.id === currentModel.id) return false;
+
+		// Must have required capabilities
+		if (!requiredCapabilities.every((cap) => m.capabilities.includes(cap))) {
+			return false;
+		}
+
+		// Must be cheaper
+		const currentCost = calculateModelCost(currentModel, 1000, 500);
+		const candidateCost = calculateModelCost(m, 1000, 500);
+		if (candidateCost >= currentCost) return false;
+
+		// Must not lose too much quality
+		const qualityLoss = calculateQualityImpact(currentModel, m);
+		if (qualityLoss > maxQualityLoss) return false;
+
+		// Must not increase latency beyond preference
+		if (options?.maxLatencyIncrease) {
+			const latencyOrder = { fast: 1, medium: 2, slow: 3 };
+			if (latencyOrder[m.latency] > latencyOrder[options.maxLatencyIncrease]) {
+				return false;
+			}
+		}
+
+		return true;
+	});
+}
+
+/**
+ * Generate a switch recommendation
+ */
+export function generateSwitchRecommendation(
+	currentModel: ModelData,
+	candidate: ModelData,
+	inputTokens: number,
+	outputTokens: number,
+): ModelSwitchRecommendation {
+	const currentCost = calculateModelCost(
+		currentModel,
+		inputTokens,
+		outputTokens,
+	);
+	const candidateCost = calculateModelCost(
+		candidate,
+		inputTokens,
+		outputTokens,
+	);
+	const savings = currentCost - candidateCost;
+	const savingsPct = (savings / currentCost) * 100;
+	const qualityLoss = calculateQualityImpact(currentModel, candidate);
+
+	return {
+		currentModel: { id: currentModel.id, providerId: currentModel.providerId },
+		recommendedModel: { id: candidate.id, providerId: candidate.providerId },
+		costSavings: Math.round(savingsPct * 10) / 10,
+		qualityImpact: Math.round(qualityLoss * 10) / 10,
+		reason: `Switch from ${currentModel.name} to ${candidate.name} saves ${savingsPct.toFixed(1)}%`,
+	};
+}
+
+/**
+ * Optimize model selection for a task
+ */
+export function optimizeModelSelection(
+	requirements: TaskRequirements,
+	models: ModelData[] = DEFAULT_MODELS,
+): OptimizedSelection[] {
+	// Filter by requirements
+	const candidates = models.filter((m) => {
+		if (m.contextWindow < requirements.minContextWindow) return false;
+		if (requirements.maxLatency) {
+			const latencyOrder = { fast: 1, medium: 2, slow: 3 };
+			if (latencyOrder[m.latency] > latencyOrder[requirements.maxLatency]) {
+				return false;
+			}
+		}
+		// Filter by required capabilities
+		if (requirements.requiredCapabilities?.length) {
+			const hasAllCaps = requirements.requiredCapabilities.every((cap) =>
+				m.capabilities.includes(cap),
+			);
+			if (!hasAllCaps) return false;
+		}
+		return true;
+	});
+
+	// Calculate scores
+	const scored = candidates.map((m) => {
+		const estimatedCost = calculateModelCost(
+			m,
+			requirements.estimatedInputTokens,
+			requirements.estimatedOutputTokens,
+		);
+		const qualityScore = getQualityScore(m, requirements.requiredCapabilities);
+
+		// Tradeoffs
+		const tradeoffs: string[] = [];
+		if (m.pricing.inputPer1M > 1) tradeoffs.push("Higher input cost");
+		if (m.qualityScore < 85) tradeoffs.push("Lower quality rating");
+
+		return {
+			model: { id: m.id, providerId: m.providerId, name: m.name },
+			estimatedCost,
+			qualityScore,
+			reason: `${m.name}: $${estimatedCost.toFixed(4)} estimated, ${qualityScore.toFixed(0)} quality`,
+			tradeoffs,
+		};
+	});
+
+	// Sort by quality-adjusted value (Pareto-optimal)
+	scored.sort((a, b) => {
+		// Quality weight 60%, cost weight 40%
+		const valueA = a.qualityScore * 0.6 - a.estimatedCost * 100 * 0.4;
+		const valueB = b.qualityScore * 0.6 - b.estimatedCost * 100 * 0.4;
+		return valueB - valueA;
+	});
+
+	return scored;
+}

package/packages/cost-optimizer/src/tracker.ts

@@ -0,0 +1,122 @@
+/**
+ * Cost Tracking Functions (RFC-0055)
+ */
+
+import type { CostEntry, CostPeriod, CostSummary } from "./types.js";
+
+let idCounter = 0;
+
+/**
+ * Generate a unique ID for cost entries
+ */
+export function generateCostId(): string {
+	return `cost_${Date.now()}_${++idCounter}`;
+}
+
+/**
+ * Create a new cost entry
+ */
+export function createCostEntry(
+	entry: Omit<CostEntry, "id" | "timestamp">,
+	id?: string,
+	timestamp?: string,
+): CostEntry {
+	return {
+		...entry,
+		id: id ?? generateCostId(),
+		timestamp: timestamp ?? new Date().toISOString(),
+	};
+}
+
+/**
+ * Get the period boundaries for a given period type
+ */
+export function getPeriodBoundaries(period: CostPeriod): {
+	start: Date;
+	end: Date;
+} {
+	const now = new Date();
+	const end = new Date(now);
+
+	switch (period.type) {
+		case "day": {
+			const start = new Date(now);
+			start.setHours(0, 0, 0, 0);
+			return { start, end };
+		}
+
+		case "week": {
+			const start = new Date(now);
+			start.setDate(now.getDate() - now.getDay());
+			start.setHours(0, 0, 0, 0);
+			return { start, end };
+		}
+
+		case "month": {
+			const start = new Date(now.getFullYear(), now.getMonth(), 1);
+			return { start, end };
+		}
+
+		case "custom": {
+			return {
+				start: new Date(period.start),
+				end: new Date(period.end),
+			};
+		}
+
+		default:
+			return { start: new Date(0), end: now };
+	}
+}
+
+/**
+ * Check if a timestamp falls within a period
+ */
+export function isInPeriod(timestamp: string, period: CostPeriod): boolean {
+	const { start, end } = getPeriodBoundaries(period);
+	const ts = new Date(timestamp);
+	return ts >= start && ts <= end;
+}
+
+/**
+ * Calculate summary from cost entries
+ */
+export function calculateSummary(
+	entries: CostEntry[],
+	period: CostPeriod,
+): CostSummary {
+	const filtered = entries.filter((e) => isInPeriod(e.timestamp, period));
+
+	const total = filtered.reduce((sum, e) => sum + e.cost, 0);
+	const currency = filtered[0]?.currency ?? "USD";
+
+	const byProvider: Record<string, number> = {};
+	const byModel: Record<string, number> = {};
+	const byJob: Record<string, number> = {};
+
+	for (const entry of filtered) {
+		byProvider[entry.providerId] =
+			(byProvider[entry.providerId] ?? 0) + entry.cost;
+		byModel[entry.modelId] = (byModel[entry.modelId] ?? 0) + entry.cost;
+		if (entry.jobId) {
+			byJob[entry.jobId] = (byJob[entry.jobId] ?? 0) + entry.cost;
+		}
+	}
+
+	return {
+		total: Math.round(total * 10000) / 10000, // higher precision for small amounts
+		currency,
+		byProvider,
+		byModel,
+		byJob,
+		period,
+	};
+}
+
+/**
+ * Get total cost for a period
+ */
+export function getTotalCost(entries: CostEntry[], period: CostPeriod): number {
+	const summary = calculateSummary(entries, period);
+	return summary.total;
+}