pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/cost-optimizer/src/selection.ts

@@ -0,0 +1,264 @@
+/**
+ * Model Selection Optimization Functions (RFC-0055)
+ */
+
+import type {
+	ModelSwitchRecommendation,
+	OptimizedSelection,
+	SwitchOptions,
+	TaskRequirements,
+} from "./types.js";
+
+// Default model data (would be injected from Model Registry)
+interface ModelData {
+	id: string;
+	providerId: string;
+	name: string;
+	contextWindow: number;
+	maxOutputTokens: number;
+	pricing: {
+		inputPer1M: number;
+		outputPer1M: number;
+	};
+	capabilities: string[];
+	latency: "fast" | "medium" | "slow";
+	qualityScore: number;
+}
+
+const DEFAULT_MODELS: ModelData[] = [
+	{
+		id: "claude-sonnet-4",
+		providerId: "anthropic",
+		name: "Claude Sonnet 4",
+		contextWindow: 200000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 3, outputPer1M: 15 },
+		capabilities: ["code_generation", "code_review", "planning", "analysis"],
+		latency: "fast",
+		qualityScore: 90,
+	},
+	{
+		id: "gpt-4o-mini",
+		providerId: "openai",
+		name: "GPT-4o Mini",
+		contextWindow: 128000,
+		maxOutputTokens: 16384,
+		pricing: { inputPer1M: 0.15, outputPer1M: 0.6 },
+		capabilities: ["code_generation", "code_review", "function_calling"],
+		latency: "fast",
+		qualityScore: 82,
+	},
+	{
+		id: "MiniMax-Text-01",
+		providerId: "minimax",
+		name: "MiniMax Text 01",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.1, outputPer1M: 0.5 },
+		capabilities: ["code_generation", "function_calling"],
+		latency: "fast",
+		qualityScore: 75,
+	},
+	{
+		id: "gemini-1.5-flash",
+		providerId: "google",
+		name: "Gemini 1.5 Flash",
+		contextWindow: 1000000,
+		maxOutputTokens: 8192,
+		pricing: { inputPer1M: 0.035, outputPer1M: 0.14 },
+		capabilities: ["code_generation", "vision"],
+		latency: "fast",
+		qualityScore: 78,
+	},
+	{
+		id: "gpt-4o",
+		providerId: "openai",
+		name: "GPT-4o",
+		contextWindow: 128000,
+		maxOutputTokens: 16384,
+		pricing: { inputPer1M: 5, outputPer1M: 15 },
+		capabilities: [
+			"code_generation",
+			"code_review",
+			"vision",
+			"function_calling",
+		],
+		latency: "medium",
+		qualityScore: 92,
+	},
+];
+
+/**
+ * Calculate cost for a model
+ */
+export function calculateModelCost(
+	model: ModelData,
+	inputTokens: number,
+	outputTokens: number,
+): number {
+	const inputCost = (inputTokens / 1_000_000) * model.pricing.inputPer1M;
+	const outputCost = (outputTokens / 1_000_000) * model.pricing.outputPer1M;
+	return Math.round((inputCost + outputCost) * 100) / 100;
+}
+
+/**
+ * Calculate quality impact between models
+ */
+export function calculateQualityImpact(
+	current: ModelData,
+	candidate: ModelData,
+): number {
+	return current.qualityScore - candidate.qualityScore;
+}
+
+/**
+ * Get quality score for a model
+ */
+export function getQualityScore(
+	model: ModelData,
+	requiredCapabilities: string[],
+): number {
+	// Base score
+	let score = model.qualityScore;
+
+	// Bonus for matching required capabilities
+	const matchedCaps = requiredCapabilities.filter((cap) =>
+		model.capabilities.includes(cap),
+	).length;
+
+	score += matchedCaps * 5;
+
+	return Math.min(score, 100);
+}
+
+/**
+ * Find cheaper alternatives for a model
+ */
+export function findCheaperAlternatives(
+	currentModel: ModelData,
+	requiredCapabilities: string[],
+	options?: SwitchOptions,
+): ModelData[] {
+	const maxQualityLoss = options?.maxQualityLoss ?? 20;
+
+	return DEFAULT_MODELS.filter((m) => {
+		// Must be different model
+		if (m.id === currentModel.id) return false;
+
+		// Must have required capabilities
+		if (!requiredCapabilities.every((cap) => m.capabilities.includes(cap))) {
+			return false;
+		}
+
+		// Must be cheaper
+		const currentCost = calculateModelCost(currentModel, 1000, 500);
+		const candidateCost = calculateModelCost(m, 1000, 500);
+		if (candidateCost >= currentCost) return false;
+
+		// Must not lose too much quality
+		const qualityLoss = calculateQualityImpact(currentModel, m);
+		if (qualityLoss > maxQualityLoss) return false;
+
+		// Must not increase latency beyond preference
+		if (options?.maxLatencyIncrease) {
+			const latencyOrder = { fast: 1, medium: 2, slow: 3 };
+			if (latencyOrder[m.latency] > latencyOrder[options.maxLatencyIncrease]) {
+				return false;
+			}
+		}
+
+		return true;
+	});
+}
+
+/**
+ * Generate a switch recommendation
+ */
+export function generateSwitchRecommendation(
+	currentModel: ModelData,
+	candidate: ModelData,
+	inputTokens: number,
+	outputTokens: number,
+): ModelSwitchRecommendation {
+	const currentCost = calculateModelCost(
+		currentModel,
+		inputTokens,
+		outputTokens,
+	);
+	const candidateCost = calculateModelCost(
+		candidate,
+		inputTokens,
+		outputTokens,
+	);
+	const savings = currentCost - candidateCost;
+	const savingsPct = (savings / currentCost) * 100;
+	const qualityLoss = calculateQualityImpact(currentModel, candidate);
+
+	return {
+		currentModel: { id: currentModel.id, providerId: currentModel.providerId },
+		recommendedModel: { id: candidate.id, providerId: candidate.providerId },
+		costSavings: Math.round(savingsPct * 10) / 10,
+		qualityImpact: Math.round(qualityLoss * 10) / 10,
+		reason: `Switch from ${currentModel.name} to ${candidate.name} saves ${savingsPct.toFixed(1)}%`,
+	};
+}
+
+/**
+ * Optimize model selection for a task
+ */
+export function optimizeModelSelection(
+	requirements: TaskRequirements,
+	models: ModelData[] = DEFAULT_MODELS,
+): OptimizedSelection[] {
+	// Filter by requirements
+	const candidates = models.filter((m) => {
+		if (m.contextWindow < requirements.minContextWindow) return false;
+		if (requirements.maxLatency) {
+			const latencyOrder = { fast: 1, medium: 2, slow: 3 };
+			if (latencyOrder[m.latency] > latencyOrder[requirements.maxLatency]) {
+				return false;
+			}
+		}
+		// Filter by required capabilities
+		if (requirements.requiredCapabilities?.length) {
+			const hasAllCaps = requirements.requiredCapabilities.every((cap) =>
+				m.capabilities.includes(cap),
+			);
+			if (!hasAllCaps) return false;
+		}
+		return true;
+	});
+
+	// Calculate scores
+	const scored = candidates.map((m) => {
+		const estimatedCost = calculateModelCost(
+			m,
+			requirements.estimatedInputTokens,
+			requirements.estimatedOutputTokens,
+		);
+		const qualityScore = getQualityScore(m, requirements.requiredCapabilities);
+
+		// Tradeoffs
+		const tradeoffs: string[] = [];
+		if (m.pricing.inputPer1M > 1) tradeoffs.push("Higher input cost");
+		if (m.qualityScore < 85) tradeoffs.push("Lower quality rating");
+
+		return {
+			model: { id: m.id, providerId: m.providerId, name: m.name },
+			estimatedCost,
+			qualityScore,
+			reason: `${m.name}: $${estimatedCost.toFixed(4)} estimated, ${qualityScore.toFixed(0)} quality`,
+			tradeoffs,
+		};
+	});
+
+	// Sort by quality-adjusted value (Pareto-optimal)
+	scored.sort((a, b) => {
+		// Quality weight 60%, cost weight 40%
+		const valueA = a.qualityScore * 0.6 - a.estimatedCost * 100 * 0.4;
+		const valueB = b.qualityScore * 0.6 - b.estimatedCost * 100 * 0.4;
+		return valueB - valueA;
+	});
+
+	return scored;
+}

package/packages/cost-optimizer/src/tracker.ts

@@ -0,0 +1,122 @@
+/**
+ * Cost Tracking Functions (RFC-0055)
+ */
+
+import type { CostEntry, CostPeriod, CostSummary } from "./types.js";
+
+let idCounter = 0;
+
+/**
+ * Generate a unique ID for cost entries
+ */
+export function generateCostId(): string {
+	return `cost_${Date.now()}_${++idCounter}`;
+}
+
+/**
+ * Create a new cost entry
+ */
+export function createCostEntry(
+	entry: Omit<CostEntry, "id" | "timestamp">,
+	id?: string,
+	timestamp?: string,
+): CostEntry {
+	return {
+		...entry,
+		id: id ?? generateCostId(),
+		timestamp: timestamp ?? new Date().toISOString(),
+	};
+}
+
+/**
+ * Get the period boundaries for a given period type
+ */
+export function getPeriodBoundaries(period: CostPeriod): {
+	start: Date;
+	end: Date;
+} {
+	const now = new Date();
+	const end = new Date(now);
+
+	switch (period.type) {
+		case "day": {
+			const start = new Date(now);
+			start.setHours(0, 0, 0, 0);
+			return { start, end };
+		}
+
+		case "week": {
+			const start = new Date(now);
+			start.setDate(now.getDate() - now.getDay());
+			start.setHours(0, 0, 0, 0);
+			return { start, end };
+		}
+
+		case "month": {
+			const start = new Date(now.getFullYear(), now.getMonth(), 1);
+			return { start, end };
+		}
+
+		case "custom": {
+			return {
+				start: new Date(period.start),
+				end: new Date(period.end),
+			};
+		}
+
+		default:
+			return { start: new Date(0), end: now };
+	}
+}
+
+/**
+ * Check if a timestamp falls within a period
+ */
+export function isInPeriod(timestamp: string, period: CostPeriod): boolean {
+	const { start, end } = getPeriodBoundaries(period);
+	const ts = new Date(timestamp);
+	return ts >= start && ts <= end;
+}
+
+/**
+ * Calculate summary from cost entries
+ */
+export function calculateSummary(
+	entries: CostEntry[],
+	period: CostPeriod,
+): CostSummary {
+	const filtered = entries.filter((e) => isInPeriod(e.timestamp, period));
+
+	const total = filtered.reduce((sum, e) => sum + e.cost, 0);
+	const currency = filtered[0]?.currency ?? "USD";
+
+	const byProvider: Record<string, number> = {};
+	const byModel: Record<string, number> = {};
+	const byJob: Record<string, number> = {};
+
+	for (const entry of filtered) {
+		byProvider[entry.providerId] =
+			(byProvider[entry.providerId] ?? 0) + entry.cost;
+		byModel[entry.modelId] = (byModel[entry.modelId] ?? 0) + entry.cost;
+		if (entry.jobId) {
+			byJob[entry.jobId] = (byJob[entry.jobId] ?? 0) + entry.cost;
+		}
+	}
+
+	return {
+		total: Math.round(total * 10000) / 10000, // higher precision for small amounts
+		currency,
+		byProvider,
+		byModel,
+		byJob,
+		period,
+	};
+}
+
+/**
+ * Get total cost for a period
+ */
+export function getTotalCost(entries: CostEntry[], period: CostPeriod): number {
+	const summary = calculateSummary(entries, period);
+	return summary.total;
+}

package/packages/cost-optimizer/src/types.ts

@@ -0,0 +1,134 @@
+/**
+ * Cost Optimizer Types (RFC-0055)
+ */
+
+// Re-export Currency type
+export type Currency = "USD" | "THB";
+
+// ─── Cost Entry Types ─────────────────────────────────────────────────
+
+export interface CostEntry {
+	id: string;
+	jobId: string;
+	taskId?: string;
+	providerId: string;
+	modelId: string;
+	inputTokens: number;
+	outputTokens: number;
+	cost: number;
+	currency: Currency;
+	timestamp: string;
+}
+
+// ─── Budget Types ─────────────────────────────────────────────────────
+
+export interface CostBudget {
+	daily?: number;
+	weekly?: number;
+	monthly?: number;
+	perJob?: number;
+	perTask?: number;
+}
+
+export interface BudgetStatus {
+	daily: { used: number; budget: number; remaining: number };
+	weekly: { used: number; budget: number; remaining: number };
+	monthly: { used: number; budget: number; remaining: number };
+	exhausted: boolean;
+	nextReset?: string;
+}
+
+export interface CostSummary {
+	total: number;
+	currency: Currency;
+	byProvider: Record<string, number>;
+	byModel: Record<string, number>;
+	byJob: Record<string, number>;
+	period: CostPeriod;
+}
+
+export interface CostPeriod {
+	start: string;
+	end: string;
+	type: "day" | "week" | "month" | "custom";
+}
+
+// ─── Optimization Types ────────────────────────────────────────────────
+
+export interface CostOptimizerConfig {
+	defaultBudget: CostBudget;
+	currency: Currency;
+	alertThreshold: number;
+	autoSwitchToCheaper: boolean;
+	maxQualityLossPercent: number;
+}
+
+export interface SwitchOptions {
+	maxQualityLoss?: number;
+	maxLatencyIncrease?: "fast" | "medium" | "slow";
+}
+
+export interface ModelSwitchRecommendation {
+	currentModel: { id: string; providerId: string };
+	recommendedModel: { id: string; providerId: string };
+	costSavings: number;
+	qualityImpact: number;
+	reason: string;
+}
+
+export interface CostForecast {
+	estimatedTotal: number;
+	byTaskType: Record<string, number>;
+	confidence: number;
+	assumptions: string[];
+}
+
+export interface TaskRequirements {
+	requiredCapabilities: string[];
+	minContextWindow: number;
+	maxLatency?: "fast" | "medium" | "slow";
+	estimatedInputTokens: number;
+	estimatedOutputTokens: number;
+}
+
+export interface OptimizedSelection {
+	model: { id: string; providerId: string; name: string };
+	estimatedCost: number;
+	qualityScore: number;
+	reason: string;
+	tradeoffs: string[];
+}
+
+// ─── Optimizer Interface ─────────────────────────────────────────────
+
+export interface CostOptimizer {
+	trackCost(entry: Omit<CostEntry, "id" | "timestamp">): CostEntry;
+	getSummary(period: CostPeriod): CostSummary;
+	getBudgetStatus(): BudgetStatus;
+	canAfford(cost: number): boolean;
+	shouldSwitchToCheaper(
+		currentModel: { id: string; providerId: string },
+		requiredCapabilities: string[],
+		options?: SwitchOptions,
+	): ModelSwitchRecommendation | null;
+	forecastCosts(jobRequirements: TaskRequirements[]): CostForecast;
+	optimizeModelSelection(
+		taskId: string,
+		requirements: TaskRequirements,
+	): OptimizedSelection[];
+}
+
+// ─── Event Types ─────────────────────────────────────────────────────
+
+export type CostOptimizerEvent =
+	| { type: "cost.tracked"; entryId: string; cost: number; total: number }
+	| { type: "cost.budget_warning"; threshold: number; percentage: number }
+	| { type: "cost.budget_exhausted"; period: string }
+	| {
+			type: "cost.switch_recommended";
+			from: string;
+			to: string;
+			savings: number;
+	  }
+	| { type: "cost.forecast"; estimated: number; confidence: number }
+	| { type: "cost.optimization_applied"; model: string; savings: number };