pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/performance-optimizer/dist/types.d.ts

@@ -0,0 +1,123 @@
+/**
+ * Performance Optimizer Types (RFC-0056)
+ *
+ * Interfaces for performance analysis and recommendations.
+ */
+export interface RuntimeMetricSnapshot {
+    jobId: string;
+    timestamp: string;
+    /** Total tokens used */
+    totalTokens: number;
+    /** Tokens used in last request */
+    lastRequestTokens: number;
+    /** Average latency in milliseconds */
+    avgLatencyMs: number;
+    /** Number of retries */
+    retryCount: number;
+    /** Number of successful calls */
+    successCount: number;
+    /** Number of failed calls */
+    failureCount: number;
+    /** Current context size in tokens */
+    currentContextTokens: number;
+    /** Max context window */
+    contextWindow: number;
+    /** Provider-specific metrics */
+    providerMetrics: Record<string, ProviderMetric>;
+}
+export interface ProviderMetric {
+    providerId: string;
+    callCount: number;
+    successCount: number;
+    failureCount: number;
+    avgLatencyMs: number;
+    avgTokensPerCall: number;
+}
+export interface ProviderRuntimeState {
+    providerId: string;
+    state: "available" | "limited" | "exhausted" | "disabled" | "unknown";
+    remainingQuota?: number;
+    avgLatencyMs?: number;
+    lastError?: string;
+}
+export interface PerformancePolicy {
+    /** Minimum context usage to consider reduction */
+    minContextReductionPercent: number;
+    /** Maximum parallelism allowed */
+    maxParallelism: number;
+    /** Minimum improvement threshold for recommendations */
+    minImprovementPercent: number;
+    /** Whether to allow provider switching */
+    allowProviderSwitch: boolean;
+    /** Mandatory stages that cannot be skipped */
+    mandatoryStages: string[];
+}
+export interface TaskGraphSnapshot {
+    nodes: Array<{
+        id: string;
+        status: string;
+    }>;
+    edges: Array<{
+        from: string;
+        to: string;
+    }>;
+}
+export interface PerformanceOptimizationRequest {
+    jobId: string;
+    taskId?: string;
+    metrics: RuntimeMetricSnapshot;
+    providerStates: ProviderRuntimeState[];
+    taskGraph: TaskGraphSnapshot;
+    policy: PerformancePolicy;
+}
+export type PerformanceRecommendation = {
+    type: "reduce_context";
+    targetTokens: number;
+    reason: string;
+} | {
+    type: "change_provider";
+    provider: string;
+    reason: string;
+} | {
+    type: "increase_parallelism";
+    limit: number;
+    reason: string;
+} | {
+    type: "decrease_parallelism";
+    limit: number;
+    reason: string;
+} | {
+    type: "reuse_checkpoint";
+    checkpointId: string;
+    reason: string;
+} | {
+    type: "skip_redundant_step";
+    taskId: string;
+    reason: string;
+};
+export interface PerformanceOptimizationPlan {
+    jobId: string;
+    recommendations: PerformanceRecommendation[];
+    expectedImpact: {
+        tokenReductionPercent?: number;
+        latencyReductionPercent?: number;
+        retryReductionPercent?: number;
+    };
+    generatedAt: string;
+}
+export type PerformanceOptimizerEvent = {
+    type: "performance.analysis.started";
+    jobId: string;
+} | {
+    type: "performance.recommendation.created";
+    jobId: string;
+    recommendation: PerformanceRecommendation;
+} | {
+    type: "performance.analysis.completed";
+    jobId: string;
+} | {
+    type: "performance.analysis.failed";
+    jobId: string;
+    error: string;
+};
+//# sourceMappingURL=types.d.ts.map

package/packages/performance-optimizer/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sCAAsC;IACtC,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,qCAAqC;IACrC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,yBAAyB;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,WAAW,GAAG,SAAS,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IACjC,kDAAkD;IAClD,0BAA0B,EAAE,MAAM,CAAC;IACnC,kCAAkC;IAClC,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,qBAAqB,EAAE,MAAM,CAAC;IAC9B,0CAA0C;IAC1C,mBAAmB,EAAE,OAAO,CAAC;IAC7B,8CAA8C;IAC9C,eAAe,EAAE,MAAM,EAAE,CAAC;CAC1B;AAID,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,KAAK,CAAC;QACZ,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,8BAA8B;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACvC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,MAAM,EAAE,iBAAiB,CAAC;CAC1B;AAID,MAAM,MAAM,yBAAyB,GAClC;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC/D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC/D;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnE,MAAM,WAAW,2BAA2B;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,yBAAyB,EAAE,CAAC;IAC7C,cAAc,EAAE;QACf,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAC/B,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;CACpB;AAID,MAAM,MAAM,yBAAyB,GAClC;IAAE,IAAI,EAAE,8BAA8B,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACvD;IACA,IAAI,EAAE,oCAAoC,CAAC;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,yBAAyB,CAAC;CACzC,GACD;IAAE,IAAI,EAAE,gCAAgC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,6BAA6B,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC"}

package/packages/performance-optimizer/dist/types.js

@@ -0,0 +1,7 @@
+/**
+ * Performance Optimizer Types (RFC-0056)
+ *
+ * Interfaces for performance analysis and recommendations.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/packages/performance-optimizer/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/packages/performance-optimizer/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@pi-harness/performance-optimizer",
+  "version": "0.6.3-beta.13",
+  "description": "Performance optimization analyzer for pi-harness runtime (RFC-0056)",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js",
+    "lint": "eslint src --ext .ts"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.5.0"
+  }
+}

package/packages/performance-optimizer/src/index.ts

@@ -0,0 +1,20 @@
+/**
+ * Performance Optimizer (RFC-0056)
+ *
+ * Re-exports all public types and classes.
+ */
+
+export {
+	PerformanceOptimizer,
+	createPerformanceOptimizer,
+} from "./optimizer.js";
+export type {
+	PerformanceOptimizationRequest,
+	PerformanceOptimizationPlan,
+	PerformanceRecommendation,
+	RuntimeMetricSnapshot,
+	ProviderMetric,
+	ProviderRuntimeState,
+	PerformancePolicy,
+	PerformanceOptimizerEvent,
+} from "./types.js";

package/packages/performance-optimizer/src/optimizer.ts

@@ -0,0 +1,428 @@
+/**
+ * Performance Optimizer (RFC-0056)
+ *
+ * Analyzes runtime metrics and produces advisory optimization decisions.
+ */
+
+import type {
+	PerformanceOptimizationRequest,
+	PerformanceOptimizationPlan,
+	PerformanceRecommendation,
+	RuntimeMetricSnapshot,
+	ProviderRuntimeState,
+	PerformancePolicy,
+	PerformanceOptimizerEvent,
+} from "./types.js";
+
+/**
+ * Detect bottlenecks from metrics
+ */
+function detectBottlenecks(metrics: RuntimeMetricSnapshot): string[] {
+	const bottlenecks: string[] = [];
+	const {
+		currentContextTokens,
+		contextWindow,
+		retryCount,
+		successCount,
+		failureCount,
+	} = metrics;
+
+	// Context usage bottleneck
+	const contextUsagePercent = (currentContextTokens / contextWindow) * 100;
+	if (contextUsagePercent > 80) {
+		bottlenecks.push("high_context_usage");
+	}
+
+	// Retry bottleneck
+	const totalCalls = successCount + failureCount;
+	if (totalCalls > 0 && retryCount / totalCalls > 0.3) {
+		bottlenecks.push("high_retry_rate");
+	}
+
+	// Provider performance
+	const avgLatency = metrics.avgLatencyMs;
+	if (avgLatency > 30000) {
+		bottlenecks.push("high_latency");
+	}
+
+	return bottlenecks;
+}
+
+/**
+ * Normalize provider metrics
+ */
+function normalizeProviderStats(
+	providerMetrics: Record<
+		string,
+		{ callCount: number; successCount: number; failureCount: number }
+	>,
+): Record<string, { successRate: number; failureRate: number }> {
+	const normalized: Record<
+		string,
+		{ successRate: number; failureRate: number }
+	> = {};
+
+	for (const [providerId, stats] of Object.entries(providerMetrics)) {
+		const total = stats.callCount;
+		if (total > 0) {
+			normalized[providerId] = {
+				successRate: stats.successCount / total,
+				failureRate: stats.failureCount / total,
+			};
+		}
+	}
+
+	return normalized;
+}
+
+/**
+ * Generate context reduction recommendation
+ */
+function suggestContextReduction(
+	metrics: RuntimeMetricSnapshot,
+	policy: PerformancePolicy,
+): PerformanceRecommendation | null {
+	const { currentContextTokens, contextWindow } = metrics;
+	const usagePercent = (currentContextTokens / contextWindow) * 100;
+
+	if (usagePercent < 50) {
+		return null; // No reduction needed
+	}
+
+	// Calculate target (reduce to 60% of current)
+	const targetTokens = Math.floor(currentContextTokens * 0.6);
+	const reductionPercent =
+		((currentContextTokens - targetTokens) / currentContextTokens) * 100;
+
+	if (reductionPercent < policy.minContextReductionPercent) {
+		return null;
+	}
+
+	return {
+		type: "reduce_context",
+		targetTokens,
+		reason: `Context usage at ${usagePercent.toFixed(1)}% (${currentContextTokens}/${contextWindow} tokens). Reduce to ~60% to improve performance.`,
+	};
+}
+
+/**
+ * Generate provider change recommendation
+ */
+function suggestProviderChange(
+	metrics: RuntimeMetricSnapshot,
+	providerStates: ProviderRuntimeState[],
+	policy: PerformancePolicy,
+): PerformanceRecommendation | null {
+	if (!policy.allowProviderSwitch) {
+		return null;
+	}
+
+	const providerStats = normalizeProviderStats(metrics.providerMetrics);
+
+	// Find worst performing provider
+	let worstProvider: string | null = null;
+	let worstFailureRate = 0;
+
+	for (const [providerId, stats] of Object.entries(providerStats)) {
+		if (stats.failureRate > worstFailureRate && stats.failureRate > 0.1) {
+			worstProvider = providerId;
+			worstFailureRate = stats.failureRate;
+		}
+	}
+
+	if (!worstProvider) {
+		return null;
+	}
+
+	// Check if there's a better available provider
+	const availableProviders = providerStates.filter(
+		(p) => p.state === "available" && p.providerId !== worstProvider,
+	);
+
+	if (availableProviders.length === 0) {
+		return null; // No alternative available
+	}
+
+	return {
+		type: "change_provider",
+		provider: availableProviders[0].providerId,
+		reason: `Provider '${worstProvider}' has ${(worstFailureRate * 100).toFixed(1)}% failure rate. Switch to '${availableProviders[0].providerId}' for better reliability.`,
+	};
+}
+
+/**
+ * Generate parallelism recommendations
+ */
+function suggestParallelismChanges(
+	metrics: RuntimeMetricSnapshot,
+	providerStates: ProviderRuntimeState[],
+	policy: PerformancePolicy,
+	taskGraph?: { nodes: Array<{ id: string; status: string }> },
+): PerformanceRecommendation[] {
+	const recommendations: PerformanceRecommendation[] = [];
+
+	// Check for blocked tasks that could be parallel
+	if (taskGraph) {
+		const runningTasks = taskGraph.nodes.filter((n) => n.status === "running");
+		const readyTasks = taskGraph.nodes.filter((n) => n.status === "ready");
+
+		// If we have many ready tasks and few running, suggest increasing parallelism
+		if (readyTasks.length > 5 && runningTasks.length < 3) {
+			const suggestedLimit = Math.min(
+				runningTasks.length + 2,
+				policy.maxParallelism,
+			);
+			if (suggestedLimit > runningTasks.length) {
+				recommendations.push({
+					type: "increase_parallelism",
+					limit: suggestedLimit,
+					reason: `${readyTasks.length} tasks waiting. Increase parallelism to ${suggestedLimit} for faster completion.`,
+				});
+			}
+		}
+
+		// If too many tasks are hitting resource limits, suggest decreasing
+		const limitedProviders = providerStates.filter(
+			(p) => p.state === "limited",
+		);
+		if (limitedProviders.length > 0 && runningTasks.length > 5) {
+			recommendations.push({
+				type: "decrease_parallelism",
+				limit: Math.max(2, runningTasks.length - 2),
+				reason: `Provider quota limited. Reduce parallelism to ${Math.max(2, runningTasks.length - 2)} to avoid quota exhaustion.`,
+			});
+		}
+	}
+
+	return recommendations;
+}
+
+/**
+ * Resolve conflicting recommendations
+ */
+function resolveConflicts(
+	recommendations: PerformanceRecommendation[],
+	policy: PerformancePolicy,
+): PerformanceRecommendation[] {
+	// Sort by type priority (reductions first, then parallelism, then provider)
+	const priority: Record<string, number> = {
+		reduce_context: 1,
+		decrease_parallelism: 2,
+		increase_parallelism: 3,
+		change_provider: 4,
+		reuse_checkpoint: 5,
+		skip_redundant_step: 6,
+	};
+
+	// Remove duplicates and conflicting recommendations
+	const seen = new Set<string>();
+	const resolved: PerformanceRecommendation[] = [];
+
+	for (const rec of recommendations.sort(
+		(a, b) => priority[a.type] - priority[b.type],
+	)) {
+		const key = rec.type;
+
+		// Check for conflicts
+		if (
+			key === "increase_parallelism" &&
+			resolved.some((r) => r.type === "decrease_parallelism")
+		) {
+			continue; // Skip increase if decrease already added
+		}
+		if (
+			key === "decrease_parallelism" &&
+			resolved.some((r) => r.type === "increase_parallelism")
+		) {
+			// Remove increase and add decrease
+			const idx = resolved.findIndex((r) => r.type === "increase_parallelism");
+			if (idx !== -1) resolved.splice(idx, 1);
+		}
+
+		if (!seen.has(key)) {
+			seen.add(key);
+			resolved.push(rec);
+		}
+	}
+
+	return resolved;
+}
+
+/**
+ * Calculate expected impact
+ */
+function calculateImpact(
+	metrics: RuntimeMetricSnapshot,
+	recommendations: PerformanceRecommendation[],
+): PerformanceOptimizationPlan["expectedImpact"] {
+	let tokenReduction = 0;
+	let latencyReduction = 0;
+	let retryReduction = 0;
+
+	for (const rec of recommendations) {
+		switch (rec.type) {
+			case "reduce_context":
+				// Estimate 15-30% latency improvement from context reduction
+				latencyReduction += 20;
+				break;
+			case "change_provider":
+				// Estimate 10-25% latency improvement from better provider
+				latencyReduction += 15;
+				break;
+			case "decrease_parallelism":
+				// Reduce contention
+				latencyReduction += 10;
+				retryReduction += 15;
+				break;
+			case "skip_redundant_step":
+				tokenReduction += 10;
+				break;
+		}
+	}
+
+	return {
+		tokenReductionPercent:
+			tokenReduction > 0 ? Math.min(tokenReduction, 50) : undefined,
+		latencyReductionPercent:
+			latencyReduction > 0 ? Math.min(latencyReduction, 40) : undefined,
+		retryReductionPercent:
+			retryReduction > 0 ? Math.min(retryReduction, 30) : undefined,
+	};
+}
+
+export class PerformanceOptimizer {
+	private policy: PerformancePolicy;
+
+	constructor(policy: Partial<PerformancePolicy> = {}) {
+		this.policy = {
+			minContextReductionPercent: 10,
+			maxParallelism: 10,
+			minImprovementPercent: 5,
+			allowProviderSwitch: true,
+			mandatoryStages: [],
+			...policy,
+		};
+	}
+
+	/**
+	 * Analyze metrics and produce optimization recommendations
+	 */
+	analyze(
+		request: PerformanceOptimizationRequest,
+	): PerformanceOptimizationPlan {
+		const { jobId, metrics, providerStates } = request;
+
+		// Emit start event
+		this.emit({ type: "performance.analysis.started", jobId });
+
+		try {
+			// 1. Detect bottlenecks
+			const bottlenecks = detectBottlenecks(metrics);
+
+			// 2. Generate recommendations
+			const recommendations: PerformanceRecommendation[] = [];
+
+			// Context reduction
+			const contextRec = suggestContextReduction(metrics, this.policy);
+			if (contextRec) {
+				recommendations.push(contextRec);
+				this.emit({
+					type: "performance.recommendation.created",
+					jobId,
+					recommendation: contextRec,
+				});
+			}
+
+			// Provider change
+			const providerRec = suggestProviderChange(
+				metrics,
+				providerStates,
+				this.policy,
+			);
+			if (providerRec) {
+				recommendations.push(providerRec);
+				this.emit({
+					type: "performance.recommendation.created",
+					jobId,
+					recommendation: providerRec,
+				});
+			}
+
+			// Parallelism changes
+			const parallelismRecs = suggestParallelismChanges(
+				metrics,
+				providerStates,
+				this.policy,
+				request.taskGraph,
+			);
+			for (const rec of parallelismRecs) {
+				recommendations.push(rec);
+				this.emit({
+					type: "performance.recommendation.created",
+					jobId,
+					recommendation: rec,
+				});
+			}
+
+			// 3. Resolve conflicts
+			const resolved = resolveConflicts(recommendations, this.policy);
+
+			// 4. Calculate impact
+			const expectedImpact = calculateImpact(metrics, resolved);
+
+			this.emit({ type: "performance.analysis.completed", jobId });
+
+			return {
+				jobId,
+				recommendations: resolved,
+				expectedImpact,
+				generatedAt: new Date().toISOString(),
+			};
+		} catch (error) {
+			const errorMessage =
+				error instanceof Error ? error.message : String(error);
+			this.emit({
+				type: "performance.analysis.failed",
+				jobId,
+				error: errorMessage,
+			});
+			return {
+				jobId,
+				recommendations: [],
+				expectedImpact: {},
+				generatedAt: new Date().toISOString(),
+			};
+		}
+	}
+
+	private eventListeners: Array<(event: PerformanceOptimizerEvent) => void> =
+		[];
+
+	/**
+	 * Subscribe to optimizer events
+	 */
+	onEvent(listener: (event: PerformanceOptimizerEvent) => void): () => void {
+		this.eventListeners.push(listener);
+		return () => {
+			this.eventListeners = this.eventListeners.filter((l) => l !== listener);
+		};
+	}
+
+	private emit(event: PerformanceOptimizerEvent): void {
+		for (const listener of this.eventListeners) {
+			try {
+				listener(event);
+			} catch {
+				// Best effort
+			}
+		}
+	}
+}
+
+// ─── Factory ───────────────────────────────────────────────────────────────
+
+export function createPerformanceOptimizer(
+	policy?: Partial<PerformancePolicy>,
+): PerformanceOptimizer {
+	return new PerformanceOptimizer(policy);
+}