pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/performance-optimizer/src/types.ts

@@ -0,0 +1,112 @@
+/**
+ * Performance Optimizer Types (RFC-0056)
+ *
+ * Interfaces for performance analysis and recommendations.
+ */
+
+// ─── Input Types ─────────────────────────────────────────────────────────────
+
+export interface RuntimeMetricSnapshot {
+	jobId: string;
+	timestamp: string;
+	/** Total tokens used */
+	totalTokens: number;
+	/** Tokens used in last request */
+	lastRequestTokens: number;
+	/** Average latency in milliseconds */
+	avgLatencyMs: number;
+	/** Number of retries */
+	retryCount: number;
+	/** Number of successful calls */
+	successCount: number;
+	/** Number of failed calls */
+	failureCount: number;
+	/** Current context size in tokens */
+	currentContextTokens: number;
+	/** Max context window */
+	contextWindow: number;
+	/** Provider-specific metrics */
+	providerMetrics: Record<string, ProviderMetric>;
+}
+
+export interface ProviderMetric {
+	providerId: string;
+	callCount: number;
+	successCount: number;
+	failureCount: number;
+	avgLatencyMs: number;
+	avgTokensPerCall: number;
+}
+
+export interface ProviderRuntimeState {
+	providerId: string;
+	state: "available" | "limited" | "exhausted" | "disabled" | "unknown";
+	remainingQuota?: number;
+	avgLatencyMs?: number;
+	lastError?: string;
+}
+
+export interface PerformancePolicy {
+	/** Minimum context usage to consider reduction */
+	minContextReductionPercent: number;
+	/** Maximum parallelism allowed */
+	maxParallelism: number;
+	/** Minimum improvement threshold for recommendations */
+	minImprovementPercent: number;
+	/** Whether to allow provider switching */
+	allowProviderSwitch: boolean;
+	/** Mandatory stages that cannot be skipped */
+	mandatoryStages: string[];
+}
+
+// ─── Local Type Definitions ────────────────────────────────────────────────────
+
+export interface TaskGraphSnapshot {
+	nodes: Array<{
+		id: string;
+		status: string;
+	}>;
+	edges: Array<{ from: string; to: string }>;
+}
+
+export interface PerformanceOptimizationRequest {
+	jobId: string;
+	taskId?: string;
+	metrics: RuntimeMetricSnapshot;
+	providerStates: ProviderRuntimeState[];
+	taskGraph: TaskGraphSnapshot;
+	policy: PerformancePolicy;
+}
+
+// ─── Output Types ─────────────────────────────────────────────────────────────
+
+export type PerformanceRecommendation =
+	| { type: "reduce_context"; targetTokens: number; reason: string }
+	| { type: "change_provider"; provider: string; reason: string }
+	| { type: "increase_parallelism"; limit: number; reason: string }
+	| { type: "decrease_parallelism"; limit: number; reason: string }
+	| { type: "reuse_checkpoint"; checkpointId: string; reason: string }
+	| { type: "skip_redundant_step"; taskId: string; reason: string };
+
+export interface PerformanceOptimizationPlan {
+	jobId: string;
+	recommendations: PerformanceRecommendation[];
+	expectedImpact: {
+		tokenReductionPercent?: number;
+		latencyReductionPercent?: number;
+		retryReductionPercent?: number;
+	};
+	generatedAt: string;
+}
+
+// ─── Runtime Events ────────────────────────────────────────────────────────────
+
+export type PerformanceOptimizerEvent =
+	| { type: "performance.analysis.started"; jobId: string }
+	| {
+			type: "performance.recommendation.created";
+			jobId: string;
+			recommendation: PerformanceRecommendation;
+	  }
+	| { type: "performance.analysis.completed"; jobId: string }
+	| { type: "performance.analysis.failed"; jobId: string; error: string };

package/packages/performance-optimizer/test/optimizer.test.ts

@@ -0,0 +1,267 @@
+/**
+ * Performance Optimizer Tests (RFC-0056)
+ */
+
+import { describe, it, expect, beforeEach } from "@jest/globals";
+import {
+	type PerformanceOptimizer,
+	createPerformanceOptimizer,
+} from "../src/index.js";
+import type {
+	PerformanceOptimizationRequest,
+	PerformancePolicy,
+} from "../src/types.js";
+
+describe("PerformanceOptimizer", () => {
+	let optimizer: PerformanceOptimizer;
+	let defaultMetrics: PerformanceOptimizationRequest["metrics"];
+	let defaultProviders: PerformanceOptimizationRequest["providerStates"];
+
+	beforeEach(() => {
+		optimizer = createPerformanceOptimizer();
+		defaultMetrics = {
+			jobId: "test-job",
+			timestamp: new Date().toISOString(),
+			totalTokens: 50000,
+			lastRequestTokens: 5000,
+			avgLatencyMs: 5000,
+			retryCount: 2,
+			successCount: 10,
+			failureCount: 1,
+			currentContextTokens: 80000,
+			contextWindow: 100000,
+			providerMetrics: {
+				"provider-1": {
+					providerId: "provider-1",
+					callCount: 10,
+					successCount: 9,
+					failureCount: 1,
+					avgLatencyMs: 5000,
+					avgTokensPerCall: 5000,
+				},
+			},
+		};
+		defaultProviders = [
+			{
+				providerId: "provider-1",
+				state: "available",
+				remainingQuota: 100000,
+			},
+		];
+	});
+
+	describe("analyze", () => {
+		it("returns no recommendations when metrics are healthy", () => {
+			const metrics = {
+				...defaultMetrics,
+				currentContextTokens: 30000, // Low context usage
+				retryCount: 0,
+				avgLatencyMs: 2000,
+			};
+
+			const plan = optimizer.analyze({
+				jobId: "test-job",
+				metrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			expect(plan.jobId).toBe("test-job");
+			expect(plan.recommendations).toBeDefined();
+			expect(Array.isArray(plan.recommendations)).toBe(true);
+		});
+
+		it("recommends context reduction when context is high", () => {
+			const metrics = {
+				...defaultMetrics,
+				currentContextTokens: 90000, // 90% usage
+				contextWindow: 100000,
+			};
+
+			const plan = optimizer.analyze({
+				jobId: "test-job",
+				metrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			const reduceContext = plan.recommendations.find(
+				(r) => r.type === "reduce_context",
+			);
+			expect(reduceContext).toBeDefined();
+			if (reduceContext && reduceContext.type === "reduce_context") {
+				expect(reduceContext.targetTokens).toBeLessThan(90000);
+				expect(reduceContext.reason.length).toBeGreaterThan(0);
+			}
+		});
+
+		it("recommends provider change when failure rate is high", () => {
+			const metrics = {
+				...defaultMetrics,
+				providerMetrics: {
+					"provider-1": {
+						providerId: "provider-1",
+						callCount: 10,
+						successCount: 3,
+						failureCount: 7,
+						avgLatencyMs: 5000,
+						avgTokensPerCall: 5000,
+					},
+				},
+			};
+
+			const providers = [
+				{ providerId: "provider-1", state: "limited" as const },
+				{ providerId: "provider-2", state: "available" as const },
+			];
+
+			const plan = optimizer.analyze({
+				jobId: "test-job",
+				metrics,
+				providerStates: providers,
+				taskGraph: { nodes: [], edges: [] },
+				policy: {
+					mandatoryStages: [],
+					allowProviderSwitch: true,
+				} as PerformancePolicy,
+			});
+
+			const changeProvider = plan.recommendations.find(
+				(r) => r.type === "change_provider",
+			);
+			expect(changeProvider).toBeDefined();
+			if (changeProvider && changeProvider.type === "change_provider") {
+				expect(changeProvider.provider).toBe("provider-2");
+			}
+		});
+
+		it("does not switch providers when not allowed by policy", () => {
+			const metrics = {
+				...defaultMetrics,
+				providerMetrics: {
+					"provider-1": {
+						providerId: "provider-1",
+						callCount: 10,
+						successCount: 2,
+						failureCount: 8,
+						avgLatencyMs: 5000,
+						avgTokensPerCall: 5000,
+					},
+				},
+			};
+
+			const plan = optimizer.analyze({
+				jobId: "test-job",
+				metrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: {
+					mandatoryStages: [],
+					allowProviderSwitch: false,
+				} as PerformancePolicy,
+			});
+
+			const changeProvider = plan.recommendations.find(
+				(r) => r.type === "change_provider",
+			);
+			expect(changeProvider).toBeUndefined();
+		});
+
+		it("produces deterministic recommendations for identical inputs", () => {
+			const request = {
+				jobId: "test-job",
+				metrics: defaultMetrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			};
+
+			const plan1 = optimizer.analyze(request);
+			const plan2 = optimizer.analyze(request);
+
+			expect(plan1.recommendations.length).toBe(plan2.recommendations.length);
+			for (let i = 0; i < plan1.recommendations.length; i++) {
+				expect(plan1.recommendations[i].type).toBe(
+					plan2.recommendations[i].type,
+				);
+			}
+		});
+
+		it("includes expected impact when recommendations are made", () => {
+			const metrics = {
+				...defaultMetrics,
+				currentContextTokens: 90000,
+			};
+
+			const plan = optimizer.analyze({
+				jobId: "test-job",
+				metrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			expect(plan.expectedImpact).toBeDefined();
+			if (plan.recommendations.length > 0) {
+				// Should have some impact estimate
+				expect(plan.generatedAt).toBeDefined();
+			}
+		});
+	});
+
+	describe("events", () => {
+		it("emits analysis started event", () => {
+			const events: any[] = [];
+			optimizer.onEvent((e) => events.push(e));
+
+			optimizer.analyze({
+				jobId: "test-job",
+				metrics: defaultMetrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			expect(
+				events.some((e) => e.type === "performance.analysis.started"),
+			).toBe(true);
+		});
+
+		it("emits analysis completed event", () => {
+			const events: any[] = [];
+			optimizer.onEvent((e) => events.push(e));
+
+			optimizer.analyze({
+				jobId: "test-job",
+				metrics: defaultMetrics,
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			expect(
+				events.some((e) => e.type === "performance.analysis.completed"),
+			).toBe(true);
+		});
+
+		it("emits recommendation created events", () => {
+			const events: any[] = [];
+			optimizer.onEvent((e) => events.push(e));
+
+			optimizer.analyze({
+				jobId: "test-job",
+				metrics: { ...defaultMetrics, currentContextTokens: 95000 },
+				providerStates: defaultProviders,
+				taskGraph: { nodes: [], edges: [] },
+				policy: { mandatoryStages: [] } as PerformancePolicy,
+			});
+
+			const recommendationEvents = events.filter(
+				(e) => e.type === "performance.recommendation.created",
+			);
+			expect(recommendationEvents.length).toBeGreaterThan(0);
+		});
+	});
+});

package/packages/performance-optimizer/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"]
+}

package/packages/project-analyzer/dist/analyzer.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Project Analyzer
+ *
+ * Main analyzer implementation that orchestrates project analysis.
+ */
+import type { AnalyzeRequest, AnalyzeResult, DetectedLanguage, FrameworkAnalyzerPlugin, ReadonlyFileSystem, TestCapability } from "./types.js";
+import { AnalysisCache } from "./cache.js";
+/**
+ * Default framework analyzer plugins.
+ * These are used when no custom plugins are provided.
+ */
+export declare const DEFAULT_PLUGINS: FrameworkAnalyzerPlugin[];
+/**
+ * Count language coverage from scanned files.
+ */
+export declare function detectLanguages(files: {
+    relativePath: string;
+    size: number;
+}[]): DetectedLanguage[];
+/**
+ * Detect test capabilities from project files.
+ */
+export declare function detectTestCapabilities(fs: ReadonlyFileSystem, framework: string): Promise<TestCapability[]>;
+/**
+ * Project analyzer implementation.
+ */
+export declare class ProjectAnalyzer {
+    private cache;
+    private plugins;
+    private defaultDetector;
+    constructor(options?: {
+        cache?: AnalysisCache;
+        plugins?: FrameworkAnalyzerPlugin[];
+    });
+    /**
+     * Get git revision for a repository.
+     */
+    private getGitRevision;
+    /**
+     * Parse JSON file safely.
+     */
+    private parseJsonFile;
+    /**
+     * Run framework detection.
+     */
+    private detectFrameworks;
+    /**
+     * Discover rules from project.
+     */
+    private discoverRules;
+    /**
+     * Discover commands from project.
+     */
+    private discoverCommands;
+    /**
+     * Analyze a project.
+     */
+    analyze(request: AnalyzeRequest): Promise<AnalyzeResult>;
+}
+/**
+ * Create a project analyzer with default configuration.
+ */
+export declare function createProjectAnalyzer(options?: {
+    cache?: AnalysisCache;
+    plugins?: FrameworkAnalyzerPlugin[];
+}): ProjectAnalyzer;
+//# sourceMappingURL=analyzer.d.ts.map

package/packages/project-analyzer/dist/analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../src/analyzer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,KAAK,EAEX,cAAc,EACd,aAAa,EAGb,gBAAgB,EAChB,uBAAuB,EACvB,kBAAkB,EAClB,cAAc,EAGd,MAAM,YAAY,CAAC;AAGpB,OAAO,EACN,aAAa,EAIb,MAAM,YAAY,CAAC;AAkBpB;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,uBAAuB,EAAO,CAAC;AA0B7D;;GAEG;AACH,wBAAgB,eAAe,CAC9B,KAAK,EAAE;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,gBAAgB,EAAE,CA8BpB;AAID;;GAEG;AACH,wBAAsB,sBAAsB,CAC3C,EAAE,EAAE,kBAAkB,EACtB,SAAS,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAgF3B;AAID;;GAEG;AACH,qBAAa,eAAe;IAC3B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,eAAe,CAA2B;gBAGjD,OAAO,GAAE;QACR,KAAK,CAAC,EAAE,aAAa,CAAC;QACtB,OAAO,CAAC,EAAE,uBAAuB,EAAE,CAAC;KAC/B;IAOP;;OAEG;YACW,cAAc;IAa5B;;OAEG;YACW,aAAa;IAS3B;;OAEG;YACW,gBAAgB;IAqF9B;;OAEG;YACW,aAAa;IAc3B;;OAEG;YACW,gBAAgB;IA+C9B;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAoQ9D;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,CAAC,EAAE;IAC/C,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,CAAC,EAAE,uBAAuB,EAAE,CAAC;CACpC,GAAG,eAAe,CAElB"}