pi-harness-runtime 0.10.12 → 0.10.13

package/RFC/0055_COST_OPTIMIZER.md

@@ -0,0 +1,285 @@
+# RFC-0055 — Cost Optimizer
+
+Status: Draft  
+Target package: `packages/cost-optimizer`  
+Depends on: RFC-0051 Capability Registry, RFC-0053 Model Registry, RFC-0054 Provider Router
+
+## 1. Problem
+
+The runtime needs intelligent cost management to minimize expenses while maintaining quality. Currently costs are not tracked or optimized. The Cost Optimizer provides budget tracking, cost forecasting, and model selection optimization.
+
+## 2. Cost Tracking
+
+```ts
+export interface CostEntry {
+  id: string;
+  jobId: string;
+  taskId?: string;
+  providerId: string;
+  modelId: string;
+  inputTokens: number;
+  outputTokens: number;
+  cost: number;
+  currency: "USD" | "THB";
+  timestamp: string;
+}
+
+export interface CostBudget {
+  daily?: number;
+  weekly?: number;
+  monthly?: number;
+  perJob?: number;
+  perTask?: number;
+}
+
+export interface CostSummary {
+  total: number;
+  currency: "USD" | "THB";
+  byProvider: Record<string, number>;
+  byModel: Record<string, number>;
+  byJob: Record<string, number>;
+  period: CostPeriod;
+}
+
+export interface CostPeriod {
+  start: string;
+  end: string;
+  type: "day" | "week" | "month" | "custom";
+}
+```
+
+## 3. Optimizer Interface
+
+```ts
+export interface CostOptimizer {
+  trackCost(entry: Omit<CostEntry, "id" | "timestamp">): CostEntry;
+  getSummary(period: CostPeriod): CostSummary;
+  getBudgetStatus(): BudgetStatus;
+  canAfford(cost: number): boolean;
+  shouldSwitchToCheaper(
+    currentModel: ModelInfo,
+    requiredCapabilities: Capability[],
+    options?: SwitchOptions,
+  ): ModelSwitchRecommendation | null;
+  forecastCosts(jobRequirements: JobRequirements): CostForecast;
+  optimizeModelSelection(
+    task: RuntimeTask,
+    requirements: TaskRequirements,
+  ): OptimizedSelection[];
+}
+
+export interface BudgetStatus {
+  daily: { used: number; budget: number; remaining: number };
+  weekly: { used: number; budget: number; remaining: number };
+  monthly: { used: number; budget: number; remaining: number };
+  exhausted: boolean;
+  nextReset?: string;
+}
+
+export interface SwitchOptions {
+  maxQualityLoss?: number; // percentage
+  maxLatencyIncrease?: "fast" | "medium" | "slow";
+}
+
+export interface ModelSwitchRecommendation {
+  currentModel: ModelInfo;
+  recommendedModel: ModelInfo;
+  costSavings: number; // percentage
+  qualityImpact: number; // percentage
+  reason: string;
+}
+
+export interface CostForecast {
+  estimatedTotal: number;
+  byTaskType: Record<string, number>;
+  confidence: number;
+  assumptions: string[];
+}
+
+export interface TaskRequirements {
+  requiredCapabilities: Capability[];
+  minContextWindow: number;
+  maxLatency?: "fast" | "medium" | "slow";
+  estimatedInputTokens: number;
+  estimatedOutputTokens: number;
+}
+
+export interface OptimizedSelection {
+  model: ModelInfo;
+  estimatedCost: number;
+  qualityScore: number;
+  reason: string;
+  tradeoffs: string[];
+}
+```
+
+## 4. Cost Tracking Algorithm
+
+```ts
+export function trackCost(
+  optimizer: CostOptimizer,
+  entry: Omit<CostEntry, "id" | "timestamp">,
+): CostEntry {
+  const costEntry: CostEntry = {
+    ...entry,
+    id: generateId(),
+    timestamp: new Date().toISOString(),
+  };
+  
+  // Store in history
+  optimizer.addToHistory(costEntry);
+  
+  // Check budget
+  const status = optimizer.getBudgetStatus();
+  if (status.exhausted) {
+    optimizer.emit({
+      type: "cost.budget_exhausted",
+      status,
+    });
+  }
+  
+  return costEntry;
+}
+
+export function canAfford(
+  optimizer: CostOptimizer,
+  cost: number,
+): boolean {
+  const status = optimizer.getBudgetStatus();
+  
+  if (cost > status.daily.remaining && status.daily.budget) return false;
+  if (cost > status.weekly.remaining && status.weekly.budget) return false;
+  if (cost > status.monthly.remaining && status.monthly.budget) return false;
+  
+  return true;
+}
+```
+
+## 5. Model Selection Optimization
+
+```ts
+export function shouldSwitchToCheaper(
+  optimizer: CostOptimizer,
+  currentModel: ModelInfo,
+  requiredCapabilities: Capability[],
+  options?: SwitchOptions,
+): ModelSwitchRecommendation | null {
+  const candidates = optimizer.findModels({
+    capabilities: requiredCapabilities,
+    status: "active",
+  });
+  
+  // Filter by quality tolerance
+  const maxQualityLoss = options?.maxQualityLoss ?? 20;
+  
+  const cheaperOptions = candidates
+    .filter(m => {
+      const currentCost = calculateModelCost(currentModel);
+      const candidateCost = calculateModelCost(m);
+      return candidateCost < currentCost;
+    })
+    .filter(m => {
+      const qualityLoss = getQualityImpact(currentModel, m);
+      return qualityLoss <= maxQualityLoss;
+    })
+    .map(m => ({
+      model: m,
+      costSavings: calculateSavings(currentModel, m),
+      qualityImpact: getQualityImpact(currentModel, m),
+    }))
+    .sort((a, b) => b.costSavings - a.costSavings);
+  
+  if (cheaperOptions.length === 0) return null;
+  
+  const best = cheaperOptions[0];
+  return {
+    currentModel,
+    recommendedModel: best.model,
+    costSavings: best.costSavings,
+    qualityImpact: best.qualityImpact,
+    reason: `Switching from ${currentModel.id} to ${best.model.id} saves ${best.costSavings.toFixed(1)}% with ${best.qualityImpact.toFixed(1)}% quality impact`,
+  };
+}
+
+export function optimizeModelSelection(
+  optimizer: CostOptimizer,
+  task: RuntimeTask,
+  requirements: TaskRequirements,
+): OptimizedSelection[] {
+  const candidates = optimizer.findModels({
+    capabilities: requirements.requiredCapabilities,
+    status: "active",
+  })
+    .filter(m => m.contextWindow >= requirements.minContextWindow)
+    .filter(m => !requirements.maxLatency || LATENCY_RANK[m.latency] <= LATENCY_RANK[requirements.maxLatency]);
+  
+  const baseTokens = requirements.estimatedInputTokens + requirements.estimatedOutputTokens;
+  
+  return candidates
+    .map(m => {
+      const estimatedCost = calculateCost(m, requirements.estimatedInputTokens, requirements.estimatedOutputTokens);
+      const qualityScore = calculateQualityScore(m, requirements.requiredCapabilities);
+      
+      return {
+        model: m,
+        estimatedCost,
+        qualityScore,
+        reason: generateReason(m, estimatedCost, qualityScore),
+        tradeoffs: generateTradeoffs(m, requirements),
+      };
+    })
+    .sort((a, b) => {
+      // Pareto-optimal: prefer lower cost at similar quality
+      const qualityAdjusted = a.qualityScore * 0.6 - a.estimatedCost * 0.4;
+      const qualityAdjustedB = b.qualityScore * 0.6 - b.estimatedCost * 0.4;
+      return qualityAdjustedB - qualityAdjusted;
+    });
+}
+```
+
+## 6. Budget Configuration
+
+```ts
+export interface CostOptimizerConfig {
+  defaultBudget: CostBudget;
+  currency: "USD" | "THB";
+  alertThreshold: number; // percentage of budget
+  autoSwitchToCheaper: boolean;
+  maxQualityLossPercent: number;
+}
+
+export const DEFAULT_CONFIG: CostOptimizerConfig = {
+  defaultBudget: {
+    daily: 10,
+    weekly: 50,
+    monthly: 200,
+  },
+  currency: "USD",
+  alertThreshold: 0.8, // 80%
+  autoSwitchToCheaper: false,
+  maxQualityLossPercent: 15,
+};
+```
+
+## 7. Events
+
+```ts
+type CostOptimizerEvent =
+  | { type: "cost.tracked"; entryId: string; cost: number; total: number }
+  | { type: "cost.budget_warning"; threshold: number; percentage: number }
+  | { type: "cost.budget_exhausted"; period: string }
+  | { type: "cost.switch_recommended"; from: string; to: string; savings: number }
+  | { type: "cost.forecast"; estimated: number; confidence: number }
+  | { type: "cost.optimization_applied"; model: string; savings: number };
+```
+
+## 8. Acceptance Criteria
+
+- Cost entries are tracked with full metadata
+- Budget status reflects current usage correctly
+- Forecast provides reasonable estimates
+- Model switch recommendations are sensible
+- Optimized selections are Pareto-optimal
+- Events are emitted for budget warnings and exhaustion
+- Unit tests cover all scenarios
+- Integration with Model Registry and Capability Registry works

package/RFC/0056_PERFORMANCE_OPTIMIZER.md

@@ -0,0 +1,118 @@
+# RFC-0056 — Performance Optimizer
+
+## 1. Purpose
+
+The Performance Optimizer improves runtime throughput and reduces waste without changing task semantics. It analyzes execution metrics and produces advisory optimization decisions for scheduling, provider selection, context size, retry policy, and concurrency.
+
+It does not directly execute tasks, modify source code, or override mandatory project policy.
+
+## 2. Inputs
+
+```ts
+export interface PerformanceOptimizationRequest {
+  jobId: string;
+  taskId?: string;
+  metrics: RuntimeMetricSnapshot;
+  providerStates: ProviderRuntimeState[];
+  taskGraph: TaskGraphSnapshot;
+  policy: PerformancePolicy;
+}
+```
+
+## 3. Outputs
+
+```ts
+export interface PerformanceOptimizationPlan {
+  jobId: string;
+  recommendations: PerformanceRecommendation[];
+  expectedImpact: {
+    tokenReductionPercent?: number;
+    latencyReductionPercent?: number;
+    retryReductionPercent?: number;
+  };
+  generatedAt: string;
+}
+```
+
+## 4. Recommendation Types
+
+```ts
+export type PerformanceRecommendation =
+  | { type: "reduce_context"; targetTokens: number; reason: string }
+  | { type: "change_provider"; provider: string; reason: string }
+  | { type: "increase_parallelism"; limit: number; reason: string }
+  | { type: "decrease_parallelism"; limit: number; reason: string }
+  | { type: "reuse_checkpoint"; checkpointId: string; reason: string }
+  | { type: "skip_redundant_step"; taskId: string; reason: string };
+```
+
+## 5. Optimization Rules
+
+The optimizer may recommend:
+
+- Reusing unchanged compiled context.
+- Reusing successful checkpoints.
+- Avoiding repeated repository scans.
+- Reducing unnecessary provider switching.
+- Limiting parallel tasks that touch overlapping files.
+- Increasing parallelism for independent read-only tasks.
+- Routing short deterministic work to lower-cost providers.
+- Escalating complex review tasks to stronger models.
+
+## 6. Safety Boundaries
+
+The optimizer must never:
+
+- Skip mandatory review or E2E stages.
+- Remove project rules.
+- Override security policy.
+- Execute `build`, `migrate`, or `commit` without approval.
+- Modify acceptance criteria.
+- Hide failed tests.
+
+## 7. Algorithm
+
+```text
+collect metrics
+ -> normalize provider/task statistics
+ -> detect bottlenecks
+ -> evaluate safe optimizations
+ -> rank recommendations
+ -> remove conflicting recommendations
+ -> emit advisory plan
+```
+
+## 8. Runtime Events
+
+```ts
+type PerformanceOptimizerEvent =
+  | { type: "performance.analysis.started"; jobId: string }
+  | { type: "performance.recommendation.created"; jobId: string; recommendation: PerformanceRecommendation }
+  | { type: "performance.analysis.completed"; jobId: string }
+  | { type: "performance.analysis.failed"; jobId: string; error: string };
+```
+
+## 9. Failure Handling
+
+- Missing metrics: return no-op recommendation set.
+- Conflicting recommendations: keep the safer option.
+- Unknown provider capability: do not route to that provider.
+- Incomplete task graph: disable concurrency recommendations.
+- Invalid policy: fail with typed configuration error.
+
+## 10. Tests
+
+- Recommends context reuse when hashes match.
+- Does not skip required E2E stage.
+- Reduces concurrency when file overlap exists.
+- Does not choose unavailable provider.
+- Produces deterministic recommendation ranking.
+- Emits no-op plan when metrics are insufficient.
+
+## 11. Acceptance Criteria
+
+- Output is advisory and typed.
+- Recommendations are deterministic for identical inputs.
+- No mandatory stage is removed.
+- Every recommendation contains a reason.
+- Policy conflicts are rejected.

package/RFC/0057_EVALUATION_ENGINE.md

@@ -0,0 +1,116 @@
+# RFC-0057 — Evaluation Engine
+
+## 1. Purpose
+
+The Evaluation Engine determines whether an agent-produced result is acceptable before the runtime advances the task. It combines test results, static analysis, policy checks, diff review, artifact validation, and task-specific acceptance criteria.
+
+## 2. Inputs
+
+```ts
+export interface EvaluationRequest {
+  jobId: string;
+  task: CompiledTask;
+  worktreePath: string;
+  outputs: TaskOutputArtifact[];
+  testResults: TestResult[];
+  policyFindings: PolicyFinding[];
+}
+```
+
+## 3. Outputs
+
+```ts
+export interface EvaluationResult {
+  status: "passed" | "failed" | "needs_review";
+  score: number;
+  findings: EvaluationFinding[];
+  evidence: EvaluationEvidence[];
+  recommendedAction: "continue" | "repair" | "human_review";
+}
+```
+
+## 4. Evaluation Dimensions
+
+- Correctness
+- Test completeness
+- Acceptance criteria coverage
+- Policy compliance
+- Code quality
+- Security risk
+- Regression risk
+- Documentation completeness
+
+## 5. Scoring
+
+```text
+final_score =
+  correctness_weight
+  + test_weight
+  + acceptance_weight
+  + quality_weight
+  - policy_penalty
+  - security_penalty
+  - regression_penalty
+```
+
+A critical policy or security finding forces failure regardless of numeric score.
+
+## 6. Evidence Contract
+
+```ts
+export interface EvaluationEvidence {
+  kind: "test" | "diff" | "file" | "policy" | "screenshot" | "trace";
+  path?: string;
+  description: string;
+  hash?: string;
+}
+```
+
+The engine must not accept an agent's self-reported completion without external evidence.
+
+## 7. State Flow
+
+```text
+pending
+ -> collecting_evidence
+ -> evaluating
+ -> passed
+      or failed
+      or needs_review
+```
+
+## 8. Failure Classification
+
+```ts
+export type EvaluationFailureClass =
+  | "test_failure"
+  | "missing_output"
+  | "acceptance_gap"
+  | "policy_violation"
+  | "security_issue"
+  | "regression_risk"
+  | "insufficient_evidence";
+```
+
+## 9. Integration
+
+- Passed: Task Graph marks task complete.
+- Failed: Repair Engine creates repair task.
+- Needs review: Human Approval Gate or reviewer agent receives evidence bundle.
+
+## 10. Tests
+
+- Fails when required artifact is missing.
+- Fails on critical policy violation.
+- Passes when all acceptance criteria are evidenced.
+- Returns needs-review for ambiguous manual criterion.
+- Preserves evidence references.
+- Produces deterministic score.
+
+## 11. Acceptance Criteria
+
+- Every result includes findings and evidence.
+- Critical policy findings force failure.
+- Self-reported completion alone is insufficient.
+- Each acceptance criterion is mapped to evidence or marked unmet.
+- Evaluation results are persisted for replay.

package/RFC/0058_LEARNING_ENGINE.md

@@ -0,0 +1,113 @@
+# RFC-0058 — Learning Engine
+
+## 1. Purpose
+
+The Learning Engine extracts reusable lessons from completed and failed jobs. Its output improves future planning, routing, prompt compilation, repair selection, and evaluation.
+
+The engine learns recommendations, not autonomous policy. Human-approved project rules remain authoritative.
+
+## 2. Inputs
+
+```ts
+export interface LearningRequest {
+  jobId: string;
+  events: RuntimeEvent[];
+  evaluation: EvaluationResult[];
+  repairHistory: RepairAttempt[];
+  providerMetrics: ProviderExecutionMetric[];
+  humanFeedback?: HumanFeedback[];
+}
+```
+
+## 3. Outputs
+
+```ts
+export interface LearnedExperience {
+  id: string;
+  scope: "global" | "framework" | "repository" | "task_type";
+  pattern: string;
+  recommendation: string;
+  confidence: number;
+  evidenceRefs: string[];
+  status: "proposed" | "approved" | "rejected";
+}
+```
+
+## 4. Learnable Patterns
+
+- Provider success by task type.
+- Prompt structures associated with fewer retries.
+- Common failure causes.
+- Effective repair strategies.
+- Framework-specific command patterns.
+- Test scenarios that catch recurring regressions.
+- Context sources frequently used in successful jobs.
+
+## 5. Non-Learnable Data
+
+The engine must not learn from or retain:
+
+- Credentials
+- Browser cookies
+- Personal data not required for engineering
+- Secret environment values
+- Unreviewed destructive commands
+
+## 6. Confidence
+
+Confidence increases with:
+
+- Repeated independent occurrences
+- Consistent positive evaluation
+- Human approval
+- Framework-specific corroboration
+
+Confidence decreases with:
+
+- Contradictory outcomes
+- Single-event evidence
+- Stale repository revisions
+- Changed project rules
+
+## 7. Approval Flow
+
+```text
+experience extracted
+ -> proposed
+ -> human or policy review
+ -> approved
+      or rejected
+```
+
+Only approved experience may affect routing or planning automatically.
+
+## 8. OKF Integration
+
+Approved lessons may be exported as OKF concepts:
+
+```yaml
+---
+type: Engineering Lesson
+title: Prefer isolated worktrees for parallel file changes
+tags: [git, worktree, concurrency]
+timestamp: 2026-07-13T00:00:00+07:00
+authority: approved
+---
+```
+
+## 9. Tests
+
+- Extracts repeated failure pattern.
+- Does not approve experience automatically.
+- Redacts secret-like values.
+- Low evidence produces low confidence.
+- Human approval raises status to approved.
+- Approved lesson exports to OKF.
+
+## 10. Acceptance Criteria
+
+- Every learned item has evidence references.
+- Automatic learning cannot override project rules.
+- Secrets are excluded.
+- Only approved lessons affect runtime behavior.
+- Lessons can be exported to OKF.