selftune 0.2.22 → 0.2.24

package/cli/selftune/evolution/validate-body.ts

@@ -3,13 +3,37 @@
  *
  * 3-gate validation for full body evolution proposals:
  *   Gate 1 (structural): Pure code — YAML frontmatter, # Title, ## Workflow Routing preserved
- *   Gate 2 (trigger accuracy): Student model YES/NO per eval entry
+ *   Gate 2 (trigger accuracy): Replay-backed or student model YES/NO per eval entry
  *   Gate 3 (quality): Student model rates body clarity/completeness 0.0-1.0
+ *
+ * Gate 2 now supports replay-backed validation (via replay engine) in addition
+ * to LLM-judge-based checking. When real host/runtime replay is available and
+ * succeeds, the replay path is preferred. Falls back to LLM judge otherwise.
  */
 
-import type { BodyEvolutionProposal, BodyValidationResult, EvalEntry } from "../types.js";
+import type {
+  BodyEvolutionProposal,
+  BodyValidationResult,
+  EvalEntry,
+  RoutingReplayEntryResult,
+  ValidationGate,
+  ValidationMode,
+} from "../types.js";
 import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
-import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
+import { runJudgeValidation } from "./engines/judge-engine.js";
+import type { ReplayValidationOptions } from "./engines/replay-engine.js";
+import { runValidationContract, type ValidationStrategy } from "./validation-contract.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface BodyValidationOptions {
+  /** Replay options for Gate 2 trigger accuracy. */
+  replay?: ReplayValidationOptions;
+  mode?: ValidationStrategy;
+  onReplayFallback?: (reason?: string) => void;
+}
 
 // ---------------------------------------------------------------------------
 // Gate 1: Structural validation (pure code, no LLM)
@@ -57,12 +81,15 @@ export function validateBodyStructure(proposedBody: string): { valid: boolean; r
 }
 
 // ---------------------------------------------------------------------------
-// Gate 2: Trigger accuracy (student model YES/NO)
+// Gate 2: Trigger accuracy (replay-backed or student model YES/NO)
 // ---------------------------------------------------------------------------
 
 /**
  * Run trigger checks on the eval set using the proposed body content.
  * Returns before/after pass rates.
+ *
+ * When replay options are provided, attempts host/runtime replay first.
+ * Falls back to LLM judge when replay is unavailable or no options given.
  */
 export async function validateBodyTriggerAccuracy(
   originalBody: string,
@@ -70,55 +97,84 @@ export async function validateBodyTriggerAccuracy(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
+  options?: BodyValidationOptions,
 ): Promise<{
   before_pass_rate: number;
   after_pass_rate: number;
   improved: boolean;
   regressions: string[];
+  validation_mode: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_fallback_reason?: string;
+  per_entry_results?: import("../types.js").RoutingReplayEntryResult[];
+  before_entry_results?: import("../types.js").RoutingReplayEntryResult[];
 }> {
   if (evalSet.length === 0) {
-    return { before_pass_rate: 0, after_pass_rate: 0, improved: false, regressions: [] };
-  }
-
-  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
-  let beforePassed = 0;
-  let afterPassed = 0;
-  const regressions: string[] = [];
-
-  for (const entry of evalSet) {
-    // Check with original body
-    const beforePrompt = buildTriggerCheckPrompt(originalBody, entry.query);
-    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
-    const beforeTriggered = parseTriggerResponse(beforeRaw);
-    const beforePass =
-      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
-
-    // Check with proposed body
-    const afterPrompt = buildTriggerCheckPrompt(proposedBody, entry.query);
-    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
-    const afterTriggered = parseTriggerResponse(afterRaw);
-    const afterPass =
-      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
-
-    if (beforePass) beforePassed++;
-    if (afterPass) afterPassed++;
-
-    // Track regressions
-    if (beforePass && !afterPass) {
-      regressions.push(entry.query);
-    }
+    return {
+      before_pass_rate: 0,
+      after_pass_rate: 0,
+      improved: false,
+      regressions: [],
+      validation_mode: "llm_judge",
+      validation_agent: agent,
+    };
   }
 
-  const total = evalSet.length;
-  const beforePassRate = beforePassed / total;
-  const afterPassRate = afterPassed / total;
+  const { result, fallbackReason } = await runValidationContract<{
+    before_pass_rate: number;
+    after_pass_rate: number;
+    improved: boolean;
+    regressions: string[];
+    validation_mode: ValidationMode;
+    validation_agent?: string;
+    validation_fixture_id?: string;
+    validation_fallback_reason?: string;
+    per_entry_results?: RoutingReplayEntryResult[];
+    before_entry_results?: RoutingReplayEntryResult[];
+  }>({
+    mode: options?.mode ?? "auto",
+    originalContent: originalBody,
+    proposedContent: proposedBody,
+    evalSet,
+    agent,
+    replayOptions: options?.replay,
+    runJudge: async () => {
+      const judgeResult = await runJudgeValidation(
+        originalBody,
+        proposedBody,
+        evalSet,
+        agent,
+        modelFlag,
+      );
+
+      return {
+        result: {
+          before_pass_rate: judgeResult.before_pass_rate,
+          after_pass_rate: judgeResult.after_pass_rate,
+          improved: judgeResult.improved,
+          regressions: judgeResult.regressions,
+          validation_mode: judgeResult.validation_mode,
+          validation_agent: judgeResult.validation_agent,
+        },
+        modeUsed: judgeResult.validation_mode,
+      };
+    },
+    onReplayFallback: options?.onReplayFallback,
+    adaptReplayResult: (replayResult) => ({
+      before_pass_rate: replayResult.before_pass_rate,
+      after_pass_rate: replayResult.after_pass_rate,
+      improved: replayResult.improved,
+      regressions: [],
+      validation_mode: replayResult.validation_mode,
+      validation_agent: replayResult.validation_agent,
+      validation_fixture_id: replayResult.validation_fixture_id,
+      per_entry_results: replayResult.per_entry_results,
+      before_entry_results: replayResult.before_entry_results,
+    }),
+  });
 
-  return {
-    before_pass_rate: beforePassRate,
-    after_pass_rate: afterPassRate,
-    improved: afterPassRate > beforePassRate,
-    regressions,
-  };
+  return fallbackReason ? { ...result, validation_fallback_reason: fallbackReason } : result;
 }
 
 // ---------------------------------------------------------------------------
@@ -190,8 +246,9 @@ export async function validateBodyProposal(
   agent: string,
   modelFlag?: string,
   qualityThreshold = QUALITY_THRESHOLD,
+  options?: BodyValidationOptions,
 ): Promise<BodyValidationResult> {
-  const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
+  const gateResults: Array<{ gate: ValidationGate; passed: boolean; reason: string }> = [];
 
   // Gate 1: Structural validation (pure code)
   const structural = validateBodyStructure(proposal.proposed_body);
@@ -214,20 +271,21 @@ export async function validateBodyProposal(
     };
   }
 
-  // Gate 2: Trigger accuracy (student model)
+  // Gate 2: Trigger accuracy (replay-backed or student model)
   const accuracy = await validateBodyTriggerAccuracy(
     proposal.original_body,
     proposal.proposed_body,
     evalSet,
     agent,
     modelFlag,
+    options,
   );
   gateResults.push({
     gate: "trigger_accuracy",
     passed: accuracy.improved,
     reason: accuracy.improved
-      ? `Improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`
-      : `Not improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`,
+      ? `Improved via ${accuracy.validation_mode}: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`
+      : `Not improved via ${accuracy.validation_mode}: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`,
   });
 
   // Gate 3: Quality assessment (student model)
@@ -252,13 +310,17 @@ export async function validateBodyProposal(
     gate_results: gateResults,
     improved: gatesPassed === 3,
     regressions: accuracy.regressions,
-    validation_mode: "llm_judge",
-    validation_agent: agent,
+    validation_mode: accuracy.validation_mode,
+    validation_agent: accuracy.validation_agent ?? agent,
+    validation_fallback_reason: accuracy.validation_fallback_reason,
+    validation_fixture_id: accuracy.validation_fixture_id,
     ...(evalSet.length > 0
       ? {
           before_pass_rate: accuracy.before_pass_rate,
           after_pass_rate: accuracy.after_pass_rate,
         }
       : {}),
+    per_entry_results: accuracy.per_entry_results,
+    before_entry_results: accuracy.before_entry_results,
   };
 }