selftune 0.2.23 → 0.2.24

package/cli/selftune/evolution/validate-body.ts

@@ -7,19 +7,22 @@
  *   Gate 3 (quality): Student model rates body clarity/completeness 0.0-1.0
  *
  * Gate 2 now supports replay-backed validation (via replay engine) in addition
- * to LLM-judge-based checking. When replay options are provided and succeed,
- * the replay path is preferred. Falls back to LLM judge otherwise.
+ * to LLM-judge-based checking. When real host/runtime replay is available and
+ * succeeds, the replay path is preferred. Falls back to LLM judge otherwise.
  */
 
 import type {
   BodyEvolutionProposal,
   BodyValidationResult,
   EvalEntry,
+  RoutingReplayEntryResult,
+  ValidationGate,
   ValidationMode,
 } from "../types.js";
 import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
 import { runJudgeValidation } from "./engines/judge-engine.js";
-import { runReplayValidation, type ReplayValidationOptions } from "./engines/replay-engine.js";
+import type { ReplayValidationOptions } from "./engines/replay-engine.js";
+import { runValidationContract, type ValidationStrategy } from "./validation-contract.js";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -28,6 +31,8 @@ import { runReplayValidation, type ReplayValidationOptions } from "./engines/rep
 export interface BodyValidationOptions {
   /** Replay options for Gate 2 trigger accuracy. */
   replay?: ReplayValidationOptions;
+  mode?: ValidationStrategy;
+  onReplayFallback?: (reason?: string) => void;
 }
 
 // ---------------------------------------------------------------------------
@@ -83,7 +88,7 @@ export function validateBodyStructure(proposedBody: string): { valid: boolean; r
  * Run trigger checks on the eval set using the proposed body content.
  * Returns before/after pass rates.
  *
- * When replay options are provided, attempts replay-backed validation first.
+ * When replay options are provided, attempts host/runtime replay first.
  * Falls back to LLM judge when replay is unavailable or no options given.
  */
 export async function validateBodyTriggerAccuracy(
@@ -99,6 +104,9 @@ export async function validateBodyTriggerAccuracy(
   improved: boolean;
   regressions: string[];
   validation_mode: ValidationMode;
+  validation_agent?: string;
+  validation_fixture_id?: string;
+  validation_fallback_reason?: string;
   per_entry_results?: import("../types.js").RoutingReplayEntryResult[];
   before_entry_results?: import("../types.js").RoutingReplayEntryResult[];
 }> {
@@ -109,48 +117,64 @@ export async function validateBodyTriggerAccuracy(
       improved: false,
       regressions: [],
       validation_mode: "llm_judge",
+      validation_agent: agent,
     };
   }
 
-  // Try replay-backed validation when options are provided
-  if (options?.replay) {
-    const replayResult = await runReplayValidation(
-      originalBody,
-      proposedBody,
-      evalSet,
-      agent,
-      options.replay,
-    );
-
-    if (replayResult) {
-      return {
-        before_pass_rate: replayResult.before_pass_rate,
-        after_pass_rate: replayResult.after_pass_rate,
-        improved: replayResult.improved,
-        regressions: [],
-        validation_mode: replayResult.validation_mode,
-        per_entry_results: replayResult.per_entry_results,
-        before_entry_results: replayResult.before_entry_results,
-      };
-    }
-  }
-
-  // Fall back to LLM judge
-  const judgeResult = await runJudgeValidation(
-    originalBody,
-    proposedBody,
+  const { result, fallbackReason } = await runValidationContract<{
+    before_pass_rate: number;
+    after_pass_rate: number;
+    improved: boolean;
+    regressions: string[];
+    validation_mode: ValidationMode;
+    validation_agent?: string;
+    validation_fixture_id?: string;
+    validation_fallback_reason?: string;
+    per_entry_results?: RoutingReplayEntryResult[];
+    before_entry_results?: RoutingReplayEntryResult[];
+  }>({
+    mode: options?.mode ?? "auto",
+    originalContent: originalBody,
+    proposedContent: proposedBody,
     evalSet,
     agent,
-    modelFlag,
-  );
+    replayOptions: options?.replay,
+    runJudge: async () => {
+      const judgeResult = await runJudgeValidation(
+        originalBody,
+        proposedBody,
+        evalSet,
+        agent,
+        modelFlag,
+      );
 
-  return {
-    before_pass_rate: judgeResult.before_pass_rate,
-    after_pass_rate: judgeResult.after_pass_rate,
-    improved: judgeResult.improved,
-    regressions: judgeResult.regressions,
-    validation_mode: judgeResult.validation_mode,
-  };
+      return {
+        result: {
+          before_pass_rate: judgeResult.before_pass_rate,
+          after_pass_rate: judgeResult.after_pass_rate,
+          improved: judgeResult.improved,
+          regressions: judgeResult.regressions,
+          validation_mode: judgeResult.validation_mode,
+          validation_agent: judgeResult.validation_agent,
+        },
+        modeUsed: judgeResult.validation_mode,
+      };
+    },
+    onReplayFallback: options?.onReplayFallback,
+    adaptReplayResult: (replayResult) => ({
+      before_pass_rate: replayResult.before_pass_rate,
+      after_pass_rate: replayResult.after_pass_rate,
+      improved: replayResult.improved,
+      regressions: [],
+      validation_mode: replayResult.validation_mode,
+      validation_agent: replayResult.validation_agent,
+      validation_fixture_id: replayResult.validation_fixture_id,
+      per_entry_results: replayResult.per_entry_results,
+      before_entry_results: replayResult.before_entry_results,
+    }),
+  });
+
+  return fallbackReason ? { ...result, validation_fallback_reason: fallbackReason } : result;
 }
 
 // ---------------------------------------------------------------------------
@@ -224,7 +248,7 @@ export async function validateBodyProposal(
   qualityThreshold = QUALITY_THRESHOLD,
   options?: BodyValidationOptions,
 ): Promise<BodyValidationResult> {
-  const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
+  const gateResults: Array<{ gate: ValidationGate; passed: boolean; reason: string }> = [];
 
   // Gate 1: Structural validation (pure code)
   const structural = validateBodyStructure(proposal.proposed_body);
@@ -287,7 +311,9 @@ export async function validateBodyProposal(
     improved: gatesPassed === 3,
     regressions: accuracy.regressions,
     validation_mode: accuracy.validation_mode,
-    validation_agent: agent,
+    validation_agent: accuracy.validation_agent ?? agent,
+    validation_fallback_reason: accuracy.validation_fallback_reason,
+    validation_fixture_id: accuracy.validation_fixture_id,
     ...(evalSet.length > 0
       ? {
           before_pass_rate: accuracy.before_pass_rate,