selftune 0.2.22 → 0.2.24

package/cli/selftune/evolution/evolve-body.ts

@@ -12,6 +12,10 @@ import { parseArgs } from "node:util";
 import { buildEvalSet } from "../eval/hooks-to-evals.js";
 import { readGradingResultsForSkill } from "../grading/results.js";
 import { getDb } from "../localdb/db.js";
+import {
+  type ReplayEntryResultInput,
+  writeReplayEntryResultsToDb,
+} from "../localdb/direct-write.js";
 import { queryQueryLog, querySkillUsageRecords } from "../localdb/queries.js";
 import type {
   BodyEvolutionProposal,
@@ -23,12 +27,11 @@ import type {
   FailurePattern,
   GradingResult,
   QueryLogRecord,
-  RoutingReplayFixture,
   SkillUsageRecord,
 } from "../types.js";
 import { CLIError, handleCLIError } from "../utils/cli-error.js";
 import type { EffortLevel, SubagentCallOptions } from "../utils/llm-call.js";
-import { callViaSubagent } from "../utils/llm-call.js";
+import { callViaSubagent, detectLlmAgent } from "../utils/llm-call.js";
 import { appendAuditEntry } from "./audit.js";
 import { checkConstitutionSizeOnly } from "./constitutional.js";
 import { parseSkillSections, replaceBody, replaceSection } from "./deploy-proposal.js";
@@ -37,12 +40,11 @@ import { extractFailurePatterns } from "./extract-patterns.js";
 import { type ExecutionContext, generateBodyProposal } from "./propose-body.js";
 import { generateRoutingProposal } from "./propose-routing.js";
 import { refineBodyProposal } from "./refine-body.js";
+import type { BodyValidationOptions } from "./validate-body.js";
 import { validateBodyProposal } from "./validate-body.js";
-import {
-  buildRoutingReplayFixture,
-  runClaudeRuntimeReplayFixture,
-} from "./validate-host-replay.js";
+import { buildRuntimeReplayValidationOptions } from "./validate-host-replay.js";
 import { validateRoutingProposal } from "./validate-routing.js";
+import { DEFAULT_VALIDATION_STRATEGY, type ValidationStrategy } from "./validation-contract.js";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -64,6 +66,7 @@ export interface EvolveBodyOptions {
   fewShotExamples?: string[];
   gradingResults?: GradingResult[];
   validationModel?: string;
+  validationMode?: ValidationStrategy;
   teacherEffort?: EffortLevel;
   /** Run evolution-reviewer subagent as Gate 4 before deployment. */
   useReviewer?: boolean;
@@ -171,6 +174,7 @@ export async function evolveBody(
   const teacherModel = options.teacherModel ?? DEFAULT_TEACHER_MODEL;
   const studentModel = options.studentModel ?? DEFAULT_STUDENT_MODEL;
   const teacherEffort = options.teacherEffort ?? DEFAULT_TEACHER_EFFORT;
+  const effectiveValidationMode = options.validationMode ?? DEFAULT_VALIDATION_STRATEGY;
 
   // Resolve injectable dependencies
   const _extractFailurePatterns = _deps.extractFailurePatterns ?? extractFailurePatterns;
@@ -463,54 +467,92 @@ export async function evolveBody(
       // Validate (validationModel overrides studentModel for validation calls)
       const validationModelFlag = options.validationModel ?? studentModel;
       let validation: BodyValidationResult;
+      let replayFallbackReason: string | undefined;
+
+      // Build replay fixture + runner for targets that can use runtime replay.
+      const replayOptions = buildRuntimeReplayValidationOptions({
+        skillName,
+        skillPath,
+        agent: studentAgent,
+        contentTarget: target === "body" ? "body" : "routing",
+      });
+      const replayFixture = replayOptions?.replayFixture;
+      const replayRunner = replayOptions?.replayRunner;
+
       if (target === "routing") {
-        const replayFixture = buildRoutingReplayFixture({
-          skillName,
-          skillPath,
-          platform: studentAgent === "codex" ? "codex" : "claude_code",
-        });
-        const replayRunner =
-          replayFixture.platform === "claude_code" && studentAgent === "claude"
-            ? async ({
-                routing,
-                evalSet,
-                fixture,
-              }: {
-                routing: string;
-                evalSet: EvalEntry[];
-                fixture: RoutingReplayFixture;
-              }) =>
-                await runClaudeRuntimeReplayFixture({
-                  routing,
-                  evalSet,
-                  fixture,
-                })
-            : undefined;
         validation = await _validateRoutingProposal(
           proposal,
           evalSet,
           studentAgent,
           validationModelFlag,
           {
-            replayFixture,
+            ...(replayFixture ? { replayFixture } : {}),
             ...(replayRunner ? { replayRunner } : {}),
+            mode: effectiveValidationMode,
+            onReplayFallback: (reason) => {
+              replayFallbackReason = reason;
+              if (reason) {
+                console.error(
+                  `[evolve-body] Replay not available (${reason}), falling back to LLM judge validation.`,
+                );
+                return;
+              }
+              console.error(
+                "[evolve-body] Replay not available, falling back to LLM judge validation.",
+              );
+            },
           },
         );
       } else {
+        const bodyReplayOptions: BodyValidationOptions = {
+          ...(replayFixture
+            ? {
+                replay: {
+                  replayFixture,
+                  ...(replayRunner ? { replayRunner } : {}),
+                },
+              }
+            : {}),
+          mode: effectiveValidationMode,
+          onReplayFallback: (reason) => {
+            replayFallbackReason = reason;
+            if (reason) {
+              console.error(
+                `[evolve-body] Replay not available (${reason}), falling back to LLM judge validation.`,
+              );
+              return;
+            }
+            console.error(
+              "[evolve-body] Replay not available, falling back to LLM judge validation.",
+            );
+          },
+        };
         validation = await _validateBodyProposal(
           proposal,
           evalSet,
           studentAgent,
           validationModelFlag,
+          undefined,
+          bodyReplayOptions,
         );
       }
+      if (replayFallbackReason && !validation.validation_fallback_reason) {
+        validation = {
+          ...validation,
+          validation_fallback_reason: replayFallbackReason,
+        };
+      }
       lastValidation = validation;
       const validatedEvidenceRef = buildValidationEvidenceRef(proposal.proposal_id, "validated");
 
       recordAudit(
         proposal.proposal_id,
         "validated",
-        `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed`,
+        `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed${
+          validation.validation_fallback_reason
+            ? ` (replay fallback: ${validation.validation_fallback_reason})`
+            : ""
+        }`,
         {
           validation_mode: validation.validation_mode,
           validation_agent: validation.validation_agent,
@@ -527,7 +569,11 @@ export async function evolveBody(
         stage: "validated",
         rationale: proposal.rationale,
         confidence: proposal.confidence,
-        details: `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed`,
+        details: `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed${
+          validation.validation_fallback_reason
+            ? ` (replay fallback: ${validation.validation_fallback_reason})`
+            : ""
+        }`,
         validation: {
           improved: validation.improved,
           gates_passed: validation.gates_passed,
@@ -539,10 +585,51 @@ export async function evolveBody(
           validation_mode: validation.validation_mode,
           validation_agent: validation.validation_agent,
           validation_fixture_id: validation.validation_fixture_id,
+          validation_fallback_reason: validation.validation_fallback_reason,
           validation_evidence_ref: validatedEvidenceRef,
         },
       });
 
+      // Persist per-entry replay results to SQLite
+      try {
+        const entryResults: ReplayEntryResultInput[] = [];
+        if (validation.before_entry_results) {
+          for (const r of validation.before_entry_results) {
+            entryResults.push({
+              proposal_id: proposal.proposal_id,
+              skill_name: skillName,
+              validation_mode: validation.validation_mode ?? "llm_judge",
+              phase: "before",
+              query: r.query,
+              should_trigger: r.should_trigger,
+              triggered: r.triggered,
+              passed: r.passed,
+              evidence: r.evidence,
+            });
+          }
+        }
+        if (validation.per_entry_results) {
+          for (const r of validation.per_entry_results) {
+            entryResults.push({
+              proposal_id: proposal.proposal_id,
+              skill_name: skillName,
+              validation_mode: validation.validation_mode ?? "llm_judge",
+              phase: "after",
+              query: r.query,
+              should_trigger: r.should_trigger,
+              triggered: r.triggered,
+              passed: r.passed,
+              evidence: r.evidence,
+            });
+          }
+        }
+        if (entryResults.length > 0) {
+          writeReplayEntryResultsToDb(entryResults);
+        }
+      } catch {
+        // Fail-open: replay entry persistence is non-blocking
+      }
+
       if (validation.improved) {
         break;
       }
@@ -550,7 +637,11 @@ export async function evolveBody(
       recordAudit(
         proposal.proposal_id,
         "rejected",
-        `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates`,
+        `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates${
+          validation.validation_fallback_reason
+            ? ` (replay fallback: ${validation.validation_fallback_reason})`
+            : ""
+        }`,
         {
           validation_mode: validation.validation_mode,
           validation_agent: validation.validation_agent,
@@ -567,7 +658,11 @@ export async function evolveBody(
         stage: "rejected",
         rationale: proposal.rationale,
         confidence: proposal.confidence,
-        details: `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates`,
+        details: `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates${
+          validation.validation_fallback_reason
+            ? ` (replay fallback: ${validation.validation_fallback_reason})`
+            : ""
+        }`,
         validation: {
           improved: validation.improved,
           gates_passed: validation.gates_passed,
@@ -579,6 +674,7 @@ export async function evolveBody(
           validation_mode: validation.validation_mode,
           validation_agent: validation.validation_agent,
           validation_fixture_id: validation.validation_fixture_id,
+          validation_fallback_reason: validation.validation_fallback_reason,
           validation_evidence_ref: buildValidationEvidenceRef(proposal.proposal_id, "rejected"),
         },
       });
@@ -678,7 +774,11 @@ export async function evolveBody(
       recordAudit(
         lastProposal.proposal_id,
         "deployed",
-        `Deployed ${target} proposal for ${skillName}`,
+        `Deployed ${target} proposal for ${skillName}${
+          lastValidation.validation_fallback_reason
+            ? ` (replay fallback: ${lastValidation.validation_fallback_reason})`
+            : ""
+        }`,
         {
           validation_mode: lastValidation.validation_mode,
           validation_agent: lastValidation.validation_agent,
@@ -707,6 +807,7 @@ export async function evolveBody(
           validation_mode: lastValidation.validation_mode,
           validation_agent: lastValidation.validation_agent,
           validation_fixture_id: lastValidation.validation_fixture_id,
+          validation_fallback_reason: lastValidation.validation_fallback_reason,
           validation_evidence_ref: buildValidationEvidenceRef(lastProposal.proposal_id, "deployed"),
         },
       });
@@ -760,6 +861,7 @@ export async function cliMain(): Promise<void> {
       "task-description": { type: "string" },
       "few-shot": { type: "string" },
       "validation-model": { type: "string" },
+      "validation-mode": { type: "string", default: DEFAULT_VALIDATION_STRATEGY },
       "teacher-effort": { type: "string", default: "high" },
       review: { type: "boolean", default: false },
       help: { type: "boolean", default: false },
@@ -788,6 +890,7 @@ Options:
   --task-description  Optional task description context
   --few-shot          Comma-separated paths to example skill files
   --validation-model  Model for trigger-check validation calls (overrides --student-model for validation)
+  --validation-mode   Validation strategy: auto, replay, or judge (default: auto)
   --teacher-effort    Effort level for teacher LLM: low, medium, high, max (default: high)
   --review            Run evolution-reviewer subagent before deployment (Gate 4)
   --help              Show this help message`);
@@ -802,15 +905,24 @@ Options:
     );
   }
 
-  const { detectAgent } = await import("../utils/llm-call.js");
-  const teacherAgent = values["teacher-agent"] ?? detectAgent() ?? "";
+  if (
+    values["validation-mode"] &&
+    !["auto", "replay", "judge"].includes(values["validation-mode"])
+  ) {
+    throw new CLIError(
+      `Invalid --validation-mode value: ${values["validation-mode"]}`,
+      "INVALID_FLAG",
+      "Use one of: auto, replay, judge",
+    );
+  }
+  const teacherAgent = values["teacher-agent"] ?? detectLlmAgent() ?? "";
   const studentAgent = values["student-agent"] ?? teacherAgent;
 
   if (!teacherAgent) {
     throw new CLIError(
-      "No agent CLI found. Install Claude Code, Codex, or OpenCode.",
+      "No agent CLI found. Install Claude Code, Codex, OpenCode, or Pi.",
       "AGENT_NOT_FOUND",
-      "Install Claude Code, Codex, or OpenCode.",
+      "Install Claude Code, Codex, OpenCode, or Pi.",
     );
   }
 
@@ -848,6 +960,8 @@ Options:
     fewShotExamples,
     gradingResults,
     validationModel: values["validation-model"],
+    validationMode:
+      (values["validation-mode"] as ValidationStrategy | undefined) ?? DEFAULT_VALIDATION_STRATEGY,
     teacherEffort: (values["teacher-effort"] as EffortLevel) ?? "high",
     useReviewer: values.review ?? false,
   });