selftune 0.2.0 → 0.2.2

package/cli/selftune/evolution/evolve.ts

@@ -13,11 +13,14 @@ import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
 import type { BaselineMeasurement } from "../eval/baseline.js";
 import { measureBaseline } from "../eval/baseline.js";
 import { buildEvalSet } from "../eval/hooks-to-evals.js";
+import { readGradingResultsForSkill } from "../grading/results.js";
 import { updateContextAfterEvolve } from "../memory/writer.js";
+import type { SyncResult } from "../sync.js";
 import type {
   EvalEntry,
   EvalPassRate,
   EvolutionAuditEntry,
+  EvolutionEvidenceEntry,
   EvolutionProposal,
   EvolveResultSummary,
   FailurePattern,
@@ -29,8 +32,10 @@ import type {
 } from "../types.js";
 import { parseFrontmatter, replaceFrontmatterDescription } from "../utils/frontmatter.js";
 import { readJsonl } from "../utils/jsonl.js";
+import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
 import { createEvolveTUI } from "../utils/tui.js";
 import { appendAuditEntry } from "./audit.js";
+import { appendEvidenceEntry } from "./evidence.js";
 import { extractFailurePatterns } from "./extract-patterns.js";
 import {
   computeInvocationScores,
@@ -68,6 +73,8 @@ export interface EvolveOptions {
   cheapLoop?: boolean;
   gateModel?: string;
   proposalModel?: string;
+  syncFirst?: boolean;
+  syncForce?: boolean;
 }
 
 export interface EvolveResult {
@@ -81,6 +88,7 @@ export interface EvolveResult {
   elapsedMs: number;
   baselineResult?: BaselineMeasurement;
   gateValidation?: ValidationResult;
+  sync_result?: SyncResult;
 }
 
 /**
@@ -98,9 +106,12 @@ export interface EvolveDeps {
   validateProposal?: typeof import("./validate-proposal.js").validateProposal;
   gateValidateProposal?: typeof import("./validate-proposal.js").validateProposal;
   appendAuditEntry?: typeof import("./audit.js").appendAuditEntry;
+  appendEvidenceEntry?: typeof import("./evidence.js").appendEvidenceEntry;
   buildEvalSet?: typeof import("../eval/hooks-to-evals.js").buildEvalSet;
   updateContextAfterEvolve?: typeof import("../memory/writer.js").updateContextAfterEvolve;
   measureBaseline?: typeof import("../eval/baseline.js").measureBaseline;
+  readSkillUsageLog?: () => SkillUsageRecord[];
+  syncSources?: typeof import("../sync.js").syncSources;
 }
 
 // ---------------------------------------------------------------------------
@@ -124,6 +135,33 @@ function createAuditEntry(
   };
 }
 
+// ---------------------------------------------------------------------------
+// Diff helper
+// ---------------------------------------------------------------------------
+
+/**
+ * Produce a simple colored diff between two text strings.
+ * Red (removed) / Green (added) lines, skipping unchanged lines.
+ */
+function formatSimpleDiff(oldText: string, newText: string): string {
+  const oldLines = oldText.split("\n");
+  const newLines = newText.split("\n");
+  const output: string[] = [];
+  const maxLen = Math.max(oldLines.length, newLines.length);
+  for (let i = 0; i < maxLen; i++) {
+    const oldLine = oldLines[i];
+    const newLine = newLines[i];
+    if (oldLine === newLine) continue;
+    if (oldLine !== undefined) {
+      output.push(`\x1b[31m- ${oldLine}\x1b[0m`);
+    }
+    if (newLine !== undefined) {
+      output.push(`\x1b[32m+ ${newLine}\x1b[0m`);
+    }
+  }
+  return output.join("\n");
+}
+
 // ---------------------------------------------------------------------------
 // Main orchestrator
 // ---------------------------------------------------------------------------
@@ -148,11 +186,14 @@ export async function evolve(
   const _validateProposal = _deps.validateProposal ?? validateProposal;
   const _gateValidateProposal = _deps.gateValidateProposal ?? validateProposal;
   const _appendAuditEntry = _deps.appendAuditEntry ?? appendAuditEntry;
+  const _appendEvidenceEntry = _deps.appendEvidenceEntry ?? appendEvidenceEntry;
   const _buildEvalSet = _deps.buildEvalSet ?? buildEvalSet;
   const _updateContextAfterEvolve = _deps.updateContextAfterEvolve ?? updateContextAfterEvolve;
   const _measureBaseline = _deps.measureBaseline ?? measureBaseline;
+  const _readSkillUsageLog = _deps.readSkillUsageLog ?? (() => readEffectiveSkillUsageRecords());
 
   const auditEntries: EvolutionAuditEntry[] = [];
+  let syncResult: SyncResult | undefined;
 
   function recordAudit(
     proposalId: string,
@@ -169,6 +210,14 @@ export async function evolve(
     }
   }
 
+  function recordEvidence(entry: EvolutionEvidenceEntry): void {
+    try {
+      _appendEvidenceEntry(entry);
+    } catch {
+      // Fail-open: evidence should not block the pipeline
+    }
+  }
+
   const pipelineStart = Date.now();
   let llmCallCount = 0;
   const tui = createEvolveTUI({ skillName, model: options.proposalModel ?? "(default)" });
@@ -182,6 +231,7 @@ export async function evolve(
     ...r,
     llmCallCount,
     elapsedMs: Date.now() - pipelineStart,
+    ...(syncResult ? { sync_result: syncResult } : {}),
   });
 
   // Hoisted so catch block can preserve partial results on error
@@ -209,21 +259,65 @@ export async function evolve(
     const currentDescription = frontmatter.description || rawContent;
     const skillVersion = frontmatter.version || undefined;
     const versionTag = skillVersion ? `, v${skillVersion}` : "";
+    const createdAuditDetails = (message: string) =>
+      `original_description:${rawContent}\n${message}`;
     tui.done(`Loaded SKILL.md (desc: ${currentDescription.length} chars${versionTag})`);
 
+    if (options.syncFirst) {
+      tui.step(`Syncing source-truth telemetry${options.syncForce ? " (force)" : ""}...`);
+      const { createDefaultSyncOptions, syncSources: realSyncSources } = await import("../sync.js");
+      const syncRunner = _deps.syncSources ?? realSyncSources;
+      syncResult = syncRunner(
+        createDefaultSyncOptions({
+          force: options.syncForce ?? false,
+        }),
+      );
+      const sourceSynced = Object.values(syncResult.sources).reduce(
+        (sum, source) => sum + source.synced,
+        0,
+      );
+      tui.done(
+        `Source sync complete (${sourceSynced} source sessions, ${syncResult.repair.repaired_records} repaired records)`,
+      );
+    }
+
     // -----------------------------------------------------------------------
     // Step 2: Load eval set
     // -----------------------------------------------------------------------
+    const skillUsage = _readSkillUsageLog();
     let evalSet: EvalEntry[];
 
     if (evalSetPath && existsSync(evalSetPath)) {
-      const raw = readFileSync(evalSetPath, "utf-8");
-      evalSet = JSON.parse(raw) as EvalEntry[];
+      try {
+        const raw = readFileSync(evalSetPath, "utf-8");
+        evalSet = JSON.parse(raw) as EvalEntry[];
+      } catch (parseErr) {
+        const msg = parseErr instanceof Error ? parseErr.message : String(parseErr);
+        tui.fail(`Failed to load eval set from ${evalSetPath}: ${msg}`);
+        finishTui();
+        return withStats({
+          proposal: null,
+          validation: null,
+          deployed: false,
+          auditEntries,
+          reason: `Failed to load eval set: ${msg}`,
+        });
+      }
+      if (!Array.isArray(evalSet)) {
+        tui.fail(`Eval set at ${evalSetPath} is not an array`);
+        finishTui();
+        return withStats({
+          proposal: null,
+          validation: null,
+          deployed: false,
+          auditEntries,
+          reason: `Eval set at ${evalSetPath} is not a JSON array`,
+        });
+      }
     } else {
       // Build from logs
-      const skillRecords = readJsonl<SkillUsageRecord>(SKILL_LOG);
       const queryRecords = readJsonl<QueryLogRecord>(QUERY_LOG);
-      evalSet = _buildEvalSet(skillRecords, queryRecords, skillName);
+      evalSet = _buildEvalSet(skillUsage, queryRecords, skillName);
     }
 
     const posCount = evalSet.filter((e) => e.should_trigger).length;
@@ -233,8 +327,6 @@ export async function evolve(
     // -----------------------------------------------------------------------
     // Step 3: Load skill usage records
     // -----------------------------------------------------------------------
-    const skillUsage = readJsonl<SkillUsageRecord>(SKILL_LOG);
-
     // -----------------------------------------------------------------------
     // Step 4: Extract failure patterns
     // -----------------------------------------------------------------------
@@ -251,17 +343,38 @@ export async function evolve(
     );
 
     // -----------------------------------------------------------------------
-    // Step 5: Early exit if no patterns
+    // Step 5: Cold-start bootstrap or early exit if no patterns
     // -----------------------------------------------------------------------
     if (failurePatterns.length === 0) {
-      finishTui();
-      return withStats({
-        proposal: null,
-        validation: null,
-        deployed: false,
-        auditEntries,
-        reason: "No failure patterns found",
-      });
+      // Cold-start: if the eval set has positive entries that the skill should
+      // match but there are zero skill usage records, treat the positive eval
+      // entries themselves as "missed queries" — they ARE the failure signal.
+      const positiveEvals = evalSet.filter((e) => e.should_trigger);
+      const hasSkillUsageHistory = skillUsage.some((record) => record.skill_name === skillName);
+      if (positiveEvals.length > 0 && !hasSkillUsageHistory) {
+        const coldStartPattern: FailurePattern = {
+          pattern_id: `fp-${skillName}-coldstart`,
+          skill_name: skillName,
+          invocation_type: "implicit",
+          missed_queries: positiveEvals.map((e) => e.query),
+          frequency: positiveEvals.length,
+          sample_sessions: [],
+          extracted_at: new Date().toISOString(),
+        };
+        failurePatterns.push(coldStartPattern);
+        tui.done(
+          `Cold-start bootstrap: ${positiveEvals.length} positive eval entries used as missed queries`,
+        );
+      } else {
+        finishTui();
+        return withStats({
+          proposal: null,
+          validation: null,
+          deployed: false,
+          auditEntries,
+          reason: "No failure patterns found",
+        });
+      }
     }
 
     // -----------------------------------------------------------------------
@@ -279,11 +392,14 @@ export async function evolve(
     const paretoEnabled = options.paretoEnabled ?? false;
     const candidateCount = options.candidateCount ?? 3;
     const tokenEfficiencyEnabled = options.tokenEfficiencyEnabled ?? false;
+    const telemetryRecords =
+      options.telemetryRecords ??
+      (tokenEfficiencyEnabled ? readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG) : undefined);
 
     // Compute token efficiency score if enabled and telemetry is available
     let tokenEffScore: number | undefined;
-    if (tokenEfficiencyEnabled && options.telemetryRecords && options.telemetryRecords.length > 0) {
-      tokenEffScore = computeTokenEfficiencyScore(skillName, options.telemetryRecords);
+    if (tokenEfficiencyEnabled && telemetryRecords && telemetryRecords.length > 0) {
+      tokenEffScore = computeTokenEfficiencyScore(skillName, telemetryRecords);
       recordAudit(
         "system",
         "created",
@@ -321,7 +437,25 @@ export async function evolve(
       // Validate each candidate
       const paretoCandidates: ParetoCandidate[] = [];
       for (const proposal of viableCandidates) {
-        recordAudit(proposal.proposal_id, "created", `Pareto candidate for ${skillName}`);
+        recordAudit(
+          proposal.proposal_id,
+          "created",
+          createdAuditDetails(`Pareto candidate for ${skillName}`),
+        );
+        recordEvidence({
+          timestamp: new Date().toISOString(),
+          proposal_id: proposal.proposal_id,
+          skill_name: skillName,
+          skill_path: skillPath,
+          target: "description",
+          stage: "created",
+          rationale: proposal.rationale,
+          confidence: proposal.confidence,
+          details: `Pareto candidate for ${skillName}`,
+          original_text: proposal.original_description,
+          proposed_text: proposal.proposed_description,
+          eval_set: evalSet,
+        });
 
         const validation = await _validateProposal(
           proposal,
@@ -334,6 +468,26 @@ export async function evolve(
           "validated",
           `Pareto validation: improved=${validation.improved}`,
         );
+        recordEvidence({
+          timestamp: new Date().toISOString(),
+          proposal_id: proposal.proposal_id,
+          skill_name: skillName,
+          skill_path: skillPath,
+          target: "description",
+          stage: "validated",
+          rationale: proposal.rationale,
+          confidence: proposal.confidence,
+          details: `Pareto validation: improved=${validation.improved}`,
+          validation: {
+            improved: validation.improved,
+            before_pass_rate: validation.before_pass_rate,
+            after_pass_rate: validation.after_pass_rate,
+            net_change: validation.net_change,
+            regressions: validation.regressions,
+            new_passes: validation.new_passes,
+            per_entry_results: validation.per_entry_results,
+          },
+        });
 
         if (validation.improved && validation.per_entry_results) {
           const invocationScores = computeInvocationScores(validation.per_entry_results);
@@ -398,8 +552,22 @@ export async function evolve(
         recordAudit(
           proposal.proposal_id,
           "created",
-          `Proposal created for ${skillName} (iteration ${iteration + 1})`,
+          createdAuditDetails(`Proposal created for ${skillName} (iteration ${iteration + 1})`),
         );
+        recordEvidence({
+          timestamp: new Date().toISOString(),
+          proposal_id: proposal.proposal_id,
+          skill_name: skillName,
+          skill_path: skillPath,
+          target: "description",
+          stage: "created",
+          rationale: proposal.rationale,
+          confidence: proposal.confidence,
+          details: `Proposal created for ${skillName} (iteration ${iteration + 1})`,
+          original_text: proposal.original_description,
+          proposed_text: proposal.proposed_description,
+          eval_set: evalSet,
+        });
 
         // Step 9: Check confidence threshold
         if (proposal.confidence < confidenceThreshold) {
@@ -409,6 +577,17 @@ export async function evolve(
             "rejected",
             `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
           );
+          recordEvidence({
+            timestamp: new Date().toISOString(),
+            proposal_id: proposal.proposal_id,
+            skill_name: skillName,
+            skill_path: skillPath,
+            target: "description",
+            stage: "rejected",
+            rationale: proposal.rationale,
+            confidence: proposal.confidence,
+            details: `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
+          });
 
           // If this is the last iteration, return early with rejection
           if (iteration === maxIterations - 1) {
@@ -455,6 +634,26 @@ export async function evolve(
           `Validation complete: improved=${validation.improved}`,
           evalSnapshot,
         );
+        recordEvidence({
+          timestamp: new Date().toISOString(),
+          proposal_id: proposal.proposal_id,
+          skill_name: skillName,
+          skill_path: skillPath,
+          target: "description",
+          stage: "validated",
+          rationale: proposal.rationale,
+          confidence: proposal.confidence,
+          details: `Validation complete: improved=${validation.improved}`,
+          validation: {
+            improved: validation.improved,
+            before_pass_rate: validation.before_pass_rate,
+            after_pass_rate: validation.after_pass_rate,
+            net_change: validation.net_change,
+            regressions: validation.regressions,
+            new_passes: validation.new_passes,
+            per_entry_results: validation.per_entry_results,
+          },
+        });
 
         // Step 12: Check validation result
         if (!validation.improved) {
@@ -464,6 +663,26 @@ export async function evolve(
             "rejected",
             `Validation failed: net_change=${validation.net_change.toFixed(3)}`,
           );
+          recordEvidence({
+            timestamp: new Date().toISOString(),
+            proposal_id: proposal.proposal_id,
+            skill_name: skillName,
+            skill_path: skillPath,
+            target: "description",
+            stage: "rejected",
+            rationale: proposal.rationale,
+            confidence: proposal.confidence,
+            details: `Validation failed: net_change=${validation.net_change.toFixed(3)}`,
+            validation: {
+              improved: validation.improved,
+              before_pass_rate: validation.before_pass_rate,
+              after_pass_rate: validation.after_pass_rate,
+              net_change: validation.net_change,
+              regressions: validation.regressions,
+              new_passes: validation.new_passes,
+              per_entry_results: validation.per_entry_results,
+            },
+          });
 
           // If this is the last iteration, return with rejection
           if (iteration === maxIterations - 1) {
@@ -583,12 +802,40 @@ export async function evolve(
       writeFileSync(skillPath, updatedContent, "utf-8");
       tui.done(`Deployed updated description to ${skillPath}`);
 
+      // Show what changed in the skill file
+      const diffOutput = formatSimpleDiff(rawContent, updatedContent);
+      if (diffOutput) {
+        console.error("\n--- Skill description diff ---");
+        console.error(diffOutput);
+        console.error("------------------------------\n");
+      }
+
       recordAudit(lastProposal.proposal_id, "deployed", `Deployed proposal for ${skillName}`, {
         total: evalSet.length,
         passed: Math.round(lastValidation.after_pass_rate * evalSet.length),
         failed: evalSet.length - Math.round(lastValidation.after_pass_rate * evalSet.length),
         pass_rate: lastValidation.after_pass_rate,
       });
+      recordEvidence({
+        timestamp: new Date().toISOString(),
+        proposal_id: lastProposal.proposal_id,
+        skill_name: skillName,
+        skill_path: skillPath,
+        target: "description",
+        stage: "deployed",
+        rationale: lastProposal.rationale,
+        confidence: lastProposal.confidence,
+        details: `Deployed proposal for ${skillName}`,
+        validation: {
+          improved: lastValidation.improved,
+          before_pass_rate: lastValidation.before_pass_rate,
+          after_pass_rate: lastValidation.after_pass_rate,
+          net_change: lastValidation.net_change,
+          regressions: lastValidation.regressions,
+          new_passes: lastValidation.new_passes,
+          per_entry_results: lastValidation.per_entry_results,
+        },
+      });
     }
 
     // -----------------------------------------------------------------------
@@ -654,9 +901,12 @@ export async function cliMain(): Promise<void> {
       "token-efficiency": { type: "boolean", default: false },
       "with-baseline": { type: "boolean", default: false },
       "validation-model": { type: "string", default: "haiku" },
-      "cheap-loop": { type: "boolean", default: false },
+      "cheap-loop": { type: "boolean", default: true },
+      "full-model": { type: "boolean", default: false },
       "gate-model": { type: "string" },
       "proposal-model": { type: "string" },
+      "sync-first": { type: "boolean", default: false },
+      "sync-force": { type: "boolean", default: false },
       verbose: { type: "boolean", default: false },
       help: { type: "boolean", default: false },
     },
@@ -682,9 +932,12 @@ Options:
   --token-efficiency  Enable 5D Pareto with token efficiency scoring
   --with-baseline     Gate deployment on baseline lift > 0.05
   --validation-model  Model for trigger-check validation calls (default: haiku)
-  --cheap-loop        Use cheap models for loop, expensive model for final gate
-  --gate-model        Model for final gate validation (default: sonnet when --cheap-loop)
+  --cheap-loop        Use cheap models for loop, expensive for gate (default: on)
+  --full-model        Use same model for all stages (disables cheap-loop)
+  --gate-model        Model for final gate validation (default: sonnet)
   --proposal-model    Model for proposal generation LLM calls
+  --sync-first        Refresh source-truth telemetry before building evals/failure patterns
+  --sync-force        Force a full rescan during --sync-first
   --verbose           Output full EvolveResult JSON (default: compact summary)
   --help              Show this help message`);
     process.exit(0);
@@ -694,6 +947,10 @@ Options:
     console.error("[ERROR] --skill and --skill-path are required");
     process.exit(1);
   }
+  if ((values["sync-force"] ?? false) && !(values["sync-first"] ?? false)) {
+    console.error("[ERROR] --sync-force requires --sync-first");
+    process.exit(1);
+  }
 
   const { detectAgent } = await import("../utils/llm-call.js");
   const requestedAgent = values.agent;
@@ -721,11 +978,60 @@ Options:
     process.exit(1);
   }
 
+  // -------------------------------------------------------------------------
+  // Pre-flight validation: catch common misconfigurations before evolve()
+  // -------------------------------------------------------------------------
+  const skillPath = values["skill-path"];
+  if (!skillPath) {
+    console.error("[ERROR] --skill-path is required.");
+    process.exit(1);
+  }
+  if (!existsSync(skillPath)) {
+    console.error(`[ERROR] SKILL.md not found at: ${skillPath}`);
+    console.error("  Verify the --skill-path argument points to an existing SKILL.md file.");
+    process.exit(1);
+  }
+
+  const evalSetPath = values["eval-set"];
+  if (evalSetPath && !existsSync(evalSetPath)) {
+    console.error(`[ERROR] Eval set file not found at: ${evalSetPath}`);
+    console.error("  Verify the --eval-set argument points to an existing JSON file.");
+    process.exit(1);
+  }
+
+  // If no eval-set provided, check that log files exist for auto-generation
+  if (!evalSetPath && !(values["sync-first"] ?? false)) {
+    const hasSkillLog = readEffectiveSkillUsageRecords().length > 0;
+    const hasQueryLog = existsSync(QUERY_LOG);
+    if (!hasSkillLog && !hasQueryLog) {
+      console.error("[ERROR] No eval set provided and no telemetry logs found.");
+      console.error(
+        "  Either pass --eval-set <path> or generate logs first by using selftune-enabled skills.",
+      );
+      console.error(`  Expected logs at: ${SKILL_LOG} and ${QUERY_LOG}`);
+      process.exit(1);
+    }
+  }
+
   const tokenEfficiencyEnabled = values["token-efficiency"] ?? false;
   let telemetryRecords: SessionTelemetryRecord[] | undefined;
-  if (tokenEfficiencyEnabled) {
+  if (tokenEfficiencyEnabled && !(values["sync-first"] ?? false)) {
     telemetryRecords = readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG);
   }
+  const gradingResults = readGradingResultsForSkill(values.skill);
+
+  if (values.verbose) {
+    console.error("[verbose] Pre-flight checks passed");
+    console.error(`[verbose] Skill: ${values.skill}`);
+    console.error(`[verbose] Skill path: ${skillPath}`);
+    console.error(`[verbose] Agent: ${agent}`);
+    console.error(`[verbose] Eval set: ${evalSetPath ?? "(auto-generated from logs)"}`);
+    console.error(`[verbose] Loaded grading results: ${gradingResults.length}`);
+    console.error(`[verbose] Cheap loop: ${values["cheap-loop"] ?? false}`);
+    console.error(`[verbose] Dry run: ${values["dry-run"] ?? false}`);
+    console.error(`[verbose] Sync first: ${values["sync-first"] ?? false}`);
+    console.error(`[verbose] Sync force: ${values["sync-force"] ?? false}`);
+  }
 
   const result = await evolve({
     skillName: values.skill,
@@ -741,9 +1047,12 @@ Options:
     telemetryRecords,
     withBaseline: values["with-baseline"] ?? false,
     validationModel: values["validation-model"],
-    cheapLoop: values["cheap-loop"] ?? false,
+    cheapLoop: (values["cheap-loop"] ?? true) && !(values["full-model"] ?? false),
     gateModel: values["gate-model"],
     proposalModel: values["proposal-model"],
+    gradingResults,
+    syncFirst: values["sync-first"] ?? false,
+    syncForce: values["sync-force"] ?? false,
   });
 
   if (values.verbose) {
@@ -769,12 +1078,49 @@ Options:
     };
     console.log(JSON.stringify(summary, null, 2));
   }
+
+  // Print human-readable status to stderr so users always see outcome
+  if (!result.deployed) {
+    console.error(`\n[NOT DEPLOYED] ${result.reason}`);
+    if (result.validation && !result.validation.improved) {
+      console.error(
+        `  Pass rate: ${(result.validation.before_pass_rate * 100).toFixed(1)}% -> ${(result.validation.after_pass_rate * 100).toFixed(1)}% (net: ${result.validation.net_change >= 0 ? "+" : ""}${(result.validation.net_change * 100).toFixed(1)}%)`,
+      );
+      if (result.validation.regressions.length > 0) {
+        console.error(`  Regressions: ${result.validation.regressions.length} entries`);
+      }
+    }
+    if (
+      result.proposal &&
+      result.proposal.confidence < Number.parseFloat(values.confidence ?? "0.6")
+    ) {
+      console.error(
+        `  Confidence ${result.proposal.confidence.toFixed(2)} below threshold ${values.confidence ?? "0.6"}`,
+      );
+    }
+    console.error("  Re-run with --verbose for full diagnostic output.");
+  } else {
+    console.error(`\n[DEPLOYED] ${result.reason}`);
+  }
+
   process.exit(result.deployed ? 0 : 1);
 }
 
 if (import.meta.main) {
   cliMain().catch((err) => {
-    console.error(`[FATAL] ${err}`);
+    const message = err instanceof Error ? err.message : String(err);
+    const stack = err instanceof Error ? err.stack : undefined;
+    console.error(`[FATAL] ${message}`);
+    if (stack && process.env.SELFTUNE_VERBOSE === "1") {
+      console.error(stack);
+    }
+    console.error(
+      "\nTroubleshooting:\n" +
+        "  - Verify --skill-path points to a valid SKILL.md file\n" +
+        "  - Ensure eval data exists (run `selftune evals` first) or pass --eval-set\n" +
+        "  - Check that ANTHROPIC_API_KEY is set if using Claude\n" +
+        "  - Re-run with --verbose for full diagnostic output",
+    );
     process.exit(1);
   });
 }