selftune 0.2.14 → 0.2.16

package/cli/selftune/evolution/unblock-suggestions.ts

@@ -0,0 +1,159 @@
+/**
+ * unblock-suggestions.ts
+ *
+ * Generates targeted, per-failure-reason suggestions when evolve doesn't deploy.
+ * Each suggestion is a concrete next CLI command or manual action that helps the
+ * agent (or user) unblock the evolution pipeline.
+ *
+ * Pure function — no I/O, no LLM calls. Depends only on EvolveResult fields and
+ * the scoreDescription heuristic.
+ */
+
+import { scoreDescription } from "./description-quality.js";
+import type { EvolveResult } from "./evolve.js";
+
+// ---------------------------------------------------------------------------
+// Quality hint helper
+// ---------------------------------------------------------------------------
+
+/**
+ * Append description quality improvement hints if the score reveals weak criteria.
+ * Only fires when composite < 0.7 to avoid noise on already-good descriptions.
+ * Skips when descriptionText is empty (no proposal was generated).
+ */
+function appendQualityHints(
+  suggestions: string[],
+  descriptionText: string,
+  skillName: string,
+): void {
+  if (!descriptionText) return;
+  const score = scoreDescription(descriptionText, skillName);
+  if (score.composite >= 0.7) return;
+
+  const weak: string[] = [];
+  if (score.criteria.trigger_context < 0.5) weak.push("add when/if/after trigger context");
+  if (score.criteria.vagueness < 0.7) weak.push("remove vague words (various, general, etc)");
+  if (score.criteria.specificity < 0.5) weak.push("add concrete action verbs");
+  if (score.criteria.length < 0.7) weak.push("adjust length (ideal: 80-300 chars)");
+  if (score.criteria.not_just_name < 0.5) weak.push("differentiate from skill name");
+
+  if (weak.length > 0) {
+    suggestions.push(
+      `Description quality: ${Math.round(score.composite * 100)}% — improve by: ${weak.join(", ")}`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main suggestion builder
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate targeted suggestions based on the specific failure reason.
+ * Each suggestion is a concrete next CLI command or manual action.
+ */
+export function buildUnblockSuggestions(result: EvolveResult, skillName: string): string[] {
+  const reason = result.reason;
+  const suggestions: string[] = [];
+  const descText = result.proposal?.original_description ?? "";
+
+  // --- Path/config failures ---
+  if (reason.includes("SKILL.md not found")) {
+    suggestions.push("Verify the --skill-path flag points to a valid SKILL.md file");
+    suggestions.push("Run: selftune init (to re-bootstrap config if paths changed)");
+    return suggestions;
+  }
+
+  if (reason.includes("Failed to load eval set") || reason.includes("not a JSON array")) {
+    suggestions.push("Run: selftune sync (to rebuild source-truth telemetry)");
+    suggestions.push(`Then: selftune evolve --skill ${skillName} (to retry with fresh evals)`);
+    return suggestions;
+  }
+
+  // --- No signal failures ---
+  if (reason.includes("No failure patterns found")) {
+    suggestions.push("This skill may already be routing well — check: selftune status");
+    suggestions.push("If undertriggering, add more sessions so evolve has signal to work with");
+    if (result.descriptionQualityBefore != null && result.descriptionQualityBefore < 0.5) {
+      suggestions.push(
+        `Description quality is ${Math.round(result.descriptionQualityBefore * 100)}% — manually improving the description may help generate patterns`,
+      );
+      appendQualityHints(suggestions, descText, skillName);
+    }
+    return suggestions;
+  }
+
+  // --- Confidence failures (specific before general) ---
+  if (reason.includes("No candidates met confidence")) {
+    suggestions.push(`Lower the threshold: selftune evolve --skill ${skillName} --confidence 0.4`);
+    suggestions.push(
+      `Or increase candidates: selftune evolve --skill ${skillName} --pareto --candidates 5`,
+    );
+    appendQualityHints(suggestions, descText, skillName);
+    return suggestions;
+  }
+  if (reason.toLowerCase().includes("confidence") && reason.includes("threshold")) {
+    suggestions.push(`Lower the threshold: selftune evolve --skill ${skillName} --confidence 0.4`);
+    suggestions.push("Or add more eval entries so the LLM has more context for proposals");
+    appendQualityHints(suggestions, descText, skillName);
+    return suggestions;
+  }
+
+  // --- Validation failures (proposals regressed) ---
+  if (reason.includes("Validation failed after")) {
+    suggestions.push(
+      `The eval set may be contradictory — review with: selftune evolve --skill ${skillName} --verbose`,
+    );
+    suggestions.push(
+      `Try: selftune evolve --skill ${skillName} --pareto --candidates 5 (more diverse proposals)`,
+    );
+    if (result.validation && result.validation.regressions.length > 0) {
+      suggestions.push(
+        `${result.validation.regressions.length} regressions detected — check if negative eval entries are too broad`,
+      );
+    }
+    appendQualityHints(suggestions, descText, skillName);
+    return suggestions;
+  }
+  if (reason.includes("No Pareto candidates improved")) {
+    suggestions.push("All candidates regressed — the eval set may need rebalancing");
+    suggestions.push(`Try: selftune sync --force && selftune evolve --skill ${skillName}`);
+    return suggestions;
+  }
+
+  // --- Gate failures ---
+  if (reason.includes("Baseline gate failed")) {
+    suggestions.push("Improvement was too marginal to justify deployment");
+    suggestions.push("Collect more session data, then retry — small gains compound over time");
+    return suggestions;
+  }
+  if (reason.includes("Gate validation failed")) {
+    suggestions.push("The gate model rejected the proposal — it may be too aggressive");
+    suggestions.push(
+      `Try: selftune evolve --skill ${skillName} --full-model (disables cheap-loop gate)`,
+    );
+    return suggestions;
+  }
+
+  // --- Constitutional rejection ---
+  if (reason.includes("Constitutional")) {
+    suggestions.push("The proposed description violated safety constraints");
+    suggestions.push("Review constitutional rules and manually adjust the description if needed");
+    return suggestions;
+  }
+
+  // --- Dry run (not really a failure) ---
+  if (reason.includes("Dry run")) {
+    suggestions.push(`Deploy: selftune evolve --skill ${skillName} (remove --dry-run to deploy)`);
+    return suggestions;
+  }
+
+  // --- Catch-all for unexpected errors ---
+  if (reason.includes("Error during evolution")) {
+    suggestions.push("Re-run with --verbose for full stack trace");
+    suggestions.push("Run: selftune doctor (to check system health)");
+    return suggestions;
+  }
+
+  return suggestions;
+}

package/cli/selftune/grading/auto-grade.ts

@@ -17,6 +17,7 @@ import { AGENT_CANDIDATES, TELEMETRY_LOG } from "../constants.js";
 import { getDb } from "../localdb/db.js";
 import { querySessionTelemetry, querySkillUsageRecords } from "../localdb/queries.js";
 import type { GradingResult, SessionTelemetryRecord, SkillUsageRecord } from "../types.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
 import { detectAgent as _detectAgent } from "../utils/llm-call.js";
 import { readExcerpt } from "../utils/transcript.js";
 import {
@@ -62,8 +63,7 @@ Options:
 
   const skill = values.skill;
   if (!skill) {
-    console.error("[ERROR] --skill is required");
-    process.exit(1);
+    throw new CLIError("--skill is required", "MISSING_FLAG", "selftune auto-grade --skill <name>");
   }
 
   // --- Determine agent ---
@@ -71,10 +71,11 @@ Options:
   const validAgents = [...AGENT_CANDIDATES];
   if (values.agent) {
     if (!validAgents.includes(values.agent)) {
-      console.error(
-        `[ERROR] Invalid --agent '${values.agent}'. Expected one of: ${validAgents.join(", ")}`,
+      throw new CLIError(
+        `Invalid --agent '${values.agent}'. Expected one of: ${validAgents.join(", ")}`,
+        "INVALID_FLAG",
+        `selftune auto-grade --skill <name> --agent ${validAgents[0]}`,
       );
-      process.exit(1);
     }
     agent = values.agent;
   } else {
@@ -82,11 +83,11 @@ Options:
   }
 
   if (!agent) {
-    console.error(
-      `[ERROR] No supported agent CLI (${AGENT_CANDIDATES.join("/")}) found in PATH.\n` +
-        "Install one of the supported agent CLIs.",
+    throw new CLIError(
+      `No supported agent CLI (${AGENT_CANDIDATES.join("/")}) found in PATH`,
+      "AGENT_NOT_FOUND",
+      "Install one of the supported agent CLIs",
     );
-    process.exit(1);
   }
 
   console.error(`[INFO] Auto-grade via agent: ${agent}`);
@@ -104,21 +105,22 @@ Options:
     sessionId = values["session-id"];
     const resolved = resolveSessionById(telRecords, sessionId);
     if (!resolved) {
-      console.error(
-        `[ERROR] Session '${sessionId}' not found in telemetry or recoverable transcript data. ` +
-          "Check the session ID or omit --session-id to auto-select the latest matching session.",
+      throw new CLIError(
+        `Session '${sessionId}' not found in telemetry or recoverable transcript data`,
+        "MISSING_DATA",
+        "Check the session ID or omit --session-id to auto-select the latest matching session",
       );
-      process.exit(1);
     }
     telemetry = resolved.telemetry;
     transcriptPath = resolved.transcriptPath;
   } else {
     const resolved = resolveLatestSessionForSkill(telRecords, skillUsageRecords, skill);
     if (!resolved) {
-      console.error(
-        `[ERROR] No session found for skill '${skill}'. Run the skill first, or pass --session-id.`,
+      throw new CLIError(
+        `No session found for skill '${skill}'`,
+        "MISSING_DATA",
+        "Run the skill first, or pass --session-id",
       );
-      process.exit(1);
     }
     telemetry = resolved.telemetry;
     sessionId = resolved.sessionId ?? "unknown";
@@ -159,8 +161,11 @@ Options:
       agent,
     });
   } catch (err) {
-    console.error(`[ERROR] ${err instanceof Error ? err.message : String(err)}`);
-    process.exit(1);
+    throw new CLIError(
+      `Grading failed: ${err instanceof Error ? err.message : String(err)}`,
+      "OPERATION_FAILED",
+      "Check agent availability and try again",
+    );
   }
 
   const outputPath = values.output ?? buildDefaultGradingOutputPath(sessionId);
@@ -193,8 +198,5 @@ Options:
 
 // Guard: only run when invoked directly
 if (import.meta.main) {
-  cliMain().catch((err) => {
-    console.error(`[FATAL] ${err}`);
-    process.exit(1);
-  });
+  cliMain().catch(handleCLIError);
 }

package/cli/selftune/grading/grade-session.ts

@@ -28,6 +28,7 @@ import type {
   SessionTelemetryRecord,
   SkillUsageRecord,
 } from "../types.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
 import {
   detectAgent as _detectAgent,
   stripMarkdownFences as _stripMarkdownFences,
@@ -743,8 +744,7 @@ Options:
 
   const skill = values.skill;
   if (!skill) {
-    console.error("[ERROR] --skill is required");
-    process.exit(1);
+    throw new CLIError("--skill is required", "MISSING_FLAG", "selftune grade --skill <name>");
   }
 
   // --- Determine agent ---
@@ -752,10 +752,11 @@ Options:
   const validAgents = [...AGENT_CANDIDATES];
   if (values.agent) {
     if (!validAgents.includes(values.agent)) {
-      console.error(
-        `[ERROR] Invalid --agent '${values.agent}'. Expected one of: ${validAgents.join(", ")}`,
+      throw new CLIError(
+        `Invalid --agent '${values.agent}'. Expected one of: ${validAgents.join(", ")}`,
+        "INVALID_FLAG",
+        `selftune grade --skill <name> --agent ${validAgents[0]}`,
       );
-      process.exit(1);
     }
     agent = values.agent;
   } else {
@@ -763,11 +764,11 @@ Options:
   }
 
   if (!agent) {
-    console.error(
-      `[ERROR] No supported agent CLI (${AGENT_CANDIDATES.join("/")}) found in PATH.\n` +
-        "Install one of the supported agent CLIs.",
+    throw new CLIError(
+      `No supported agent CLI (${AGENT_CANDIDATES.join("/")}) found in PATH`,
+      "AGENT_NOT_FOUND",
+      "Install claude, codex, or opencode CLI, then retry",
     );
-    process.exit(1);
   }
 
   console.error(`[INFO] Grading via agent: ${agent}`);
@@ -777,8 +778,11 @@ Options:
   if (values["evals-json"] && values["eval-id"] != null) {
     const evalIdNum = Number(values["eval-id"]);
     if (!Number.isFinite(evalIdNum) || !Number.isInteger(evalIdNum)) {
-      console.error(`[ERROR] --eval-id must be a finite integer, got: ${values["eval-id"]}`);
-      process.exit(1);
+      throw new CLIError(
+        `--eval-id must be a finite integer, got: ${values["eval-id"]}`,
+        "INVALID_FLAG",
+        "selftune grade --eval-id <integer>",
+      );
     }
     expectations = loadExpectationsFromEvalsJson(values["evals-json"], evalIdNum);
   } else if (values.expectations?.length) {
@@ -863,8 +867,11 @@ Options:
       agent,
     });
   } catch (err) {
-    console.error(`[ERROR] ${err instanceof Error ? err.message : String(err)}`);
-    process.exit(1);
+    throw new CLIError(
+      `Grading failed: ${err instanceof Error ? err.message : String(err)}`,
+      "OPERATION_FAILED",
+      "Check agent availability and try again",
+    );
   }
 
   const outputPath = values.output ?? buildDefaultGradingOutputPath(sessionId);
@@ -888,8 +895,5 @@ Options:
 
 // Guard: only run when invoked directly
 if (import.meta.main) {
-  cliMain().catch((err) => {
-    console.error(`[FATAL] ${err}`);
-    process.exit(1);
-  });
+  cliMain().catch(handleCLIError);
 }

package/cli/selftune/hooks/auto-activate.ts

@@ -179,9 +179,18 @@ if (import.meta.main) {
       const statePath = sessionStatePath(sessionId);
       const suggestions = evaluateRules(DEFAULT_RULES, ctx, statePath);
 
-      for (const s of suggestions) {
-        // Output to stderr — Claude Code shows stderr as system messages
-        process.stderr.write(`[selftune] 💡 Suggestion: ${s}\n`);
+      if (suggestions.length > 0) {
+        // Output as JSON with additionalContext — Claude Code adds this to
+        // Claude's context on UserPromptSubmit (more reliable than stderr)
+        const context = suggestions.map((s) => `[selftune] Suggestion: ${s}`).join("\n");
+        process.stdout.write(
+          JSON.stringify({
+            hookSpecificOutput: {
+              hookEventName: "UserPromptSubmit",
+              additionalContext: context,
+            },
+          }),
+        );
       }
     }
   } catch {

package/cli/selftune/hooks/prompt-log.ts

@@ -154,7 +154,13 @@ export async function processPrompt(
   promptStatePath?: string,
   _signalLogPath?: string,
 ): Promise<QueryLogRecord | null> {
-  const query = (payload.user_prompt ?? "").trim();
+  const rawPrompt =
+    typeof payload.prompt === "string"
+      ? payload.prompt
+      : typeof payload.user_prompt === "string"
+        ? payload.user_prompt
+        : "";
+  const query = rawPrompt.trim();
 
   if (!query) return null;
 

package/cli/selftune/index.ts

@@ -29,6 +29,11 @@
  *   selftune hook <name>        — Run a hook by name (prompt-log, session-stop, etc.)
  */
 
+import { CLIError, handleCLIError } from "./utils/cli-error.js";
+
+process.on("uncaughtException", handleCLIError);
+process.on("unhandledRejection", handleCLIError);
+
 const command = process.argv[2];
 
 if (command === "--help" || command === "-h") {
@@ -84,6 +89,7 @@ if (!command) {
   // Show status by default — same as `selftune status`
   const { cliMain: statusMain } = await import("./status.js");
   statusMain();
+  process.exit(0);
 }
 
 // Route to the appropriate subcommand module.
@@ -142,10 +148,11 @@ Run 'selftune ingest <agent> --help' for agent-specific options.`);
         break;
       }
       default:
-        console.error(
-          `Unknown ingest agent: ${sub}\nRun 'selftune ingest --help' for available agents.`,
+        throw new CLIError(
+          `Unknown ingest agent: ${sub}`,
+          "UNKNOWN_COMMAND",
+          "selftune ingest --help",
         );
-        process.exit(1);
     }
     break;
   }
@@ -182,10 +189,11 @@ Run 'selftune grade <subcommand> --help' for subcommand-specific options.`);
           break;
         }
         default:
-          console.error(
-            `Unknown grade mode: ${sub}\nRun 'selftune grade --help' for available modes.`,
+          throw new CLIError(
+            `Unknown grade mode: ${sub}`,
+            "UNKNOWN_COMMAND",
+            "selftune grade --help",
           );
-          process.exit(1);
       }
     }
     break;
@@ -223,10 +231,11 @@ Run 'selftune evolve <subcommand> --help' for subcommand-specific options.`);
           break;
         }
         default:
-          console.error(
-            `Unknown evolve target: ${sub}\nRun 'selftune evolve --help' for available targets.`,
+          throw new CLIError(
+            `Unknown evolve target: ${sub}`,
+            "UNKNOWN_COMMAND",
+            "selftune evolve --help",
           );
-          process.exit(1);
       }
     }
     break;
@@ -289,13 +298,18 @@ Run 'selftune eval <action> --help' for action-specific options.`);
           }));
         } catch (error) {
           const message = error instanceof Error ? error.message : String(error);
-          console.error(`Invalid arguments: ${message}`);
-          console.error("Run 'selftune eval composability --help' for usage.");
-          process.exit(1);
+          throw new CLIError(
+            `Invalid arguments: ${message}`,
+            "INVALID_FLAG",
+            "selftune eval composability --help",
+          );
         }
         if (!values.skill) {
-          console.error("[ERROR] --skill <name> is required.");
-          process.exit(1);
+          throw new CLIError(
+            "--skill <name> is required.",
+            "MISSING_FLAG",
+            "selftune eval composability --skill <name>",
+          );
         }
         const logPath = values["telemetry-log"] ?? TELEMETRY_LOG;
         let telemetry: unknown[];
@@ -316,8 +330,11 @@ Run 'selftune eval <action> --help' for action-specific options.`);
         }
         const rawWindow = values.window as string | undefined;
         if (rawWindow !== undefined && !/^[1-9]\d*$/.test(rawWindow)) {
-          console.error("Invalid --window value. Use a positive integer number of days.");
-          process.exit(1);
+          throw new CLIError(
+            "Invalid --window value. Use a positive integer number of days.",
+            "INVALID_FLAG",
+            "selftune eval composability --skill <name> --window 30",
+          );
         }
         const windowSize = rawWindow === undefined ? undefined : Number(rawWindow);
         const report = analyzeComposability(values.skill, telemetry, windowSize);
@@ -325,10 +342,11 @@ Run 'selftune eval <action> --help' for action-specific options.`);
         break;
       }
       default:
-        console.error(
-          `Unknown eval action: ${sub}\nRun 'selftune eval --help' for available actions.`,
+        throw new CLIError(
+          `Unknown eval action: ${sub}`,
+          "UNKNOWN_COMMAND",
+          "selftune eval --help",
         );
-        process.exit(1);
     }
     break;
   }
@@ -457,10 +475,11 @@ Run 'selftune cron <subcommand> --help' for subcommand-specific options.`);
         break;
       }
       default:
-        console.error(
-          `Unknown cron subcommand: ${sub}\nRun 'selftune cron --help' for available subcommands.`,
+        throw new CLIError(
+          `Unknown cron subcommand: ${sub}`,
+          "UNKNOWN_COMMAND",
+          "selftune cron --help",
         );
-        process.exit(1);
     }
     break;
   }
@@ -505,9 +524,7 @@ Run 'selftune cron <subcommand> --help' for subcommand-specific options.`);
       }));
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      console.error(`Invalid arguments: ${message}`);
-      console.error("Run 'selftune export --help' for usage.");
-      process.exit(1);
+      throw new CLIError(`Invalid arguments: ${message}`, "INVALID_FLAG", "selftune export --help");
     }
     if (values.help) {
       console.log(`selftune export — Export SQLite data to JSONL files
@@ -544,9 +561,7 @@ Options:
       }
     } catch (err: unknown) {
       const message = err instanceof Error ? err.message : String(err);
-      console.error(`Export failed: ${message}`);
-      console.error("Ensure the SQLite database exists. Run 'selftune sync' first if needed.");
-      process.exit(1);
+      throw new CLIError(`Export failed: ${message}`, "OPERATION_FAILED", "selftune sync");
     }
     break;
   }
@@ -590,8 +605,11 @@ Run 'selftune alpha <subcommand> --help' for subcommand-specific options.`);
           }));
         } catch (error) {
           const message = error instanceof Error ? error.message : String(error);
-          console.error(`Invalid arguments: ${message}`);
-          process.exit(1);
+          throw new CLIError(
+            `Invalid arguments: ${message}`,
+            "INVALID_FLAG",
+            "selftune alpha upload --help",
+          );
         }
         if (values.help) {
           console.log(`selftune alpha upload — Run a manual alpha data upload cycle
@@ -619,44 +637,20 @@ Output:
         const identity = readAlphaIdentity(SELFTUNE_CONFIG_PATH);
         if (!identity?.enrolled) {
           const guidance = getAlphaGuidance(identity);
-          console.log(
-            JSON.stringify(
-              {
-                enrolled: false,
-                prepared: 0,
-                sent: 0,
-                failed: 0,
-                skipped: 0,
-                guidance,
-              },
-              null,
-              2,
-            ),
+          throw new CLIError(
+            `[alpha upload] ${guidance.message}`,
+            "OPERATION_FAILED",
+            guidance.next_command,
           );
-          console.error(`[alpha upload] ${guidance.message}`);
-          console.error(`[alpha upload] Next: ${guidance.next_command}`);
-          process.exit(1);
         }
 
         if (!identity.user_id?.trim() || !identity.api_key?.trim()) {
           const guidance = getAlphaGuidance(identity);
-          console.log(
-            JSON.stringify(
-              {
-                enrolled: true,
-                prepared: 0,
-                sent: 0,
-                failed: 0,
-                skipped: 0,
-                guidance,
-              },
-              null,
-              2,
-            ),
+          throw new CLIError(
+            `[alpha upload] ${guidance.message}`,
+            "OPERATION_FAILED",
+            guidance.next_command,
           );
-          console.error(`[alpha upload] ${guidance.message}`);
-          console.error(`[alpha upload] Next: ${guidance.next_command}`);
-          process.exit(1);
         }
 
         const db = getDb();
@@ -742,10 +736,11 @@ Output:
         break;
       }
       default:
-        console.error(
-          `Unknown alpha subcommand: ${sub}\nRun 'selftune alpha --help' for available subcommands.`,
+        throw new CLIError(
+          `Unknown alpha subcommand: ${sub}`,
+          "UNKNOWN_COMMAND",
+          "selftune alpha --help",
         );
-        process.exit(1);
     }
     break;
   }
@@ -767,8 +762,11 @@ Output:
     };
     if (!hookName || !HOOK_MAP[hookName]) {
       const available = Object.keys(HOOK_MAP).join(", ");
-      console.error(`Unknown hook: ${hookName ?? "(none)"}\nAvailable hooks: ${available}`);
-      process.exit(1);
+      throw new CLIError(
+        `Unknown hook: ${hookName ?? "(none)"}. Available: ${available}`,
+        "UNKNOWN_COMMAND",
+        "selftune hook prompt-log",
+      );
     }
     const { resolve, dirname } = await import("node:path");
     const { fileURLToPath } = await import("node:url");
@@ -782,6 +780,5 @@ Output:
     break;
   }
   default:
-    console.error(`Unknown command: ${command}\nRun 'selftune --help' for available commands.`);
-    process.exit(1);
+    throw new CLIError(`Unknown command: ${command}`, "UNKNOWN_COMMAND", "selftune --help");
 }