selftune 0.2.21 → 0.2.23

package/cli/selftune/evolution/validate-routing.ts

@@ -3,6 +3,9 @@
  *
  * Validates a routing table evolution proposal by checking structural validity
  * and running trigger accuracy checks against an eval set.
+ *
+ * Delegates replay-based and judge-based validation to dedicated engines
+ * (engines/replay-engine.ts and engines/judge-engine.ts).
  */
 
 import type {
@@ -10,28 +13,20 @@ import type {
   BodyValidationResult,
   EvalEntry,
   RoutingReplayEntryResult,
-  RoutingReplayFixture,
   ValidationMode,
 } from "../types.js";
-import { callLlm } from "../utils/llm-call.js";
-import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
-import { runHostReplayFixture } from "./validate-host-replay.js";
-
-export interface RoutingReplayRunnerInput {
-  routing: string;
-  evalSet: EvalEntry[];
-  agent: string;
-  fixture: RoutingReplayFixture;
-}
-
-export type RoutingReplayRunner = (
-  input: RoutingReplayRunnerInput,
-) => Promise<RoutingReplayEntryResult[]>;
-
-export interface RoutingValidationOptions {
-  replayFixture?: RoutingReplayFixture;
-  replayRunner?: RoutingReplayRunner;
-}
+import { runJudgeValidation } from "./engines/judge-engine.js";
+import {
+  runReplayValidation,
+  type ReplayRunner,
+  type ReplayRunnerInput,
+  type ReplayValidationOptions,
+} from "./engines/replay-engine.js";
+
+// Re-export engine types for backward compatibility
+export type { ReplayRunnerInput as RoutingReplayRunnerInput };
+export type { ReplayRunner as RoutingReplayRunner };
+export type { ReplayValidationOptions as RoutingValidationOptions };
 
 export interface RoutingTriggerAccuracyResult {
   before_pass_rate: number;
@@ -41,6 +36,7 @@ export interface RoutingTriggerAccuracyResult {
   validation_agent: string;
   validation_fixture_id?: string;
   per_entry_results?: RoutingReplayEntryResult[];
+  before_entry_results?: RoutingReplayEntryResult[];
 }
 
 // ---------------------------------------------------------------------------
@@ -104,6 +100,9 @@ export function validateRoutingStructure(routing: string): { valid: boolean; rea
 /**
  * Run before/after trigger checks on the eval set using the routing content.
  * Returns pass rates for comparison.
+ *
+ * Prefers replay-backed validation when a fixture is available,
+ * falls back to LLM judge otherwise.
  */
 export async function validateRoutingTriggerAccuracy(
   originalRouting: string,
@@ -111,7 +110,7 @@ export async function validateRoutingTriggerAccuracy(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
-  options: RoutingValidationOptions = {},
+  options: ReplayValidationOptions = {},
 ): Promise<RoutingTriggerAccuracyResult> {
   if (evalSet.length === 0) {
     return {
@@ -123,93 +122,34 @@ export async function validateRoutingTriggerAccuracy(
     };
   }
 
-  if (options.replayFixture && options.replayRunner) {
-    const beforeResults = await options.replayRunner({
-      routing: originalRouting,
-      evalSet,
-      agent,
-      fixture: options.replayFixture,
-    });
-    const afterResults = await options.replayRunner({
-      routing: proposedRouting,
-      evalSet,
-      agent,
-      fixture: options.replayFixture,
-    });
-    const beforePassed = beforeResults.filter((result) => result.passed).length;
-    const afterPassed = afterResults.filter((result) => result.passed).length;
-    const total = evalSet.length;
-
-    return {
-      before_pass_rate: beforePassed / total,
-      after_pass_rate: afterPassed / total,
-      improved: afterPassed > beforePassed,
-      validation_mode: "host_replay",
-      validation_agent: agent,
-      validation_fixture_id: options.replayFixture.fixture_id,
-      per_entry_results: afterResults,
-    };
-  }
-
-  if (options.replayFixture) {
-    const beforeResults = runHostReplayFixture({
-      routing: originalRouting,
-      evalSet,
-      fixture: options.replayFixture,
-    });
-    const afterResults = runHostReplayFixture({
-      routing: proposedRouting,
-      evalSet,
-      fixture: options.replayFixture,
-    });
-    const beforePassed = beforeResults.filter((result) => result.passed).length;
-    const afterPassed = afterResults.filter((result) => result.passed).length;
-    const total = evalSet.length;
-
-    return {
-      before_pass_rate: beforePassed / total,
-      after_pass_rate: afterPassed / total,
-      improved: afterPassed > beforePassed,
-      validation_mode: "host_replay",
-      validation_agent: agent,
-      validation_fixture_id: options.replayFixture.fixture_id,
-      per_entry_results: afterResults,
-    };
-  }
-
-  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
-  let beforePassed = 0;
-  let afterPassed = 0;
-
-  for (const entry of evalSet) {
-    // Check with original routing
-    const beforePrompt = buildTriggerCheckPrompt(originalRouting, entry.query);
-    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
-    const beforeTriggered = parseTriggerResponse(beforeRaw);
-    const beforePass =
-      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
-
-    // Check with proposed routing
-    const afterPrompt = buildTriggerCheckPrompt(proposedRouting, entry.query);
-    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
-    const afterTriggered = parseTriggerResponse(afterRaw);
-    const afterPass =
-      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
+  // Try replay-backed validation first
+  const replayResult = await runReplayValidation(
+    originalRouting,
+    proposedRouting,
+    evalSet,
+    agent,
+    options,
+  );
 
-    if (beforePass) beforePassed++;
-    if (afterPass) afterPassed++;
+  if (replayResult) {
+    return replayResult;
   }
 
-  const total = evalSet.length;
-  const beforePassRate = beforePassed / total;
-  const afterPassRate = afterPassed / total;
+  // Fall back to LLM judge
+  const judgeResult = await runJudgeValidation(
+    originalRouting,
+    proposedRouting,
+    evalSet,
+    agent,
+    modelFlag,
+  );
 
   return {
-    before_pass_rate: beforePassRate,
-    after_pass_rate: afterPassRate,
-    improved: afterPassRate > beforePassRate,
-    validation_mode: "llm_judge",
-    validation_agent: agent,
+    before_pass_rate: judgeResult.before_pass_rate,
+    after_pass_rate: judgeResult.after_pass_rate,
+    improved: judgeResult.improved,
+    validation_mode: judgeResult.validation_mode,
+    validation_agent: judgeResult.validation_agent,
   };
 }
 
@@ -223,7 +163,7 @@ export async function validateRoutingProposal(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
-  options: RoutingValidationOptions = {},
+  options: ReplayValidationOptions = {},
 ): Promise<BodyValidationResult> {
   const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
 
@@ -280,5 +220,6 @@ export async function validateRoutingProposal(
     before_pass_rate: accuracy.before_pass_rate,
     after_pass_rate: accuracy.after_pass_rate,
     per_entry_results: accuracy.per_entry_results,
+    before_entry_results: accuracy.before_entry_results,
   };
 }

package/cli/selftune/hooks/auto-activate.ts

@@ -147,6 +147,37 @@ export function evaluateRules(
   return suggestions;
 }
 
+// ---------------------------------------------------------------------------
+// Reusable auto-activate orchestration
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluate activation rules for a session and return suggestion strings.
+ * Checks PAI coexistence and session state dedup internally.
+ * Returns an empty array when PAI is active or no rules fire.
+ */
+export async function processAutoActivate(
+  sessionId: string,
+  settingsPath?: string,
+): Promise<string[]> {
+  // Only check PAI coexistence when a settings path is provided (platform-specific)
+  if (settingsPath && checkPaiCoexistence(settingsPath)) return [];
+
+  const { DEFAULT_RULES } = await import("../activation-rules.js");
+
+  const ctx: ActivationContext = {
+    session_id: sessionId,
+    query_log_path: QUERY_LOG,
+    telemetry_log_path: TELEMETRY_LOG,
+    evolution_audit_log_path: EVOLUTION_AUDIT_LOG,
+    selftune_dir: SELFTUNE_CONFIG_DIR,
+    settings_path: settingsPath ?? "",
+  };
+
+  const statePath = sessionStatePath(sessionId);
+  return evaluateRules(DEFAULT_RULES, ctx, statePath);
+}
+
 // ---------------------------------------------------------------------------
 // stdin main (only when executed directly, not when imported)
 // ---------------------------------------------------------------------------
@@ -155,43 +186,18 @@ if (import.meta.main) {
   try {
     const payload: PromptSubmitPayload = JSON.parse(await Bun.stdin.text());
     const sessionId = payload.session_id ?? "unknown";
-
-    // Dynamically import default rules (keeps hook file lightweight)
-    const { DEFAULT_RULES } = await import("../activation-rules.js");
-
-    /**
-     * The *_log_path fields exist for test overrides only; default code paths
-     * in activation-rules.ts read from SQLite when the path matches the
-     * constant (QUERY_LOG, EVOLUTION_AUDIT_LOG, etc.).
-     */
-    const ctx: ActivationContext = {
-      session_id: sessionId,
-      query_log_path: QUERY_LOG,
-      telemetry_log_path: TELEMETRY_LOG,
-      evolution_audit_log_path: EVOLUTION_AUDIT_LOG,
-      selftune_dir: SELFTUNE_CONFIG_DIR,
-      settings_path: CLAUDE_SETTINGS_PATH,
-    };
-
-    // Check PAI coexistence — if PAI is active, skip selftune suggestions
-    // (PAI handles skill-level activation; selftune handles observability)
-    if (!checkPaiCoexistence(CLAUDE_SETTINGS_PATH)) {
-      const statePath = sessionStatePath(sessionId);
-      const suggestions = evaluateRules(DEFAULT_RULES, ctx, statePath);
-
-      if (suggestions.length > 0) {
-        // Output as JSON with additionalContext — Claude Code adds this to
-        // Claude's context on UserPromptSubmit (more reliable than stderr)
-        const context = suggestions.map((s) => `[selftune] Suggestion: ${s}`).join("\n");
-        process.stdout.write(
-          JSON.stringify({
-            hookSpecificOutput: {
-              hookEventName: "UserPromptSubmit",
-              additionalContext: context,
-            },
-          }),
-        );
-      }
+    const suggestions = await processAutoActivate(sessionId, CLAUDE_SETTINGS_PATH);
+
+    if (suggestions.length > 0) {
+      const context = suggestions.map((s) => `[selftune] Suggestion: ${s}`).join("\n");
+      process.stdout.write(
+        JSON.stringify({
+          hookSpecificOutput: {
+            hookEventName: "UserPromptSubmit",
+            additionalContext: context,
+          },
+        }),
+      );
     }
   } catch {
     // silent — hooks must never block Claude

package/cli/selftune/hooks/skill-eval.ts

@@ -25,7 +25,7 @@ import {
   getLatestPromptIdentity,
 } from "../normalization.js";
 import type { PostToolUsePayload, SkillUsageRecord } from "../types.js";
-import { classifySkillPath } from "../utils/skill-discovery.js";
+import { classifySkillPath, isTestFixturePath } from "../utils/skill-discovery.js";
 import { getLastUserMessage } from "../utils/transcript.js";
 
 /**
@@ -122,6 +122,7 @@ export async function processToolUse(
   const skillName = extractSkillName(filePath);
 
   if (skillName === null) return null;
+  if (isTestFixturePath(filePath)) return null;
 
   const transcriptPath = payload.transcript_path ?? "";
   const sessionId = payload.session_id ?? "unknown";

package/cli/selftune/hooks-shared/git-metadata.ts

@@ -0,0 +1,149 @@
+/**
+ * Shared git metadata extraction for hooks.
+ *
+ * Extracted from duplicated logic in session-stop.ts (branch/remote extraction)
+ * and commit-track.ts (commit detection, remote scrubbing, branch fallback).
+ *
+ * All functions are fail-open: git errors return undefined/null, never throw.
+ */
+
+import { execSync } from "node:child_process";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Git metadata extracted from a working directory. */
+export interface GitMetadata {
+  branch?: string;
+  repoRemote?: string;
+}
+
+/** Parsed commit information from git output. */
+export interface ParsedCommit {
+  sha?: string;
+  title?: string;
+  branch?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Pre-compiled regex patterns
+// ---------------------------------------------------------------------------
+
+/** Matches git commands that produce commits: commit, merge, cherry-pick, revert. */
+const GIT_COMMIT_CMD_RE = /\bgit\s+(commit|merge|cherry-pick|revert)\b/;
+
+/**
+ * Matches standard git commit output: [branch SHA] title
+ * Supports optional parenthetical like (root-commit).
+ * Branch names can contain word chars, slashes, dots, hyphens, plus signs.
+ */
+const COMMIT_OUTPUT_RE = /\[([\w/.+-]+)(?:\s+\([^)]+\))?\s+([a-f0-9]{7,40})\]\s+(.+)/;
+
+// ---------------------------------------------------------------------------
+// Git metadata extraction
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract git branch and remote URL from a working directory.
+ *
+ * Uses short-timeout execSync calls. Returns partial results if one
+ * command fails (e.g., branch succeeds but remote is not configured).
+ * Returns empty object if cwd is not a git repo.
+ *
+ * @param cwd  Working directory to inspect
+ */
+export function extractGitMetadata(cwd: string): GitMetadata {
+  if (!cwd) return {};
+
+  const result: GitMetadata = {};
+
+  try {
+    result.branch =
+      execSync("git rev-parse --abbrev-ref HEAD", {
+        cwd,
+        timeout: 3000,
+        stdio: ["ignore", "pipe", "ignore"],
+      })
+        .toString()
+        .trim() || undefined;
+  } catch {
+    /* not a git repo or git not available */
+  }
+
+  try {
+    const rawRemote =
+      execSync("git remote get-url origin", {
+        cwd,
+        timeout: 3000,
+        stdio: ["ignore", "pipe", "ignore"],
+      })
+        .toString()
+        .trim() || undefined;
+    if (rawRemote) {
+      result.repoRemote = scrubRemoteUrl(rawRemote);
+    }
+  } catch {
+    /* no remote configured */
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// URL scrubbing
+// ---------------------------------------------------------------------------
+
+/**
+ * Scrub credentials from a git remote URL.
+ *
+ * HTTP(S) URLs have username/password stripped. SSH URLs and other formats
+ * are returned as-is (they don't embed credentials in the URL structure).
+ *
+ * @param rawUrl  Raw remote URL from `git remote get-url`
+ * @returns       Scrubbed URL, or undefined for empty input
+ */
+export function scrubRemoteUrl(rawUrl: string): string | undefined {
+  if (!rawUrl) return undefined;
+  try {
+    const parsed = new URL(rawUrl);
+    parsed.username = "";
+    parsed.password = "";
+    return `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
+  } catch {
+    // SSH or non-URL format -- safe as-is
+    return rawUrl;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Commit detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a command string contains a git commit-producing operation.
+ * Detects: git commit, git merge, git cherry-pick, git revert.
+ */
+export function containsGitCommitCommand(command: string): boolean {
+  return GIT_COMMIT_CMD_RE.test(command);
+}
+
+/**
+ * Parse commit metadata from git's standard output format.
+ *
+ * Expects output like: `[main abc1234] Fix the bug`
+ * or with root-commit: `[main (root-commit) abc1234] Initial commit`
+ *
+ * @param stdout  The stdout from a git commit/merge/cherry-pick/revert command
+ * @returns       Parsed commit info, or null if output doesn't match
+ */
+export function parseCommitFromOutput(stdout: string): ParsedCommit | null {
+  const match = stdout.match(COMMIT_OUTPUT_RE);
+  if (!match) return null;
+
+  return {
+    branch: match[1],
+    sha: match[2],
+    title: match[3].trim(),
+  };
+}

package/cli/selftune/hooks-shared/hook-output.ts

@@ -0,0 +1,105 @@
+/**
+ * Shared output helpers for platform-agnostic hook responses.
+ *
+ * Hooks communicate with their host agent through stdout (context injection),
+ * stderr (advisory messages), and exit codes (allow/block decisions). The
+ * exact format varies by platform — this module abstracts those differences.
+ *
+ * Claude Code conventions (the primary platform):
+ *   - stdout JSON with hookSpecificOutput.additionalContext for context injection
+ *   - stderr for advisory suggestions
+ *   - exit 0 = allow, exit 2 = block with message
+ */
+
+import type { HookPlatform, HookResponse } from "./types.js";
+
+/**
+ * Format a HookResponse for a specific platform's expected output format.
+ *
+ * For Claude Code: produces JSON with hookSpecificOutput wrapper.
+ * For other platforms: produces a simplified JSON response.
+ *
+ * @param response  The platform-agnostic hook response
+ * @param platform  The target platform
+ * @returns         Formatted string ready to write to stdout
+ */
+export function formatResponseForPlatform(response: HookResponse, platform: HookPlatform): string {
+  if (platform === "claude-code" || platform === "codex") {
+    // Claude Code / Codex use hookSpecificOutput wrapper
+    const output: Record<string, unknown> = {};
+
+    if (response.context) {
+      output.hookSpecificOutput = {
+        additionalContext: response.context,
+      };
+    }
+
+    if (response.updated_input) {
+      output.updatedInput = response.updated_input;
+    }
+
+    if (response.decision) {
+      output.decision = response.decision;
+    }
+
+    return JSON.stringify(output);
+  }
+
+  // Generic JSON format for other platforms
+  return JSON.stringify({
+    modified: response.modified,
+    decision: response.decision,
+    message: response.message,
+    context: response.context,
+    updated_input: response.updated_input,
+  });
+}
+
+/**
+ * Write an advisory suggestion to stderr.
+ *
+ * Stderr messages appear as system messages to the host agent in Claude Code.
+ * Other platforms may handle stderr differently, but writing to stderr is
+ * universally safe (it never affects the hook's exit code or stdout response).
+ *
+ * @param message  The suggestion text to display
+ */
+export function writeSuggestion(message: string): void {
+  process.stderr.write(`${message}\n`);
+}
+
+/**
+ * Write context injection to stdout.
+ *
+ * For Claude Code, this injects content into Claude's context via the
+ * hookSpecificOutput.additionalContext mechanism. The output is JSON-formatted
+ * to match Claude Code's expected hook output schema.
+ *
+ * @param context  The context string to inject
+ */
+export function writeContext(context: string): void {
+  process.stdout.write(
+    JSON.stringify({
+      hookSpecificOutput: {
+        additionalContext: context,
+      },
+    }),
+  );
+}
+
+/**
+ * Exit with a platform-appropriate exit code for allow/block decisions.
+ *
+ * Claude Code convention:
+ *   - exit 0 = allow the tool call
+ *   - exit 2 = block the tool call (with stderr message)
+ *
+ * Other platforms use the same convention unless they specify otherwise.
+ *
+ * @param decision  "allow" or "block"
+ * @param _platform  The target platform (reserved for future per-platform codes)
+ */
+export function exitWithDecision(decision: "allow" | "block", _platform: HookPlatform): never {
+  const code = decision === "block" ? 2 : 0;
+  process.exit(code);
+}