selftune 0.2.20 → 0.2.22

package/cli/selftune/evolution/validate-host-replay.ts

@@ -1,5 +1,16 @@
-import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs";
-import { basename, dirname, join } from "node:path";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, isAbsolute, join } from "node:path";
 
 import type { EvalEntry, RoutingReplayEntryResult, RoutingReplayFixture } from "../types.js";
 import { parseFrontmatter } from "../utils/frontmatter.js";
@@ -10,6 +21,7 @@ import {
   jaccardSimilarity,
   tokenizeText,
 } from "../utils/text-similarity.js";
+import { replaceSection } from "./deploy-proposal.js";
 
 interface ReplaySkillSurface {
   skillName: string;
@@ -17,12 +29,41 @@ interface ReplaySkillSurface {
   whenToUseTokens: Set<string>;
 }
 
+interface ReplayWorkspace {
+  rootDir: string;
+  targetSkillPath: string;
+  competingSkillPaths: string[];
+}
+
+export interface ClaudeRuntimeReplayInvokerInput {
+  query: string;
+  workspaceRoot: string;
+  targetSkillName: string;
+  targetSkillPath: string;
+  competingSkillPaths: string[];
+}
+
+export interface ClaudeRuntimeReplayObservation {
+  invokedSkillNames: string[];
+  readSkillPaths: string[];
+  rawOutput: string;
+  sessionId?: string;
+  runtimeError?: string;
+}
+
+export type ClaudeRuntimeReplayInvoker = (
+  input: ClaudeRuntimeReplayInvokerInput,
+) => Promise<ClaudeRuntimeReplayObservation>;
+
 /**
  * Minimum score needed before replay treats routing text or skill-surface overlap
  * as a real match. Tuned to suppress weak false positives without killing recall
  * for short routing phrases and sparse skill surfaces.
  */
 const HOST_REPLAY_MATCH_THRESHOLD = 0.18;
+const CLAUDE_RUNTIME_REPLAY_TIMEOUT_MS = 30_000;
+const CLAUDE_RUNTIME_ROUTING_PROMPT =
+  "You are being evaluated only on skill routing. Do not solve the user's task. If a local project skill is relevant, invoke exactly one skill immediately. If no local project skill fits, respond with NO_SKILL and do not browse unrelated files.";
 
 function resolveReplayPath(path: string): string {
   try {
@@ -32,6 +73,10 @@ function resolveReplayPath(path: string): string {
   }
 }
 
+function resolveObservedReplayPath(path: string, workspaceRoot: string): string {
+  return resolveReplayPath(isAbsolute(path) ? path : join(workspaceRoot, path));
+}
+
 function listCompetingSkillPaths(targetSkillPath: string): string[] {
   const normalizedTargetPath = resolveReplayPath(targetSkillPath);
   const targetSkillDir = dirname(normalizedTargetPath);
@@ -82,6 +127,304 @@ export function buildRoutingReplayFixture(options: {
   };
 }
 
+function buildRuntimeReplayTargetContent(skillPath: string, routing: string): string {
+  const currentContent = readFileSync(skillPath, "utf8");
+  return replaceSection(currentContent, "Workflow Routing", routing.trim());
+}
+
+function stageReplaySkill(
+  registryDir: string,
+  sourceSkillPath: string,
+  overrideContent?: string,
+): string {
+  const skillDirName = basename(dirname(sourceSkillPath)) || "unknown-skill";
+  const destinationDir = join(registryDir, skillDirName);
+  mkdirSync(destinationDir, { recursive: true });
+  const destinationPath = join(destinationDir, "SKILL.md");
+  const content = overrideContent ?? readFileSync(sourceSkillPath, "utf8");
+  writeFileSync(destinationPath, content, "utf8");
+  return destinationPath;
+}
+
+function buildRuntimeReplayWorkspace(
+  fixture: RoutingReplayFixture,
+  routing: string,
+): ReplayWorkspace {
+  const rootDir = mkdtempSync(join(tmpdir(), "selftune-runtime-replay-"));
+  try {
+    const registryDir = join(rootDir, ".claude", "skills");
+    mkdirSync(join(rootDir, ".git"), { recursive: true });
+    mkdirSync(registryDir, { recursive: true });
+
+    const targetSkillPath = stageReplaySkill(
+      registryDir,
+      fixture.target_skill_path,
+      buildRuntimeReplayTargetContent(fixture.target_skill_path, routing),
+    );
+    const competingSkillPaths = fixture.competing_skill_paths.map((skillPath) =>
+      stageReplaySkill(registryDir, skillPath),
+    );
+
+    return {
+      rootDir,
+      targetSkillPath,
+      competingSkillPaths,
+    };
+  } catch (error) {
+    rmSync(rootDir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+function cleanupRuntimeReplayWorkspace(workspace: ReplayWorkspace): void {
+  rmSync(workspace.rootDir, { recursive: true, force: true });
+}
+
+function parseClaudeRuntimeReplayOutput(rawOutput: string): ClaudeRuntimeReplayObservation {
+  const invokedSkillNames = new Set<string>();
+  const readSkillPaths = new Set<string>();
+  let sessionId: string | undefined;
+  let runtimeError: string | undefined;
+
+  for (const line of rawOutput.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    let parsed: Record<string, unknown>;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+
+    const maybeSessionId = parsed.session_id;
+    if (typeof maybeSessionId === "string" && maybeSessionId) {
+      sessionId = maybeSessionId;
+    }
+
+    if (typeof parsed.error === "string" && parsed.error) {
+      runtimeError = parsed.error;
+    }
+
+    const assistantMessage =
+      parsed.type === "assistant" && typeof parsed.message === "object" && parsed.message !== null
+        ? (parsed.message as Record<string, unknown>)
+        : undefined;
+    const content = assistantMessage?.content;
+    if (!Array.isArray(content)) continue;
+
+    for (const block of content) {
+      if (typeof block !== "object" || block === null) continue;
+      const typedBlock = block as Record<string, unknown>;
+      if (typedBlock.type !== "tool_use") continue;
+
+      const toolName = typedBlock.name;
+      const input =
+        typeof typedBlock.input === "object" && typedBlock.input !== null
+          ? (typedBlock.input as Record<string, unknown>)
+          : {};
+
+      if (toolName === "Skill") {
+        const skillName = input.skill;
+        if (typeof skillName === "string" && skillName.trim()) {
+          invokedSkillNames.add(skillName.trim());
+        }
+      }
+
+      if (toolName === "Read") {
+        const filePath = input.file_path;
+        if (typeof filePath === "string" && filePath.trim()) {
+          readSkillPaths.add(resolveReplayPath(filePath.trim()));
+        }
+      }
+    }
+  }
+
+  return {
+    invokedSkillNames: [...invokedSkillNames],
+    readSkillPaths: [...readSkillPaths],
+    rawOutput,
+    ...(sessionId ? { sessionId } : {}),
+    ...(runtimeError ? { runtimeError } : {}),
+  };
+}
+
+async function invokeClaudeRuntimeReplay(
+  input: ClaudeRuntimeReplayInvokerInput,
+): Promise<ClaudeRuntimeReplayObservation> {
+  const command = [
+    "claude",
+    "-p",
+    "--verbose",
+    "--output-format",
+    "stream-json",
+    "--dangerously-skip-permissions",
+    "--no-session-persistence",
+    "--setting-sources",
+    "project,local",
+    "--tools",
+    "Skill,Read",
+    "--max-turns",
+    "1",
+    "--append-system-prompt",
+    CLAUDE_RUNTIME_ROUTING_PROMPT,
+    input.query,
+  ];
+
+  const proc = Bun.spawn(command, {
+    cwd: input.workspaceRoot,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: { ...process.env, CLAUDECODE: "" },
+  });
+  const timeout = setTimeout(() => proc.kill(), CLAUDE_RUNTIME_REPLAY_TIMEOUT_MS);
+
+  const [stdoutText, stderrText, exitCode] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ]);
+  clearTimeout(timeout);
+
+  const observation = parseClaudeRuntimeReplayOutput(stdoutText);
+  const combinedError = [observation.runtimeError, stderrText.trim()].filter(Boolean).join(" | ");
+  const hasRoutingSignal =
+    observation.invokedSkillNames.length > 0 || observation.readSkillPaths.length > 0;
+
+  if (exitCode !== 0 && !hasRoutingSignal) {
+    throw new Error(combinedError || `claude runtime replay exited with code ${exitCode}`);
+  }
+
+  return {
+    ...observation,
+    ...(combinedError ? { runtimeError: combinedError } : {}),
+  };
+}
+
+function prefixReplayEvidence(
+  results: RoutingReplayEntryResult[],
+  prefix: string,
+): RoutingReplayEntryResult[] {
+  return results.map((result) => ({
+    ...result,
+    evidence: result.evidence ? `${prefix}; ${result.evidence}` : prefix,
+  }));
+}
+
+function evaluateRuntimeReplayObservation(
+  entry: EvalEntry,
+  fixture: RoutingReplayFixture,
+  observation: ClaudeRuntimeReplayObservation,
+  workspace: ReplayWorkspace,
+): RoutingReplayEntryResult {
+  const normalizedReadPaths = new Set(
+    observation.readSkillPaths.map((path) => resolveObservedReplayPath(path, workspace.rootDir)),
+  );
+  const allowedReadPaths = new Set([
+    resolveReplayPath(workspace.targetSkillPath),
+    ...workspace.competingSkillPaths.map(resolveReplayPath),
+  ]);
+  const targetSkillName = fixture.target_skill_name.trim();
+  const targetInvoked = observation.invokedSkillNames.includes(targetSkillName);
+  const competingInvoked = observation.invokedSkillNames.find((skillName) =>
+    fixture.competing_skill_paths.some(
+      (skillPath) => basename(dirname(skillPath)).trim() === skillName.trim(),
+    ),
+  );
+  const unrelatedInvoked = observation.invokedSkillNames.find(
+    (skillName) => skillName.trim() !== targetSkillName && skillName.trim() !== competingInvoked,
+  );
+  const unrelatedReadPaths = [...normalizedReadPaths].filter((path) => !allowedReadPaths.has(path));
+  const targetRead = normalizedReadPaths.has(resolveReplayPath(workspace.targetSkillPath));
+  const competingRead = workspace.competingSkillPaths.find((skillPath) =>
+    normalizedReadPaths.has(resolveReplayPath(skillPath)),
+  );
+  const sessionPrefix = observation.sessionId
+    ? `runtime replay session ${observation.sessionId}`
+    : "runtime replay";
+  if (observation.invokedSkillNames.length > 1) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: false,
+      evidence: `${sessionPrefix} invoked multiple skills: ${observation.invokedSkillNames.join(", ")}`,
+    };
+  }
+
+  if (targetInvoked) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: true,
+      passed: entry.should_trigger,
+      evidence: `${sessionPrefix} invoked target skill: ${targetSkillName}`,
+    };
+  }
+
+  if (competingInvoked) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: !entry.should_trigger,
+      evidence: `${sessionPrefix} invoked competing skill: ${competingInvoked}`,
+    };
+  }
+
+  if (unrelatedInvoked) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: false,
+      evidence: `${sessionPrefix} invoked unrelated skill: ${unrelatedInvoked}`,
+    };
+  }
+
+  if (unrelatedReadPaths.length > 0) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: false,
+      evidence: `${sessionPrefix} read files outside staged skill set: ${unrelatedReadPaths.join(", ")}`,
+    };
+  }
+
+  if (targetRead) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: !entry.should_trigger,
+      evidence: `${sessionPrefix} only read the target skill without invoking it`,
+    };
+  }
+
+  if (competingRead) {
+    return {
+      query: entry.query,
+      should_trigger: entry.should_trigger,
+      triggered: false,
+      passed: !entry.should_trigger,
+      evidence: `${sessionPrefix} only read a competing skill without invoking it`,
+    };
+  }
+
+  if (observation.runtimeError) {
+    throw new Error(`${sessionPrefix} did not reach a skill decision: ${observation.runtimeError}`);
+  }
+
+  return {
+    query: entry.query,
+    should_trigger: entry.should_trigger,
+    triggered: false,
+    passed: !entry.should_trigger,
+    evidence: `${sessionPrefix} did not invoke any local project skill`,
+  };
+}
+
 function loadReplaySkillSurface(skillPath: string): ReplaySkillSurface {
   const fallbackName = basename(dirname(skillPath)) || "unknown-skill";
   try {
@@ -234,3 +577,48 @@ export function runHostReplayFixture(options: {
     };
   });
 }
+
+export async function runClaudeRuntimeReplayFixture(options: {
+  routing: string;
+  evalSet: EvalEntry[];
+  fixture: RoutingReplayFixture;
+  runtimeInvoker?: ClaudeRuntimeReplayInvoker;
+}): Promise<RoutingReplayEntryResult[]> {
+  const fallbackReason = (reason: string) =>
+    `runtime replay unavailable; fell back to fixture simulation (${reason})`;
+
+  if (options.fixture.platform !== "claude_code") {
+    return prefixReplayEvidence(
+      runHostReplayFixture(options),
+      fallbackReason(`unsupported platform ${options.fixture.platform}`),
+    );
+  }
+
+  const invokeRuntime = options.runtimeInvoker ?? invokeClaudeRuntimeReplay;
+  let workspace: ReplayWorkspace | undefined;
+
+  try {
+    workspace = buildRuntimeReplayWorkspace(options.fixture, options.routing);
+    const results: RoutingReplayEntryResult[] = [];
+
+    for (const entry of options.evalSet) {
+      const observation = await invokeRuntime({
+        query: entry.query,
+        workspaceRoot: workspace.rootDir,
+        targetSkillName: options.fixture.target_skill_name,
+        targetSkillPath: workspace.targetSkillPath,
+        competingSkillPaths: workspace.competingSkillPaths,
+      });
+      results.push(
+        evaluateRuntimeReplayObservation(entry, options.fixture, observation, workspace),
+      );
+    }
+
+    return results;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return prefixReplayEvidence(runHostReplayFixture(options), fallbackReason(message));
+  } finally {
+    if (workspace) cleanupRuntimeReplayWorkspace(workspace);
+  }
+}

package/cli/selftune/hooks/auto-activate.ts

@@ -147,6 +147,37 @@ export function evaluateRules(
   return suggestions;
 }
 
+// ---------------------------------------------------------------------------
+// Reusable auto-activate orchestration
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluate activation rules for a session and return suggestion strings.
+ * Checks PAI coexistence and session state dedup internally.
+ * Returns an empty array when PAI is active or no rules fire.
+ */
+export async function processAutoActivate(
+  sessionId: string,
+  settingsPath?: string,
+): Promise<string[]> {
+  // Only check PAI coexistence when a settings path is provided (platform-specific)
+  if (settingsPath && checkPaiCoexistence(settingsPath)) return [];
+
+  const { DEFAULT_RULES } = await import("../activation-rules.js");
+
+  const ctx: ActivationContext = {
+    session_id: sessionId,
+    query_log_path: QUERY_LOG,
+    telemetry_log_path: TELEMETRY_LOG,
+    evolution_audit_log_path: EVOLUTION_AUDIT_LOG,
+    selftune_dir: SELFTUNE_CONFIG_DIR,
+    settings_path: settingsPath ?? "",
+  };
+
+  const statePath = sessionStatePath(sessionId);
+  return evaluateRules(DEFAULT_RULES, ctx, statePath);
+}
+
 // ---------------------------------------------------------------------------
 // stdin main (only when executed directly, not when imported)
 // ---------------------------------------------------------------------------
@@ -155,43 +186,18 @@ if (import.meta.main) {
   try {
     const payload: PromptSubmitPayload = JSON.parse(await Bun.stdin.text());
     const sessionId = payload.session_id ?? "unknown";
-
-    // Dynamically import default rules (keeps hook file lightweight)
-    const { DEFAULT_RULES } = await import("../activation-rules.js");
-
-    /**
-     * The *_log_path fields exist for test overrides only; default code paths
-     * in activation-rules.ts read from SQLite when the path matches the
-     * constant (QUERY_LOG, EVOLUTION_AUDIT_LOG, etc.).
-     */
-    const ctx: ActivationContext = {
-      session_id: sessionId,
-      query_log_path: QUERY_LOG,
-      telemetry_log_path: TELEMETRY_LOG,
-      evolution_audit_log_path: EVOLUTION_AUDIT_LOG,
-      selftune_dir: SELFTUNE_CONFIG_DIR,
-      settings_path: CLAUDE_SETTINGS_PATH,
-    };
-
-    // Check PAI coexistence — if PAI is active, skip selftune suggestions
-    // (PAI handles skill-level activation; selftune handles observability)
-    if (!checkPaiCoexistence(CLAUDE_SETTINGS_PATH)) {
-      const statePath = sessionStatePath(sessionId);
-      const suggestions = evaluateRules(DEFAULT_RULES, ctx, statePath);
-
-      if (suggestions.length > 0) {
-        // Output as JSON with additionalContext — Claude Code adds this to
-        // Claude's context on UserPromptSubmit (more reliable than stderr)
-        const context = suggestions.map((s) => `[selftune] Suggestion: ${s}`).join("\n");
-        process.stdout.write(
-          JSON.stringify({
-            hookSpecificOutput: {
-              hookEventName: "UserPromptSubmit",
-              additionalContext: context,
-            },
-          }),
-        );
-      }
+    const suggestions = await processAutoActivate(sessionId, CLAUDE_SETTINGS_PATH);
+
+    if (suggestions.length > 0) {
+      const context = suggestions.map((s) => `[selftune] Suggestion: ${s}`).join("\n");
+      process.stdout.write(
+        JSON.stringify({
+          hookSpecificOutput: {
+            hookEventName: "UserPromptSubmit",
+            additionalContext: context,
+          },
+        }),
+      );
     }
   } catch {
     // silent — hooks must never block Claude

package/cli/selftune/hooks-shared/git-metadata.ts

@@ -0,0 +1,149 @@
+/**
+ * Shared git metadata extraction for hooks.
+ *
+ * Extracted from duplicated logic in session-stop.ts (branch/remote extraction)
+ * and commit-track.ts (commit detection, remote scrubbing, branch fallback).
+ *
+ * All functions are fail-open: git errors return undefined/null, never throw.
+ */
+
+import { execSync } from "node:child_process";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Git metadata extracted from a working directory. */
+export interface GitMetadata {
+  branch?: string;
+  repoRemote?: string;
+}
+
+/** Parsed commit information from git output. */
+export interface ParsedCommit {
+  sha?: string;
+  title?: string;
+  branch?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Pre-compiled regex patterns
+// ---------------------------------------------------------------------------
+
+/** Matches git commands that produce commits: commit, merge, cherry-pick, revert. */
+const GIT_COMMIT_CMD_RE = /\bgit\s+(commit|merge|cherry-pick|revert)\b/;
+
+/**
+ * Matches standard git commit output: [branch SHA] title
+ * Supports optional parenthetical like (root-commit).
+ * Branch names can contain word chars, slashes, dots, hyphens, plus signs.
+ */
+const COMMIT_OUTPUT_RE = /\[([\w/.+-]+)(?:\s+\([^)]+\))?\s+([a-f0-9]{7,40})\]\s+(.+)/;
+
+// ---------------------------------------------------------------------------
+// Git metadata extraction
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract git branch and remote URL from a working directory.
+ *
+ * Uses short-timeout execSync calls. Returns partial results if one
+ * command fails (e.g., branch succeeds but remote is not configured).
+ * Returns empty object if cwd is not a git repo.
+ *
+ * @param cwd  Working directory to inspect
+ */
+export function extractGitMetadata(cwd: string): GitMetadata {
+  if (!cwd) return {};
+
+  const result: GitMetadata = {};
+
+  try {
+    result.branch =
+      execSync("git rev-parse --abbrev-ref HEAD", {
+        cwd,
+        timeout: 3000,
+        stdio: ["ignore", "pipe", "ignore"],
+      })
+        .toString()
+        .trim() || undefined;
+  } catch {
+    /* not a git repo or git not available */
+  }
+
+  try {
+    const rawRemote =
+      execSync("git remote get-url origin", {
+        cwd,
+        timeout: 3000,
+        stdio: ["ignore", "pipe", "ignore"],
+      })
+        .toString()
+        .trim() || undefined;
+    if (rawRemote) {
+      result.repoRemote = scrubRemoteUrl(rawRemote);
+    }
+  } catch {
+    /* no remote configured */
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// URL scrubbing
+// ---------------------------------------------------------------------------
+
+/**
+ * Scrub credentials from a git remote URL.
+ *
+ * HTTP(S) URLs have username/password stripped. SSH URLs and other formats
+ * are returned as-is (they don't embed credentials in the URL structure).
+ *
+ * @param rawUrl  Raw remote URL from `git remote get-url`
+ * @returns       Scrubbed URL, or undefined for empty input
+ */
+export function scrubRemoteUrl(rawUrl: string): string | undefined {
+  if (!rawUrl) return undefined;
+  try {
+    const parsed = new URL(rawUrl);
+    parsed.username = "";
+    parsed.password = "";
+    return `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
+  } catch {
+    // SSH or non-URL format -- safe as-is
+    return rawUrl;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Commit detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a command string contains a git commit-producing operation.
+ * Detects: git commit, git merge, git cherry-pick, git revert.
+ */
+export function containsGitCommitCommand(command: string): boolean {
+  return GIT_COMMIT_CMD_RE.test(command);
+}
+
+/**
+ * Parse commit metadata from git's standard output format.
+ *
+ * Expects output like: `[main abc1234] Fix the bug`
+ * or with root-commit: `[main (root-commit) abc1234] Initial commit`
+ *
+ * @param stdout  The stdout from a git commit/merge/cherry-pick/revert command
+ * @returns       Parsed commit info, or null if output doesn't match
+ */
+export function parseCommitFromOutput(stdout: string): ParsedCommit | null {
+  const match = stdout.match(COMMIT_OUTPUT_RE);
+  if (!match) return null;
+
+  return {
+    branch: match[1],
+    sha: match[2],
+    title: match[3].trim(),
+  };
+}