selftune 0.1.4 → 0.2.1

package/cli/selftune/grading/pre-gates.ts

@@ -0,0 +1,104 @@
+/**
+ * pre-gates.ts
+ *
+ * Deterministic pre-gate checks that resolve grading expectations without LLM.
+ * Each gate matches an expectation text pattern and resolves it using telemetry data.
+ */
+
+import type { GradingExpectation, SessionTelemetryRecord } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Gate definitions
+// ---------------------------------------------------------------------------
+
+export interface PreGate {
+  name: string;
+  pattern: RegExp;
+  check: (ctx: PreGateContext) => boolean;
+}
+
+export interface PreGateContext {
+  telemetry: SessionTelemetryRecord;
+  skillName: string;
+  transcriptExcerpt?: string;
+}
+
+export interface PreGateResult {
+  resolved: GradingExpectation[];
+  remaining: string[];
+}
+
+/** Default set of pre-gates. */
+export const DEFAULT_GATES: PreGate[] = [
+  {
+    name: "skill_md_read",
+    pattern: /(read.*skill\.md|skill\.md.*read)/i,
+    check: (ctx) => {
+      // Check if skills_triggered contains the skill name
+      const triggered = ctx.telemetry.skills_triggered ?? [];
+      if (triggered.includes(ctx.skillName)) return true;
+      // Also check if transcript mentions reading SKILL.md
+      if (ctx.transcriptExcerpt && /Read.*SKILL\.md/i.test(ctx.transcriptExcerpt)) return true;
+      return false;
+    },
+  },
+  {
+    name: "expected_tools_called",
+    pattern: /tool[s]?\s+(were\s+)?called/i,
+    check: (ctx) => (ctx.telemetry.total_tool_calls ?? 0) > 0,
+  },
+  {
+    name: "error_count",
+    pattern: /error[s]?\s*(count|encountered)/i,
+    check: (ctx) => (ctx.telemetry.errors_encountered ?? 0) <= 2,
+  },
+  {
+    name: "session_completed",
+    pattern: /session\s*(completed|finished)/i,
+    check: (ctx) => (ctx.telemetry.assistant_turns ?? 0) > 0,
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Pre-gate runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run pre-gate checks against expectations. Returns resolved expectations
+ * (with source: "pre-gate" and score: 1.0 or 0.0) and remaining expectation
+ * texts that need LLM grading.
+ */
+export function runPreGates(
+  expectations: string[],
+  ctx: PreGateContext,
+  gates: PreGate[] = DEFAULT_GATES,
+): PreGateResult {
+  const resolved: GradingExpectation[] = [];
+  const remaining: string[] = [];
+
+  for (const text of expectations) {
+    let matched = false;
+    for (const gate of gates) {
+      if (gate.pattern.global || gate.pattern.sticky) {
+        gate.pattern.lastIndex = 0;
+      }
+      if (gate.pattern.test(text)) {
+        const passed = gate.check(ctx);
+        resolved.push({
+          text,
+          passed,
+          evidence: `Pre-gate "${gate.name}": ${passed ? "PASS" : "FAIL"}`,
+          score: passed ? 1.0 : 0.0,
+          source: "pre-gate",
+        });
+        matched = true;
+        break; // first matching gate wins
+      }
+    }
+    if (!matched) {
+      remaining.push(text);
+    }
+  }
+
+  return { resolved, remaining };
+}

package/cli/selftune/grading/results.ts

@@ -0,0 +1,42 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { SELFTUNE_CONFIG_DIR } from "../constants.js";
+import type { GradingResult } from "../types.js";
+
+export const DEFAULT_GRADING_DIR = join(SELFTUNE_CONFIG_DIR, "grading");
+
+export function readGradingResults(gradingDir: string = DEFAULT_GRADING_DIR): GradingResult[] {
+  if (!existsSync(gradingDir)) return [];
+
+  const results: GradingResult[] = [];
+
+  for (const entry of readdirSync(gradingDir).sort()) {
+    if (!entry.startsWith("result-") || !entry.endsWith(".json")) continue;
+
+    try {
+      const parsed = JSON.parse(
+        readFileSync(join(gradingDir, entry), "utf-8"),
+      ) as Partial<GradingResult>;
+      if (typeof parsed?.session_id !== "string" || typeof parsed?.skill_name !== "string")
+        continue;
+      results.push(parsed as GradingResult);
+    } catch {
+      // Ignore malformed grading artifacts.
+    }
+  }
+
+  return results.sort((a, b) => b.graded_at.localeCompare(a.graded_at));
+}
+
+export function readGradingResultsForSkill(
+  skillName: string,
+  gradingDir: string = DEFAULT_GRADING_DIR,
+): GradingResult[] {
+  const normalizedSkill = skillName.trim().toLowerCase();
+  if (!normalizedSkill) return [];
+
+  return readGradingResults(gradingDir).filter(
+    (result) => result.skill_name.trim().toLowerCase() === normalizedSkill,
+  );
+}

package/cli/selftune/hooks/auto-activate.ts

@@ -0,0 +1,185 @@
+#!/usr/bin/env bun
+/**
+ * Claude Code UserPromptSubmit hook: auto-activate.ts
+ *
+ * Evaluates activation rules against the current session context and
+ * outputs suggestions to stderr (shown to Claude as system messages).
+ * Suggestions are advisory — exit code is always 0.
+ *
+ * Session state is tracked to avoid repeated nags within a session.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import {
+  CLAUDE_SETTINGS_PATH,
+  EVOLUTION_AUDIT_LOG,
+  QUERY_LOG,
+  SELFTUNE_CONFIG_DIR,
+  sessionStatePath,
+  TELEMETRY_LOG,
+} from "../constants.js";
+import type {
+  ActivationContext,
+  ActivationRule,
+  PromptSubmitPayload,
+  SessionState,
+} from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Session state persistence
+// ---------------------------------------------------------------------------
+
+export function loadSessionState(path: string, sessionId: string): SessionState {
+  if (!existsSync(path)) {
+    return { session_id: sessionId, suggestions_shown: [], updated_at: new Date().toISOString() };
+  }
+
+  try {
+    const data = JSON.parse(readFileSync(path, "utf-8")) as SessionState;
+    if (data.session_id === sessionId && Array.isArray(data.suggestions_shown)) {
+      return data;
+    }
+  } catch {
+    // corrupt file — start fresh
+  }
+
+  return { session_id: sessionId, suggestions_shown: [], updated_at: new Date().toISOString() };
+}
+
+export function saveSessionState(path: string, state: SessionState): void {
+  const dir = dirname(path);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  writeFileSync(path, JSON.stringify(state, null, 2), "utf-8");
+}
+
+// ---------------------------------------------------------------------------
+// PAI coexistence check
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if PAI's skill-activation-prompt hook is registered in settings.
+ * If so, selftune defers skill-level suggestions.
+ */
+export function checkPaiCoexistence(settingsPath: string): boolean {
+  if (!existsSync(settingsPath)) return false;
+
+  try {
+    const settings = JSON.parse(readFileSync(settingsPath, "utf-8")) as {
+      hooks?: Record<string, Array<{ command?: string; hooks?: Array<{ command?: string }> }>>;
+    };
+
+    if (!settings.hooks) return false;
+
+    // Search all hook entries for skill-activation-prompt
+    for (const hookEntries of Object.values(settings.hooks)) {
+      if (!Array.isArray(hookEntries)) continue;
+      for (const entry of hookEntries) {
+        // Check flat entry.command
+        if (
+          typeof entry.command === "string" &&
+          entry.command.includes("skill-activation-prompt")
+        ) {
+          return true;
+        }
+        // Check nested entry.hooks[].command
+        if (entry.hooks && Array.isArray(entry.hooks)) {
+          for (const hook of entry.hooks) {
+            if (
+              typeof hook.command === "string" &&
+              hook.command.includes("skill-activation-prompt")
+            ) {
+              return true;
+            }
+          }
+        }
+      }
+    }
+  } catch {
+    // fail-open
+  }
+
+  return false;
+}
+
+// ---------------------------------------------------------------------------
+// Rule evaluation engine
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluate all rules against the current context, respecting session state.
+ * Returns array of suggestion strings for rules that fired.
+ */
+export function evaluateRules(
+  rules: ActivationRule[],
+  ctx: ActivationContext,
+  statePath: string,
+): string[] {
+  const state = loadSessionState(statePath, ctx.session_id);
+  const suggestions: string[] = [];
+  const newlyShown: string[] = [];
+
+  for (const rule of rules) {
+    // Skip rules already shown this session
+    if (state.suggestions_shown.includes(rule.id)) continue;
+
+    try {
+      const suggestion = rule.evaluate(ctx);
+      if (suggestion !== null) {
+        suggestions.push(suggestion);
+        newlyShown.push(rule.id);
+      }
+    } catch {
+      // fail-open: skip rules that throw
+    }
+  }
+
+  // Persist updated session state
+  if (newlyShown.length > 0) {
+    state.suggestions_shown.push(...newlyShown);
+    state.updated_at = new Date().toISOString();
+    saveSessionState(statePath, state);
+  }
+
+  return suggestions;
+}
+
+// ---------------------------------------------------------------------------
+// stdin main (only when executed directly, not when imported)
+// ---------------------------------------------------------------------------
+
+if (import.meta.main) {
+  try {
+    const payload: PromptSubmitPayload = JSON.parse(await Bun.stdin.text());
+    const sessionId = payload.session_id ?? "unknown";
+
+    // Dynamically import default rules (keeps hook file lightweight)
+    const { DEFAULT_RULES } = await import("../activation-rules.js");
+
+    const ctx: ActivationContext = {
+      session_id: sessionId,
+      query_log_path: QUERY_LOG,
+      telemetry_log_path: TELEMETRY_LOG,
+      evolution_audit_log_path: EVOLUTION_AUDIT_LOG,
+      selftune_dir: SELFTUNE_CONFIG_DIR,
+      settings_path: CLAUDE_SETTINGS_PATH,
+    };
+
+    // Check PAI coexistence — if PAI is active, skip selftune suggestions
+    // (PAI handles skill-level activation; selftune handles observability)
+    if (!checkPaiCoexistence(CLAUDE_SETTINGS_PATH)) {
+      const statePath = sessionStatePath(sessionId);
+      const suggestions = evaluateRules(DEFAULT_RULES, ctx, statePath);
+
+      for (const s of suggestions) {
+        // Output to stderr — Claude Code shows stderr as system messages
+        process.stderr.write(`[selftune] 💡 Suggestion: ${s}\n`);
+      }
+    }
+  } catch {
+    // silent — hooks must never block Claude
+  }
+  process.exit(0);
+}

package/cli/selftune/hooks/evolution-guard.ts

@@ -0,0 +1,165 @@
+#!/usr/bin/env bun
+/**
+ * Claude Code PreToolUse hook: evolution-guard.ts
+ *
+ * Fires before Write/Edit tool calls. If the target is a SKILL.md file
+ * that has a deployed evolution (i.e., is under active monitoring), and
+ * no recent `selftune watch` snapshot exists, this hook BLOCKS the write
+ * with exit code 2 and a message suggesting to run watch first.
+ *
+ * Exit codes:
+ *   0 = allow (not a SKILL.md, not monitored, or watch is recent)
+ *   2 = block with message (Claude Code convention for PreToolUse hooks)
+ *
+ * Fail-open: any error → exit 0 (never block accidentally).
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { EVOLUTION_AUDIT_LOG, SELFTUNE_CONFIG_DIR } from "../constants.js";
+import type { PreToolUsePayload } from "../types.js";
+import { readJsonl } from "../utils/jsonl.js";
+
+// ---------------------------------------------------------------------------
+// Detection helpers (same pattern as skill-change-guard)
+// ---------------------------------------------------------------------------
+
+function isSkillMdWrite(toolName: string, filePath: string): boolean {
+  if (toolName !== "Write" && toolName !== "Edit") return false;
+  return basename(filePath).toUpperCase() === "SKILL.MD";
+}
+
+function extractSkillName(filePath: string): string {
+  return basename(dirname(filePath)) || "unknown";
+}
+
+// ---------------------------------------------------------------------------
+// Active monitoring check (reads audit log directly — no evolution imports)
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a skill has an active deployed evolution (meaning it's under monitoring).
+ * Reads the evolution audit JSONL directly to respect architecture lint rules.
+ *
+ * A skill is "actively monitored" if its last audit action is "deployed".
+ * If the last action is "rolled_back", it's no longer monitored.
+ */
+export function checkActiveMonitoring(skillName: string, auditLogPath: string): boolean {
+  const entries = readJsonl<{
+    skill_name?: string;
+    action: string;
+  }>(auditLogPath);
+
+  // Filter entries for this skill by skill_name field
+  const skillEntries = entries.filter((e) => e.skill_name === skillName);
+  if (skillEntries.length === 0) return false;
+
+  const lastEntry = skillEntries[skillEntries.length - 1];
+  return lastEntry.action === "deployed";
+}
+
+// ---------------------------------------------------------------------------
+// Recent watch snapshot check (reads monitoring dir directly)
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if there's a recent monitoring snapshot for the given skill.
+ * "Recent" means within `maxAgeHours` hours.
+ */
+export function hasRecentWatchSnapshot(
+  skillName: string,
+  selftuneDir: string,
+  maxAgeHours: number,
+): boolean {
+  const snapshotPath = join(selftuneDir, "monitoring", "latest-snapshot.json");
+  if (!existsSync(snapshotPath)) return false;
+
+  try {
+    const snapshot = JSON.parse(readFileSync(snapshotPath, "utf-8")) as {
+      timestamp: string;
+      skill_name?: string;
+    };
+
+    // Must be for the same skill
+    if (snapshot.skill_name !== skillName) return false;
+
+    // Must be recent
+    const snapshotAge = Date.now() - new Date(snapshot.timestamp).getTime();
+    const maxAgeMs = maxAgeHours * 60 * 60 * 1000;
+    return snapshotAge <= maxAgeMs;
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Guard result type
+// ---------------------------------------------------------------------------
+
+export interface GuardResult {
+  exitCode: number;
+  message: string;
+}
+
+// ---------------------------------------------------------------------------
+// Core processing logic
+// ---------------------------------------------------------------------------
+
+export interface GuardOptions {
+  auditLogPath: string;
+  selftuneDir: string;
+  maxSnapshotAgeHours?: number;
+}
+
+/**
+ * Process a PreToolUse payload. Returns null if the write should be allowed,
+ * or a GuardResult with exitCode 2 if the write should be blocked.
+ */
+export function processEvolutionGuard(
+  payload: PreToolUsePayload,
+  options: GuardOptions,
+): GuardResult | null {
+  const filePath =
+    typeof payload.tool_input?.file_path === "string" ? payload.tool_input.file_path : "";
+
+  if (!isSkillMdWrite(payload.tool_name, filePath)) return null;
+
+  const skillName = extractSkillName(filePath);
+  const { auditLogPath, selftuneDir, maxSnapshotAgeHours = 24 } = options;
+
+  // Check if this skill is under active monitoring
+  if (!checkActiveMonitoring(skillName, auditLogPath)) return null;
+
+  // Check if there's a recent watch snapshot
+  if (hasRecentWatchSnapshot(skillName, selftuneDir, maxSnapshotAgeHours)) return null;
+
+  // Block: skill is monitored but no recent watch
+  return {
+    exitCode: 2,
+    message: `[selftune] Skill "${skillName}" has a deployed evolution and is under active monitoring. Run \`selftune watch --skill ${skillName}\` before modifying SKILL.md to check current health.`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// stdin main (only when executed directly, not when imported)
+// ---------------------------------------------------------------------------
+
+if (import.meta.main) {
+  try {
+    const payload: PreToolUsePayload = JSON.parse(await Bun.stdin.text());
+
+    const result = processEvolutionGuard(payload, {
+      auditLogPath: EVOLUTION_AUDIT_LOG,
+      selftuneDir: SELFTUNE_CONFIG_DIR,
+    });
+
+    if (result) {
+      // Exit code 2 = block with message
+      process.stderr.write(`${result.message}\n`);
+      process.exit(2);
+    }
+  } catch {
+    // Fail-open: any error → allow the write
+  }
+  process.exit(0);
+}

package/cli/selftune/hooks/prompt-log.ts

@@ -8,10 +8,141 @@
  * a skill — the raw material for false-negative eval entries.
  */
 
-import { QUERY_LOG, SKIP_PREFIXES } from "../constants.js";
-import type { PromptSubmitPayload, QueryLogRecord } from "../types.js";
+import { readdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { CANONICAL_LOG, QUERY_LOG, SIGNAL_LOG, SKIP_PREFIXES } from "../constants.js";
+import {
+  appendCanonicalRecord,
+  buildCanonicalPrompt,
+  type CanonicalBaseInput,
+  classifyIsActionable,
+  reservePromptIdentity,
+} from "../normalization.js";
+import type { ImprovementSignalRecord, PromptSubmitPayload, QueryLogRecord } from "../types.js";
 import { appendJsonl } from "../utils/jsonl.js";
 
+// ---------------------------------------------------------------------------
+// Installed skill name cache
+// ---------------------------------------------------------------------------
+
+let cachedSkillNames: string[] | null = null;
+
+/**
+ * Read directory names from ~/.claude/skills. Cached after first call.
+ * Returns empty array on any error (graceful degradation).
+ */
+export function getInstalledSkillNames(): string[] {
+  if (cachedSkillNames !== null) return cachedSkillNames;
+  try {
+    const skillsDir = join(homedir(), ".claude", "skills");
+    const entries = readdirSync(skillsDir, { withFileTypes: true });
+    cachedSkillNames = entries.filter((e) => e.isDirectory()).map((e) => e.name);
+  } catch {
+    cachedSkillNames = [];
+  }
+  return cachedSkillNames;
+}
+
+// ---------------------------------------------------------------------------
+// Signal detection patterns
+// ---------------------------------------------------------------------------
+
+interface SignalPattern {
+  regex: RegExp;
+  signal_type: "correction" | "explicit_request";
+  /** Named capture group index for the skill name. */
+  skillGroup: string;
+}
+
+const SIGNAL_PATTERNS: SignalPattern[] = [
+  // "why didn't you use/run/invoke X" → correction
+  {
+    regex: /why\s+didn['']t\s+you\s+(?:use|run|invoke)\s+(?:the\s+)?(?<skill>[\w-]+)/i,
+    signal_type: "correction",
+    skillGroup: "skill",
+  },
+  // "you should have used X" → correction
+  {
+    regex: /you\s+should\s+have\s+used\s+(?:the\s+)?(?<skill>[\w-]+)/i,
+    signal_type: "correction",
+    skillGroup: "skill",
+  },
+  // "next time use X" → correction
+  {
+    regex: /next\s+time\s+use\s+(?:the\s+)?(?<skill>[\w-]+)/i,
+    signal_type: "correction",
+    skillGroup: "skill",
+  },
+  // "forgot to use X" → correction
+  {
+    regex: /forgot\s+to\s+use\s+(?:the\s+)?(?<skill>[\w-]+)/i,
+    signal_type: "correction",
+    skillGroup: "skill",
+  },
+  // "please use X skill" / "please use the X skill" → explicit_request
+  {
+    regex: /please\s+use\s+(?:the\s+)?(?<skill>[\w-]+)\s+skill/i,
+    signal_type: "explicit_request",
+    skillGroup: "skill",
+  },
+  // "use the X skill" → explicit_request (must have "the" and "skill" to avoid false positives)
+  {
+    regex: /\buse\s+the\s+(?<skill>[\w-]+)\s+skill/i,
+    signal_type: "explicit_request",
+    skillGroup: "skill",
+  },
+];
+
+/**
+ * Detect whether a user query contains an improvement signal.
+ * Pure regex — no LLM, no network.
+ */
+export function detectImprovementSignal(
+  query: string,
+  sessionId: string,
+  installedSkills?: string[],
+): ImprovementSignalRecord | null {
+  const skills = installedSkills ?? getInstalledSkillNames();
+  const skillsLower = skills.map((s) => s.toLowerCase());
+
+  for (const pattern of SIGNAL_PATTERNS) {
+    const match = query.match(pattern.regex);
+    if (!match?.groups?.[pattern.skillGroup]) continue;
+
+    const rawSkill = match.groups[pattern.skillGroup];
+
+    // Skip generic words that aren't skill names
+    const genericWords = new Set(["strict", "git", "the", "a", "an", "this", "that", "it", "my"]);
+    if (genericWords.has(rawSkill.toLowerCase())) continue;
+
+    // Try to match against installed skills (case-insensitive)
+    let mentionedSkill: string | undefined;
+    const idx = skillsLower.indexOf(rawSkill.toLowerCase());
+    if (idx !== -1) {
+      mentionedSkill = skills[idx];
+    } else {
+      // Use the raw captured name if it looks like a skill (capitalized or known)
+      mentionedSkill = rawSkill;
+    }
+
+    return {
+      timestamp: new Date().toISOString(),
+      session_id: sessionId,
+      query,
+      signal_type: pattern.signal_type,
+      mentioned_skill: mentionedSkill,
+      consumed: false,
+    };
+  }
+
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Core prompt processing
+// ---------------------------------------------------------------------------
+
 /**
  * Core processing logic, exported for testability.
  * Returns the record that was appended, or null if skipped.
@@ -19,6 +150,9 @@ import { appendJsonl } from "../utils/jsonl.js";
 export function processPrompt(
   payload: PromptSubmitPayload,
   logPath: string = QUERY_LOG,
+  canonicalLogPath: string = CANONICAL_LOG,
+  promptStatePath?: string,
+  signalLogPath: string = SIGNAL_LOG,
 ): QueryLogRecord | null {
   const query = (payload.user_prompt ?? "").trim();
 
@@ -37,6 +171,42 @@ export function processPrompt(
   };
 
   appendJsonl(logPath, record);
+
+  // Emit canonical prompt record (additive)
+  const baseInput: CanonicalBaseInput = {
+    platform: "claude_code",
+    capture_mode: "hook",
+    source_session_kind: "interactive",
+    session_id: record.session_id,
+    raw_source_ref: { event_type: "UserPromptSubmit" },
+  };
+  const isActionable = classifyIsActionable(query);
+  const promptIdentity = reservePromptIdentity(
+    record.session_id,
+    isActionable,
+    promptStatePath,
+    canonicalLogPath,
+  );
+  const canonical = buildCanonicalPrompt({
+    ...baseInput,
+    prompt_id: promptIdentity.prompt_id,
+    occurred_at: record.timestamp,
+    prompt_text: query,
+    prompt_index: promptIdentity.prompt_index,
+    is_actionable: isActionable,
+  });
+  appendCanonicalRecord(canonical, canonicalLogPath);
+
+  // Detect and log improvement signals (never throws)
+  try {
+    const signal = detectImprovementSignal(query, record.session_id);
+    if (signal) {
+      appendJsonl(signalLogPath, signal);
+    }
+  } catch {
+    // silent — hooks must never block Claude
+  }
+
   return record;
 }