selftune 0.1.2 → 0.2.0

package/cli/selftune/hooks/skill-change-guard.ts

@@ -0,0 +1,112 @@
+#!/usr/bin/env bun
+/**
+ * Claude Code PreToolUse hook: skill-change-guard.ts
+ *
+ * Fires before Write/Edit tool calls. If the target is a SKILL.md file,
+ * outputs a suggestion to run `selftune watch --skill <name>` to monitor
+ * the impact of the change.
+ *
+ * This is advisory only — exit code is always 0, never blocking.
+ * Uses session state to avoid repeating suggestions for the same skill.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, dirname } from "node:path";
+import { SESSION_STATE_DIR } from "../constants.js";
+import type { PreToolUsePayload } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Detection helpers
+// ---------------------------------------------------------------------------
+
+/** Check if a tool call is a Write or Edit targeting a SKILL.md file. */
+export function isSkillMdWrite(toolName: string, filePath: string): boolean {
+  if (toolName !== "Write" && toolName !== "Edit") return false;
+  return basename(filePath).toUpperCase() === "SKILL.MD";
+}
+
+/** Extract the skill folder name from a path ending in SKILL.md. */
+export function extractSkillNameFromPath(filePath: string): string {
+  return basename(dirname(filePath)) || "unknown";
+}
+
+// ---------------------------------------------------------------------------
+// Session state (minimal — just tracks which skills we've already warned about)
+// ---------------------------------------------------------------------------
+
+interface GuardState {
+  session_id: string;
+  warned_skills: string[];
+}
+
+function loadGuardState(path: string, sessionId: string): GuardState {
+  if (!existsSync(path)) {
+    return { session_id: sessionId, warned_skills: [] };
+  }
+  try {
+    const data = JSON.parse(readFileSync(path, "utf-8")) as GuardState;
+    if (data.session_id === sessionId && Array.isArray(data.warned_skills)) {
+      return data;
+    }
+  } catch {
+    // corrupt — start fresh
+  }
+  return { session_id: sessionId, warned_skills: [] };
+}
+
+function saveGuardState(path: string, state: GuardState): void {
+  const dir = dirname(path);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  writeFileSync(path, JSON.stringify(state, null, 2), "utf-8");
+}
+
+// ---------------------------------------------------------------------------
+// Core processing logic
+// ---------------------------------------------------------------------------
+
+/**
+ * Process a PreToolUse payload and return a suggestion string if the tool
+ * call is writing to a SKILL.md file that hasn't been warned about yet.
+ */
+export function processPreToolUse(payload: PreToolUsePayload, statePath: string): string | null {
+  const filePath =
+    typeof payload.tool_input?.file_path === "string" ? payload.tool_input.file_path : "";
+
+  if (!isSkillMdWrite(payload.tool_name, filePath)) return null;
+
+  const skillName = extractSkillNameFromPath(filePath);
+  const sessionId = payload.session_id ?? "unknown";
+
+  // Check if we've already warned about this skill in this session
+  const state = loadGuardState(statePath, sessionId);
+  if (state.warned_skills.includes(skillName)) return null;
+
+  // Record that we warned about this skill
+  state.warned_skills.push(skillName);
+  saveGuardState(statePath, state);
+
+  return `Run \`selftune watch --skill ${skillName}\` to monitor the impact of this SKILL.md change.`;
+}
+
+// ---------------------------------------------------------------------------
+// stdin main (only when executed directly, not when imported)
+// ---------------------------------------------------------------------------
+
+if (import.meta.main) {
+  try {
+    const payload: PreToolUsePayload = JSON.parse(await Bun.stdin.text());
+    const sessionId = payload.session_id ?? "unknown";
+    const safe = sessionId.replace(/[^a-zA-Z0-9_-]/g, "_");
+    const statePath = `${SESSION_STATE_DIR}/guard-state-${safe}.json`;
+
+    const suggestion = processPreToolUse(payload, statePath);
+    if (suggestion) {
+      process.stderr.write(`[selftune] 💡 Suggestion: ${suggestion}\n`);
+    }
+  } catch {
+    // silent — hooks must never block Claude
+  }
+  process.exit(0);
+}

package/cli/selftune/index.ts

@@ -8,14 +8,23 @@
  *   selftune grade [options]          — Grade a skill session
  *   selftune ingest-codex [options]   — Ingest Codex rollout logs
  *   selftune ingest-opencode [options] — Ingest OpenCode sessions
+ *   selftune ingest-openclaw [options] — Ingest OpenClaw sessions
  *   selftune wrap-codex [options]     — Wrap codex exec with telemetry
+ *   selftune replay [options]         — Replay Claude Code transcripts into logs
+ *   selftune contribute [options]     — Export anonymized skill data for community
  *   selftune evolve [options]         — Evolve a skill description via failure patterns
+ *   selftune evolve-body [options]    — Evolve a skill body or routing table
  *   selftune rollback [options]       — Rollback a skill to its pre-evolution state
  *   selftune watch [options]          — Monitor post-deploy skill health
  *   selftune doctor                   — Run health checks
  *   selftune status                   — Show skill health summary
  *   selftune last                     — Show last session details
  *   selftune dashboard [options]      — Open visual data dashboard
+ *   selftune cron [options]           — Manage OpenClaw cron jobs (setup, list, remove)
+ *   selftune baseline [options]      — Measure skill value vs. no-skill baseline
+ *   selftune composability [options]  — Analyze skill co-occurrence conflicts
+ *   selftune unit-test [options]     — Run or generate skill unit tests
+ *   selftune import-skillsbench [options] — Import SkillsBench task corpus as eval entries
  */
 
 const command = process.argv[2];
@@ -32,14 +41,24 @@ Commands:
   grade              Grade a skill session
   ingest-codex       Ingest Codex rollout logs
   ingest-opencode    Ingest OpenCode sessions
+  ingest-openclaw    Ingest OpenClaw sessions
   wrap-codex         Wrap codex exec with telemetry
+  replay             Replay Claude Code transcripts into logs
+  contribute         Export anonymized skill data for community
   evolve             Evolve a skill description via failure patterns
+  evolve-body        Evolve a skill body or routing table
   rollback           Rollback a skill to its pre-evolution state
   watch              Monitor post-deploy skill health
   doctor             Run health checks
   status             Show skill health summary
   last               Show last session details
   dashboard          Open visual data dashboard
+  cron               Manage OpenClaw cron jobs (setup, list, remove)
+  badge              Generate skill health badges for READMEs
+  baseline           Measure skill value vs. no-skill baseline
+  composability      Analyze skill co-occurrence conflicts
+  unit-test          Run or generate skill unit tests
+  import-skillsbench Import SkillsBench task corpus as eval entries
 
 Run 'selftune <command> --help' for command-specific options.`);
   process.exit(0);
@@ -77,16 +96,41 @@ switch (command) {
     cliMain();
     break;
   }
+  case "ingest-openclaw": {
+    const { cliMain } = await import("./ingestors/openclaw-ingest.js");
+    cliMain();
+    break;
+  }
   case "wrap-codex": {
     const { cliMain } = await import("./ingestors/codex-wrapper.js");
     await cliMain();
     break;
   }
+  case "replay": {
+    const { cliMain } = await import("./ingestors/claude-replay.js");
+    cliMain();
+    break;
+  }
+  case "contribute": {
+    const { cliMain } = await import("./contribute/contribute.js");
+    await cliMain();
+    break;
+  }
   case "evolve": {
     const { cliMain } = await import("./evolution/evolve.js");
     await cliMain();
     break;
   }
+  case "evolve-body": {
+    const { cliMain } = await import("./evolution/evolve-body.js");
+    await cliMain();
+    break;
+  }
+  case "baseline": {
+    const { cliMain } = await import("./eval/baseline.js");
+    await cliMain();
+    break;
+  }
   case "rollback": {
     const { cliMain } = await import("./evolution/rollback.js");
     await cliMain();
@@ -116,9 +160,53 @@ switch (command) {
   }
   case "dashboard": {
     const { cliMain } = await import("./dashboard.js");
+    await cliMain();
+    break;
+  }
+  case "cron": {
+    const { cliMain } = await import("./cron/setup.js");
+    await cliMain();
+    break;
+  }
+  case "badge": {
+    const { cliMain } = await import("./badge/badge.js");
+    cliMain();
+    break;
+  }
+  case "unit-test": {
+    const { cliMain } = await import("./eval/unit-test-cli.js");
+    await cliMain();
+    break;
+  }
+  case "import-skillsbench": {
+    const { cliMain } = await import("./eval/import-skillsbench.js");
     cliMain();
     break;
   }
+  case "composability": {
+    const { parseArgs } = await import("node:util");
+    const { readJsonl } = await import("./utils/jsonl.js");
+    const { TELEMETRY_LOG } = await import("./constants.js");
+    const { analyzeComposability } = await import("./eval/composability.js");
+    const { values } = parseArgs({
+      options: {
+        skill: { type: "string" },
+        window: { type: "string" },
+        "telemetry-log": { type: "string" },
+      },
+      strict: true,
+    });
+    if (!values.skill) {
+      console.error("[ERROR] --skill <name> is required.");
+      process.exit(1);
+    }
+    const logPath = values["telemetry-log"] ?? TELEMETRY_LOG;
+    const telemetry = readJsonl(logPath);
+    const windowSize = values.window ? Number.parseInt(values.window, 10) : undefined;
+    const report = analyzeComposability(values.skill, telemetry, windowSize);
+    console.log(JSON.stringify(report, null, 2));
+    break;
+  }
   default:
     console.error(`Unknown command: ${command}\nRun 'selftune --help' for available commands.`);
     process.exit(1);

package/cli/selftune/ingestors/claude-replay.ts

@@ -0,0 +1,351 @@
+#!/usr/bin/env bun
+/**
+ * Claude Code transcript ingestor: claude-replay.ts
+ *
+ * Retroactively ingests Claude Code session transcripts into our shared
+ * skill eval log format.
+ *
+ * Claude Code saves transcripts to:
+ *   ~/.claude/projects/<hash>/<session-id>.jsonl
+ *
+ * This script scans those files and populates:
+ *   ~/.claude/all_queries_log.jsonl
+ *   ~/.claude/session_telemetry_log.jsonl
+ *   ~/.claude/skill_usage_log.jsonl
+ *
+ * Usage:
+ *   bun claude-replay.ts
+ *   bun claude-replay.ts --since 2026-01-01
+ *   bun claude-replay.ts --projects-dir /custom/path
+ *   bun claude-replay.ts --dry-run
+ *   bun claude-replay.ts --force
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+import { parseArgs } from "node:util";
+import {
+  CLAUDE_CODE_MARKER,
+  CLAUDE_CODE_PROJECTS_DIR,
+  QUERY_LOG,
+  SKILL_LOG,
+  SKIP_PREFIXES,
+  TELEMETRY_LOG,
+} from "../constants.js";
+import type {
+  QueryLogRecord,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+  TranscriptMetrics,
+} from "../types.js";
+import { appendJsonl, loadMarker, saveMarker } from "../utils/jsonl.js";
+import { parseTranscript } from "../utils/transcript.js";
+
+export interface ParsedSession {
+  transcript_path: string;
+  session_id: string;
+  timestamp: string;
+  metrics: TranscriptMetrics;
+  user_queries: Array<{ query: string; timestamp: string }>;
+}
+
+/**
+ * Find all .jsonl transcript files under projectsDir/<hash>/<session>.jsonl.
+ * If `since` is given, only return files with mtime >= since.
+ */
+export function findTranscriptFiles(projectsDir: string, since?: Date): string[] {
+  if (!existsSync(projectsDir)) return [];
+
+  const files: string[] = [];
+
+  let hashDirs: string[];
+  try {
+    hashDirs = readdirSync(projectsDir).sort();
+  } catch {
+    return [];
+  }
+
+  for (const hashEntry of hashDirs) {
+    const hashDir = join(projectsDir, hashEntry);
+    try {
+      if (!statSync(hashDir).isDirectory()) continue;
+    } catch {
+      continue;
+    }
+
+    let sessionFiles: string[];
+    try {
+      sessionFiles = readdirSync(hashDir).sort();
+    } catch {
+      continue;
+    }
+
+    for (const file of sessionFiles) {
+      if (!file.endsWith(".jsonl")) continue;
+
+      const filePath = join(hashDir, file);
+      if (since) {
+        try {
+          const mtime = statSync(filePath).mtime;
+          if (mtime < since) continue;
+        } catch {
+          continue;
+        }
+      }
+
+      files.push(filePath);
+    }
+  }
+
+  return files.sort();
+}
+
+/**
+ * Extract all user queries from a Claude Code transcript JSONL.
+ *
+ * Handles two transcript variants:
+ *   Variant A: {"type": "user", "message": {"role": "user", "content": [...]}}
+ *   Variant B: {"role": "user", "content": "..."}
+ *
+ * Filters out messages matching SKIP_PREFIXES and queries < 4 chars.
+ */
+export function extractAllUserQueries(
+  transcriptPath: string,
+): Array<{ query: string; timestamp: string }> {
+  if (!existsSync(transcriptPath)) return [];
+
+  let content: string;
+  try {
+    content = readFileSync(transcriptPath, "utf-8");
+  } catch {
+    return [];
+  }
+
+  const results: Array<{ query: string; timestamp: string }> = [];
+
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    // Normalise: unwrap nested message if present
+    const msg = (entry.message as Record<string, unknown>) ?? entry;
+    const role = (msg.role as string) ?? (entry.role as string) ?? "";
+
+    if (role !== "user") continue;
+
+    const entryContent = msg.content ?? entry.content ?? "";
+    let text = "";
+
+    if (typeof entryContent === "string") {
+      text = entryContent.trim();
+    } else if (Array.isArray(entryContent)) {
+      const texts = entryContent
+        .filter(
+          (p): p is Record<string, unknown> =>
+            typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
+        )
+        .map((p) => (p.text as string) ?? "")
+        .filter(Boolean);
+      text = texts.join(" ").trim();
+    }
+
+    if (!text) continue;
+
+    // Apply SKIP_PREFIXES filter
+    const shouldSkip = SKIP_PREFIXES.some((prefix) => text.startsWith(prefix));
+    if (shouldSkip) continue;
+
+    // Apply 4-char minimum length filter
+    if (text.length < 4) continue;
+
+    // Extract timestamp from entry if present, else empty string
+    const timestamp = (entry.timestamp as string) ?? (msg.timestamp as string) ?? "";
+
+    results.push({ query: text, timestamp });
+  }
+
+  return results;
+}
+
+/**
+ * Parse a Claude Code session transcript into a ParsedSession.
+ * Returns null if no user queries are found after filtering.
+ */
+export function parseSession(transcriptPath: string): ParsedSession | null {
+  const metrics = parseTranscript(transcriptPath);
+  const userQueries = extractAllUserQueries(transcriptPath);
+
+  if (userQueries.length === 0) return null;
+
+  const sessionId = basename(transcriptPath, ".jsonl");
+
+  // Determine timestamp: use first query's timestamp, or file mtime as fallback
+  let timestamp = userQueries[0].timestamp;
+  if (!timestamp) {
+    try {
+      timestamp = statSync(transcriptPath).mtime.toISOString();
+    } catch {
+      timestamp = new Date().toISOString();
+    }
+  }
+
+  return {
+    transcript_path: transcriptPath,
+    session_id: sessionId,
+    timestamp,
+    metrics,
+    user_queries: userQueries,
+  };
+}
+
+/**
+ * Write parsed session data to shared JSONL logs.
+ * Writes ONE QueryLogRecord per user query, ONE SessionTelemetryRecord per session,
+ * and ONE SkillUsageRecord per triggered skill.
+ */
+export function writeSession(
+  session: ParsedSession,
+  dryRun = false,
+  queryLogPath: string = QUERY_LOG,
+  telemetryLogPath: string = TELEMETRY_LOG,
+  skillLogPath: string = SKILL_LOG,
+): void {
+  if (dryRun) {
+    console.log(
+      `  [DRY RUN] Would ingest: session=${session.session_id.slice(0, 12)}... ` +
+        `turns=${session.metrics.assistant_turns} queries=${session.user_queries.length} ` +
+        `skills=${JSON.stringify(session.metrics.skills_triggered)}`,
+    );
+    return;
+  }
+
+  // Write ONE query record per user query
+  for (const uq of session.user_queries) {
+    const queryRecord: QueryLogRecord = {
+      timestamp: uq.timestamp || session.timestamp,
+      session_id: session.session_id,
+      query: uq.query,
+      source: "claude_code_replay",
+    };
+    appendJsonl(queryLogPath, queryRecord, "all_queries");
+  }
+
+  // Write ONE telemetry record per session
+  const telemetry: SessionTelemetryRecord = {
+    timestamp: session.timestamp,
+    session_id: session.session_id,
+    cwd: "",
+    transcript_path: session.transcript_path,
+    tool_calls: session.metrics.tool_calls,
+    total_tool_calls: session.metrics.total_tool_calls,
+    bash_commands: session.metrics.bash_commands,
+    skills_triggered: session.metrics.skills_triggered,
+    assistant_turns: session.metrics.assistant_turns,
+    errors_encountered: session.metrics.errors_encountered,
+    transcript_chars: session.metrics.transcript_chars,
+    last_user_query: session.metrics.last_user_query,
+    source: "claude_code_replay",
+  };
+  appendJsonl(telemetryLogPath, telemetry, "session_telemetry");
+
+  // Write ONE skill record per triggered skill
+  for (const skillName of session.metrics.skills_triggered) {
+    const skillRecord: SkillUsageRecord = {
+      timestamp: session.timestamp,
+      session_id: session.session_id,
+      skill_name: skillName,
+      skill_path: `(claude_code:${skillName})`,
+      query: session.metrics.last_user_query,
+      triggered: true,
+      source: "claude_code_replay",
+    };
+    appendJsonl(skillLogPath, skillRecord, "skill_usage");
+  }
+}
+
+// --- CLI main ---
+export function cliMain(): void {
+  const { values } = parseArgs({
+    options: {
+      "projects-dir": { type: "string", default: CLAUDE_CODE_PROJECTS_DIR },
+      since: { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+      verbose: { type: "boolean", short: "v", default: false },
+    },
+    strict: true,
+  });
+
+  const projectsDir = values["projects-dir"] ?? CLAUDE_CODE_PROJECTS_DIR;
+  let since: Date | undefined;
+  if (values.since) {
+    since = new Date(values.since);
+    if (Number.isNaN(since.getTime())) {
+      console.error(
+        `Error: Invalid --since date: "${values.since}". Use a valid date format (e.g., 2026-01-01).`,
+      );
+      process.exit(1);
+    }
+  }
+
+  const transcriptFiles = findTranscriptFiles(projectsDir, since);
+  if (transcriptFiles.length === 0) {
+    console.log(`No transcript files found under ${projectsDir}/`);
+    console.log("Make sure you've run some Claude Code sessions.");
+    process.exit(0);
+  }
+
+  const alreadyIngested = values.force ? new Set<string>() : loadMarker(CLAUDE_CODE_MARKER);
+  const newIngested = new Set<string>();
+
+  const pending = transcriptFiles.filter((f) => !alreadyIngested.has(f));
+  console.log(
+    `Found ${transcriptFiles.length} transcript files, ${pending.length} not yet ingested.`,
+  );
+
+  if (since) {
+    console.log(`  Filtering to sessions from ${values.since} onward.`);
+  }
+
+  let ingestedCount = 0;
+  let skippedCount = 0;
+
+  for (const transcriptFile of pending) {
+    const session = parseSession(transcriptFile);
+    if (session === null) {
+      if (values.verbose) {
+        console.log(`  SKIP (empty/no queries): ${basename(transcriptFile)}`);
+      }
+      skippedCount += 1;
+      continue;
+    }
+
+    if (values.verbose || values["dry-run"]) {
+      console.log(`  ${values["dry-run"] ? "[DRY] " : ""}Ingesting: ${basename(transcriptFile)}`);
+    }
+
+    writeSession(session, values["dry-run"]);
+    newIngested.add(transcriptFile);
+    ingestedCount += 1;
+  }
+
+  if (!values["dry-run"]) {
+    saveMarker(CLAUDE_CODE_MARKER, new Set([...alreadyIngested, ...newIngested]));
+  }
+
+  console.log(`\nDone. Ingested ${ingestedCount} sessions, skipped ${skippedCount}.`);
+  if (newIngested.size > 0 && !values["dry-run"]) {
+    console.log(`Marker updated: ${CLAUDE_CODE_MARKER}`);
+  }
+}
+
+if (import.meta.main) {
+  cliMain();
+}

package/cli/selftune/ingestors/codex-rollout.ts

@@ -21,7 +21,7 @@
  *   bun codex-rollout.ts --force
  */
 
-import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { basename, join } from "node:path";
 import { parseArgs } from "node:util";