selftune 0.1.0

package/cli/selftune/init.ts

@@ -0,0 +1,297 @@
+#!/usr/bin/env bun
+/**
+ * selftune init — Bootstrap agent identity and write config.
+ *
+ * Detects the coding agent environment, resolves the CLI path,
+ * determines LLM mode, checks hook installation, and writes
+ * the result to ~/.selftune/config.json.
+ *
+ * Usage:
+ *   selftune init [--agent <type>] [--cli-path <path>] [--llm-mode <mode>] [--force]
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+import { SELFTUNE_CONFIG_DIR, SELFTUNE_CONFIG_PATH } from "./constants.js";
+import type { SelftuneConfig } from "./types.js";
+import { detectAgent } from "./utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// Agent type detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect which coding agent environment we are running inside.
+ *
+ * Detection order:
+ *   1. Claude Code — ~/.claude/ directory exists AND (`which claude` OR env signals)
+ *   2. Codex — $CODEX_HOME set OR `which codex`
+ *   3. OpenCode — ~/.local/share/opencode/opencode.db exists OR `which opencode`
+ *   4. "unknown" fallback
+ */
+const VALID_AGENT_TYPES: SelftuneConfig["agent_type"][] = [
+  "claude_code",
+  "codex",
+  "opencode",
+  "unknown",
+];
+
+export function detectAgentType(
+  override?: string,
+  homeOverride?: string,
+): SelftuneConfig["agent_type"] {
+  if (override) {
+    if (VALID_AGENT_TYPES.includes(override as SelftuneConfig["agent_type"])) {
+      return override as SelftuneConfig["agent_type"];
+    }
+    console.error(`[WARN] Unknown agent type "${override}", falling back to detection`);
+  }
+
+  const home = homeOverride ?? homedir();
+
+  // Claude Code: .claude directory + claude binary
+  const claudeDir = join(home, ".claude");
+  if (existsSync(claudeDir)) {
+    if (Bun.which("claude") || process.env.CLAUDE_CODE_ENTRYPOINT) {
+      return "claude_code";
+    }
+  }
+
+  // Codex: env var or binary
+  if (process.env.CODEX_HOME || Bun.which("codex")) {
+    return "codex";
+  }
+
+  // OpenCode: db file or binary
+  const opencodeDb = join(home, ".local", "share", "opencode", "opencode.db");
+  if (existsSync(opencodeDb) || Bun.which("opencode")) {
+    return "opencode";
+  }
+
+  return "unknown";
+}
+
+// ---------------------------------------------------------------------------
+// CLI path resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the absolute path to cli/selftune/index.ts.
+ * Uses the directory of this file (init.ts lives alongside index.ts).
+ */
+export function determineCliPath(override?: string): string {
+  if (override) return override;
+  return resolve(dirname(import.meta.path), "index.ts");
+}
+
+// ---------------------------------------------------------------------------
+// LLM mode determination
+// ---------------------------------------------------------------------------
+
+/**
+ * Determine LLM mode and agent CLI based on available signals.
+ */
+export function determineLlmMode(
+  agentCli: string | null,
+  hasApiKey?: boolean,
+  modeOverride?: string,
+): { llm_mode: "agent" | "api"; agent_cli: string | null } {
+  const detectedAgent = agentCli;
+  const validModes = ["agent", "api"] as const;
+  if (modeOverride && !validModes.includes(modeOverride as (typeof validModes)[number])) {
+    throw new Error(
+      `Invalid --llm-mode "${modeOverride}". Allowed values: ${validModes.join(", ")}`,
+    );
+  }
+  const resolvedMode = modeOverride as "agent" | "api" | undefined;
+
+  if (resolvedMode) {
+    return { llm_mode: resolvedMode, agent_cli: detectedAgent };
+  }
+
+  if (detectedAgent) {
+    return { llm_mode: "agent", agent_cli: detectedAgent };
+  }
+
+  if (hasApiKey) {
+    return { llm_mode: "api", agent_cli: null };
+  }
+
+  // Fallback: agent mode with null cli (will need setup)
+  return { llm_mode: "agent", agent_cli: null };
+}
+
+// ---------------------------------------------------------------------------
+// Hook detection (Claude Code only)
+// ---------------------------------------------------------------------------
+
+const REQUIRED_HOOK_KEYS = ["prompt-submit", "post-tool-use", "session-stop"] as const;
+
+/**
+ * Check if the selftune hooks are configured in Claude Code settings.
+ */
+export function checkClaudeCodeHooks(settingsPath: string): boolean {
+  if (!existsSync(settingsPath)) return false;
+
+  try {
+    const raw = readFileSync(settingsPath, "utf-8");
+    const settings = JSON.parse(raw);
+    const hooks = settings?.hooks;
+    if (!hooks || typeof hooks !== "object") return false;
+
+    for (const key of REQUIRED_HOOK_KEYS) {
+      const entries = hooks[key];
+      if (!Array.isArray(entries) || entries.length === 0) return false;
+      // Check that at least one entry references selftune
+      const hasSelftune = entries.some(
+        (e: { command?: string }) =>
+          typeof e.command === "string" && e.command.includes("selftune"),
+      );
+      if (!hasSelftune) return false;
+    }
+
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Init options (for testability)
+// ---------------------------------------------------------------------------
+
+export interface InitOptions {
+  configDir: string;
+  configPath: string;
+  force: boolean;
+  agentOverride?: string;
+  cliPathOverride?: string;
+  llmModeOverride?: string;
+  homeDir?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Core init logic
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the init flow. Returns the written (or existing) config.
+ * Extracted as a pure function for testability.
+ */
+export function runInit(opts: InitOptions): SelftuneConfig {
+  const { configDir, configPath, force } = opts;
+
+  // If config exists and no --force, return existing
+  if (!force && existsSync(configPath)) {
+    const raw = readFileSync(configPath, "utf-8");
+    try {
+      return JSON.parse(raw) as SelftuneConfig;
+    } catch (err) {
+      throw new Error(
+        `Config file at ${configPath} contains invalid JSON. Delete it or use --force to reinitialize. Cause: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  // Detect agent type
+  const agentType = detectAgentType(opts.agentOverride, opts.homeDir);
+
+  // Resolve CLI path
+  const cliPath = determineCliPath(opts.cliPathOverride);
+
+  // Detect agent CLI
+  const agentCli = detectAgent();
+
+  // Determine LLM mode
+  const hasApiKey = Boolean(process.env.ANTHROPIC_API_KEY);
+  const { llm_mode, agent_cli } = determineLlmMode(agentCli, hasApiKey, opts.llmModeOverride);
+
+  // Check hooks (Claude Code only)
+  const home = opts.homeDir ?? homedir();
+  const settingsPath = join(home, ".claude", "settings.json");
+  const hooksInstalled = agentType === "claude_code" ? checkClaudeCodeHooks(settingsPath) : false;
+
+  const config: SelftuneConfig = {
+    agent_type: agentType,
+    cli_path: cliPath,
+    llm_mode,
+    agent_cli,
+    hooks_installed: hooksInstalled,
+    initialized_at: new Date().toISOString(),
+  };
+
+  // Write config
+  mkdirSync(configDir, { recursive: true });
+  writeFileSync(configPath, JSON.stringify(config, null, 2), "utf-8");
+
+  return config;
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      agent: { type: "string" },
+      "cli-path": { type: "string" },
+      "llm-mode": { type: "string" },
+      force: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  const configDir = SELFTUNE_CONFIG_DIR;
+  const configPath = SELFTUNE_CONFIG_PATH;
+  const force = values.force ?? false;
+
+  // Check for existing config without force
+  if (!force && existsSync(configPath)) {
+    try {
+      const raw = readFileSync(configPath, "utf-8");
+      const existing = JSON.parse(raw) as SelftuneConfig;
+      console.log(JSON.stringify(existing, null, 2));
+      console.error("Already initialized. Use --force to reinitialize.");
+      process.exit(0);
+    } catch (err) {
+      console.error(
+        `[WARN] Config at ${configPath} is corrupted: ${err instanceof Error ? err.message : String(err)}. Reinitializing...`,
+      );
+    }
+  }
+
+  const config = runInit({
+    configDir,
+    configPath,
+    force,
+    agentOverride: values.agent,
+    cliPathOverride: values["cli-path"],
+    llmModeOverride: values["llm-mode"],
+  });
+
+  console.log(JSON.stringify(config, null, 2));
+
+  // Run doctor as post-check
+  const { doctor } = await import("./observability.js");
+  const doctorResult = doctor();
+  console.error(
+    `\n[doctor] ${doctorResult.summary.pass}/${doctorResult.summary.total} checks pass`,
+  );
+}
+
+// Guard: only run when invoked directly
+const isMain =
+  (import.meta as Record<string, unknown>).main === true ||
+  process.argv[1] === fileURLToPath(import.meta.url);
+
+if (isMain) {
+  cliMain().catch((err) => {
+    console.error(`[FATAL] ${err}`);
+    process.exit(1);
+  });
+}

package/cli/selftune/monitoring/watch.ts

@@ -0,0 +1,328 @@
+/**
+ * Post-deploy monitoring: compute snapshots and detect regressions (TASK-16).
+ *
+ * Exports:
+ *  - computeMonitoringSnapshot  (pure function, deterministic)
+ *  - watch                      (reads log files, computes snapshot, optionally rolls back)
+ */
+
+import { parseArgs } from "node:util";
+
+import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
+import { getLastDeployedProposal } from "../evolution/audit.js";
+import type {
+  InvocationType,
+  MonitoringSnapshot,
+  QueryLogRecord,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+} from "../types.js";
+import { readJsonl } from "../utils/jsonl.js";
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface WatchOptions {
+  skillName: string;
+  skillPath: string;
+  windowSessions: number;
+  regressionThreshold: number;
+  autoRollback: boolean;
+  /** Injected log paths for testing (override defaults). */
+  _telemetryLogPath?: string;
+  _skillLogPath?: string;
+  _queryLogPath?: string;
+  _auditLogPath?: string;
+  /** Injected rollback function for testing. */
+  _rollbackFn?: (opts: {
+    skillName: string;
+    skillPath: string;
+    proposalId?: string;
+  }) => Promise<{ rolledBack: boolean; restoredDescription: string; reason: string }>;
+}
+
+export interface WatchResult {
+  snapshot: MonitoringSnapshot;
+  alert: string | null;
+  rolledBack: boolean;
+  recommendation: string;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const DEFAULT_BASELINE_PASS_RATE = 0.5;
+const DEFAULT_REGRESSION_THRESHOLD = 0.1;
+
+// ---------------------------------------------------------------------------
+// computeMonitoringSnapshot - pure function
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute a monitoring snapshot from raw log records.
+ *
+ * The function windows telemetry to the last `windowSessions` entries, then
+ * scopes skill and query records to those sessions. If telemetry is empty or
+ * no records match the windowed session IDs, all provided skill/query records
+ * are used directly (unfiltered by session).
+ *
+ * @param skillName        - The skill to monitor
+ * @param telemetry        - All session telemetry records
+ * @param skillRecords     - All skill usage records
+ * @param queryRecords     - All query log records
+ * @param windowSessions   - Max number of recent sessions to consider
+ * @param baselinePassRate - The baseline pass rate for regression detection
+ * @param regressionThreshold - Drop below baseline minus this triggers regression (default 0.10)
+ */
+export function computeMonitoringSnapshot(
+  skillName: string,
+  telemetry: SessionTelemetryRecord[],
+  skillRecords: SkillUsageRecord[],
+  queryRecords: QueryLogRecord[],
+  windowSessions: number,
+  baselinePassRate: number,
+  regressionThreshold: number = DEFAULT_REGRESSION_THRESHOLD,
+): MonitoringSnapshot {
+  // 1. Window the telemetry to the last N sessions (by array order, assumed chronological)
+  const windowedTelemetry = telemetry.slice(-windowSessions);
+  const windowedSessionIds = new Set(windowedTelemetry.map((t) => t.session_id));
+
+  // 2. Filter skill records by skill name first
+  const skillNameFiltered = skillRecords.filter((r) => r.skill_name === skillName);
+
+  // 3. Apply session ID windowing only if telemetry is present and overlaps
+  const hasSessionOverlap =
+    windowedSessionIds.size > 0 &&
+    (skillNameFiltered.some((r) => windowedSessionIds.has(r.session_id)) ||
+      queryRecords.some((r) => windowedSessionIds.has(r.session_id)));
+
+  const filteredSkillRecords = hasSessionOverlap
+    ? skillNameFiltered.filter((r) => windowedSessionIds.has(r.session_id))
+    : skillNameFiltered;
+
+  const filteredQueryRecords = hasSessionOverlap
+    ? queryRecords.filter((r) => windowedSessionIds.has(r.session_id))
+    : queryRecords;
+
+  // 4. Compute pass rate: triggered_count / total_query_count
+  const triggeredCount = filteredSkillRecords.filter((r) => r.triggered).length;
+  const totalQueries = filteredQueryRecords.length;
+  const passRate = totalQueries === 0 ? 1.0 : triggeredCount / totalQueries;
+
+  // 5. Compute false negative rate from skill usage records
+  const totalSkillChecks = filteredSkillRecords.length;
+  const falseNegatives = filteredSkillRecords.filter((r) => !r.triggered).length;
+  const falseNegativeRate = totalSkillChecks === 0 ? 0 : falseNegatives / totalSkillChecks;
+
+  // 6. by_invocation_type: MVP classifies everything as "implicit"
+  const byInvocationType: Record<InvocationType, { passed: number; total: number }> = {
+    explicit: { passed: 0, total: 0 },
+    implicit: { passed: triggeredCount, total: totalSkillChecks },
+    contextual: { passed: 0, total: 0 },
+    negative: { passed: 0, total: 0 },
+  };
+
+  // 7. Regression detection: pass_rate < baseline - threshold
+  // Use rounding to avoid floating-point boundary issues (e.g. 0.8 - 0.1 = 0.7000000000000001)
+  const precision = 1e10;
+  const adjustedThreshold =
+    Math.round((baselinePassRate - regressionThreshold) * precision) / precision;
+  const roundedPassRate = Math.round(passRate * precision) / precision;
+  const regressionDetected = roundedPassRate < adjustedThreshold;
+
+  return {
+    timestamp: new Date().toISOString(),
+    skill_name: skillName,
+    window_sessions: windowSessions,
+    pass_rate: passRate,
+    false_negative_rate: falseNegativeRate,
+    by_invocation_type: byInvocationType,
+    regression_detected: regressionDetected,
+    baseline_pass_rate: baselinePassRate,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// watch - reads logs, computes snapshot, optionally rolls back
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the post-deploy monitoring check for a skill.
+ */
+export async function watch(options: WatchOptions): Promise<WatchResult> {
+  const {
+    skillName,
+    skillPath,
+    windowSessions = 20,
+    regressionThreshold = DEFAULT_REGRESSION_THRESHOLD,
+    autoRollback = false,
+    _telemetryLogPath = TELEMETRY_LOG,
+    _skillLogPath = SKILL_LOG,
+    _queryLogPath = QUERY_LOG,
+    _auditLogPath,
+    _rollbackFn,
+  } = options;
+
+  // 1. Read log files
+  const telemetry = readJsonl<SessionTelemetryRecord>(_telemetryLogPath);
+  const skillRecords = readJsonl<SkillUsageRecord>(_skillLogPath);
+  const queryRecords = readJsonl<QueryLogRecord>(_queryLogPath);
+
+  // 2. Determine baseline pass rate from last deployed audit entry
+  const lastDeployed = getLastDeployedProposal(skillName, _auditLogPath);
+  const baselinePassRate = lastDeployed?.eval_snapshot?.pass_rate ?? DEFAULT_BASELINE_PASS_RATE;
+
+  // 3. Compute the monitoring snapshot (includes regression detection)
+  const snapshot = computeMonitoringSnapshot(
+    skillName,
+    telemetry,
+    skillRecords,
+    queryRecords,
+    windowSessions,
+    baselinePassRate,
+    regressionThreshold,
+  );
+
+  // 4. Build alert and recommendation
+  let alert: string | null = null;
+  let rolledBack = false;
+  let recommendation: string;
+
+  if (snapshot.regression_detected) {
+    alert = `regression detected for "${skillName}": pass_rate=${snapshot.pass_rate.toFixed(2)} below baseline=${baselinePassRate.toFixed(2)} minus threshold=${regressionThreshold.toFixed(2)}`;
+
+    // 5. Auto-rollback if enabled
+    if (autoRollback) {
+      const rollbackFn = _rollbackFn ?? (await loadRollbackFn());
+      const proposalId = lastDeployed?.proposal_id;
+      const rollbackResult = await rollbackFn({
+        skillName,
+        skillPath,
+        proposalId,
+      });
+      rolledBack = rollbackResult.rolledBack;
+    }
+
+    recommendation = rolledBack
+      ? `Rolled back "${skillName}" to previous version. Monitor to confirm recovery.`
+      : `Consider running: selftune rollback --skill "${skillName}" --skill-path "${skillPath}"`;
+  } else {
+    recommendation = `Skill "${skillName}" is stable. Pass rate ${snapshot.pass_rate.toFixed(2)} is within acceptable range of baseline ${baselinePassRate.toFixed(2)}.`;
+  }
+
+  return {
+    snapshot,
+    alert,
+    rolledBack,
+    recommendation,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Lazy rollback loader (avoids import if rollback.ts doesn't exist yet)
+// ---------------------------------------------------------------------------
+
+async function loadRollbackFn(): Promise<
+  (opts: {
+    skillName: string;
+    skillPath: string;
+    proposalId?: string;
+  }) => Promise<{ rolledBack: boolean; restoredDescription: string; reason: string }>
+> {
+  try {
+    const mod = await import("../evolution/rollback.js");
+    return mod.rollback;
+  } catch (error: unknown) {
+    // Only suppress module-resolution failures; rethrow syntax/runtime errors
+    const code = (error as NodeJS.ErrnoException)?.code;
+    if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") {
+      return async () => ({
+        rolledBack: false,
+        restoredDescription: "",
+        reason: "Rollback module not available",
+      });
+    }
+    throw error;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      "skill-path": { type: "string" },
+      window: { type: "string", default: "20" },
+      threshold: { type: "string", default: "0.1" },
+      "auto-rollback": { type: "boolean", default: false },
+      help: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune watch — Monitor post-deploy skill health
+
+Usage:
+  selftune watch --skill <name> --skill-path <path> [options]
+
+Options:
+  --skill             Skill name (required)
+  --skill-path        Path to SKILL.md (required)
+  --window            Number of recent sessions to consider (default: 20)
+  --threshold         Regression threshold below baseline (default: 0.1)
+  --auto-rollback     Automatically rollback on regression detection
+  --help              Show this help message`);
+    process.exit(0);
+  }
+
+  if (!values.skill || !values["skill-path"]) {
+    console.error("[ERROR] --skill and --skill-path are required");
+    process.exit(1);
+  }
+
+  const rawWindow = values.window ?? "20";
+  if (!/^\d+$/.test(rawWindow)) {
+    console.error("[ERROR] --window must be a positive integer >= 1");
+    process.exit(1);
+  }
+  const windowSessions = Number.parseInt(rawWindow, 10);
+  if (windowSessions < 1) {
+    console.error("[ERROR] --window must be a positive integer >= 1");
+    process.exit(1);
+  }
+
+  const rawThreshold = values.threshold ?? "0.1";
+  if (!/^\d+(\.\d+)?$/.test(rawThreshold)) {
+    console.error("[ERROR] --threshold must be a finite number between 0 and 1");
+    process.exit(1);
+  }
+  const regressionThreshold = Number.parseFloat(rawThreshold);
+  if (regressionThreshold < 0 || regressionThreshold > 1) {
+    console.error("[ERROR] --threshold must be a finite number between 0 and 1");
+    process.exit(1);
+  }
+
+  const result = await watch({
+    skillName: values.skill,
+    skillPath: values["skill-path"],
+    windowSessions,
+    regressionThreshold,
+    autoRollback: values["auto-rollback"] ?? false,
+  });
+
+  console.log(JSON.stringify(result, null, 2));
+  process.exit(result.alert ? 1 : 0);
+}
+
+if (import.meta.main) {
+  cliMain().catch((err) => {
+    console.error(`[FATAL] ${err}`);
+    process.exit(1);
+  });
+}