selftune 0.1.0

package/cli/selftune/evolution/deploy-proposal.ts

@@ -0,0 +1,244 @@
+/**
+ * deploy-proposal.ts
+ *
+ * Deploys a validated evolution proposal by updating SKILL.md, creating a
+ * backup, building a commit message with metrics, and optionally creating
+ * a git branch and PR via `gh pr create`.
+ */
+
+import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+import type { EvolutionProposal } from "../types.js";
+import type { ValidationResult } from "./validate-proposal.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface DeployOptions {
+  proposal: EvolutionProposal;
+  validation: ValidationResult;
+  skillPath: string;
+  createPr: boolean;
+  branchPrefix?: string; // default "selftune/evolve"
+}
+
+export interface DeployResult {
+  skillMdUpdated: boolean;
+  backupPath: string | null;
+  branchName: string | null;
+  commitMessage: string;
+}
+
+// ---------------------------------------------------------------------------
+// SKILL.md reading
+// ---------------------------------------------------------------------------
+
+/** Read the contents of a SKILL.md file. Throws if the file does not exist. */
+export function readSkillMd(skillPath: string): string {
+  if (!existsSync(skillPath)) {
+    throw new Error(`SKILL.md not found at ${skillPath}`);
+  }
+  return readFileSync(skillPath, "utf-8");
+}
+
+// ---------------------------------------------------------------------------
+// Description replacement
+// ---------------------------------------------------------------------------
+
+/**
+ * Replace the description section of a SKILL.md file.
+ *
+ * The description is defined as the content between the first `#` heading
+ * and the first `##` heading. If no `##` heading exists, the entire body
+ * after the first heading is replaced.
+ */
+export function replaceDescription(currentContent: string, newDescription: string): string {
+  const lines = currentContent.split("\n");
+
+  // Find the first # heading line
+  let headingIndex = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i].startsWith("# ") && !lines[i].startsWith("## ")) {
+      headingIndex = i;
+      break;
+    }
+  }
+
+  // If no heading found, just prepend the description
+  if (headingIndex === -1) {
+    return `${newDescription}\n${currentContent}`;
+  }
+
+  // Find the first ## heading after the main heading
+  let subHeadingIndex = -1;
+  for (let i = headingIndex + 1; i < lines.length; i++) {
+    if (lines[i].startsWith("## ")) {
+      subHeadingIndex = i;
+      break;
+    }
+  }
+
+  // Build the new content, preserving any preamble before the first heading
+  const preamble = headingIndex > 0 ? `${lines.slice(0, headingIndex).join("\n")}\n` : "";
+  const headingLine = lines[headingIndex];
+  const descriptionBlock = newDescription.length > 0 ? `\n${newDescription}\n` : "\n";
+
+  if (subHeadingIndex === -1) {
+    // No sub-heading: preamble + heading + new description + trailing newline
+    return `${preamble}${headingLine}\n${descriptionBlock}\n`;
+  }
+
+  // Preamble + heading + description + everything from the first ## onward
+  const afterSubHeading = lines.slice(subHeadingIndex).join("\n");
+  return `${preamble}${headingLine}\n${descriptionBlock}\n${afterSubHeading}`;
+}
+
+// ---------------------------------------------------------------------------
+// Commit message builder
+// ---------------------------------------------------------------------------
+
+/** Build a commit message that includes the skill name and pass rate change. */
+export function buildCommitMessage(
+  proposal: EvolutionProposal,
+  validation: ValidationResult,
+): string {
+  const changePercent = Math.round(validation.net_change * 100);
+  const sign = changePercent >= 0 ? "+" : "";
+  const passRateStr = `${sign}${changePercent}% pass rate`;
+
+  return `evolve(${proposal.skill_name}): ${passRateStr}`;
+}
+
+// ---------------------------------------------------------------------------
+// Git/GH operations (PR creation)
+// ---------------------------------------------------------------------------
+
+/** Sanitize a string for use in a git branch name. */
+function sanitizeForGitRef(name: string): string {
+  return name
+    .replace(/[^a-zA-Z0-9._-]/g, "-")
+    .replace(/\.{2,}/g, ".")
+    .replace(/^[.-]|[.-]$/g, "")
+    .replace(/-{2,}/g, "-");
+}
+
+/** Generate a branch name from the prefix and skill name. */
+function makeBranchName(prefix: string, skillName: string): string {
+  const timestamp = Date.now();
+  const safeName = sanitizeForGitRef(skillName) || "untitled";
+  return `${prefix}/${safeName}-${timestamp}`;
+}
+
+/**
+ * Run a git/gh command via Bun.spawn. Returns stdout on success.
+ * Throws on non-zero exit code or if the command exceeds timeoutMs.
+ */
+async function runCommand(args: string[], cwd?: string, timeoutMs = 30_000): Promise<string> {
+  const proc = Bun.spawn(args, {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    proc.kill();
+  }, timeoutMs);
+
+  try {
+    // Read stdout and stderr concurrently to avoid deadlock when both pipes fill.
+    const [stdout, stderr] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+    ]);
+    const exitCode = await proc.exited;
+
+    if (timedOut) {
+      throw new Error(`Command timed out after ${timeoutMs}ms: ${args.join(" ")}`);
+    }
+
+    if (exitCode !== 0) {
+      throw new Error(`Command failed (exit ${exitCode}): ${args.join(" ")}\n${stderr}`);
+    }
+
+    return stdout.trim();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main deploy function
+// ---------------------------------------------------------------------------
+
+/** Deploy a validated evolution proposal to SKILL.md and optionally create a PR. */
+export async function deployProposal(options: DeployOptions): Promise<DeployResult> {
+  const { proposal, validation, skillPath, createPr, branchPrefix = "selftune/evolve" } = options;
+
+  // Step 1: Read current SKILL.md
+  const currentContent = readSkillMd(skillPath);
+
+  // Step 2: Create backup (unique per deploy to avoid overwriting previous backups)
+  const backupTimestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const backupPath = `${skillPath}.${backupTimestamp}.bak`;
+  copyFileSync(skillPath, backupPath);
+
+  // Step 3: Replace description and write
+  const updatedContent = replaceDescription(currentContent, proposal.proposed_description);
+  writeFileSync(skillPath, updatedContent, "utf-8");
+
+  // Step 4: Build commit message
+  const commitMessage = buildCommitMessage(proposal, validation);
+
+  // Step 5: Optionally create branch and PR
+  let branchName: string | null = null;
+
+  if (createPr) {
+    branchName = makeBranchName(branchPrefix, proposal.skill_name);
+
+    try {
+      // Create and checkout branch
+      await runCommand(["git", "checkout", "-b", branchName]);
+
+      // Stage the SKILL.md
+      await runCommand(["git", "add", skillPath]);
+
+      // Commit
+      await runCommand(["git", "commit", "-m", commitMessage]);
+
+      // Push
+      await runCommand(["git", "push", "-u", "origin", branchName]);
+
+      // Create PR
+      await runCommand([
+        "gh",
+        "pr",
+        "create",
+        "--title",
+        commitMessage,
+        "--body",
+        `Proposal: ${proposal.proposal_id}\nRationale: ${proposal.rationale}\nNet change: ${validation.net_change > 0 ? "+" : ""}${Math.round(validation.net_change * 100)}%`,
+      ]);
+    } catch (err) {
+      // Git/GH operations are best-effort in test environments.
+      // The branch name is still returned for tracking.
+      console.error(`[WARN] Git/GH operation failed: ${err instanceof Error ? err.message : err}`);
+    }
+  }
+
+  return {
+    skillMdUpdated: true,
+    backupPath,
+    branchName,
+    commitMessage,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry guard
+// ---------------------------------------------------------------------------
+
+if (import.meta.main) {
+  console.log("deploy-proposal: use deployProposal() programmatically or via evolve CLI");
+}

package/cli/selftune/evolution/evolve.ts

@@ -0,0 +1,406 @@
+/**
+ * evolve.ts
+ *
+ * Evolution orchestrator: coordinates failure pattern extraction, proposal
+ * generation, validation, and deployment into a single pipeline with retry
+ * logic and comprehensive audit tracking.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+import { QUERY_LOG, SKILL_LOG } from "../constants.js";
+import { buildEvalSet } from "../eval/hooks-to-evals.js";
+import type {
+  EvalEntry,
+  EvalPassRate,
+  EvolutionAuditEntry,
+  EvolutionProposal,
+  QueryLogRecord,
+  SkillUsageRecord,
+} from "../types.js";
+import { readJsonl } from "../utils/jsonl.js";
+import { appendAuditEntry } from "./audit.js";
+import { extractFailurePatterns } from "./extract-patterns.js";
+import { generateProposal } from "./propose-description.js";
+import { validateProposal } from "./validate-proposal.js";
+import type { ValidationResult } from "./validate-proposal.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface EvolveOptions {
+  skillName: string;
+  skillPath: string;
+  evalSetPath?: string;
+  mode: "agent" | "api";
+  agent?: string;
+  dryRun: boolean;
+  confidenceThreshold: number; // default 0.6
+  maxIterations: number; // default 3
+}
+
+export interface EvolveResult {
+  proposal: EvolutionProposal | null;
+  validation: ValidationResult | null;
+  deployed: boolean;
+  auditEntries: EvolutionAuditEntry[];
+  reason: string;
+}
+
+/**
+ * Injectable dependencies for evolve(). When omitted, the real module
+ * imports are used. Pass overrides in tests to avoid mock.module().
+ */
+export interface EvolveDeps {
+  extractFailurePatterns?: typeof import("./extract-patterns.js").extractFailurePatterns;
+  generateProposal?: typeof import("./propose-description.js").generateProposal;
+  validateProposal?: typeof import("./validate-proposal.js").validateProposal;
+  appendAuditEntry?: typeof import("./audit.js").appendAuditEntry;
+  buildEvalSet?: typeof import("../eval/hooks-to-evals.js").buildEvalSet;
+}
+
+// ---------------------------------------------------------------------------
+// Audit helper
+// ---------------------------------------------------------------------------
+
+function createAuditEntry(
+  proposalId: string,
+  action: EvolutionAuditEntry["action"],
+  details: string,
+  evalSnapshot?: EvalPassRate,
+): EvolutionAuditEntry {
+  return {
+    timestamp: new Date().toISOString(),
+    proposal_id: proposalId,
+    action,
+    details,
+    ...(evalSnapshot ? { eval_snapshot: evalSnapshot } : {}),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main orchestrator
+// ---------------------------------------------------------------------------
+
+export async function evolve(
+  options: EvolveOptions,
+  _deps: EvolveDeps = {},
+): Promise<EvolveResult> {
+  const {
+    skillName,
+    skillPath,
+    evalSetPath,
+    mode,
+    agent,
+    dryRun,
+    confidenceThreshold,
+    maxIterations,
+  } = options;
+
+  // Resolve injectable dependencies with real-import fallbacks
+  const _extractFailurePatterns = _deps.extractFailurePatterns ?? extractFailurePatterns;
+  const _generateProposal = _deps.generateProposal ?? generateProposal;
+  const _validateProposal = _deps.validateProposal ?? validateProposal;
+  const _appendAuditEntry = _deps.appendAuditEntry ?? appendAuditEntry;
+  const _buildEvalSet = _deps.buildEvalSet ?? buildEvalSet;
+
+  const auditEntries: EvolutionAuditEntry[] = [];
+
+  function recordAudit(
+    proposalId: string,
+    action: EvolutionAuditEntry["action"],
+    details: string,
+    evalSnapshot?: EvalPassRate,
+  ): void {
+    const entry = createAuditEntry(proposalId, action, details, evalSnapshot);
+    auditEntries.push(entry);
+    try {
+      _appendAuditEntry(entry);
+    } catch {
+      // Fail-open: audit write failures should not break the pipeline
+    }
+  }
+
+  try {
+    // -----------------------------------------------------------------------
+    // Step 1: Read current SKILL.md
+    // -----------------------------------------------------------------------
+    if (!existsSync(skillPath)) {
+      return {
+        proposal: null,
+        validation: null,
+        deployed: false,
+        auditEntries,
+        reason: `SKILL.md not found at ${skillPath}`,
+      };
+    }
+
+    const currentDescription = readFileSync(skillPath, "utf-8");
+
+    // -----------------------------------------------------------------------
+    // Step 2: Load eval set
+    // -----------------------------------------------------------------------
+    let evalSet: EvalEntry[];
+
+    if (evalSetPath && existsSync(evalSetPath)) {
+      const raw = readFileSync(evalSetPath, "utf-8");
+      evalSet = JSON.parse(raw) as EvalEntry[];
+    } else {
+      // Build from logs
+      const skillRecords = readJsonl<SkillUsageRecord>(SKILL_LOG);
+      const queryRecords = readJsonl<QueryLogRecord>(QUERY_LOG);
+      evalSet = _buildEvalSet(skillRecords, queryRecords, skillName);
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 3: Load skill usage records
+    // -----------------------------------------------------------------------
+    const skillUsage = readJsonl<SkillUsageRecord>(SKILL_LOG);
+
+    // -----------------------------------------------------------------------
+    // Step 4: Extract failure patterns
+    // -----------------------------------------------------------------------
+    const failurePatterns = _extractFailurePatterns(evalSet, skillUsage, skillName);
+
+    // -----------------------------------------------------------------------
+    // Step 5: Early exit if no patterns
+    // -----------------------------------------------------------------------
+    if (failurePatterns.length === 0) {
+      return {
+        proposal: null,
+        validation: null,
+        deployed: false,
+        auditEntries,
+        reason: "No failure patterns found",
+      };
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 6: Collect all missed queries
+    // -----------------------------------------------------------------------
+    const missedQueries = failurePatterns.flatMap((p) => p.missed_queries);
+
+    // -----------------------------------------------------------------------
+    // Steps 7-12: Retry loop for proposal generation and validation
+    // -----------------------------------------------------------------------
+    let lastProposal: EvolutionProposal | null = null;
+    let lastValidation: ValidationResult | null = null;
+    let feedbackReason = "";
+
+    for (let iteration = 0; iteration < maxIterations; iteration++) {
+      // Step 7: Generate proposal
+      const effectiveMissedQueries = feedbackReason
+        ? [...missedQueries, `[Previous attempt failed: ${feedbackReason}]`]
+        : missedQueries;
+
+      const proposal = await _generateProposal(
+        currentDescription,
+        failurePatterns,
+        effectiveMissedQueries,
+        skillName,
+        skillPath,
+        mode,
+        agent,
+      );
+
+      lastProposal = proposal;
+
+      // Step 8: Audit "created"
+      recordAudit(
+        proposal.proposal_id,
+        "created",
+        `Proposal created for ${skillName} (iteration ${iteration + 1})`,
+      );
+
+      // Step 9: Check confidence threshold
+      if (proposal.confidence < confidenceThreshold) {
+        feedbackReason = `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`;
+        recordAudit(
+          proposal.proposal_id,
+          "rejected",
+          `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
+        );
+
+        // If this is the last iteration, return early with rejection
+        if (iteration === maxIterations - 1) {
+          return {
+            proposal: lastProposal,
+            validation: null,
+            deployed: false,
+            auditEntries,
+            reason: `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
+          };
+        }
+
+        continue;
+      }
+
+      // Step 10: Validate against eval set
+      const validation = await _validateProposal(proposal, evalSet, mode, agent);
+      lastValidation = validation;
+
+      // Step 11: Audit "validated"
+      const evalSnapshot: EvalPassRate = {
+        total: evalSet.length,
+        passed: Math.round(validation.after_pass_rate * evalSet.length),
+        failed: evalSet.length - Math.round(validation.after_pass_rate * evalSet.length),
+        pass_rate: validation.after_pass_rate,
+      };
+      recordAudit(
+        proposal.proposal_id,
+        "validated",
+        `Validation complete: improved=${validation.improved}`,
+        evalSnapshot,
+      );
+
+      // Step 12: Check validation result
+      if (!validation.improved) {
+        feedbackReason = `Validation failed: net_change=${validation.net_change.toFixed(3)}, improved=false`;
+        recordAudit(
+          proposal.proposal_id,
+          "rejected",
+          `Validation failed: net_change=${validation.net_change.toFixed(3)}`,
+        );
+
+        // If this is the last iteration, return with rejection
+        if (iteration === maxIterations - 1) {
+          return {
+            proposal: lastProposal,
+            validation: lastValidation,
+            deployed: false,
+            auditEntries,
+            reason: `Validation failed after ${maxIterations} iterations: net_change=${validation.net_change.toFixed(3)}`,
+          };
+        }
+
+        continue;
+      }
+
+      // Validation passed - break out of retry loop
+      break;
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 13: Dry run check
+    // -----------------------------------------------------------------------
+    if (dryRun) {
+      return {
+        proposal: lastProposal,
+        validation: lastValidation,
+        deployed: false,
+        auditEntries,
+        reason: "Dry run - proposal validated but not deployed",
+      };
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 14: Deploy (actual deploy wired in TASK-14)
+    // -----------------------------------------------------------------------
+    if (lastProposal) {
+      recordAudit(
+        lastProposal.proposal_id,
+        "deployed",
+        `Deployed proposal for ${skillName}`,
+        lastValidation
+          ? {
+              total: evalSet.length,
+              passed: Math.round(lastValidation.after_pass_rate * evalSet.length),
+              failed: evalSet.length - Math.round(lastValidation.after_pass_rate * evalSet.length),
+              pass_rate: lastValidation.after_pass_rate,
+            }
+          : undefined,
+      );
+    }
+
+    // -----------------------------------------------------------------------
+    // Step 15-16: Return complete result
+    // -----------------------------------------------------------------------
+    return {
+      proposal: lastProposal,
+      validation: lastValidation,
+      deployed: true,
+      auditEntries,
+      reason: "Evolution deployed successfully",
+    };
+  } catch (error) {
+    // Robust error handling: catch any unexpected errors and return gracefully
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    return {
+      proposal: null,
+      validation: null,
+      deployed: false,
+      auditEntries,
+      reason: `Error during evolution: ${errorMessage}`,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      "skill-path": { type: "string" },
+      "eval-set": { type: "string" },
+      mode: { type: "string", default: "agent" },
+      agent: { type: "string" },
+      "dry-run": { type: "boolean", default: false },
+      confidence: { type: "string", default: "0.6" },
+      "max-iterations": { type: "string", default: "3" },
+      help: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune evolve — Evolve a skill description via failure patterns
+
+Usage:
+  selftune evolve --skill <name> --skill-path <path> [options]
+
+Options:
+  --skill             Skill name (required)
+  --skill-path        Path to SKILL.md (required)
+  --eval-set          Path to eval set JSON (optional, builds from logs if omitted)
+  --mode              Execution mode: "agent" or "api" (default: "agent")
+  --agent             Agent CLI to use (claude, codex, opencode)
+  --dry-run           Validate proposal without deploying
+  --confidence        Confidence threshold 0.0-1.0 (default: 0.6)
+  --max-iterations    Max retry iterations (default: 3)
+  --help              Show this help message`);
+    process.exit(0);
+  }
+
+  if (!values.skill || !values["skill-path"]) {
+    console.error("[ERROR] --skill and --skill-path are required");
+    process.exit(1);
+  }
+
+  const mode = values.mode === "api" ? "api" : "agent";
+
+  const result = await evolve({
+    skillName: values.skill,
+    skillPath: values["skill-path"],
+    evalSetPath: values["eval-set"],
+    mode,
+    agent: values.agent,
+    dryRun: values["dry-run"] ?? false,
+    confidenceThreshold: Number.parseFloat(values.confidence ?? "0.6"),
+    maxIterations: Number.parseInt(values["max-iterations"] ?? "3", 10),
+  });
+
+  console.log(JSON.stringify(result, null, 2));
+  process.exit(result.deployed ? 0 : 1);
+}
+
+if (import.meta.main) {
+  cliMain().catch((err) => {
+    console.error(`[FATAL] ${err}`);
+    process.exit(1);
+  });
+}