selftune 0.1.4 → 0.2.1

package/cli/selftune/eval/unit-test-cli.ts

@@ -0,0 +1,152 @@
+/**
+ * CLI entrypoint for skill unit tests.
+ *
+ * Usage:
+ *   selftune eval unit-test --skill <name> --tests <path> [--run-agent] [--generate]
+ *
+ *   --skill <name>    Skill name (required)
+ *   --tests <path>    Path to unit test JSON file (default: ~/.selftune/unit-tests/<skill>.json)
+ *   --run-agent       Actually run tests through an agent (otherwise dry-run with static checks)
+ *   --generate        Generate tests from skill content using LLM (requires agent)
+ *   --skill-path <p>  Path to skill file (used with --generate for content)
+ *   --eval-set <p>    Path to eval set JSON (used with --generate for failure context)
+ *   --model <m>       Model flag for LLM calls
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+
+import { SELFTUNE_CONFIG_DIR } from "../constants.js";
+import type { EvalEntry } from "../types.js";
+import { callLlm, detectAgent } from "../utils/llm-call.js";
+import { generateUnitTests } from "./generate-unit-tests.js";
+import type { AgentRunner } from "./unit-test.js";
+import { loadUnitTests, runUnitTestSuite } from "./unit-test.js";
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      tests: { type: "string" },
+      "run-agent": { type: "boolean", default: false },
+      generate: { type: "boolean", default: false },
+      "skill-path": { type: "string" },
+      "eval-set": { type: "string" },
+      model: { type: "string" },
+    },
+    strict: true,
+  });
+
+  if (!values.skill) {
+    console.error("[ERROR] --skill <name> is required.");
+    process.exit(1);
+  }
+
+  const skillName = values.skill;
+  const unitTestDir = join(SELFTUNE_CONFIG_DIR, "unit-tests");
+  const defaultTestsPath = join(unitTestDir, `${skillName}.json`);
+  const testsPath = values.tests ?? defaultTestsPath;
+
+  // --generate: create tests from skill content
+  if (values.generate) {
+    const agent = detectAgent();
+    if (!agent) {
+      console.error("[ERROR] No agent CLI found (claude/codex/opencode). Cannot generate tests.");
+      process.exit(1);
+    }
+
+    let skillContent = `Skill: ${skillName}`;
+    if (values["skill-path"] && existsSync(values["skill-path"])) {
+      skillContent = readFileSync(values["skill-path"], "utf-8");
+    } else if (values["skill-path"]) {
+      console.warn(`[WARN] Skill path not found: ${values["skill-path"]}. Using skill name only.`);
+    }
+
+    let evalFailures: EvalEntry[] = [];
+    if (values["eval-set"] && existsSync(values["eval-set"])) {
+      try {
+        const raw = readFileSync(values["eval-set"], "utf-8");
+        const entries: EvalEntry[] = JSON.parse(raw);
+        evalFailures = entries.filter((e) => e.should_trigger);
+      } catch {
+        console.warn("[WARN] Failed to parse eval set. Proceeding without failure context.");
+      }
+    }
+
+    const modelFlag = values.model;
+    const llmCaller = (systemPrompt: string, userPrompt: string) =>
+      callLlm(systemPrompt, userPrompt, agent, modelFlag);
+
+    console.log(`Generating unit tests for skill '${skillName}'...`);
+    const tests = await generateUnitTests(skillName, skillContent, evalFailures, llmCaller);
+
+    if (tests.length === 0) {
+      console.error("[ERROR] No tests generated. Check agent/LLM availability.");
+      process.exit(1);
+    }
+
+    // Ensure output directory exists
+    mkdirSync(unitTestDir, { recursive: true });
+    writeFileSync(testsPath, JSON.stringify(tests, null, 2), "utf-8");
+    console.log(`Generated ${tests.length} unit tests -> ${testsPath}`);
+    return;
+  }
+
+  // Load and run tests
+  const tests = loadUnitTests(testsPath);
+  if (tests.length === 0) {
+    console.error(`[ERROR] No tests found at ${testsPath}`);
+    console.error("  Use --generate to create tests, or provide --tests <path>.");
+    process.exit(1);
+  }
+
+  console.log(`Loaded ${tests.length} unit tests for skill '${skillName}'`);
+
+  let agentRunner: AgentRunner;
+
+  if (values["run-agent"]) {
+    const agent = detectAgent();
+    if (!agent) {
+      console.error("[ERROR] No agent CLI found. Cannot run agent-based tests.");
+      process.exit(1);
+    }
+    const modelFlag = values.model;
+    agentRunner = async (query: string): Promise<string> => {
+      return callLlm("You are a helpful assistant.", query, agent, modelFlag);
+    };
+  } else {
+    // Dry-run: use query as transcript (only static assertions like contains work meaningfully)
+    console.log("(dry-run mode — use --run-agent for full agent execution)\n");
+    agentRunner = async (query: string): Promise<string> => query;
+  }
+
+  const suite = await runUnitTestSuite(tests, skillName, agentRunner);
+
+  // Print results
+  console.log(`\nResults for '${suite.skill_name}':`);
+  console.log(`  Total: ${suite.total}  Passed: ${suite.passed}  Failed: ${suite.failed}`);
+  console.log(`  Pass rate: ${(suite.pass_rate * 100).toFixed(1)}%`);
+
+  if (suite.failed > 0) {
+    console.log("\nFailed tests:");
+    for (const r of suite.results.filter((r) => !r.passed)) {
+      console.log(`  [FAIL] ${r.test_id} (${r.duration_ms}ms)`);
+      if (r.error) {
+        console.log(`         Error: ${r.error}`);
+      }
+      for (const a of r.assertion_results.filter((a) => !a.passed)) {
+        console.log(
+          `         - ${a.assertion.type}: expected "${a.assertion.value}", got "${a.actual}"`,
+        );
+      }
+    }
+  }
+
+  console.log(`\n${JSON.stringify(suite, null, 2)}`);
+  process.exit(suite.failed > 0 ? 1 : 0);
+}

package/cli/selftune/eval/unit-test.ts

@@ -0,0 +1,196 @@
+/**
+ * Skill unit test runner.
+ *
+ * Loads, runs, and reports on skill-level unit tests.
+ * Tests are stored as JSON arrays of SkillUnitTest objects.
+ *
+ * Assertion types:
+ *   - contains / not_contains: check transcript for substring
+ *   - regex: check transcript against a regex pattern
+ *   - tool_called / tool_not_called: check transcript for tool usage
+ *   - json_path: check key=value in parsed JSON from transcript
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import type {
+  SkillAssertion,
+  SkillUnitTest,
+  UnitTestResult,
+  UnitTestSuiteResult,
+} from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Assertion checker (deterministic, no agent needed)
+// ---------------------------------------------------------------------------
+
+/** Check a single assertion against a transcript string. */
+export function checkAssertion(
+  assertion: SkillAssertion,
+  transcript: string,
+): { passed: boolean; actual?: string } {
+  switch (assertion.type) {
+    case "contains":
+      return {
+        passed: transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? assertion.value : "(not found)",
+      };
+
+    case "not_contains":
+      return {
+        passed: !transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? `found: ${assertion.value}` : "(absent)",
+      };
+
+    case "regex": {
+      const re = new RegExp(assertion.value);
+      const match = re.exec(transcript);
+      return {
+        passed: match !== null,
+        actual: match ? match[0] : "(no match)",
+      };
+    }
+
+    case "tool_called":
+      return {
+        passed: transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? assertion.value : "(tool not found)",
+      };
+
+    case "tool_not_called":
+      return {
+        passed: !transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? `found: ${assertion.value}` : "(absent)",
+      };
+
+    case "json_path": {
+      // Simple key=value check: "status=ok" looks for {"status":"ok"} in transcript
+      const eqIdx = assertion.value.indexOf("=");
+      if (eqIdx < 0) {
+        return { passed: false, actual: "invalid json_path format (expected key=value)" };
+      }
+      const key = assertion.value.slice(0, eqIdx);
+      const expected = assertion.value.slice(eqIdx + 1);
+      try {
+        const parsed = JSON.parse(transcript);
+        const actual = String(parsed[key] ?? "");
+        return { passed: actual === expected, actual };
+      } catch {
+        // Try to find JSON in the transcript
+        const jsonMatch = transcript.match(/\{[^}]+\}/);
+        if (jsonMatch) {
+          try {
+            const parsed = JSON.parse(jsonMatch[0]);
+            const actual = String(parsed[key] ?? "");
+            return { passed: actual === expected, actual };
+          } catch {
+            return { passed: false, actual: "(json parse error)" };
+          }
+        }
+        return { passed: false, actual: "(no json found)" };
+      }
+    }
+
+    default:
+      return { passed: false, actual: `unknown assertion type: ${assertion.type}` };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Load unit tests from JSON file
+// ---------------------------------------------------------------------------
+
+/** Load unit tests from a JSON file. Returns empty array on error. */
+export function loadUnitTests(testsPath: string): SkillUnitTest[] {
+  try {
+    if (!existsSync(testsPath)) {
+      console.warn(`[WARN] Unit test file not found: ${testsPath}`);
+      return [];
+    }
+    const raw = readFileSync(testsPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) {
+      console.warn(`[WARN] Unit test file is not an array: ${testsPath}`);
+      return [];
+    }
+    return parsed as SkillUnitTest[];
+  } catch (err) {
+    console.warn(`[WARN] Failed to load unit tests from ${testsPath}:`, err);
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Run a single unit test
+// ---------------------------------------------------------------------------
+
+/** Agent function type: takes a query, returns transcript text. */
+export type AgentRunner = (query: string) => Promise<string>;
+
+/** Run a single unit test against an agent runner. */
+export async function runUnitTest(
+  test: SkillUnitTest,
+  agent: AgentRunner,
+): Promise<UnitTestResult> {
+  const start = Date.now();
+
+  try {
+    const transcript = await agent(test.query);
+    const assertionResults = test.assertions.map((assertion) => {
+      const result = checkAssertion(assertion, transcript);
+      return { assertion, passed: result.passed, actual: result.actual };
+    });
+
+    const allPassed = assertionResults.every((r) => r.passed);
+
+    return {
+      test_id: test.id,
+      passed: allPassed,
+      assertion_results: assertionResults,
+      duration_ms: Date.now() - start,
+    };
+  } catch (err) {
+    return {
+      test_id: test.id,
+      passed: false,
+      assertion_results: test.assertions.map((assertion) => ({
+        assertion,
+        passed: false,
+        actual: "error",
+      })),
+      duration_ms: Date.now() - start,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Run a full unit test suite
+// ---------------------------------------------------------------------------
+
+/** Run all unit tests and return aggregated results. */
+export async function runUnitTestSuite(
+  tests: SkillUnitTest[],
+  skillName: string,
+  agent: AgentRunner,
+): Promise<UnitTestSuiteResult> {
+  const results: UnitTestResult[] = [];
+
+  for (const t of tests) {
+    const result = await runUnitTest(t, agent);
+    results.push(result);
+  }
+
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.filter((r) => !r.passed).length;
+  const total = results.length;
+
+  return {
+    skill_name: skillName,
+    total,
+    passed,
+    failed,
+    pass_rate: total > 0 ? passed / total : 0,
+    results,
+    run_at: new Date().toISOString(),
+  };
+}

package/cli/selftune/evolution/deploy-proposal.ts

@@ -7,7 +7,7 @@
  */
 
 import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
-import type { EvolutionProposal } from "../types.js";
+import type { EvolutionProposal, SkillSections } from "../types.js";
 import type { ValidationResult } from "./validate-proposal.js";
 
 // ---------------------------------------------------------------------------
@@ -93,6 +93,147 @@ export function replaceDescription(currentContent: string, newDescription: strin
   return `${preamble}${headingLine}\n${descriptionBlock}\n${afterSubHeading}`;
 }
 
+// ---------------------------------------------------------------------------
+// Structured SKILL.md parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a SKILL.md file into named sections.
+ *
+ * Splits the content into:
+ *   - frontmatter: YAML frontmatter block (if present, including delimiters)
+ *   - title: the first `# Heading` line
+ *   - description: content between the title and the first `## ` heading
+ *   - sections: map of `## Name` -> content (up to next `##` or EOF)
+ */
+export function parseSkillSections(content: string): SkillSections {
+  const lines = content.split("\n");
+  let idx = 0;
+
+  // --- frontmatter ---
+  let frontmatter = "";
+  if (lines[0]?.trim() === "---") {
+    const endIdx = lines.indexOf("---", 1);
+    if (endIdx > 0) {
+      frontmatter = lines.slice(0, endIdx + 1).join("\n");
+      idx = endIdx + 1;
+      // skip blank line after frontmatter
+      if (idx < lines.length && lines[idx].trim() === "") idx++;
+    }
+  }
+
+  // --- title ---
+  let title = "";
+  while (idx < lines.length) {
+    if (lines[idx].startsWith("# ") && !lines[idx].startsWith("## ")) {
+      title = lines[idx];
+      idx++;
+      break;
+    }
+    idx++;
+  }
+
+  // --- description (between title and first ## heading) ---
+  const descLines: string[] = [];
+  while (idx < lines.length && !lines[idx].startsWith("## ")) {
+    descLines.push(lines[idx]);
+    idx++;
+  }
+  // Trim leading/trailing blank lines from description
+  const description = descLines.join("\n").trim();
+
+  // --- remaining ## sections ---
+  const sections: Record<string, string> = {};
+  let currentSection = "";
+  const sectionLines: string[] = [];
+
+  while (idx < lines.length) {
+    if (lines[idx].startsWith("## ")) {
+      // Flush previous section
+      if (currentSection) {
+        sections[currentSection] = sectionLines.join("\n").trim();
+        sectionLines.length = 0;
+      }
+      currentSection = lines[idx].replace(/^## /, "").trim();
+      idx++;
+    } else {
+      sectionLines.push(lines[idx]);
+      idx++;
+    }
+  }
+  // Flush last section
+  if (currentSection) {
+    sections[currentSection] = sectionLines.join("\n").trim();
+  }
+
+  return { frontmatter, title, description, sections };
+}
+
+// ---------------------------------------------------------------------------
+// Section replacement
+// ---------------------------------------------------------------------------
+
+/**
+ * Replace a named `## Section` block in a SKILL.md file.
+ *
+ * If the section does not exist, appends it at the end.
+ */
+export function replaceSection(content: string, sectionName: string, newContent: string): string {
+  const lines = content.split("\n");
+  const heading = `## ${sectionName}`;
+  let startIdx = -1;
+  let endIdx = lines.length;
+
+  for (let i = 0; i < lines.length; i++) {
+    if (
+      lines[i].startsWith(heading) &&
+      (lines[i].length === heading.length || lines[i][heading.length] === " ")
+    ) {
+      startIdx = i;
+      // Find end: next ## heading or EOF
+      for (let j = i + 1; j < lines.length; j++) {
+        if (lines[j].startsWith("## ")) {
+          endIdx = j;
+          break;
+        }
+      }
+      break;
+    }
+  }
+
+  if (startIdx === -1) {
+    // Section not found — append
+    const trimmed = content.trimEnd();
+    return `${trimmed}\n\n${heading}\n\n${newContent}\n`;
+  }
+
+  const before = lines.slice(0, startIdx);
+  const after = lines.slice(endIdx);
+  return [...before, heading, "", newContent, "", ...after].join("\n");
+}
+
+/**
+ * Replace the entire body below frontmatter with a proposed body.
+ *
+ * Preserves frontmatter (if present) and the `# Title` line intact.
+ */
+export function replaceBody(currentContent: string, proposedBody: string): string {
+  const parsed = parseSkillSections(currentContent);
+  const parts: string[] = [];
+
+  if (parsed.frontmatter) {
+    parts.push(parsed.frontmatter);
+    parts.push("");
+  }
+  if (parsed.title) {
+    parts.push(parsed.title);
+    parts.push("");
+  }
+  parts.push(proposedBody);
+
+  return `${parts.join("\n").trimEnd()}\n`;
+}
+
 // ---------------------------------------------------------------------------
 // Commit message builder
 // ---------------------------------------------------------------------------

package/cli/selftune/evolution/evidence.ts

@@ -0,0 +1,26 @@
+/**
+ * Evolution evidence trail: append and read proposal/eval artifacts that power
+ * explainable dashboard drill-downs.
+ */
+
+import { EVOLUTION_EVIDENCE_LOG } from "../constants.js";
+import type { EvolutionEvidenceEntry } from "../types.js";
+import { appendJsonl, readJsonl } from "../utils/jsonl.js";
+
+/** Append a structured evidence artifact to the evolution evidence log. */
+export function appendEvidenceEntry(
+  entry: EvolutionEvidenceEntry,
+  logPath: string = EVOLUTION_EVIDENCE_LOG,
+): void {
+  appendJsonl(logPath, entry);
+}
+
+/** Read all evidence entries, optionally filtered by exact skill name. */
+export function readEvidenceTrail(
+  skillName?: string,
+  logPath: string = EVOLUTION_EVIDENCE_LOG,
+): EvolutionEvidenceEntry[] {
+  const entries = readJsonl<EvolutionEvidenceEntry>(logPath);
+  if (!skillName) return entries;
+  return entries.filter((entry) => entry.skill_name === skillName);
+}