selftune 0.1.2 → 0.2.0

package/cli/selftune/eval/unit-test.ts

@@ -0,0 +1,196 @@
+/**
+ * Skill unit test runner.
+ *
+ * Loads, runs, and reports on skill-level unit tests.
+ * Tests are stored as JSON arrays of SkillUnitTest objects.
+ *
+ * Assertion types:
+ *   - contains / not_contains: check transcript for substring
+ *   - regex: check transcript against a regex pattern
+ *   - tool_called / tool_not_called: check transcript for tool usage
+ *   - json_path: check key=value in parsed JSON from transcript
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import type {
+  SkillAssertion,
+  SkillUnitTest,
+  UnitTestResult,
+  UnitTestSuiteResult,
+} from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Assertion checker (deterministic, no agent needed)
+// ---------------------------------------------------------------------------
+
+/** Check a single assertion against a transcript string. */
+export function checkAssertion(
+  assertion: SkillAssertion,
+  transcript: string,
+): { passed: boolean; actual?: string } {
+  switch (assertion.type) {
+    case "contains":
+      return {
+        passed: transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? assertion.value : "(not found)",
+      };
+
+    case "not_contains":
+      return {
+        passed: !transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? `found: ${assertion.value}` : "(absent)",
+      };
+
+    case "regex": {
+      const re = new RegExp(assertion.value);
+      const match = re.exec(transcript);
+      return {
+        passed: match !== null,
+        actual: match ? match[0] : "(no match)",
+      };
+    }
+
+    case "tool_called":
+      return {
+        passed: transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? assertion.value : "(tool not found)",
+      };
+
+    case "tool_not_called":
+      return {
+        passed: !transcript.includes(assertion.value),
+        actual: transcript.includes(assertion.value) ? `found: ${assertion.value}` : "(absent)",
+      };
+
+    case "json_path": {
+      // Simple key=value check: "status=ok" looks for {"status":"ok"} in transcript
+      const eqIdx = assertion.value.indexOf("=");
+      if (eqIdx < 0) {
+        return { passed: false, actual: "invalid json_path format (expected key=value)" };
+      }
+      const key = assertion.value.slice(0, eqIdx);
+      const expected = assertion.value.slice(eqIdx + 1);
+      try {
+        const parsed = JSON.parse(transcript);
+        const actual = String(parsed[key] ?? "");
+        return { passed: actual === expected, actual };
+      } catch {
+        // Try to find JSON in the transcript
+        const jsonMatch = transcript.match(/\{[^}]+\}/);
+        if (jsonMatch) {
+          try {
+            const parsed = JSON.parse(jsonMatch[0]);
+            const actual = String(parsed[key] ?? "");
+            return { passed: actual === expected, actual };
+          } catch {
+            return { passed: false, actual: "(json parse error)" };
+          }
+        }
+        return { passed: false, actual: "(no json found)" };
+      }
+    }
+
+    default:
+      return { passed: false, actual: `unknown assertion type: ${assertion.type}` };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Load unit tests from JSON file
+// ---------------------------------------------------------------------------
+
+/** Load unit tests from a JSON file. Returns empty array on error. */
+export function loadUnitTests(testsPath: string): SkillUnitTest[] {
+  try {
+    if (!existsSync(testsPath)) {
+      console.warn(`[WARN] Unit test file not found: ${testsPath}`);
+      return [];
+    }
+    const raw = readFileSync(testsPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) {
+      console.warn(`[WARN] Unit test file is not an array: ${testsPath}`);
+      return [];
+    }
+    return parsed as SkillUnitTest[];
+  } catch (err) {
+    console.warn(`[WARN] Failed to load unit tests from ${testsPath}:`, err);
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Run a single unit test
+// ---------------------------------------------------------------------------
+
+/** Agent function type: takes a query, returns transcript text. */
+export type AgentRunner = (query: string) => Promise<string>;
+
+/** Run a single unit test against an agent runner. */
+export async function runUnitTest(
+  test: SkillUnitTest,
+  agent: AgentRunner,
+): Promise<UnitTestResult> {
+  const start = Date.now();
+
+  try {
+    const transcript = await agent(test.query);
+    const assertionResults = test.assertions.map((assertion) => {
+      const result = checkAssertion(assertion, transcript);
+      return { assertion, passed: result.passed, actual: result.actual };
+    });
+
+    const allPassed = assertionResults.every((r) => r.passed);
+
+    return {
+      test_id: test.id,
+      passed: allPassed,
+      assertion_results: assertionResults,
+      duration_ms: Date.now() - start,
+    };
+  } catch (err) {
+    return {
+      test_id: test.id,
+      passed: false,
+      assertion_results: test.assertions.map((assertion) => ({
+        assertion,
+        passed: false,
+        actual: "error",
+      })),
+      duration_ms: Date.now() - start,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Run a full unit test suite
+// ---------------------------------------------------------------------------
+
+/** Run all unit tests and return aggregated results. */
+export async function runUnitTestSuite(
+  tests: SkillUnitTest[],
+  skillName: string,
+  agent: AgentRunner,
+): Promise<UnitTestSuiteResult> {
+  const results: UnitTestResult[] = [];
+
+  for (const t of tests) {
+    const result = await runUnitTest(t, agent);
+    results.push(result);
+  }
+
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.filter((r) => !r.passed).length;
+  const total = results.length;
+
+  return {
+    skill_name: skillName,
+    total,
+    passed,
+    failed,
+    pass_rate: total > 0 ? passed / total : 0,
+    results,
+    run_at: new Date().toISOString(),
+  };
+}

package/cli/selftune/evolution/deploy-proposal.ts

@@ -7,7 +7,7 @@
  */
 
 import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
-import type { EvolutionProposal } from "../types.js";
+import type { EvolutionProposal, SkillSections } from "../types.js";
 import type { ValidationResult } from "./validate-proposal.js";
 
 // ---------------------------------------------------------------------------
@@ -93,6 +93,147 @@ export function replaceDescription(currentContent: string, newDescription: strin
   return `${preamble}${headingLine}\n${descriptionBlock}\n${afterSubHeading}`;
 }
 
+// ---------------------------------------------------------------------------
+// Structured SKILL.md parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a SKILL.md file into named sections.
+ *
+ * Splits the content into:
+ *   - frontmatter: YAML frontmatter block (if present, including delimiters)
+ *   - title: the first `# Heading` line
+ *   - description: content between the title and the first `## ` heading
+ *   - sections: map of `## Name` -> content (up to next `##` or EOF)
+ */
+export function parseSkillSections(content: string): SkillSections {
+  const lines = content.split("\n");
+  let idx = 0;
+
+  // --- frontmatter ---
+  let frontmatter = "";
+  if (lines[0]?.trim() === "---") {
+    const endIdx = lines.indexOf("---", 1);
+    if (endIdx > 0) {
+      frontmatter = lines.slice(0, endIdx + 1).join("\n");
+      idx = endIdx + 1;
+      // skip blank line after frontmatter
+      if (idx < lines.length && lines[idx].trim() === "") idx++;
+    }
+  }
+
+  // --- title ---
+  let title = "";
+  while (idx < lines.length) {
+    if (lines[idx].startsWith("# ") && !lines[idx].startsWith("## ")) {
+      title = lines[idx];
+      idx++;
+      break;
+    }
+    idx++;
+  }
+
+  // --- description (between title and first ## heading) ---
+  const descLines: string[] = [];
+  while (idx < lines.length && !lines[idx].startsWith("## ")) {
+    descLines.push(lines[idx]);
+    idx++;
+  }
+  // Trim leading/trailing blank lines from description
+  const description = descLines.join("\n").trim();
+
+  // --- remaining ## sections ---
+  const sections: Record<string, string> = {};
+  let currentSection = "";
+  const sectionLines: string[] = [];
+
+  while (idx < lines.length) {
+    if (lines[idx].startsWith("## ")) {
+      // Flush previous section
+      if (currentSection) {
+        sections[currentSection] = sectionLines.join("\n").trim();
+        sectionLines.length = 0;
+      }
+      currentSection = lines[idx].replace(/^## /, "").trim();
+      idx++;
+    } else {
+      sectionLines.push(lines[idx]);
+      idx++;
+    }
+  }
+  // Flush last section
+  if (currentSection) {
+    sections[currentSection] = sectionLines.join("\n").trim();
+  }
+
+  return { frontmatter, title, description, sections };
+}
+
+// ---------------------------------------------------------------------------
+// Section replacement
+// ---------------------------------------------------------------------------
+
+/**
+ * Replace a named `## Section` block in a SKILL.md file.
+ *
+ * If the section does not exist, appends it at the end.
+ */
+export function replaceSection(content: string, sectionName: string, newContent: string): string {
+  const lines = content.split("\n");
+  const heading = `## ${sectionName}`;
+  let startIdx = -1;
+  let endIdx = lines.length;
+
+  for (let i = 0; i < lines.length; i++) {
+    if (
+      lines[i].startsWith(heading) &&
+      (lines[i].length === heading.length || lines[i][heading.length] === " ")
+    ) {
+      startIdx = i;
+      // Find end: next ## heading or EOF
+      for (let j = i + 1; j < lines.length; j++) {
+        if (lines[j].startsWith("## ")) {
+          endIdx = j;
+          break;
+        }
+      }
+      break;
+    }
+  }
+
+  if (startIdx === -1) {
+    // Section not found — append
+    const trimmed = content.trimEnd();
+    return `${trimmed}\n\n${heading}\n\n${newContent}\n`;
+  }
+
+  const before = lines.slice(0, startIdx);
+  const after = lines.slice(endIdx);
+  return [...before, heading, "", newContent, "", ...after].join("\n");
+}
+
+/**
+ * Replace the entire body below frontmatter with a proposed body.
+ *
+ * Preserves frontmatter (if present) and the `# Title` line intact.
+ */
+export function replaceBody(currentContent: string, proposedBody: string): string {
+  const parsed = parseSkillSections(currentContent);
+  const parts: string[] = [];
+
+  if (parsed.frontmatter) {
+    parts.push(parsed.frontmatter);
+    parts.push("");
+  }
+  if (parsed.title) {
+    parts.push(parsed.title);
+    parts.push("");
+  }
+  parts.push(proposedBody);
+
+  return `${parts.join("\n").trimEnd()}\n`;
+}
+
 // ---------------------------------------------------------------------------
 // Commit message builder
 // ---------------------------------------------------------------------------