selftune 0.1.4 → 0.2.0

package/cli/selftune/eval/import-skillsbench.ts

@@ -0,0 +1,221 @@
+#!/usr/bin/env bun
+/**
+ * import-skillsbench.ts
+ *
+ * Imports task definitions from a SkillsBench-style corpus directory and
+ * converts them into EvalEntry arrays for use with selftune eval/grading.
+ *
+ * Expected directory structure:
+ *   <dir>/tasks/<task-id>/instruction.md   — task description (query text)
+ *   <dir>/tasks/<task-id>/task.toml        — metadata (difficulty, category, tags, etc.)
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import type { EvalEntry, SkillsBenchTask } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Minimal TOML parser (handles the subset used by SkillsBench task.toml files)
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse the subset of TOML used by SkillsBench task.toml files.
+ *
+ * Supports: single-line key = "value", flat string arrays ["a", "b"],
+ * bare values (numbers, booleans).
+ *
+ * Does NOT support: multi-line / triple-quoted strings, inline tables,
+ * nested arrays, or section headers ([table]).
+ */
+function parseSimpleToml(content: string): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+  for (const rawLine of content.split("\n")) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) continue;
+
+    const eqIdx = line.indexOf("=");
+    if (eqIdx === -1) continue;
+
+    const key = line.slice(0, eqIdx).trim();
+    const rawValue = line.slice(eqIdx + 1).trim();
+
+    if (rawValue.startsWith("[")) {
+      // Array value — parse simple string arrays like ["a", "b", "c"]
+      const arrayContent = rawValue.slice(1, rawValue.lastIndexOf("]"));
+      const items: string[] = [];
+      for (const item of arrayContent.split(",")) {
+        const trimmed = item.trim().replace(/^["']|["']$/g, "");
+        if (trimmed) items.push(trimmed);
+      }
+      result[key] = items;
+    } else if (rawValue.startsWith('"') || rawValue.startsWith("'")) {
+      // String value
+      result[key] = rawValue.replace(/^["']|["']$/g, "");
+    } else {
+      // Bare value (number, boolean, etc.)
+      result[key] = rawValue;
+    }
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Parse SkillsBench directory
+// ---------------------------------------------------------------------------
+
+export function parseSkillsBenchDir(dirPath: string): SkillsBenchTask[] {
+  const tasksDir = join(dirPath, "tasks");
+  if (!existsSync(tasksDir)) return [];
+
+  const tasks: SkillsBenchTask[] = [];
+
+  let entries: ReturnType<typeof readdirSync>;
+  try {
+    entries = readdirSync(tasksDir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+
+    const taskDir = join(tasksDir, entry.name);
+    const instructionPath = join(taskDir, "instruction.md");
+
+    if (!existsSync(instructionPath)) continue;
+
+    const query = readFileSync(instructionPath, "utf-8").trim();
+    if (!query) continue;
+
+    // Parse optional task.toml
+    const tomlPath = join(taskDir, "task.toml");
+    let metadata: Record<string, unknown> = {};
+    if (existsSync(tomlPath)) {
+      metadata = parseSimpleToml(readFileSync(tomlPath, "utf-8"));
+    }
+
+    const difficulty = metadata.difficulty as SkillsBenchTask["difficulty"] | undefined;
+
+    const task: SkillsBenchTask = {
+      task_id: entry.name,
+      category: (metadata.category as string) ?? "general",
+      query,
+      difficulty:
+        difficulty && ["easy", "medium", "hard"].includes(difficulty) ? difficulty : "medium",
+    };
+
+    if (metadata.expected_skill) {
+      task.expected_skill = metadata.expected_skill as string;
+    }
+    if (metadata.expected_tools && Array.isArray(metadata.expected_tools)) {
+      task.expected_tools = metadata.expected_tools as string[];
+    }
+    if (metadata.tags && Array.isArray(metadata.tags)) {
+      task.tags = metadata.tags as string[];
+    }
+
+    tasks.push(task);
+  }
+
+  return tasks;
+}
+
+// ---------------------------------------------------------------------------
+// Convert tasks to EvalEntries
+// ---------------------------------------------------------------------------
+
+export function convertToEvalEntries(
+  tasks: SkillsBenchTask[],
+  targetSkill: string,
+  matchStrategy: "exact" | "fuzzy" = "exact",
+): EvalEntry[] {
+  const entries: EvalEntry[] = [];
+
+  for (const task of tasks) {
+    let matches = false;
+
+    if (matchStrategy === "exact") {
+      matches = task.expected_skill === targetSkill;
+    } else {
+      // Fuzzy: check if targetSkill appears as substring in category, tags, or expected_skill
+      const skillLower = targetSkill.toLowerCase();
+      const searchable = [task.category, task.expected_skill, ...(task.tags ?? [])]
+        .filter(Boolean)
+        .map((s) => (s as string).toLowerCase());
+
+      matches = searchable.some((s) => s.includes(skillLower) || skillLower.includes(s));
+    }
+
+    if (matches) {
+      entries.push({
+        query: task.query,
+        should_trigger: true,
+      });
+    }
+  }
+
+  return entries;
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export function cliMain(): void {
+  const { values } = parseArgs({
+    options: {
+      dir: { type: "string" },
+      skill: { type: "string" },
+      output: { type: "string" },
+      "match-strategy": { type: "string", default: "exact" },
+    },
+    strict: true,
+  });
+
+  if (!values.dir) {
+    console.error("[ERROR] --dir required (path to SkillsBench corpus directory)");
+    process.exit(1);
+  }
+
+  if (!values.skill) {
+    console.error("[ERROR] --skill required (target skill name)");
+    process.exit(1);
+  }
+
+  const matchStrategy = values["match-strategy"] === "fuzzy" ? "fuzzy" : "exact";
+
+  const tasks = parseSkillsBenchDir(values.dir);
+
+  if (tasks.length === 0) {
+    console.error(`[WARN] No tasks found in ${values.dir}/tasks/`);
+    console.error("Expected structure: <dir>/tasks/<task-id>/instruction.md");
+    process.exit(1);
+  }
+
+  console.log(`Parsed ${tasks.length} tasks from ${values.dir}`);
+
+  const entries = convertToEvalEntries(tasks, values.skill, matchStrategy);
+
+  if (entries.length === 0) {
+    console.log(
+      `[WARN] No tasks matched skill '${values.skill}' with strategy '${matchStrategy}'.`,
+    );
+    console.log("Available expected_skills:");
+    const skills = [...new Set(tasks.map((t) => t.expected_skill).filter(Boolean))].sort();
+    for (const s of skills) {
+      console.log(`  ${s}`);
+    }
+    if (matchStrategy === "exact") {
+      console.log("\nTip: try --match-strategy fuzzy for keyword-based matching.");
+    }
+  }
+
+  const outputPath = values.output ?? `${values.skill}_skillsbench_eval.json`;
+  writeFileSync(outputPath, JSON.stringify(entries, null, 2), "utf-8");
+  console.log(`Wrote ${entries.length} eval entries to ${outputPath}`);
+}
+
+if (import.meta.main) {
+  cliMain();
+}

package/cli/selftune/eval/synthetic-evals.ts

@@ -0,0 +1,172 @@
+/**
+ * synthetic-evals.ts
+ *
+ * Generates eval queries from a SKILL.md using an LLM, without requiring
+ * real session logs. Solves the cold-start problem for new skills that
+ * have no telemetry data yet.
+ */
+
+import { readFileSync } from "node:fs";
+
+import type { EvalEntry, InvocationType } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+import { classifyInvocation } from "./hooks-to-evals.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface SyntheticEvalOptions {
+  maxPositives?: number;
+  maxNegatives?: number;
+  modelFlag?: string;
+}
+
+interface RawSyntheticEntry {
+  query: string;
+  should_trigger: boolean;
+  invocation_type?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Prompt building
+// ---------------------------------------------------------------------------
+
+export function buildSyntheticPrompt(
+  skillContent: string,
+  skillName: string,
+  maxPositives: number,
+  maxNegatives: number,
+): { system: string; user: string } {
+  const system = `You are generating test queries for a coding agent skill. Given the skill description below, generate realistic user queries.
+
+For POSITIVE queries (should trigger this skill):
+- Generate a mix of:
+  - Explicit: directly names the skill or uses $${skillName} syntax
+  - Implicit: describes the task without naming the skill
+  - Contextual: natural language with domain context, proper nouns, dates, filenames
+- Vary phrasing, formality, and specificity
+
+For NEGATIVE queries (should NOT trigger this skill):
+- Queries that are topically adjacent but wrong intent
+- Queries for different skills that share keywords
+- Generic queries unrelated to this skill
+
+Output as JSON array with no surrounding text:
+[{"query": "...", "should_trigger": true, "invocation_type": "explicit|implicit|contextual|negative"}]`;
+
+  const user = `Skill name: ${skillName}
+
+Skill content:
+${skillContent}
+
+Generate exactly ${maxPositives} positive queries (should_trigger: true) and ${maxNegatives} negative queries (should_trigger: false). Return ONLY the JSON array.`;
+
+  return { system, user };
+}
+
+// ---------------------------------------------------------------------------
+// Response parsing
+// ---------------------------------------------------------------------------
+
+export function parseSyntheticResponse(raw: string, skillName: string): EvalEntry[] {
+  let text = raw.trim();
+
+  // Strip markdown fences manually for array-first JSON
+  // (stripMarkdownFences slices to first '{' which breaks '[' arrays)
+  const fenceMatch = text.match(/^```\w*\n/);
+  if (fenceMatch) {
+    text = text.slice(fenceMatch[0].length);
+    const closingIdx = text.lastIndexOf("```");
+    if (closingIdx >= 0) {
+      text = text.slice(0, closingIdx);
+    }
+    text = text.trim();
+  }
+
+  // Find the JSON array start
+  const bracketIdx = text.indexOf("[");
+  if (bracketIdx < 0) {
+    // No array found — try stripMarkdownFences as fallback for edge cases
+    const cleaned = stripMarkdownFences(raw);
+    const retryIdx = cleaned.indexOf("[");
+    if (retryIdx >= 0) {
+      text = cleaned.slice(retryIdx);
+    } else {
+      throw new Error(`Failed to parse synthetic eval response as JSON: ${text.slice(0, 200)}`);
+    }
+  } else {
+    text = text.slice(bracketIdx);
+  }
+
+  // Trim trailing content after the array closes
+  const lastBracket = text.lastIndexOf("]");
+  if (lastBracket >= 0) {
+    text = text.slice(0, lastBracket + 1);
+  }
+
+  const jsonText = text;
+
+  let entries: RawSyntheticEntry[];
+  try {
+    entries = JSON.parse(jsonText);
+  } catch {
+    throw new Error(`Failed to parse synthetic eval response as JSON: ${jsonText.slice(0, 200)}`);
+  }
+
+  if (!Array.isArray(entries)) {
+    throw new Error("Synthetic eval response is not a JSON array");
+  }
+
+  const result: EvalEntry[] = [];
+  for (const entry of entries) {
+    if (!entry || typeof entry.query !== "string" || typeof entry.should_trigger !== "boolean") {
+      continue;
+    }
+
+    const query = entry.query.trim();
+    if (!query) continue;
+
+    // For positives, use classifyInvocation to verify/override the LLM's type
+    let invocationType: InvocationType;
+    if (entry.should_trigger) {
+      invocationType = classifyInvocation(query, skillName);
+    } else {
+      invocationType = "negative";
+    }
+
+    result.push({
+      query,
+      should_trigger: entry.should_trigger,
+      invocation_type: invocationType,
+    });
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function generateSyntheticEvals(
+  skillPath: string,
+  skillName: string,
+  agent: string,
+  options: SyntheticEvalOptions = {},
+): Promise<EvalEntry[]> {
+  const maxPositives = options.maxPositives ?? 15;
+  const maxNegatives = options.maxNegatives ?? 10;
+
+  const skillContent = readFileSync(skillPath, "utf-8");
+
+  const { system, user } = buildSyntheticPrompt(
+    skillContent,
+    skillName,
+    maxPositives,
+    maxNegatives,
+  );
+
+  const raw = await callLlm(system, user, agent, options.modelFlag);
+  return parseSyntheticResponse(raw, skillName);
+}

package/cli/selftune/eval/unit-test-cli.ts

@@ -0,0 +1,152 @@
+/**
+ * CLI entrypoint for skill unit tests.
+ *
+ * Usage:
+ *   selftune unit-test --skill <name> --tests <path> [--run-agent] [--generate]
+ *
+ *   --skill <name>    Skill name (required)
+ *   --tests <path>    Path to unit test JSON file (default: ~/.selftune/unit-tests/<skill>.json)
+ *   --run-agent       Actually run tests through an agent (otherwise dry-run with static checks)
+ *   --generate        Generate tests from skill content using LLM (requires agent)
+ *   --skill-path <p>  Path to skill file (used with --generate for content)
+ *   --eval-set <p>    Path to eval set JSON (used with --generate for failure context)
+ *   --model <m>       Model flag for LLM calls
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+
+import { SELFTUNE_CONFIG_DIR } from "../constants.js";
+import type { EvalEntry } from "../types.js";
+import { callLlm, detectAgent } from "../utils/llm-call.js";
+import { generateUnitTests } from "./generate-unit-tests.js";
+import type { AgentRunner } from "./unit-test.js";
+import { loadUnitTests, runUnitTestSuite } from "./unit-test.js";
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      tests: { type: "string" },
+      "run-agent": { type: "boolean", default: false },
+      generate: { type: "boolean", default: false },
+      "skill-path": { type: "string" },
+      "eval-set": { type: "string" },
+      model: { type: "string" },
+    },
+    strict: true,
+  });
+
+  if (!values.skill) {
+    console.error("[ERROR] --skill <name> is required.");
+    process.exit(1);
+  }
+
+  const skillName = values.skill;
+  const unitTestDir = join(SELFTUNE_CONFIG_DIR, "unit-tests");
+  const defaultTestsPath = join(unitTestDir, `${skillName}.json`);
+  const testsPath = values.tests ?? defaultTestsPath;
+
+  // --generate: create tests from skill content
+  if (values.generate) {
+    const agent = detectAgent();
+    if (!agent) {
+      console.error("[ERROR] No agent CLI found (claude/codex/opencode). Cannot generate tests.");
+      process.exit(1);
+    }
+
+    let skillContent = `Skill: ${skillName}`;
+    if (values["skill-path"] && existsSync(values["skill-path"])) {
+      skillContent = readFileSync(values["skill-path"], "utf-8");
+    } else if (values["skill-path"]) {
+      console.warn(`[WARN] Skill path not found: ${values["skill-path"]}. Using skill name only.`);
+    }
+
+    let evalFailures: EvalEntry[] = [];
+    if (values["eval-set"] && existsSync(values["eval-set"])) {
+      try {
+        const raw = readFileSync(values["eval-set"], "utf-8");
+        const entries: EvalEntry[] = JSON.parse(raw);
+        evalFailures = entries.filter((e) => e.should_trigger);
+      } catch {
+        console.warn("[WARN] Failed to parse eval set. Proceeding without failure context.");
+      }
+    }
+
+    const modelFlag = values.model;
+    const llmCaller = (systemPrompt: string, userPrompt: string) =>
+      callLlm(systemPrompt, userPrompt, agent, modelFlag);
+
+    console.log(`Generating unit tests for skill '${skillName}'...`);
+    const tests = await generateUnitTests(skillName, skillContent, evalFailures, llmCaller);
+
+    if (tests.length === 0) {
+      console.error("[ERROR] No tests generated. Check agent/LLM availability.");
+      process.exit(1);
+    }
+
+    // Ensure output directory exists
+    mkdirSync(unitTestDir, { recursive: true });
+    writeFileSync(testsPath, JSON.stringify(tests, null, 2), "utf-8");
+    console.log(`Generated ${tests.length} unit tests -> ${testsPath}`);
+    return;
+  }
+
+  // Load and run tests
+  const tests = loadUnitTests(testsPath);
+  if (tests.length === 0) {
+    console.error(`[ERROR] No tests found at ${testsPath}`);
+    console.error("  Use --generate to create tests, or provide --tests <path>.");
+    process.exit(1);
+  }
+
+  console.log(`Loaded ${tests.length} unit tests for skill '${skillName}'`);
+
+  let agentRunner: AgentRunner;
+
+  if (values["run-agent"]) {
+    const agent = detectAgent();
+    if (!agent) {
+      console.error("[ERROR] No agent CLI found. Cannot run agent-based tests.");
+      process.exit(1);
+    }
+    const modelFlag = values.model;
+    agentRunner = async (query: string): Promise<string> => {
+      return callLlm("You are a helpful assistant.", query, agent, modelFlag);
+    };
+  } else {
+    // Dry-run: use query as transcript (only static assertions like contains work meaningfully)
+    console.log("(dry-run mode — use --run-agent for full agent execution)\n");
+    agentRunner = async (query: string): Promise<string> => query;
+  }
+
+  const suite = await runUnitTestSuite(tests, skillName, agentRunner);
+
+  // Print results
+  console.log(`\nResults for '${suite.skill_name}':`);
+  console.log(`  Total: ${suite.total}  Passed: ${suite.passed}  Failed: ${suite.failed}`);
+  console.log(`  Pass rate: ${(suite.pass_rate * 100).toFixed(1)}%`);
+
+  if (suite.failed > 0) {
+    console.log("\nFailed tests:");
+    for (const r of suite.results.filter((r) => !r.passed)) {
+      console.log(`  [FAIL] ${r.test_id} (${r.duration_ms}ms)`);
+      if (r.error) {
+        console.log(`         Error: ${r.error}`);
+      }
+      for (const a of r.assertion_results.filter((a) => !a.passed)) {
+        console.log(
+          `         - ${a.assertion.type}: expected "${a.assertion.value}", got "${a.actual}"`,
+        );
+      }
+    }
+  }
+
+  console.log(`\n${JSON.stringify(suite, null, 2)}`);
+  process.exit(suite.failed > 0 ? 1 : 0);
+}