selftune 0.1.4 → 0.2.1

package/cli/selftune/eval/hooks-to-evals.ts

@@ -26,7 +26,15 @@ import type {
   SkillUsageRecord,
 } from "../types.js";
 import { readJsonl } from "../utils/jsonl.js";
+import { detectAgent } from "../utils/llm-call.js";
+import {
+  filterActionableQueryRecords,
+  filterActionableSkillUsageRecords,
+} from "../utils/query-filter.js";
 import { seededShuffle } from "../utils/seeded-random.js";
+import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
+import { isHighConfidencePositiveSkillRecord } from "../utils/skill-usage-confidence.js";
+import { generateSyntheticEvals } from "./synthetic-evals.js";
 
 // ---------------------------------------------------------------------------
 // Query truncation
@@ -114,14 +122,16 @@ export function buildEvalSet(
   seed = 42,
   annotateTaxonomy = true,
 ): EvalEntry[] {
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
   const effectiveMaxPerSide = Number.isNaN(maxPerSide) || maxPerSide <= 0 ? 50 : maxPerSide;
   const effectiveSeed = Number.isNaN(seed) ? 42 : seed;
 
   // Build set of positive query texts (for exclusion from negatives)
   const positiveQueries = new Set<string>();
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     if (!r || typeof r.skill_name !== "string" || typeof r.query !== "string") continue;
-    if (r.skill_name === skillName) {
+    if (isHighConfidencePositiveSkillRecord(r, skillName)) {
       const q = (r.query ?? "").trim();
       if (q && q !== "(query not found)") {
         positiveQueries.add(q);
@@ -132,9 +142,9 @@ export function buildEvalSet(
   // Build deduplicated positives with taxonomy classification
   const seen = new Set<string>();
   const positives: EvalEntry[] = [];
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     if (!r || typeof r.skill_name !== "string" || typeof r.query !== "string") continue;
-    if (r.skill_name !== skillName) continue;
+    if (!isHighConfidencePositiveSkillRecord(r, skillName)) continue;
     const q = (r.query ?? "").trim();
     if (!q || q === "(query not found)" || seen.has(q)) continue;
     seen.add(q);
@@ -151,7 +161,7 @@ export function buildEvalSet(
   if (includeNegatives) {
     const negCandidates: string[] = [];
     const negSeen = new Set<string>();
-    for (const r of queryRecords) {
+    for (const r of actionableQueryRecords) {
       if (!r || typeof r.query !== "string") continue;
       const q = (r.query ?? "").trim();
       if (!q || positiveQueries.has(q) || negSeen.has(q)) continue;
@@ -196,13 +206,17 @@ export function listSkills(
   queryRecords: QueryLogRecord[],
   telemetryRecords: SessionTelemetryRecord[],
 ): void {
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
   const counts = new Map<string, number>();
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     const name = r.skill_name ?? "unknown";
     counts.set(name, (counts.get(name) ?? 0) + 1);
   }
 
-  console.log(`Skill triggers in skill_usage_log (${skillRecords.length} total records):`);
+  console.log(
+    `Skill triggers in skill_usage_log (${actionableSkillRecords.length} actionable records):`,
+  );
   if (counts.size > 0) {
     const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);
     for (const [name, count] of sorted) {
@@ -212,8 +226,8 @@ export function listSkills(
     console.log("  (none yet -- trigger some skills in Claude Code to populate)");
   }
 
-  console.log(`\nAll queries in all_queries_log: ${queryRecords.length}`);
-  if (queryRecords.length === 0) {
+  console.log(`\nActionable queries in all_queries_log: ${actionableQueryRecords.length}`);
+  if (actionableQueryRecords.length === 0) {
     console.log("  (none yet -- make sure prompt_log_hook is installed)");
   }
 
@@ -301,14 +315,16 @@ export function printEvalStats(
 ): void {
   const pos = evalSet.filter((e) => e.should_trigger);
   const neg = evalSet.filter((e) => !e.should_trigger);
-  const totalTriggers = skillRecords.filter((r) => r.skill_name === skillName).length;
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
+  const totalTriggers = actionableSkillRecords.filter((r) => r.skill_name === skillName).length;
 
   console.log(`Wrote ${evalSet.length} eval entries to ${outputPath}`);
   console.log(
     `  Positives (should_trigger=true) : ${pos.length}  (from ${totalTriggers} logged triggers)`,
   );
   console.log(
-    `  Negatives (should_trigger=false): ${neg.length}  (from ${queryRecords.length} total logged queries)`,
+    `  Negatives (should_trigger=false): ${neg.length}  (from ${actionableQueryRecords.length} actionable logged queries)`,
   );
 
   if (annotateTaxonomy && pos.length > 0) {
@@ -334,7 +350,7 @@ export function printEvalStats(
   console.log();
   if (pos.length === 0) {
     console.log(`[WARN] No positives for skill '${skillName}'.`);
-    const names = [...new Set(skillRecords.map((r) => r.skill_name))].sort();
+    const names = [...new Set(actionableSkillRecords.map((r) => r.skill_name))].sort();
     if (names.length > 0) {
       console.log(`       Known skills: ${names.join(", ")}`);
     }
@@ -359,11 +375,12 @@ export function printEvalStats(
 // CLI entry point
 // ---------------------------------------------------------------------------
 
-export function cliMain(): void {
+export async function cliMain(): Promise<void> {
   const { values } = parseArgs({
     options: {
       skill: { type: "string" },
       output: { type: "string" },
+      out: { type: "string" },
       max: { type: "string", default: "50" },
       seed: { type: "string", default: "42" },
       "list-skills": { type: "boolean", default: false },
@@ -373,11 +390,76 @@ export function cliMain(): void {
       "skill-log": { type: "string", default: SKILL_LOG },
       "query-log": { type: "string", default: QUERY_LOG },
       "telemetry-log": { type: "string", default: TELEMETRY_LOG },
+      synthetic: { type: "boolean", default: false },
+      "skill-path": { type: "string" },
+      model: { type: "string" },
     },
     strict: true,
   });
 
-  const skillRecords = readJsonl<SkillUsageRecord>(values["skill-log"] ?? SKILL_LOG);
+  // --- Synthetic mode: generate evals from SKILL.md via LLM ---
+  if (values.synthetic) {
+    if (!values.skill) {
+      console.error("[ERROR] --skill required with --synthetic");
+      process.exit(1);
+    }
+    if (!values["skill-path"]) {
+      console.error("[ERROR] --skill-path required with --synthetic");
+      process.exit(1);
+    }
+
+    const agent = detectAgent();
+    if (!agent) {
+      console.error("[ERROR] No agent CLI found (claude/codex/opencode). Install one first.");
+      process.exit(1);
+    }
+
+    const maxPerSide = Number.parseInt(values.max ?? "50", 10);
+    const effectiveMax = Number.isNaN(maxPerSide) || maxPerSide <= 0 ? 50 : maxPerSide;
+
+    console.log(`Generating synthetic evals for skill '${values.skill}'...`);
+    const evalSet = await generateSyntheticEvals(values["skill-path"], values.skill, agent, {
+      maxPositives: effectiveMax,
+      maxNegatives: effectiveMax,
+      modelFlag: values.model,
+    });
+
+    const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
+    writeFileSync(outputPath, JSON.stringify(evalSet, null, 2), "utf-8");
+
+    const pos = evalSet.filter((e) => e.should_trigger);
+    const neg = evalSet.filter((e) => !e.should_trigger);
+
+    console.log(`Wrote ${evalSet.length} synthetic eval entries to ${outputPath}`);
+    console.log(`  Positives (should_trigger=true) : ${pos.length}`);
+    console.log(`  Negatives (should_trigger=false): ${neg.length}`);
+
+    if (pos.length > 0) {
+      const types = new Map<string, number>();
+      for (const e of pos) {
+        const t = e.invocation_type ?? "?";
+        types.set(t, (types.get(t) ?? 0) + 1);
+      }
+      console.log("\n  Positive invocation types:");
+      for (const [t, c] of [...types.entries()].sort()) {
+        console.log(`    ${t.padEnd(15)}  ${c}`);
+      }
+    }
+
+    console.log("\nNext steps:");
+    console.log("  bun run cli/selftune/eval/run-eval.ts \\");
+    console.log(`    --eval-set ${outputPath} \\`);
+    console.log(`    --skill-path ${values["skill-path"]} \\`);
+    console.log("    --runs-per-query 3 --verbose");
+    return;
+  }
+
+  // --- Log-based mode (original behavior) ---
+  const skillLogPath = values["skill-log"] ?? SKILL_LOG;
+  const skillRecords =
+    skillLogPath === SKILL_LOG
+      ? readEffectiveSkillUsageRecords()
+      : readJsonl<SkillUsageRecord>(skillLogPath);
   const queryRecords = readJsonl<QueryLogRecord>(values["query-log"] ?? QUERY_LOG);
   const telemetryRecords = readJsonl<SessionTelemetryRecord>(
     values["telemetry-log"] ?? TELEMETRY_LOG,
@@ -412,11 +494,14 @@ export function cliMain(): void {
     annotateTaxonomy,
   );
 
-  const outputPath = values.output ?? `${values.skill}_trigger_eval.json`;
+  const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
   writeFileSync(outputPath, JSON.stringify(evalSet, null, 2), "utf-8");
   printEvalStats(evalSet, values.skill, outputPath, skillRecords, queryRecords, annotateTaxonomy);
 }
 
 if (import.meta.main) {
-  cliMain();
+  cliMain().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
 }

package/cli/selftune/eval/import-skillsbench.ts

@@ -0,0 +1,221 @@
+#!/usr/bin/env bun
+/**
+ * import-skillsbench.ts
+ *
+ * Imports task definitions from a SkillsBench-style corpus directory and
+ * converts them into EvalEntry arrays for use with selftune eval/grading.
+ *
+ * Expected directory structure:
+ *   <dir>/tasks/<task-id>/instruction.md   — task description (query text)
+ *   <dir>/tasks/<task-id>/task.toml        — metadata (difficulty, category, tags, etc.)
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import type { EvalEntry, SkillsBenchTask } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Minimal TOML parser (handles the subset used by SkillsBench task.toml files)
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse the subset of TOML used by SkillsBench task.toml files.
+ *
+ * Supports: single-line key = "value", flat string arrays ["a", "b"],
+ * bare values (numbers, booleans).
+ *
+ * Does NOT support: multi-line / triple-quoted strings, inline tables,
+ * nested arrays, or section headers ([table]).
+ */
+function parseSimpleToml(content: string): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+  for (const rawLine of content.split("\n")) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) continue;
+
+    const eqIdx = line.indexOf("=");
+    if (eqIdx === -1) continue;
+
+    const key = line.slice(0, eqIdx).trim();
+    const rawValue = line.slice(eqIdx + 1).trim();
+
+    if (rawValue.startsWith("[")) {
+      // Array value — parse simple string arrays like ["a", "b", "c"]
+      const arrayContent = rawValue.slice(1, rawValue.lastIndexOf("]"));
+      const items: string[] = [];
+      for (const item of arrayContent.split(",")) {
+        const trimmed = item.trim().replace(/^["']|["']$/g, "");
+        if (trimmed) items.push(trimmed);
+      }
+      result[key] = items;
+    } else if (rawValue.startsWith('"') || rawValue.startsWith("'")) {
+      // String value
+      result[key] = rawValue.replace(/^["']|["']$/g, "");
+    } else {
+      // Bare value (number, boolean, etc.)
+      result[key] = rawValue;
+    }
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Parse SkillsBench directory
+// ---------------------------------------------------------------------------
+
+export function parseSkillsBenchDir(dirPath: string): SkillsBenchTask[] {
+  const tasksDir = join(dirPath, "tasks");
+  if (!existsSync(tasksDir)) return [];
+
+  const tasks: SkillsBenchTask[] = [];
+
+  let entries: ReturnType<typeof readdirSync>;
+  try {
+    entries = readdirSync(tasksDir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+
+    const taskDir = join(tasksDir, entry.name);
+    const instructionPath = join(taskDir, "instruction.md");
+
+    if (!existsSync(instructionPath)) continue;
+
+    const query = readFileSync(instructionPath, "utf-8").trim();
+    if (!query) continue;
+
+    // Parse optional task.toml
+    const tomlPath = join(taskDir, "task.toml");
+    let metadata: Record<string, unknown> = {};
+    if (existsSync(tomlPath)) {
+      metadata = parseSimpleToml(readFileSync(tomlPath, "utf-8"));
+    }
+
+    const difficulty = metadata.difficulty as SkillsBenchTask["difficulty"] | undefined;
+
+    const task: SkillsBenchTask = {
+      task_id: entry.name,
+      category: (metadata.category as string) ?? "general",
+      query,
+      difficulty:
+        difficulty && ["easy", "medium", "hard"].includes(difficulty) ? difficulty : "medium",
+    };
+
+    if (metadata.expected_skill) {
+      task.expected_skill = metadata.expected_skill as string;
+    }
+    if (metadata.expected_tools && Array.isArray(metadata.expected_tools)) {
+      task.expected_tools = metadata.expected_tools as string[];
+    }
+    if (metadata.tags && Array.isArray(metadata.tags)) {
+      task.tags = metadata.tags as string[];
+    }
+
+    tasks.push(task);
+  }
+
+  return tasks;
+}
+
+// ---------------------------------------------------------------------------
+// Convert tasks to EvalEntries
+// ---------------------------------------------------------------------------
+
+export function convertToEvalEntries(
+  tasks: SkillsBenchTask[],
+  targetSkill: string,
+  matchStrategy: "exact" | "fuzzy" = "exact",
+): EvalEntry[] {
+  const entries: EvalEntry[] = [];
+
+  for (const task of tasks) {
+    let matches = false;
+
+    if (matchStrategy === "exact") {
+      matches = task.expected_skill === targetSkill;
+    } else {
+      // Fuzzy: check if targetSkill appears as substring in category, tags, or expected_skill
+      const skillLower = targetSkill.toLowerCase();
+      const searchable = [task.category, task.expected_skill, ...(task.tags ?? [])]
+        .filter(Boolean)
+        .map((s) => (s as string).toLowerCase());
+
+      matches = searchable.some((s) => s.includes(skillLower) || skillLower.includes(s));
+    }
+
+    if (matches) {
+      entries.push({
+        query: task.query,
+        should_trigger: true,
+      });
+    }
+  }
+
+  return entries;
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+export function cliMain(): void {
+  const { values } = parseArgs({
+    options: {
+      dir: { type: "string" },
+      skill: { type: "string" },
+      output: { type: "string" },
+      "match-strategy": { type: "string", default: "exact" },
+    },
+    strict: true,
+  });
+
+  if (!values.dir) {
+    console.error("[ERROR] --dir required (path to SkillsBench corpus directory)");
+    process.exit(1);
+  }
+
+  if (!values.skill) {
+    console.error("[ERROR] --skill required (target skill name)");
+    process.exit(1);
+  }
+
+  const matchStrategy = values["match-strategy"] === "fuzzy" ? "fuzzy" : "exact";
+
+  const tasks = parseSkillsBenchDir(values.dir);
+
+  if (tasks.length === 0) {
+    console.error(`[WARN] No tasks found in ${values.dir}/tasks/`);
+    console.error("Expected structure: <dir>/tasks/<task-id>/instruction.md");
+    process.exit(1);
+  }
+
+  console.log(`Parsed ${tasks.length} tasks from ${values.dir}`);
+
+  const entries = convertToEvalEntries(tasks, values.skill, matchStrategy);
+
+  if (entries.length === 0) {
+    console.log(
+      `[WARN] No tasks matched skill '${values.skill}' with strategy '${matchStrategy}'.`,
+    );
+    console.log("Available expected_skills:");
+    const skills = [...new Set(tasks.map((t) => t.expected_skill).filter(Boolean))].sort();
+    for (const s of skills) {
+      console.log(`  ${s}`);
+    }
+    if (matchStrategy === "exact") {
+      console.log("\nTip: try --match-strategy fuzzy for keyword-based matching.");
+    }
+  }
+
+  const outputPath = values.output ?? `${values.skill}_skillsbench_eval.json`;
+  writeFileSync(outputPath, JSON.stringify(entries, null, 2), "utf-8");
+  console.log(`Wrote ${entries.length} eval entries to ${outputPath}`);
+}
+
+if (import.meta.main) {
+  cliMain();
+}

package/cli/selftune/eval/synthetic-evals.ts

@@ -0,0 +1,172 @@
+/**
+ * synthetic-evals.ts
+ *
+ * Generates eval queries from a SKILL.md using an LLM, without requiring
+ * real session logs. Solves the cold-start problem for new skills that
+ * have no telemetry data yet.
+ */
+
+import { readFileSync } from "node:fs";
+
+import type { EvalEntry, InvocationType } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+import { classifyInvocation } from "./hooks-to-evals.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface SyntheticEvalOptions {
+  maxPositives?: number;
+  maxNegatives?: number;
+  modelFlag?: string;
+}
+
+interface RawSyntheticEntry {
+  query: string;
+  should_trigger: boolean;
+  invocation_type?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Prompt building
+// ---------------------------------------------------------------------------
+
+export function buildSyntheticPrompt(
+  skillContent: string,
+  skillName: string,
+  maxPositives: number,
+  maxNegatives: number,
+): { system: string; user: string } {
+  const system = `You are generating test queries for a coding agent skill. Given the skill description below, generate realistic user queries.
+
+For POSITIVE queries (should trigger this skill):
+- Generate a mix of:
+  - Explicit: directly names the skill or uses $${skillName} syntax
+  - Implicit: describes the task without naming the skill
+  - Contextual: natural language with domain context, proper nouns, dates, filenames
+- Vary phrasing, formality, and specificity
+
+For NEGATIVE queries (should NOT trigger this skill):
+- Queries that are topically adjacent but wrong intent
+- Queries for different skills that share keywords
+- Generic queries unrelated to this skill
+
+Output as JSON array with no surrounding text:
+[{"query": "...", "should_trigger": true, "invocation_type": "explicit|implicit|contextual|negative"}]`;
+
+  const user = `Skill name: ${skillName}
+
+Skill content:
+${skillContent}
+
+Generate exactly ${maxPositives} positive queries (should_trigger: true) and ${maxNegatives} negative queries (should_trigger: false). Return ONLY the JSON array.`;
+
+  return { system, user };
+}
+
+// ---------------------------------------------------------------------------
+// Response parsing
+// ---------------------------------------------------------------------------
+
+export function parseSyntheticResponse(raw: string, skillName: string): EvalEntry[] {
+  let text = raw.trim();
+
+  // Strip markdown fences manually for array-first JSON
+  // (stripMarkdownFences slices to first '{' which breaks '[' arrays)
+  const fenceMatch = text.match(/^```\w*\n/);
+  if (fenceMatch) {
+    text = text.slice(fenceMatch[0].length);
+    const closingIdx = text.lastIndexOf("```");
+    if (closingIdx >= 0) {
+      text = text.slice(0, closingIdx);
+    }
+    text = text.trim();
+  }
+
+  // Find the JSON array start
+  const bracketIdx = text.indexOf("[");
+  if (bracketIdx < 0) {
+    // No array found — try stripMarkdownFences as fallback for edge cases
+    const cleaned = stripMarkdownFences(raw);
+    const retryIdx = cleaned.indexOf("[");
+    if (retryIdx >= 0) {
+      text = cleaned.slice(retryIdx);
+    } else {
+      throw new Error(`Failed to parse synthetic eval response as JSON: ${text.slice(0, 200)}`);
+    }
+  } else {
+    text = text.slice(bracketIdx);
+  }
+
+  // Trim trailing content after the array closes
+  const lastBracket = text.lastIndexOf("]");
+  if (lastBracket >= 0) {
+    text = text.slice(0, lastBracket + 1);
+  }
+
+  const jsonText = text;
+
+  let entries: RawSyntheticEntry[];
+  try {
+    entries = JSON.parse(jsonText);
+  } catch {
+    throw new Error(`Failed to parse synthetic eval response as JSON: ${jsonText.slice(0, 200)}`);
+  }
+
+  if (!Array.isArray(entries)) {
+    throw new Error("Synthetic eval response is not a JSON array");
+  }
+
+  const result: EvalEntry[] = [];
+  for (const entry of entries) {
+    if (!entry || typeof entry.query !== "string" || typeof entry.should_trigger !== "boolean") {
+      continue;
+    }
+
+    const query = entry.query.trim();
+    if (!query) continue;
+
+    // For positives, use classifyInvocation to verify/override the LLM's type
+    let invocationType: InvocationType;
+    if (entry.should_trigger) {
+      invocationType = classifyInvocation(query, skillName);
+    } else {
+      invocationType = "negative";
+    }
+
+    result.push({
+      query,
+      should_trigger: entry.should_trigger,
+      invocation_type: invocationType,
+    });
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+export async function generateSyntheticEvals(
+  skillPath: string,
+  skillName: string,
+  agent: string,
+  options: SyntheticEvalOptions = {},
+): Promise<EvalEntry[]> {
+  const maxPositives = options.maxPositives ?? 15;
+  const maxNegatives = options.maxNegatives ?? 10;
+
+  const skillContent = readFileSync(skillPath, "utf-8");
+
+  const { system, user } = buildSyntheticPrompt(
+    skillContent,
+    skillName,
+    maxPositives,
+    maxNegatives,
+  );
+
+  const raw = await callLlm(system, user, agent, options.modelFlag);
+  return parseSyntheticResponse(raw, skillName);
+}