selftune 0.2.18 → 0.2.19

package/cli/selftune/eval/family-overlap.ts

@@ -0,0 +1,395 @@
+#!/usr/bin/env bun
+
+import { parseArgs } from "node:util";
+
+import { getDb } from "../localdb/db.js";
+import { queryQueryLog, querySkillUsageRecords } from "../localdb/queries.js";
+import type {
+  QueryLogRecord,
+  SkillFamilyOverlapMember,
+  SkillFamilyOverlapPair,
+  SkillFamilyOverlapReport,
+  SkillFamilyRefactorProposal,
+  SkillUsageRecord,
+} from "../types.js";
+import { CLIError } from "../utils/cli-error.js";
+import {
+  findInstalledSkillNames,
+  findInstalledSkillPath,
+  findRepositoryClaudeSkillDirs,
+  findRepositorySkillDirs,
+} from "../utils/skill-discovery.js";
+import { buildEvalSet } from "./hooks-to-evals.js";
+
+const DEFAULT_MIN_OVERLAP = 0.3;
+const DEFAULT_MIN_SHARED = 2;
+const DEFAULT_MAX_SHARED = 10;
+
+interface FamilyOverlapOptions {
+  familyPrefix?: string;
+  parentSkillName?: string;
+  minOverlapPct?: number;
+  minSharedQueries?: number;
+  maxSharedQueries?: number;
+  searchDirs?: string[];
+}
+
+function getEvalSkillSearchDirs(): string[] {
+  const cwd = process.cwd();
+  const homeDir = process.env.HOME ?? "";
+  const codexHome = process.env.CODEX_HOME ?? `${homeDir}/.codex`;
+  return [
+    ...findRepositorySkillDirs(cwd),
+    ...findRepositoryClaudeSkillDirs(cwd),
+    `${homeDir}/.agents/skills`,
+    `${homeDir}/.claude/skills`,
+    `${codexHome}/skills`,
+  ];
+}
+
+function normalizeQuery(value: string): string {
+  return value.trim().replace(/\s+/g, " ").toLowerCase();
+}
+
+function inferFamilyPrefix(skills: string[]): string | undefined {
+  if (skills.length < 2) return undefined;
+  const firstPrefixes = skills.map((skill) => {
+    const hyphen = skill.indexOf("-");
+    return hyphen === -1 ? skill : skill.slice(0, hyphen + 1);
+  });
+  const candidate = firstPrefixes[0];
+  return firstPrefixes.every((prefix) => prefix === candidate) ? candidate : undefined;
+}
+
+function inferParentSkillName(
+  skills: string[],
+  explicitParent?: string,
+  familyPrefix?: string,
+): string {
+  if (explicitParent?.trim()) return explicitParent.trim();
+  const inferredPrefix = familyPrefix ?? inferFamilyPrefix(skills) ?? "family";
+  return inferredPrefix.endsWith("-") ? inferredPrefix.slice(0, -1) : inferredPrefix;
+}
+
+function toWorkflowName(skillName: string, familyPrefix?: string): string {
+  const stripped =
+    familyPrefix && skillName.startsWith(familyPrefix)
+      ? skillName.slice(familyPrefix.length)
+      : skillName;
+  return stripped.trim() || "default";
+}
+
+function buildPositiveQuerySet(
+  skillName: string,
+  skillRecords: SkillUsageRecord[],
+  queryRecords: QueryLogRecord[],
+): Set<string> {
+  const evalEntries = buildEvalSet(
+    skillRecords,
+    queryRecords,
+    skillName,
+    Number.MAX_SAFE_INTEGER,
+    false,
+    42,
+    false,
+  );
+  return new Set(
+    evalEntries
+      .filter((entry) => entry.should_trigger)
+      .map((entry) => normalizeQuery(entry.query))
+      .filter(Boolean),
+  );
+}
+
+function buildMember(
+  skillName: string,
+  positiveQueries: Set<string>,
+  searchDirs: string[],
+): SkillFamilyOverlapMember {
+  return {
+    skill_name: skillName,
+    skill_path: findInstalledSkillPath(skillName, searchDirs),
+    positive_query_count: positiveQueries.size,
+  };
+}
+
+function scoreConsolidationPressure(overlapPct: number): "low" | "medium" | "high" {
+  if (overlapPct >= 0.6) return "high";
+  if (overlapPct >= 0.4) return "medium";
+  return "low";
+}
+
+function buildRefactorProposal(
+  skills: string[],
+  familyPrefix: string | undefined,
+  parentSkillName: string,
+): SkillFamilyRefactorProposal {
+  const workflows = skills.map((skillName) => {
+    const workflowName = toWorkflowName(skillName, familyPrefix);
+    return {
+      workflow_name: workflowName,
+      source_skill: skillName,
+      suggested_path: `Workflows/${workflowName}.md`,
+    };
+  });
+
+  return {
+    parent_skill_name: parentSkillName,
+    family_prefix: familyPrefix,
+    internal_workflows: workflows,
+    compatibility_aliases: workflows.map((workflow) => ({
+      skill_name: workflow.source_skill,
+      target_workflow: workflow.workflow_name,
+    })),
+    migration_notes: [
+      `Create a parent skill \`${parentSkillName}\` whose SKILL.md routes into internal workflows instead of exposing each family member as a primary top-level trigger surface.`,
+      "Keep the existing sibling skills as thin compatibility aliases for at least one release cycle while usage shifts to the parent skill.",
+      "Move execution-specific instructions into internal Workflows/ or references/ files so the parent SKILL.md stays focused on routing and progressive disclosure.",
+      "Use the compatibility aliases to measure whether trigger quality improves before removing the old skill entry points.",
+    ],
+  };
+}
+
+export function analyzeSkillFamilyOverlap(
+  skills: string[],
+  skillRecords: SkillUsageRecord[],
+  queryRecords: QueryLogRecord[],
+  options: FamilyOverlapOptions = {},
+): SkillFamilyOverlapReport {
+  if (skills.length < 2) {
+    throw new CLIError(
+      "Skill family overlap analysis requires at least 2 skills.",
+      "INVALID_FLAG",
+      "selftune eval family-overlap --skills skill-a,skill-b",
+    );
+  }
+
+  const searchDirs = options.searchDirs ?? getEvalSkillSearchDirs();
+  const familyPrefix = options.familyPrefix ?? inferFamilyPrefix(skills);
+  const minOverlapPct = options.minOverlapPct ?? DEFAULT_MIN_OVERLAP;
+  const minSharedQueries = options.minSharedQueries ?? DEFAULT_MIN_SHARED;
+  const maxSharedQueries = options.maxSharedQueries ?? DEFAULT_MAX_SHARED;
+
+  const positiveQueriesBySkill = new Map<string, Set<string>>();
+  const members: SkillFamilyOverlapMember[] = [];
+  for (const skillName of skills) {
+    const positives = buildPositiveQuerySet(skillName, skillRecords, queryRecords);
+    positiveQueriesBySkill.set(skillName, positives);
+    members.push(buildMember(skillName, positives, searchDirs));
+  }
+
+  const pairs: SkillFamilyOverlapPair[] = [];
+  for (let i = 0; i < skills.length; i++) {
+    for (let j = i + 1; j < skills.length; j++) {
+      const skillA = skills[i];
+      const skillB = skills[j];
+      const positivesA = positiveQueriesBySkill.get(skillA) ?? new Set<string>();
+      const positivesB = positiveQueriesBySkill.get(skillB) ?? new Set<string>();
+      if (positivesA.size === 0 || positivesB.size === 0) continue;
+
+      const sharedQueries = [...positivesA].filter((query) => positivesB.has(query));
+      const overlapPct = sharedQueries.length / Math.min(positivesA.size, positivesB.size);
+      if (sharedQueries.length < minSharedQueries || overlapPct < minOverlapPct) continue;
+
+      pairs.push({
+        skill_a: skillA,
+        skill_b: skillB,
+        overlap_pct: overlapPct,
+        shared_query_count: sharedQueries.length,
+        shared_queries: sharedQueries.slice(0, maxSharedQueries),
+        consolidation_pressure: scoreConsolidationPressure(overlapPct),
+      });
+    }
+  }
+
+  pairs.sort(
+    (a, b) => b.overlap_pct - a.overlap_pct || b.shared_query_count - a.shared_query_count,
+  );
+
+  const totalPairsAnalyzed = (skills.length * (skills.length - 1)) / 2;
+  const overlapCount = pairs.length;
+  const overlapDensity = totalPairsAnalyzed > 0 ? overlapCount / totalPairsAnalyzed : 0;
+  const averageOverlapPct =
+    overlapCount > 0 ? pairs.reduce((sum, pair) => sum + pair.overlap_pct, 0) / overlapCount : 0;
+  const readySkillCount = members.filter(
+    (member) => member.positive_query_count >= minSharedQueries,
+  ).length;
+  const consolidationCandidate =
+    readySkillCount >= 2 &&
+    skills.length >= 3 &&
+    (overlapCount >= 2 || (overlapCount >= 1 && overlapDensity >= 0.5));
+
+  const parentSkillName = inferParentSkillName(skills, options.parentSkillName, familyPrefix);
+  const rationale = [
+    `${skills.length} sibling skills analyzed with ${totalPairsAnalyzed} pairwise boundary checks.`,
+    overlapCount === 0
+      ? "No exact-query overlap crossed the current consolidation threshold."
+      : `${overlapCount} skill pairs share at least ${Math.round(minOverlapPct * 100)}% of their trusted positive queries.`,
+  ];
+
+  if (pairs.some((pair) => pair.consolidation_pressure === "high")) {
+    rationale.push(
+      "High-overlap pairs suggest the current top-level routing surfaces are competing for the same real user intent.",
+    );
+  }
+
+  if (readySkillCount < 2) {
+    rationale.push(
+      `Only ${readySkillCount} sibling skills currently have enough trusted positives to make a packaging call. Generate cold-start evals and gather real usage before treating this as evidence against consolidation.`,
+    );
+  }
+
+  if (consolidationCandidate) {
+    rationale.push(
+      "This family looks like a packaging problem, not just a wording problem. Test a parent skill with internal workflows before continuing standalone description optimization.",
+    );
+  }
+
+  return {
+    family_prefix: familyPrefix,
+    analyzed_skills: skills,
+    members,
+    pairs,
+    total_pairs_analyzed: totalPairsAnalyzed,
+    overlap_count: overlapCount,
+    overlap_density: overlapDensity,
+    average_overlap_pct: averageOverlapPct,
+    consolidation_candidate: consolidationCandidate,
+    recommendation:
+      readySkillCount < 2
+        ? "Insufficient trusted telemetry to make a family-packaging call yet. Use cold-start evals plus a few days of real usage before deciding whether to consolidate."
+        : consolidationCandidate
+          ? `Consider consolidating this family under a parent skill like \`${parentSkillName}\`.`
+          : "Keep the skills separate for now and continue improving boundaries at the description/workflow level.",
+    rationale,
+    refactor_proposal: consolidationCandidate
+      ? buildRefactorProposal(skills, familyPrefix, parentSkillName)
+      : undefined,
+    generated_at: new Date().toISOString(),
+  };
+}
+
+function parseSkillList(raw: string | undefined): string[] {
+  if (!raw) return [];
+  return raw
+    .split(",")
+    .map((value) => value.trim())
+    .filter(Boolean);
+}
+
+function resolveFamilySkills(
+  explicitSkills: string[],
+  familyPrefix: string | undefined,
+  skillRecords: SkillUsageRecord[],
+  searchDirs: string[],
+): string[] {
+  if (explicitSkills.length > 0)
+    return [...new Set(explicitSkills)].sort((a, b) => a.localeCompare(b));
+
+  if (!familyPrefix) {
+    throw new CLIError(
+      "Pass either --skills <a,b,c> or --prefix <family->.",
+      "MISSING_FLAG",
+      "selftune eval family-overlap --prefix sc-",
+    );
+  }
+
+  const installedNames = findInstalledSkillNames(searchDirs);
+  const observedNames = new Set<string>(
+    skillRecords.map((record) => record.skill_name).filter(Boolean),
+  );
+  const familySkills = new Set<string>();
+  for (const name of [...installedNames, ...observedNames]) {
+    if (name.startsWith(familyPrefix)) familySkills.add(name);
+  }
+
+  return [...familySkills].sort((a, b) => a.localeCompare(b));
+}
+
+export async function cliMain(): Promise<void> {
+  let values: ReturnType<typeof parseArgs>["values"];
+  try {
+    ({ values } = parseArgs({
+      options: {
+        help: { type: "boolean", short: "h", default: false },
+        prefix: { type: "string" },
+        skills: { type: "string" },
+        "parent-skill": { type: "string" },
+        "min-overlap": { type: "string" },
+        "min-shared": { type: "string" },
+      },
+      strict: true,
+    }));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new CLIError(
+      `Invalid arguments: ${message}`,
+      "INVALID_FLAG",
+      "selftune eval family-overlap --help",
+    );
+  }
+
+  if (values.help) {
+    console.log(`Usage:
+  selftune eval family-overlap --skills skill-a,skill-b[,skill-c]
+  selftune eval family-overlap --prefix sc-
+
+Options:
+  --skills <a,b,c>       Explicit skill names
+  --prefix <family->     Analyze installed or observed skills with this prefix
+  --parent-skill <name>  Override the inferred parent skill name
+  --min-overlap <0-1>    Minimum overlap percentage (default: 0.3)
+  --min-shared <n>       Minimum shared queries (default: 2)
+  -h, --help             Show this help
+`);
+    return;
+  }
+
+  const rawMinOverlap = values["min-overlap"] as string | undefined;
+  const rawMinShared = values["min-shared"] as string | undefined;
+  const minOverlapPct =
+    rawMinOverlap === undefined ? DEFAULT_MIN_OVERLAP : Number.parseFloat(rawMinOverlap);
+  const minSharedQueries =
+    rawMinShared === undefined ? DEFAULT_MIN_SHARED : Number.parseInt(rawMinShared, 10);
+
+  if (!Number.isFinite(minOverlapPct) || minOverlapPct <= 0 || minOverlapPct > 1) {
+    throw new CLIError(
+      "Invalid --min-overlap value. Use a number between 0 and 1.",
+      "INVALID_FLAG",
+      "selftune eval family-overlap --prefix sc- --min-overlap 0.3",
+    );
+  }
+
+  if (!Number.isFinite(minSharedQueries) || minSharedQueries < 1) {
+    throw new CLIError(
+      "Invalid --min-shared value. Use a positive integer.",
+      "INVALID_FLAG",
+      "selftune eval family-overlap --prefix sc- --min-shared 2",
+    );
+  }
+
+  const searchDirs = getEvalSkillSearchDirs();
+  const db = getDb();
+  const skillRecords = querySkillUsageRecords(db) as SkillUsageRecord[];
+  const queryRecords = queryQueryLog(db) as QueryLogRecord[];
+  const familyPrefix = (values.prefix as string | undefined)?.trim() || undefined;
+  const explicitSkills = parseSkillList(values.skills as string | undefined);
+  const skills = resolveFamilySkills(explicitSkills, familyPrefix, skillRecords, searchDirs);
+
+  if (skills.length < 2) {
+    throw new CLIError(
+      `Need at least 2 skills to analyze, found ${skills.length}.`,
+      "INVALID_FLAG",
+      "selftune eval family-overlap --prefix sc-",
+    );
+  }
+
+  const report = analyzeSkillFamilyOverlap(skills, skillRecords, queryRecords, {
+    familyPrefix,
+    parentSkillName: (values["parent-skill"] as string | undefined)?.trim() || undefined,
+    minOverlapPct,
+    minSharedQueries,
+    searchDirs,
+  });
+  console.log(JSON.stringify(report, null, 2));
+}

package/cli/selftune/eval/hooks-to-evals.ts

@@ -2,7 +2,8 @@
 /**
  * hooks-to-evals.ts
  *
- * Converts hook logs into trigger eval sets compatible with run_eval / run_loop.
+ * Converts hook logs into trigger eval sets compatible with the current
+ * eval-generate -> evolve --dry-run validation loop.
  *
  * Default read path is SQLite (via localdb/queries). JSONL fallback is used only
  * when custom --skill-log / --query-log / --telemetry-log paths are supplied
@@ -43,6 +44,13 @@ import {
   filterActionableSkillUsageRecords,
 } from "../utils/query-filter.js";
 import { seededShuffle } from "../utils/seeded-random.js";
+import {
+  escapeRegExp,
+  findInstalledSkillNames,
+  findInstalledSkillPath,
+  findRepositoryClaudeSkillDirs,
+  findRepositorySkillDirs,
+} from "../utils/skill-discovery.js";
 import { isHighConfidencePositiveSkillRecord } from "../utils/skill-usage-confidence.js";
 import { generateSyntheticEvals } from "./synthetic-evals.js";
 
@@ -78,14 +86,14 @@ export function classifyInvocation(query: string, skillName: string): Invocation
   // Handle hyphenated skill names: check if all parts appear
   if (skillLower.includes("-")) {
     const parts = skillLower.split("-");
-    if (parts.every((part) => qLower.includes(part))) {
+    if (parts.every((part) => new RegExp(`\\b${escapeRegExp(part)}\\b`, "i").test(query))) {
       return "explicit";
     }
   }
 
   // Convert skill-name to camelCase and check
   const camelCase = skillLower.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
-  if (camelCase !== skillLower && qLower.includes(camelCase)) {
+  if (camelCase !== skillLower && qLower.includes(camelCase.toLowerCase())) {
     return "explicit";
   }
 
@@ -207,6 +215,78 @@ export function buildEvalSet(
   return [...shuffledPositives, ...negatives];
 }
 
+// ---------------------------------------------------------------------------
+// Installed skill discovery / readiness
+// ---------------------------------------------------------------------------
+
+export interface EvalSkillReadiness {
+  name: string;
+  trusted_trigger_count: number;
+  raw_trigger_count: number;
+  trusted_session_count: number;
+  raw_session_count: number;
+  installed: boolean;
+  skill_path?: string;
+  readiness: "log_ready" | "cold_start_ready" | "telemetry_only";
+}
+
+function getEvalSkillSearchDirs(): string[] {
+  const cwd = process.cwd();
+  const homeDir = process.env.HOME ?? "";
+  const codexHome = process.env.CODEX_HOME ?? `${homeDir}/.codex`;
+  return [
+    ...findRepositorySkillDirs(cwd),
+    ...findRepositoryClaudeSkillDirs(cwd),
+    `${homeDir}/.agents/skills`,
+    `${homeDir}/.claude/skills`,
+    `${codexHome}/skills`,
+  ];
+}
+
+export function listEvalSkillReadiness(
+  skillRecords: SkillUsageRecord[],
+  searchDirs: string[] = getEvalSkillSearchDirs(),
+): EvalSkillReadiness[] {
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const rawTriggerCounts = new Map<string, number>();
+  const rawSessionCounts = new Map<string, Set<string>>();
+  const trustedTriggerCounts = new Map<string, number>();
+  const trustedSessionCounts = new Map<string, Set<string>>();
+  for (const r of actionableSkillRecords) {
+    const name = r.skill_name ?? "unknown";
+    rawTriggerCounts.set(name, (rawTriggerCounts.get(name) ?? 0) + 1);
+    if (!rawSessionCounts.has(name)) rawSessionCounts.set(name, new Set<string>());
+    if (r.session_id) rawSessionCounts.get(name)?.add(r.session_id);
+
+    if (!isHighConfidencePositiveSkillRecord(r, name)) continue;
+    trustedTriggerCounts.set(name, (trustedTriggerCounts.get(name) ?? 0) + 1);
+    if (!trustedSessionCounts.has(name)) trustedSessionCounts.set(name, new Set<string>());
+    if (r.session_id) trustedSessionCounts.get(name)?.add(r.session_id);
+  }
+
+  const installedNames = findInstalledSkillNames(searchDirs);
+  const allNames = new Set<string>([...rawTriggerCounts.keys(), ...installedNames]);
+
+  return [...allNames]
+    .sort((a, b) => a.localeCompare(b))
+    .map((name) => {
+      const trustedTriggerCount = trustedTriggerCounts.get(name) ?? 0;
+      const rawTriggerCount = rawTriggerCounts.get(name) ?? 0;
+      const installed = installedNames.has(name);
+      return {
+        name,
+        trusted_trigger_count: trustedTriggerCount,
+        raw_trigger_count: rawTriggerCount,
+        trusted_session_count: trustedSessionCounts.get(name)?.size ?? 0,
+        raw_session_count: rawSessionCounts.get(name)?.size ?? 0,
+        installed,
+        skill_path: installed ? findInstalledSkillPath(name, searchDirs) : undefined,
+        readiness:
+          trustedTriggerCount > 0 ? "log_ready" : installed ? "cold_start_ready" : "telemetry_only",
+      } satisfies EvalSkillReadiness;
+    });
+}
+
 // ---------------------------------------------------------------------------
 // List skills
 // ---------------------------------------------------------------------------
@@ -216,24 +296,37 @@ export function listSkills(
   queryRecords: QueryLogRecord[],
   telemetryRecords: SessionTelemetryRecord[],
 ): void {
-  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
   const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
-  const counts = new Map<string, number>();
-  for (const r of actionableSkillRecords) {
-    const name = r.skill_name ?? "unknown";
-    counts.set(name, (counts.get(name) ?? 0) + 1);
-  }
-
-  console.log(
-    `Skill triggers in skill_usage_log (${actionableSkillRecords.length} actionable records):`,
-  );
-  if (counts.size > 0) {
-    const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);
-    for (const [name, count] of sorted) {
-      console.log(`  ${name.padEnd(30)}  ${String(count).padStart(4)} triggers`);
+  const readiness = listEvalSkillReadiness(skillRecords);
+
+  console.log(`Skills with eval readiness (${readiness.length} total):`);
+  if (readiness.length > 0) {
+    for (const skill of readiness) {
+      const readinessLabel =
+        skill.readiness === "log_ready"
+          ? "log-ready"
+          : skill.readiness === "cold_start_ready"
+            ? "cold-start"
+            : "telemetry-only";
+      const installLabel = skill.installed ? "installed" : "not installed";
+      const trustedLabel = `${String(skill.trusted_trigger_count).padStart(3)} trusted`;
+      const rawLabel =
+        skill.raw_trigger_count !== skill.trusted_trigger_count
+          ? ` / ${String(skill.raw_trigger_count).padStart(3)} raw`
+          : "";
+      console.log(
+        `  ${skill.name.padEnd(30)}  ${trustedLabel}${rawLabel}  ${String(skill.trusted_session_count).padStart(3)} trusted sessions  ${readinessLabel} / ${installLabel}`,
+      );
     }
+    console.log("");
+    console.log("Legend:");
+    console.log("  log-ready    real triggers exist; run eval generate normally");
+    console.log(
+      "  cold-start   installed locally but no trusted triggers yet; use --auto-synthetic",
+    );
+    console.log("  telemetry-only  trigger data exists but local SKILL.md was not found");
   } else {
-    console.log("  (none yet -- trigger some skills in Claude Code to populate)");
+    console.log("  (none yet -- install skills or sync source data first)");
   }
 
   console.log(`\nActionable queries in all_queries_log: ${actionableQueryRecords.length}`);
@@ -370,15 +463,25 @@ export function printEvalStats(
   }
 
   console.log("Next steps:");
-  console.log("  bun run cli/selftune/eval/run-eval.ts \\");
+  console.log(`  selftune evolve --skill ${skillName} \\`);
+  console.log(`    --skill-path /path/to/skills/${skillName}/SKILL.md \\`);
   console.log(`    --eval-set ${outputPath} \\`);
-  console.log(`    --skill-path /path/to/skills/${skillName} \\`);
-  console.log("    --runs-per-query 3 --verbose");
+  console.log("    --dry-run --verbose");
   console.log();
-  console.log("  bun run cli/selftune/eval/run-loop.ts \\");
-  console.log(`    --eval-set ${outputPath} \\`);
-  console.log(`    --skill-path /path/to/skills/${skillName} \\`);
-  console.log("    --max-iterations 5 --verbose");
+  console.log(`  selftune evolve --skill ${skillName} \\`);
+  console.log(`    --skill-path /path/to/skills/${skillName}/SKILL.md \\`);
+  console.log(`    --eval-set ${outputPath}`);
+}
+
+function printSyntheticFallbackHint(skillName: string, skillPath: string): void {
+  console.log("");
+  console.log(`[TIP] No trusted trigger data found yet for '${skillName}'.`);
+  console.log(
+    "      This skill is installed locally, so you can still generate a cold-start eval set:",
+  );
+  console.log(
+    `      selftune eval generate --skill ${skillName} --auto-synthetic --skill-path ${skillPath}`,
+  );
 }
 
 // ---------------------------------------------------------------------------
@@ -401,6 +504,7 @@ export async function cliMain(): Promise<void> {
       "query-log": { type: "string", default: QUERY_LOG },
       "telemetry-log": { type: "string", default: TELEMETRY_LOG },
       synthetic: { type: "boolean", default: false },
+      "auto-synthetic": { type: "boolean", default: false },
       "skill-path": { type: "string" },
       model: { type: "string" },
     },
@@ -466,10 +570,10 @@ export async function cliMain(): Promise<void> {
     }
 
     console.log("\nNext steps:");
-    console.log("  bun run cli/selftune/eval/run-eval.ts \\");
-    console.log(`    --eval-set ${outputPath} \\`);
+    console.log(`  selftune evolve --skill ${values.skill} \\`);
     console.log(`    --skill-path ${values["skill-path"]} \\`);
-    console.log("    --runs-per-query 3 --verbose");
+    console.log(`    --eval-set ${outputPath} \\`);
+    console.log("    --dry-run --verbose");
     return;
   }
 
@@ -504,6 +608,8 @@ export async function cliMain(): Promise<void> {
   const maxPerSide = Number.parseInt(values.max ?? "50", 10);
   const seed = Number.parseInt(values.seed ?? "42", 10);
   const annotateTaxonomy = !values["no-taxonomy"];
+  const searchDirs = getEvalSkillSearchDirs();
+  const detectedSkillPath = findInstalledSkillPath(values.skill, searchDirs);
 
   const evalSet = buildEvalSet(
     skillRecords,
@@ -515,9 +621,57 @@ export async function cliMain(): Promise<void> {
     annotateTaxonomy,
   );
 
+  const positiveCount = evalSet.filter((entry) => entry.should_trigger).length;
+  if (positiveCount === 0 && values["auto-synthetic"]) {
+    const skillPath = values["skill-path"] ?? detectedSkillPath;
+    if (!skillPath) {
+      throw new CLIError(
+        `No trusted triggers found for '${values.skill}', and no SKILL.md path could be resolved for synthetic fallback.`,
+        "FILE_NOT_FOUND",
+        `Run 'selftune eval generate --list-skills' or rerun with --skill-path /path/to/SKILL.md`,
+      );
+    }
+
+    const agent = detectAgent();
+    if (!agent) {
+      throw new CLIError(
+        "No agent CLI found (claude/codex/opencode)",
+        "AGENT_NOT_FOUND",
+        "Install one of the supported agent CLIs",
+      );
+    }
+
+    console.log(
+      `No trusted triggers found for '${values.skill}'. Falling back to synthetic cold-start eval generation...`,
+    );
+    const effectiveMax = Number.isNaN(maxPerSide) || maxPerSide <= 0 ? 50 : maxPerSide;
+    const syntheticEvalSet = await generateSyntheticEvals(skillPath, values.skill, agent, {
+      maxPositives: effectiveMax,
+      maxNegatives: effectiveMax,
+      modelFlag: values.model,
+    });
+    const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
+    writeFileSync(outputPath, JSON.stringify(syntheticEvalSet, null, 2), "utf-8");
+    const pos = syntheticEvalSet.filter((e) => e.should_trigger);
+    const neg = syntheticEvalSet.filter((e) => !e.should_trigger);
+
+    console.log(`Wrote ${syntheticEvalSet.length} synthetic eval entries to ${outputPath}`);
+    console.log(`  Positives (should_trigger=true) : ${pos.length}`);
+    console.log(`  Negatives (should_trigger=false): ${neg.length}`);
+    console.log("\nNext steps:");
+    console.log(`  selftune evolve --skill ${values.skill} \\`);
+    console.log(`    --skill-path ${skillPath} \\`);
+    console.log(`    --eval-set ${outputPath} \\`);
+    console.log("    --dry-run --verbose");
+    return;
+  }
+
   const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
   writeFileSync(outputPath, JSON.stringify(evalSet, null, 2), "utf-8");
   printEvalStats(evalSet, values.skill, outputPath, skillRecords, queryRecords, annotateTaxonomy);
+  if (positiveCount === 0 && detectedSkillPath) {
+    printSyntheticFallbackHint(values.skill, detectedSkillPath);
+  }
 }
 
 if (import.meta.main) {