selftune 0.2.0 → 0.2.1

package/cli/selftune/eval/hooks-to-evals.ts

@@ -27,7 +27,13 @@ import type {
 } from "../types.js";
 import { readJsonl } from "../utils/jsonl.js";
 import { detectAgent } from "../utils/llm-call.js";
+import {
+  filterActionableQueryRecords,
+  filterActionableSkillUsageRecords,
+} from "../utils/query-filter.js";
 import { seededShuffle } from "../utils/seeded-random.js";
+import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
+import { isHighConfidencePositiveSkillRecord } from "../utils/skill-usage-confidence.js";
 import { generateSyntheticEvals } from "./synthetic-evals.js";
 
 // ---------------------------------------------------------------------------
@@ -116,14 +122,16 @@ export function buildEvalSet(
   seed = 42,
   annotateTaxonomy = true,
 ): EvalEntry[] {
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
   const effectiveMaxPerSide = Number.isNaN(maxPerSide) || maxPerSide <= 0 ? 50 : maxPerSide;
   const effectiveSeed = Number.isNaN(seed) ? 42 : seed;
 
   // Build set of positive query texts (for exclusion from negatives)
   const positiveQueries = new Set<string>();
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     if (!r || typeof r.skill_name !== "string" || typeof r.query !== "string") continue;
-    if (r.skill_name === skillName) {
+    if (isHighConfidencePositiveSkillRecord(r, skillName)) {
       const q = (r.query ?? "").trim();
       if (q && q !== "(query not found)") {
         positiveQueries.add(q);
@@ -134,9 +142,9 @@ export function buildEvalSet(
   // Build deduplicated positives with taxonomy classification
   const seen = new Set<string>();
   const positives: EvalEntry[] = [];
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     if (!r || typeof r.skill_name !== "string" || typeof r.query !== "string") continue;
-    if (r.skill_name !== skillName) continue;
+    if (!isHighConfidencePositiveSkillRecord(r, skillName)) continue;
     const q = (r.query ?? "").trim();
     if (!q || q === "(query not found)" || seen.has(q)) continue;
     seen.add(q);
@@ -153,7 +161,7 @@ export function buildEvalSet(
   if (includeNegatives) {
     const negCandidates: string[] = [];
     const negSeen = new Set<string>();
-    for (const r of queryRecords) {
+    for (const r of actionableQueryRecords) {
       if (!r || typeof r.query !== "string") continue;
       const q = (r.query ?? "").trim();
       if (!q || positiveQueries.has(q) || negSeen.has(q)) continue;
@@ -198,13 +206,17 @@ export function listSkills(
   queryRecords: QueryLogRecord[],
   telemetryRecords: SessionTelemetryRecord[],
 ): void {
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
   const counts = new Map<string, number>();
-  for (const r of skillRecords) {
+  for (const r of actionableSkillRecords) {
     const name = r.skill_name ?? "unknown";
     counts.set(name, (counts.get(name) ?? 0) + 1);
   }
 
-  console.log(`Skill triggers in skill_usage_log (${skillRecords.length} total records):`);
+  console.log(
+    `Skill triggers in skill_usage_log (${actionableSkillRecords.length} actionable records):`,
+  );
   if (counts.size > 0) {
     const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);
     for (const [name, count] of sorted) {
@@ -214,8 +226,8 @@ export function listSkills(
     console.log("  (none yet -- trigger some skills in Claude Code to populate)");
   }
 
-  console.log(`\nAll queries in all_queries_log: ${queryRecords.length}`);
-  if (queryRecords.length === 0) {
+  console.log(`\nActionable queries in all_queries_log: ${actionableQueryRecords.length}`);
+  if (actionableQueryRecords.length === 0) {
     console.log("  (none yet -- make sure prompt_log_hook is installed)");
   }
 
@@ -303,14 +315,16 @@ export function printEvalStats(
 ): void {
   const pos = evalSet.filter((e) => e.should_trigger);
   const neg = evalSet.filter((e) => !e.should_trigger);
-  const totalTriggers = skillRecords.filter((r) => r.skill_name === skillName).length;
+  const actionableSkillRecords = filterActionableSkillUsageRecords(skillRecords);
+  const actionableQueryRecords = filterActionableQueryRecords(queryRecords);
+  const totalTriggers = actionableSkillRecords.filter((r) => r.skill_name === skillName).length;
 
   console.log(`Wrote ${evalSet.length} eval entries to ${outputPath}`);
   console.log(
     `  Positives (should_trigger=true) : ${pos.length}  (from ${totalTriggers} logged triggers)`,
   );
   console.log(
-    `  Negatives (should_trigger=false): ${neg.length}  (from ${queryRecords.length} total logged queries)`,
+    `  Negatives (should_trigger=false): ${neg.length}  (from ${actionableQueryRecords.length} actionable logged queries)`,
   );
 
   if (annotateTaxonomy && pos.length > 0) {
@@ -336,7 +350,7 @@ export function printEvalStats(
   console.log();
   if (pos.length === 0) {
     console.log(`[WARN] No positives for skill '${skillName}'.`);
-    const names = [...new Set(skillRecords.map((r) => r.skill_name))].sort();
+    const names = [...new Set(actionableSkillRecords.map((r) => r.skill_name))].sort();
     if (names.length > 0) {
       console.log(`       Known skills: ${names.join(", ")}`);
     }
@@ -366,6 +380,7 @@ export async function cliMain(): Promise<void> {
     options: {
       skill: { type: "string" },
       output: { type: "string" },
+      out: { type: "string" },
       max: { type: "string", default: "50" },
       seed: { type: "string", default: "42" },
       "list-skills": { type: "boolean", default: false },
@@ -409,7 +424,7 @@ export async function cliMain(): Promise<void> {
       modelFlag: values.model,
     });
 
-    const outputPath = values.output ?? `${values.skill}_trigger_eval.json`;
+    const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
     writeFileSync(outputPath, JSON.stringify(evalSet, null, 2), "utf-8");
 
     const pos = evalSet.filter((e) => e.should_trigger);
@@ -440,7 +455,11 @@ export async function cliMain(): Promise<void> {
   }
 
   // --- Log-based mode (original behavior) ---
-  const skillRecords = readJsonl<SkillUsageRecord>(values["skill-log"] ?? SKILL_LOG);
+  const skillLogPath = values["skill-log"] ?? SKILL_LOG;
+  const skillRecords =
+    skillLogPath === SKILL_LOG
+      ? readEffectiveSkillUsageRecords()
+      : readJsonl<SkillUsageRecord>(skillLogPath);
   const queryRecords = readJsonl<QueryLogRecord>(values["query-log"] ?? QUERY_LOG);
   const telemetryRecords = readJsonl<SessionTelemetryRecord>(
     values["telemetry-log"] ?? TELEMETRY_LOG,
@@ -475,7 +494,7 @@ export async function cliMain(): Promise<void> {
     annotateTaxonomy,
   );
 
-  const outputPath = values.output ?? `${values.skill}_trigger_eval.json`;
+  const outputPath = values.output ?? values.out ?? `${values.skill}_trigger_eval.json`;
   writeFileSync(outputPath, JSON.stringify(evalSet, null, 2), "utf-8");
   printEvalStats(evalSet, values.skill, outputPath, skillRecords, queryRecords, annotateTaxonomy);
 }

package/cli/selftune/eval/unit-test-cli.ts

@@ -2,7 +2,7 @@
  * CLI entrypoint for skill unit tests.
  *
  * Usage:
- *   selftune unit-test --skill <name> --tests <path> [--run-agent] [--generate]
+ *   selftune eval unit-test --skill <name> --tests <path> [--run-agent] [--generate]
  *
  *   --skill <name>    Skill name (required)
  *   --tests <path>    Path to unit test JSON file (default: ~/.selftune/unit-tests/<skill>.json)

package/cli/selftune/evolution/evidence.ts

@@ -0,0 +1,26 @@
+/**
+ * Evolution evidence trail: append and read proposal/eval artifacts that power
+ * explainable dashboard drill-downs.
+ */
+
+import { EVOLUTION_EVIDENCE_LOG } from "../constants.js";
+import type { EvolutionEvidenceEntry } from "../types.js";
+import { appendJsonl, readJsonl } from "../utils/jsonl.js";
+
+/** Append a structured evidence artifact to the evolution evidence log. */
+export function appendEvidenceEntry(
+  entry: EvolutionEvidenceEntry,
+  logPath: string = EVOLUTION_EVIDENCE_LOG,
+): void {
+  appendJsonl(logPath, entry);
+}
+
+/** Read all evidence entries, optionally filtered by exact skill name. */
+export function readEvidenceTrail(
+  skillName?: string,
+  logPath: string = EVOLUTION_EVIDENCE_LOG,
+): EvolutionEvidenceEntry[] {
+  const entries = readJsonl<EvolutionEvidenceEntry>(logPath);
+  if (!skillName) return entries;
+  return entries.filter((entry) => entry.skill_name === skillName);
+}

package/cli/selftune/evolution/evolve-body.ts

@@ -9,13 +9,15 @@
 import { existsSync, readFileSync } from "node:fs";
 import { parseArgs } from "node:util";
 
-import { QUERY_LOG, SKILL_LOG } from "../constants.js";
+import { QUERY_LOG } from "../constants.js";
 import { buildEvalSet } from "../eval/hooks-to-evals.js";
+import { readGradingResultsForSkill } from "../grading/results.js";
 import type {
   BodyEvolutionProposal,
   BodyValidationResult,
   EvalEntry,
   EvolutionAuditEntry,
+  EvolutionEvidenceEntry,
   EvolutionTarget,
   FailurePattern,
   GradingResult,
@@ -23,8 +25,10 @@ import type {
   SkillUsageRecord,
 } from "../types.js";
 import { readJsonl } from "../utils/jsonl.js";
+import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
 import { appendAuditEntry } from "./audit.js";
 import { parseSkillSections, replaceBody, replaceSection } from "./deploy-proposal.js";
+import { appendEvidenceEntry } from "./evidence.js";
 import { extractFailurePatterns } from "./extract-patterns.js";
 import { generateBodyProposal } from "./propose-body.js";
 import { generateRoutingProposal } from "./propose-routing.js";
@@ -79,7 +83,9 @@ export interface EvolveBodyDeps {
   validateRoutingProposal?: typeof import("./validate-routing.js").validateRoutingProposal;
   refineBodyProposal?: typeof import("./refine-body.js").refineBodyProposal;
   appendAuditEntry?: typeof import("./audit.js").appendAuditEntry;
+  appendEvidenceEntry?: typeof import("./evidence.js").appendEvidenceEntry;
   buildEvalSet?: typeof import("../eval/hooks-to-evals.js").buildEvalSet;
+  readEffectiveSkillUsageRecords?: typeof import("../utils/skill-log.js").readEffectiveSkillUsageRecords;
   readFileSync?: typeof readFileSync;
   writeFileSync?: (path: string, data: string, encoding: string) => void;
 }
@@ -134,7 +140,10 @@ export async function evolveBody(
   const _validateRoutingProposal = _deps.validateRoutingProposal ?? validateRoutingProposal;
   const _refineBodyProposal = _deps.refineBodyProposal ?? refineBodyProposal;
   const _appendAuditEntry = _deps.appendAuditEntry ?? appendAuditEntry;
+  const _appendEvidenceEntry = _deps.appendEvidenceEntry ?? appendEvidenceEntry;
   const _buildEvalSet = _deps.buildEvalSet ?? buildEvalSet;
+  const _readEffectiveSkillUsageRecords =
+    _deps.readEffectiveSkillUsageRecords ?? readEffectiveSkillUsageRecords;
   const _readFileSync = _deps.readFileSync ?? readFileSync;
   const _writeFileSync = _deps.writeFileSync ?? (await import("node:fs")).writeFileSync;
 
@@ -154,6 +163,14 @@ export async function evolveBody(
     }
   }
 
+  function recordEvidence(entry: EvolutionEvidenceEntry): void {
+    try {
+      _appendEvidenceEntry(entry);
+    } catch {
+      // Fail-open
+    }
+  }
+
   try {
     // Step 1: Read current SKILL.md
     if (!existsSync(skillPath)) {
@@ -168,6 +185,8 @@ export async function evolveBody(
 
     const currentContent = _readFileSync(skillPath, "utf-8");
     const parsed = parseSkillSections(currentContent);
+    const createdAuditDetails = (): string => `original_description:${currentContent}`;
+    const skillUsage = _readEffectiveSkillUsageRecords();
 
     // Step 2: Load eval set
     let evalSet: EvalEntry[];
@@ -179,13 +198,11 @@ export async function evolveBody(
       }
       evalSet = parsed as EvalEntry[];
     } else {
-      const skillRecords = readJsonl<SkillUsageRecord>(SKILL_LOG);
       const queryRecords = readJsonl<QueryLogRecord>(QUERY_LOG);
-      evalSet = _buildEvalSet(skillRecords, queryRecords, skillName);
+      evalSet = _buildEvalSet(skillUsage, queryRecords, skillName);
     }
 
     // Step 3: Load skill usage and extract failure patterns
-    const skillUsage = readJsonl<SkillUsageRecord>(SKILL_LOG);
     const failurePatterns = _extractFailurePatterns(
       evalSet,
       skillUsage,
@@ -252,11 +269,21 @@ export async function evolveBody(
 
       lastProposal = proposal;
 
-      recordAudit(
-        proposal.proposal_id,
-        "created",
-        `${target} proposal created for ${skillName} (iteration ${iteration + 1})`,
-      );
+      recordAudit(proposal.proposal_id, "created", createdAuditDetails());
+      recordEvidence({
+        timestamp: new Date().toISOString(),
+        proposal_id: proposal.proposal_id,
+        skill_name: skillName,
+        skill_path: skillPath,
+        target,
+        stage: "created",
+        rationale: proposal.rationale,
+        confidence: proposal.confidence,
+        details: `${target} proposal created for ${skillName} (iteration ${iteration + 1})`,
+        original_text: proposal.original_body,
+        proposed_text: proposal.proposed_body,
+        eval_set: evalSet,
+      });
 
       // Check confidence threshold
       if (proposal.confidence < confidenceThreshold) {
@@ -265,6 +292,17 @@ export async function evolveBody(
           "rejected",
           `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
         );
+        recordEvidence({
+          timestamp: new Date().toISOString(),
+          proposal_id: proposal.proposal_id,
+          skill_name: skillName,
+          skill_path: skillPath,
+          target,
+          stage: "rejected",
+          rationale: proposal.rationale,
+          confidence: proposal.confidence,
+          details: `Confidence ${proposal.confidence} below threshold ${confidenceThreshold}`,
+        });
 
         if (iteration === maxIterations - 1) {
           return {
@@ -303,6 +341,24 @@ export async function evolveBody(
         "validated",
         `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed`,
       );
+      recordEvidence({
+        timestamp: new Date().toISOString(),
+        proposal_id: proposal.proposal_id,
+        skill_name: skillName,
+        skill_path: skillPath,
+        target,
+        stage: "validated",
+        rationale: proposal.rationale,
+        confidence: proposal.confidence,
+        details: `Validation: ${validation.gates_passed}/${validation.gates_total} gates passed`,
+        validation: {
+          improved: validation.improved,
+          gates_passed: validation.gates_passed,
+          gates_total: validation.gates_total,
+          gate_results: validation.gate_results,
+          regressions: validation.regressions,
+        },
+      });
 
       if (validation.improved) {
         break;
@@ -313,6 +369,24 @@ export async function evolveBody(
         "rejected",
         `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates`,
       );
+      recordEvidence({
+        timestamp: new Date().toISOString(),
+        proposal_id: proposal.proposal_id,
+        skill_name: skillName,
+        skill_path: skillPath,
+        target,
+        stage: "rejected",
+        rationale: proposal.rationale,
+        confidence: proposal.confidence,
+        details: `Validation failed: ${validation.gates_passed}/${validation.gates_total} gates`,
+        validation: {
+          improved: validation.improved,
+          gates_passed: validation.gates_passed,
+          gates_total: validation.gates_total,
+          gate_results: validation.gate_results,
+          regressions: validation.regressions,
+        },
+      });
 
       if (iteration === maxIterations - 1) {
         return {
@@ -355,6 +429,24 @@ export async function evolveBody(
         "deployed",
         `Deployed ${target} proposal for ${skillName}`,
       );
+      recordEvidence({
+        timestamp: new Date().toISOString(),
+        proposal_id: lastProposal.proposal_id,
+        skill_name: skillName,
+        skill_path: skillPath,
+        target,
+        stage: "deployed",
+        rationale: lastProposal.rationale,
+        confidence: lastProposal.confidence,
+        details: `Deployed ${target} proposal for ${skillName}`,
+        validation: {
+          improved: lastValidation.improved,
+          gates_passed: lastValidation.gates_passed,
+          gates_total: lastValidation.gates_total,
+          gate_results: lastValidation.gate_results,
+          regressions: lastValidation.regressions,
+        },
+      });
 
       return {
         proposal: lastProposal,
@@ -411,10 +503,10 @@ export async function cliMain(): Promise<void> {
   });
 
   if (values.help) {
-    console.log(`selftune evolve-body — Evolve a skill body or routing table
+    console.log(`selftune evolve body — Evolve a skill body or routing table
 
 Usage:
-  selftune evolve-body --skill <name> --skill-path <path> [options]
+  selftune evolve body --skill <name> --skill-path <path> [options]
 
 Options:
   --skill             Skill name (required)
@@ -462,6 +554,7 @@ Options:
     const paths = values["few-shot"].split(",").map((p) => p.trim());
     fewShotExamples = paths.filter((p) => existsSync(p)).map((p) => readFileSync(p, "utf-8"));
   }
+  const gradingResults = readGradingResultsForSkill(values.skill);
 
   const result = await evolveBody({
     skillName: values.skill,
@@ -477,6 +570,7 @@ Options:
     confidenceThreshold: Number.parseFloat(values.confidence ?? "0.6"),
     taskDescription: values["task-description"],
     fewShotExamples,
+    gradingResults,
     validationModel: values["validation-model"],
   });