selftune 0.1.4 → 0.2.1

package/cli/selftune/evolution/extract-patterns.ts

@@ -6,7 +6,16 @@
  * similar queries together using Jaccard similarity.
  */
 
-import type { EvalEntry, FailurePattern, InvocationType, SkillUsageRecord } from "../types.js";
+import type {
+  EvalEntry,
+  FailureFeedback,
+  FailurePattern,
+  GradingResult,
+  InvocationType,
+  SkillUsageRecord,
+} from "../types.js";
+import { filterActionableSkillUsageRecords } from "../utils/query-filter.js";
+import { isHighConfidencePositiveSkillRecord } from "../utils/skill-usage-confidence.js";
 
 // ---------------------------------------------------------------------------
 // Jaccard similarity
@@ -93,37 +102,39 @@ export function extractFailurePatterns(
   evalEntries: EvalEntry[],
   skillUsage: SkillUsageRecord[],
   skillName: string,
+  gradingResults?: GradingResult[],
 ): FailurePattern[] {
-  // 1. Build a set of triggered queries from skillUsage for the given skillName
+  const actionableSkillUsage = filterActionableSkillUsageRecords(skillUsage);
   const triggeredQueries = new Set<string>();
-  for (const record of skillUsage) {
-    if (record.skill_name === skillName && record.triggered) {
-      triggeredQueries.add(record.query);
-    }
+  const skillUsageBySession = new Map<string, SkillUsageRecord[]>();
+
+  for (const record of actionableSkillUsage) {
+    if (!isHighConfidencePositiveSkillRecord(record, skillName)) continue;
+    triggeredQueries.add(record.query);
+    const sessionRecords = skillUsageBySession.get(record.session_id) ?? [];
+    sessionRecords.push(record);
+    skillUsageBySession.set(record.session_id, sessionRecords);
   }
 
-  // 2. Find missed queries: should_trigger === true but NOT in the triggered set
   const missedByType = new Map<InvocationType, string[]>();
-
   for (const entry of evalEntries) {
     if (!entry.should_trigger) continue;
     if (triggeredQueries.has(entry.query)) continue;
 
     const invType = entry.invocation_type ?? "implicit";
-    if (!missedByType.has(invType)) {
-      missedByType.set(invType, []);
-    }
-    missedByType.get(invType)?.push(entry.query);
+    const queries = missedByType.get(invType) ?? [];
+    queries.push(entry.query);
+    missedByType.set(invType, queries);
   }
 
-  // 3. For each group, cluster similar queries
   const now = new Date().toISOString();
   const allPatterns: FailurePattern[] = [];
   let index = 0;
+  const feedbackMap = new Map<string, FailureFeedback[]>();
+  const sampleSessionsByQuery = new Map<string, Set<string>>();
 
   for (const [invType, queries] of missedByType) {
     const clusters = clusterQueries(queries);
-
     for (const cluster of clusters) {
       allPatterns.push({
         pattern_id: `fp-${skillName}-${index}`,
@@ -138,8 +149,86 @@ export function extractFailurePatterns(
     }
   }
 
-  // 4. Sort by frequency descending
-  allPatterns.sort((a, b) => b.frequency - a.frequency);
+  if (gradingResults && gradingResults.length > 0) {
+    for (const result of gradingResults) {
+      const hasExplicitFeedback = (result.failure_feedback?.length ?? 0) > 0;
+      const hasFailedSummary = (result.summary.failed ?? 0) > 0;
+      if (result.skill_name !== skillName || (!hasExplicitFeedback && !hasFailedSummary)) continue;
+
+      const failedQueries = new Set<string>();
+
+      if (result.failure_feedback) {
+        const sessionRecords = skillUsageBySession.get(result.session_id) ?? [];
+        for (const feedback of result.failure_feedback) {
+          if (!feedback.query) continue;
+          const existing = feedbackMap.get(feedback.query) ?? [];
+          existing.push(feedback);
+          feedbackMap.set(feedback.query, existing);
+          if (sessionRecords.some((record) => record.query === feedback.query)) {
+            failedQueries.add(feedback.query);
+            const sessions = sampleSessionsByQuery.get(feedback.query) ?? new Set<string>();
+            sessions.add(result.session_id);
+            sampleSessionsByQuery.set(feedback.query, sessions);
+          }
+        }
+      }
 
+      if (failedQueries.size === 0) {
+        const sessionRecords = skillUsageBySession.get(result.session_id) ?? [];
+        const failedExpectations = result.expectations.filter((expectation) => !expectation.passed);
+        for (const record of sessionRecords) {
+          failedQueries.add(record.query);
+          const sessions = sampleSessionsByQuery.get(record.query) ?? new Set<string>();
+          sessions.add(result.session_id);
+          sampleSessionsByQuery.set(record.query, sessions);
+
+          if (failedExpectations.length > 0) {
+            const feedback = feedbackMap.get(record.query) ?? [];
+            for (const expectation of failedExpectations) {
+              feedback.push({
+                query: record.query,
+                failure_reason: expectation.evidence || expectation.text,
+                improvement_hint: expectation.text,
+                invocation_type: "contextual",
+              });
+            }
+            feedbackMap.set(record.query, feedback);
+          }
+        }
+      }
+    }
+
+    const contextualQueries = [...sampleSessionsByQuery.keys()];
+    if (contextualQueries.length > 0) {
+      const clusters = clusterQueries(contextualQueries);
+      for (const cluster of clusters) {
+        allPatterns.push({
+          pattern_id: `fp-${skillName}-${index}`,
+          skill_name: skillName,
+          invocation_type: "contextual",
+          missed_queries: cluster,
+          frequency: cluster.length,
+          sample_sessions: [
+            ...new Set(cluster.flatMap((query) => [...(sampleSessionsByQuery.get(query) ?? [])])),
+          ],
+          extracted_at: now,
+          feedback: cluster.flatMap((query) => feedbackMap.get(query) ?? []),
+        });
+        index++;
+      }
+    }
+  }
+
+  for (const pattern of allPatterns) {
+    if (pattern.feedback && pattern.feedback.length > 0) continue;
+    const matchingFeedback = pattern.missed_queries.flatMap(
+      (query) => feedbackMap.get(query) ?? [],
+    );
+    if (matchingFeedback.length > 0) {
+      pattern.feedback = matchingFeedback;
+    }
+  }
+
+  allPatterns.sort((a, b) => b.frequency - a.frequency);
   return allPatterns;
 }

package/cli/selftune/evolution/pareto.ts

@@ -0,0 +1,314 @@
+/**
+ * pareto.ts
+ *
+ * Pareto frontier computation for multi-candidate evolution.
+ * All functions are pure — no I/O, no LLM calls.
+ */
+
+import type {
+  InvocationType,
+  InvocationTypeScores,
+  ParetoCandidate,
+  SessionTelemetryRecord,
+  TokenUsageMetrics,
+} from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Score computation
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute per-invocation-type scores from per-entry validation results.
+ */
+export function computeInvocationScores(
+  perEntryResults: Array<{ entry: { invocation_type?: InvocationType }; after_pass: boolean }>,
+): InvocationTypeScores {
+  const dims: InvocationType[] = ["explicit", "implicit", "contextual", "negative"];
+  const counts: Record<string, { passed: number; total: number }> = {};
+
+  for (const dim of dims) {
+    counts[dim] = { passed: 0, total: 0 };
+  }
+
+  for (const r of perEntryResults) {
+    const type = r.entry.invocation_type ?? "implicit";
+    counts[type].total++;
+    if (r.after_pass) counts[type].passed++;
+  }
+
+  const result: Record<string, { passed: number; total: number; pass_rate: number }> = {};
+  for (const dim of dims) {
+    const { passed, total } = counts[dim];
+    result[dim] = { passed, total, pass_rate: total > 0 ? passed / total : 0 };
+  }
+
+  return result as unknown as InvocationTypeScores;
+}
+
+// ---------------------------------------------------------------------------
+// Token efficiency scoring
+// ---------------------------------------------------------------------------
+
+/**
+ * Clamp a value to [min, max].
+ */
+function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+/**
+ * Compute token usage metrics from telemetry records.
+ */
+export function computeTokenUsageMetrics(records: SessionTelemetryRecord[]): TokenUsageMetrics {
+  let input = 0;
+  let output = 0;
+  for (const r of records) {
+    input += r.input_tokens ?? 0;
+    output += r.output_tokens ?? 0;
+  }
+  return {
+    input_tokens: input,
+    output_tokens: output,
+    total_tokens: input + output,
+  };
+}
+
+/**
+ * Compute a token efficiency score for a skill.
+ *
+ * Compares average total tokens for sessions WITH the skill triggered
+ * vs sessions WITHOUT it. Returns `clamp(baseline_avg / with_skill_avg, 0, 1)`.
+ * Values near 1.0 indicate the baseline uses more tokens than sessions with the
+ * skill (i.e. the skill is efficient). Values near 0.0 indicate the skill uses
+ * more tokens than the baseline.
+ *
+ * Returns 0.5 (neutral) when there is insufficient data in either group.
+ */
+export function computeTokenEfficiencyScore(
+  skillName: string,
+  telemetry: SessionTelemetryRecord[],
+): number {
+  const withSkill: number[] = [];
+  const withoutSkill: number[] = [];
+
+  for (const record of telemetry) {
+    const total = (record.input_tokens ?? 0) + (record.output_tokens ?? 0);
+    if (total <= 0) continue;
+
+    if (record.skills_triggered.includes(skillName)) {
+      withSkill.push(total);
+    } else {
+      withoutSkill.push(total);
+    }
+  }
+
+  if (withSkill.length === 0 || withoutSkill.length === 0) {
+    return 0.5; // neutral when insufficient data
+  }
+
+  const avgWithSkill = withSkill.reduce((a, b) => a + b, 0) / withSkill.length;
+  const avgBaseline = withoutSkill.reduce((a, b) => a + b, 0) / withoutSkill.length;
+
+  if (avgWithSkill === 0) return 1; // zero-token skill usage is maximally efficient
+
+  return clamp(avgBaseline / avgWithSkill, 0, 1);
+}
+
+// ---------------------------------------------------------------------------
+// Pareto dominance
+// ---------------------------------------------------------------------------
+
+const DIMS: InvocationType[] = ["explicit", "implicit", "contextual", "negative"];
+
+/**
+ * Returns true if candidate A dominates candidate B:
+ * A >= B on all dimensions AND A > B on at least one.
+ *
+ * When token efficiency scores are provided for BOTH candidates,
+ * a 5th dimension is added to the comparison.
+ */
+export function dominates(
+  a: InvocationTypeScores,
+  b: InvocationTypeScores,
+  aTokenEfficiency?: number,
+  bTokenEfficiency?: number,
+): boolean {
+  let strictlyBetterOnAny = false;
+
+  for (const dim of DIMS) {
+    const aRate = a[dim].pass_rate;
+    const bRate = b[dim].pass_rate;
+
+    if (aRate < bRate) return false; // A is worse on this dim
+    if (aRate > bRate) strictlyBetterOnAny = true;
+  }
+
+  // 5th dimension: token efficiency (only when both have data)
+  if (aTokenEfficiency !== undefined && bTokenEfficiency !== undefined) {
+    if (aTokenEfficiency < bTokenEfficiency) return false;
+    if (aTokenEfficiency > bTokenEfficiency) strictlyBetterOnAny = true;
+  }
+
+  return strictlyBetterOnAny;
+}
+
+/**
+ * Compute the dimensions where candidate A dominates candidate B.
+ */
+export function getDominatedDimensions(
+  a: InvocationTypeScores,
+  b: InvocationTypeScores,
+): InvocationType[] {
+  const result: InvocationType[] = [];
+  for (const dim of DIMS) {
+    if (a[dim].pass_rate > b[dim].pass_rate) {
+      result.push(dim);
+    }
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Pareto frontier
+// ---------------------------------------------------------------------------
+
+/**
+ * Filter candidates to the Pareto frontier (non-dominated set).
+ * Also sets `dominates_on` for each frontier member.
+ *
+ * When candidates have `token_efficiency_score` set, the 5th dimension
+ * is used in dominance checks.
+ */
+export function computeParetoFrontier(candidates: ParetoCandidate[]): ParetoCandidate[] {
+  if (candidates.length === 0) return [];
+
+  const frontier: ParetoCandidate[] = [];
+
+  for (const candidate of candidates) {
+    // Check if any existing frontier member dominates this candidate
+    let isDominated = false;
+    for (const member of frontier) {
+      if (
+        dominates(
+          member.invocation_scores,
+          candidate.invocation_scores,
+          member.token_efficiency_score,
+          candidate.token_efficiency_score,
+        )
+      ) {
+        isDominated = true;
+        break;
+      }
+    }
+
+    if (!isDominated) {
+      // Remove frontier members that this candidate dominates
+      for (let i = frontier.length - 1; i >= 0; i--) {
+        if (
+          dominates(
+            candidate.invocation_scores,
+            frontier[i].invocation_scores,
+            candidate.token_efficiency_score,
+            frontier[i].token_efficiency_score,
+          )
+        ) {
+          frontier.splice(i, 1);
+        }
+      }
+      frontier.push(candidate);
+    }
+  }
+
+  // Set dominates_on for each frontier member (compared to others in frontier)
+  for (const member of frontier) {
+    const allDominatedDims = new Set<InvocationType>();
+    for (const other of frontier) {
+      if (other === member) continue;
+      for (const dim of getDominatedDimensions(member.invocation_scores, other.invocation_scores)) {
+        allDominatedDims.add(dim);
+      }
+    }
+    member.dominates_on = [...allDominatedDims];
+  }
+
+  return frontier;
+}
+
+// ---------------------------------------------------------------------------
+// Merge prompt
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a merge prompt for complementary frontier candidates.
+ * Returns null if <= 1 candidate or no complementarity detected.
+ */
+export function buildMergePrompt(
+  frontier: ParetoCandidate[],
+  originalDescription: string,
+): string | null {
+  if (frontier.length <= 1) return null;
+
+  // Check for complementarity: different candidates dominate on different dimensions
+  const hasComplementarity = frontier.some((c) => c.dominates_on.length > 0);
+  if (!hasComplementarity) return null;
+
+  const candidateDescriptions = frontier
+    .map((c, i) => {
+      const strengths =
+        c.dominates_on.length > 0
+          ? `Strengths: ${c.dominates_on.join(", ")}`
+          : "No unique strengths";
+      return `Candidate ${i + 1} (${c.proposal.proposal_id}):\nDescription: ${c.proposal.proposed_description}\n${strengths}\nOverall pass rate: ${(c.validation.after_pass_rate * 100).toFixed(1)}%`;
+    })
+    .join("\n\n");
+
+  return `You are merging multiple skill descriptions that each excel on different invocation types.
+
+Original description:
+${originalDescription}
+
+Candidates:
+${candidateDescriptions}
+
+Create a single merged description that combines the strengths of all candidates.
+Output ONLY valid JSON with:
+- "proposed_description": the merged description
+- "rationale": explanation of what was combined
+- "confidence": 0.0-1.0`;
+}
+
+// ---------------------------------------------------------------------------
+// Selection
+// ---------------------------------------------------------------------------
+
+/**
+ * Select the best candidate from a Pareto frontier.
+ * Returns the best single candidate and whether a merge should be attempted.
+ */
+export function selectFromFrontier(frontier: ParetoCandidate[]): {
+  best: ParetoCandidate;
+  shouldMerge: boolean;
+  mergePrompt: string | null;
+} {
+  if (frontier.length === 0) {
+    throw new Error("Cannot select from empty frontier");
+  }
+
+  // Sort by overall after_pass_rate descending, then by number of new_passes
+  const sorted = [...frontier].sort((a, b) => {
+    const rateDiff = b.validation.after_pass_rate - a.validation.after_pass_rate;
+    if (Math.abs(rateDiff) > 0.001) return rateDiff;
+    return b.validation.new_passes.length - a.validation.new_passes.length;
+  });
+
+  const best = sorted[0];
+  const shouldMerge = frontier.length > 1 && frontier.some((c) => c.dominates_on.length > 0);
+
+  return {
+    best,
+    shouldMerge,
+    mergePrompt: shouldMerge
+      ? buildMergePrompt(frontier, best.proposal.original_description)
+      : null,
+  };
+}

package/cli/selftune/evolution/propose-body.ts

@@ -0,0 +1,171 @@
+/**
+ * propose-body.ts
+ *
+ * Generates full body proposals for SKILL.md files using a teacher LLM.
+ * The teacher analyzes current content, failure patterns, and missed queries
+ * to produce an improved skill body.
+ */
+
+import type { BodyEvolutionProposal, EvolutionTarget, FailurePattern } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+
+/** System prompt for the body generator (teacher) LLM. */
+export const BODY_GENERATOR_SYSTEM = `You are an expert skill document author for an AI agent routing system.
+
+Your task is to generate an improved SKILL.md body that better covers the semantic
+space of queries that the skill should handle. The body includes everything after
+the title line: the description, workflow routing table, instructions, examples, etc.
+
+Rules:
+- Preserve the overall structure: description paragraph, ## Workflow Routing table, and other ## sections.
+- The ## Workflow Routing table must be a valid markdown table with | Trigger | Workflow | columns.
+- Cover the semantic space of the missed queries without being too broad.
+- Maintain the original intent and scope of the skill.
+- Be specific and actionable in instructions.
+- Output ONLY valid JSON with exactly these fields:
+  - "proposed_body" (string): the complete improved skill body (markdown, everything below the title)
+  - "rationale" (string): explanation of what changed and why
+  - "confidence" (number): 0.0-1.0 how confident you are this improves the skill
+
+Do NOT include any text outside the JSON object.`;
+
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+
+/** Build the user prompt for full body generation. */
+export function buildBodyGenerationPrompt(
+  currentContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  fewShotExamples?: string[],
+): string {
+  const patternLines = failurePatterns.map((p) => {
+    const queries = p.missed_queries.map((q) => `    - "${q}"`).join("\n");
+    return `  Pattern ${p.pattern_id} (frequency: ${p.frequency}, type: ${p.invocation_type}):\n${queries}`;
+  });
+
+  const missedLines = missedQueries.map((q) => `  - "${q}"`).join("\n");
+
+  // Build failure feedback section if any patterns have feedback
+  const feedbackLines: string[] = [];
+  for (const p of failurePatterns) {
+    if (p.feedback && p.feedback.length > 0) {
+      for (const fb of p.feedback) {
+        feedbackLines.push(`  Query: "${fb.query}"`);
+        feedbackLines.push(`    Failure reason: ${fb.failure_reason}`);
+        feedbackLines.push(`    Improvement hint: ${fb.improvement_hint}`);
+      }
+    }
+  }
+  const feedbackSection =
+    feedbackLines.length > 0 ? `\n\nStructured Failure Analysis:\n${feedbackLines.join("\n")}` : "";
+
+  // Build few-shot examples section if provided
+  const fewShotSection =
+    fewShotExamples && fewShotExamples.length > 0
+      ? `\n\nReference Examples (other well-written skills):\n${fewShotExamples.map((ex, i) => `--- Example ${i + 1} ---\n${ex}`).join("\n\n")}`
+      : "";
+
+  return `Skill Name: ${skillName}
+
+Current Skill Content:
+${currentContent}
+
+Failure Patterns:
+${patternLines.join("\n\n")}
+
+All Missed Queries:
+${missedLines}${feedbackSection}${fewShotSection}
+
+Generate an improved full body for the "${skillName}" skill that would correctly handle the missed queries listed above. The body should include everything below the # Title line: description, ## Workflow Routing table, and any other sections. Output ONLY a JSON object with "proposed_body", "rationale", and "confidence" fields.`;
+}
+
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+
+/** Parse LLM response text into structured body proposal data. */
+export function parseBodyProposalResponse(raw: string): {
+  proposed_body: string;
+  rationale: string;
+  confidence: number;
+} {
+  const cleaned = stripMarkdownFences(raw);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse LLM response as JSON: ${cleaned.slice(0, 200)}`);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new Error("LLM response is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  if (typeof obj.proposed_body !== "string") {
+    throw new Error("Missing or invalid 'proposed_body' field in LLM response");
+  }
+  if (typeof obj.rationale !== "string") {
+    throw new Error("Missing or invalid 'rationale' field in LLM response");
+  }
+  if (typeof obj.confidence !== "number") {
+    throw new Error("Missing or invalid 'confidence' field in LLM response");
+  }
+
+  const confidence = Math.max(0.0, Math.min(1.0, obj.confidence));
+
+  return {
+    proposed_body: obj.proposed_body,
+    rationale: obj.rationale,
+    confidence,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Proposal generator
+// ---------------------------------------------------------------------------
+
+/** Generate a full body evolution proposal using teacher LLM. */
+export async function generateBodyProposal(
+  currentContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  skillPath: string,
+  agent: string,
+  modelFlag?: string,
+  fewShotExamples?: string[],
+): Promise<BodyEvolutionProposal> {
+  const prompt = buildBodyGenerationPrompt(
+    currentContent,
+    failurePatterns,
+    missedQueries,
+    skillName,
+    fewShotExamples,
+  );
+  const rawResponse = await callLlm(BODY_GENERATOR_SYSTEM, prompt, agent, modelFlag);
+  const { proposed_body, rationale, confidence } = parseBodyProposalResponse(rawResponse);
+
+  return {
+    proposal_id: `evo-body-${skillName}-${Date.now()}`,
+    skill_name: skillName,
+    skill_path: skillPath,
+    original_body: currentContent,
+    proposed_body,
+    rationale,
+    target: "body" as EvolutionTarget,
+    failure_patterns: failurePatterns.map((p) => p.pattern_id),
+    confidence,
+    created_at: new Date().toISOString(),
+    status: "pending",
+  };
+}