selftune 0.1.4 → 0.2.1

package/cli/selftune/evolution/propose-description.ts

@@ -50,6 +50,23 @@ export function buildProposalPrompt(
 
   const missedLines = missedQueries.map((q) => `  - "${q}"`).join("\n");
 
+  // Build failure feedback section if any patterns have feedback
+  const feedbackLines: string[] = [];
+  for (const p of failurePatterns) {
+    if (p.feedback && p.feedback.length > 0) {
+      for (const fb of p.feedback) {
+        feedbackLines.push(`  Query: "${fb.query}"`);
+        feedbackLines.push(`    Failure reason: ${fb.failure_reason}`);
+        feedbackLines.push(`    Improvement hint: ${fb.improvement_hint}`);
+        if (fb.invocation_type) {
+          feedbackLines.push(`    Invocation type: ${fb.invocation_type}`);
+        }
+      }
+    }
+  }
+  const feedbackSection =
+    feedbackLines.length > 0 ? `\n\nStructured Failure Analysis:\n${feedbackLines.join("\n")}` : "";
+
   return `Skill Name: ${skillName}
 
 Current Description:
@@ -59,7 +76,7 @@ Failure Patterns:
 ${patternLines.join("\n\n")}
 
 All Missed Queries:
-${missedLines}
+${missedLines}${feedbackSection}
 
 Propose an improved description for the "${skillName}" skill that would correctly route the missed queries listed above. Output ONLY a JSON object with "proposed_description", "rationale", and "confidence" fields.`;
 }
@@ -113,6 +130,86 @@ export function parseProposalResponse(raw: string): {
 // Proposal generator
 // ---------------------------------------------------------------------------
 
+/**
+ * Generate multiple proposals in parallel, each biased toward a different invocation type.
+ */
+export async function generateMultipleProposals(
+  currentDescription: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  skillPath: string,
+  agent: string,
+  count = 3,
+  modelFlag?: string,
+): Promise<EvolutionProposal[]> {
+  const variations = buildPromptVariations(
+    currentDescription,
+    failurePatterns,
+    missedQueries,
+    skillName,
+    count,
+  );
+
+  const proposals = await Promise.all(
+    variations.map(async (prompt, i) => {
+      const rawResponse = await callLlm(PROPOSER_SYSTEM, prompt, agent, modelFlag);
+      const { proposed_description, rationale, confidence } = parseProposalResponse(rawResponse);
+
+      return {
+        proposal_id: `evo-${skillName}-${Date.now()}-${i}`,
+        skill_name: skillName,
+        skill_path: skillPath,
+        original_description: currentDescription,
+        proposed_description,
+        rationale,
+        failure_patterns: failurePatterns.map((p) => p.pattern_id),
+        eval_results: {
+          before: { total: 0, passed: 0, failed: 0, pass_rate: 0 },
+          after: { total: 0, passed: 0, failed: 0, pass_rate: 0 },
+        },
+        confidence,
+        created_at: new Date().toISOString(),
+        status: "pending" as const,
+      };
+    }),
+  );
+
+  return proposals;
+}
+
+/**
+ * Build prompt variations, each biased toward a different invocation type.
+ */
+export function buildPromptVariations(
+  currentDescription: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  count: number,
+): string[] {
+  const biases: string[] = [
+    "Focus especially on improving explicit invocation (direct mentions of the skill).",
+    "Focus especially on improving implicit invocation (indirect references to skill capabilities).",
+    "Focus especially on improving contextual invocation (where the context implies the skill is needed).",
+  ];
+
+  const basePrompt = buildProposalPrompt(
+    currentDescription,
+    failurePatterns,
+    missedQueries,
+    skillName,
+  );
+  const variations: string[] = [];
+
+  for (let i = 0; i < count; i++) {
+    const bias = biases[i % biases.length];
+    variations.push(`${basePrompt}\n\nAdditional focus: ${bias}`);
+  }
+
+  return variations;
+}
+
 /** Generate a complete evolution proposal using LLM. */
 export async function generateProposal(
   currentDescription: string,
@@ -121,9 +218,10 @@ export async function generateProposal(
   skillName: string,
   skillPath: string,
   agent: string,
+  modelFlag?: string,
 ): Promise<EvolutionProposal> {
   const prompt = buildProposalPrompt(currentDescription, failurePatterns, missedQueries, skillName);
-  const rawResponse = await callLlm(PROPOSER_SYSTEM, prompt, agent);
+  const rawResponse = await callLlm(PROPOSER_SYSTEM, prompt, agent, modelFlag);
   const { proposed_description, rationale, confidence } = parseProposalResponse(rawResponse);
 
   return {

package/cli/selftune/evolution/propose-routing.ts

@@ -0,0 +1,166 @@
+/**
+ * propose-routing.ts
+ *
+ * Generates improved routing table proposals using LLM analysis of failure
+ * patterns. Targets the `## Workflow Routing` section of a SKILL.md file.
+ */
+
+import type { BodyEvolutionProposal, EvolutionTarget, FailurePattern } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+
+/** System prompt for the routing table proposer LLM. */
+export const ROUTING_PROPOSER_SYSTEM = `You are a workflow routing optimizer for an AI agent skill system.
+
+Your task is to analyze the current routing table and its failure patterns,
+then propose an improved routing table that would correctly route missed queries
+while preserving correct routing for existing queries.
+
+Rules:
+- The routing table must be a valid markdown table with | Trigger | Workflow | columns.
+- Each row maps a trigger pattern to the workflow it should activate.
+- Cover the semantic space of the missed queries without being too broad.
+- Maintain the original intent and scope of the skill routing.
+- Output ONLY valid JSON with exactly these fields:
+  - "proposed_routing" (string): the improved routing table in markdown format
+  - "rationale" (string): explanation of what changed and why
+  - "confidence" (number): 0.0-1.0 how confident you are this improves routing
+
+Do NOT include any text outside the JSON object.`;
+
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+
+/** Build the user prompt for routing table proposal. */
+export function buildRoutingProposalPrompt(
+  currentRouting: string,
+  fullSkillContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+): string {
+  const patternLines = failurePatterns.map((p) => {
+    const queries = p.missed_queries.map((q) => `    - "${q}"`).join("\n");
+    return `  Pattern ${p.pattern_id} (frequency: ${p.frequency}, type: ${p.invocation_type}):\n${queries}`;
+  });
+
+  const missedLines = missedQueries.map((q) => `  - "${q}"`).join("\n");
+
+  // Build failure feedback section if any patterns have feedback
+  const feedbackLines: string[] = [];
+  for (const p of failurePatterns) {
+    if (p.feedback && p.feedback.length > 0) {
+      for (const fb of p.feedback) {
+        feedbackLines.push(`  Query: "${fb.query}"`);
+        feedbackLines.push(`    Failure reason: ${fb.failure_reason}`);
+        feedbackLines.push(`    Improvement hint: ${fb.improvement_hint}`);
+      }
+    }
+  }
+  const feedbackSection =
+    feedbackLines.length > 0 ? `\n\nStructured Failure Analysis:\n${feedbackLines.join("\n")}` : "";
+
+  return `Skill Name: ${skillName}
+
+Current Routing Table:
+${currentRouting}
+
+Full Skill Content:
+${fullSkillContent}
+
+Failure Patterns:
+${patternLines.join("\n\n")}
+
+All Missed Queries:
+${missedLines}${feedbackSection}
+
+Propose an improved routing table for the "${skillName}" skill that would correctly route the missed queries listed above. Output ONLY a JSON object with "proposed_routing", "rationale", and "confidence" fields.`;
+}
+
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+
+/** Parse LLM response text into structured routing proposal data. */
+export function parseRoutingProposalResponse(raw: string): {
+  proposed_routing: string;
+  rationale: string;
+  confidence: number;
+} {
+  const cleaned = stripMarkdownFences(raw);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse LLM response as JSON: ${cleaned.slice(0, 200)}`);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new Error("LLM response is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  if (typeof obj.proposed_routing !== "string") {
+    throw new Error("Missing or invalid 'proposed_routing' field in LLM response");
+  }
+  if (typeof obj.rationale !== "string") {
+    throw new Error("Missing or invalid 'rationale' field in LLM response");
+  }
+  if (typeof obj.confidence !== "number") {
+    throw new Error("Missing or invalid 'confidence' field in LLM response");
+  }
+
+  const confidence = Math.max(0.0, Math.min(1.0, obj.confidence));
+
+  return {
+    proposed_routing: obj.proposed_routing,
+    rationale: obj.rationale,
+    confidence,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Proposal generator
+// ---------------------------------------------------------------------------
+
+/** Generate a routing table evolution proposal using LLM. */
+export async function generateRoutingProposal(
+  currentRouting: string,
+  fullSkillContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  skillPath: string,
+  agent: string,
+  modelFlag?: string,
+): Promise<BodyEvolutionProposal> {
+  const prompt = buildRoutingProposalPrompt(
+    currentRouting,
+    fullSkillContent,
+    failurePatterns,
+    missedQueries,
+    skillName,
+  );
+  const rawResponse = await callLlm(ROUTING_PROPOSER_SYSTEM, prompt, agent, modelFlag);
+  const { proposed_routing, rationale, confidence } = parseRoutingProposalResponse(rawResponse);
+
+  return {
+    proposal_id: `evo-routing-${skillName}-${Date.now()}`,
+    skill_name: skillName,
+    skill_path: skillPath,
+    original_body: currentRouting,
+    proposed_body: proposed_routing,
+    rationale,
+    target: "routing" as EvolutionTarget,
+    failure_patterns: failurePatterns.map((p) => p.pattern_id),
+    confidence,
+    created_at: new Date().toISOString(),
+    status: "pending",
+  };
+}

package/cli/selftune/evolution/refine-body.ts

@@ -0,0 +1,141 @@
+/**
+ * refine-body.ts
+ *
+ * Takes failure feedback from a validation pass and asks the teacher LLM
+ * to revise specific sections of a body proposal.
+ */
+
+import type { BodyEvolutionProposal, BodyValidationResult } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+
+/** System prompt for the body refiner (teacher) LLM. */
+export const BODY_REFINER_SYSTEM = `You are an expert skill document refiner for an AI agent routing system.
+
+You are given a proposed SKILL.md body that failed one or more validation gates.
+Your task is to revise the body to address the specific failures while preserving
+the parts that passed validation.
+
+Rules:
+- Address each failure reason specifically.
+- Preserve structural elements: ## Workflow Routing table, ## sections.
+- Keep the routing table as a valid markdown table with | Trigger | Workflow | columns.
+- Do not make unnecessary changes to parts that passed validation.
+- Output ONLY valid JSON with exactly these fields:
+  - "refined_body" (string): the revised skill body (markdown, everything below the title)
+  - "changes_made" (string): summary of what was changed
+  - "confidence" (number): 0.0-1.0 how confident you are this addresses the failures
+
+Do NOT include any text outside the JSON object.`;
+
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+
+/** Build the refinement prompt from validation feedback. */
+export function buildRefinementPrompt(
+  proposedBody: string,
+  validationResult: BodyValidationResult,
+  skillName: string,
+  regressionQueries?: string[],
+): string {
+  const failedGates = validationResult.gate_results
+    .filter((g) => !g.passed)
+    .map((g) => `  - ${g.gate}: ${g.reason}`)
+    .join("\n");
+
+  const regressionSection =
+    regressionQueries && regressionQueries.length > 0
+      ? `\n\nRegression Queries (these worked before but broke after):\n${regressionQueries.map((q) => `  - "${q}"`).join("\n")}`
+      : "";
+
+  return `Skill Name: ${skillName}
+
+Current Proposed Body:
+${proposedBody}
+
+Failed Validation Gates:
+${failedGates}
+${regressionSection}
+
+Revise the proposed body to address the failed validation gates. Preserve what works, fix what doesn't. Output ONLY a JSON object with "refined_body", "changes_made", and "confidence" fields.`;
+}
+
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+
+/** Parse LLM response text into structured refinement data. */
+export function parseRefinementResponse(raw: string): {
+  refined_body: string;
+  changes_made: string;
+  confidence: number;
+} {
+  const cleaned = stripMarkdownFences(raw);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse LLM response as JSON: ${cleaned.slice(0, 200)}`);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new Error("LLM response is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  if (typeof obj.refined_body !== "string") {
+    throw new Error("Missing or invalid 'refined_body' field in LLM response");
+  }
+  if (typeof obj.changes_made !== "string") {
+    throw new Error("Missing or invalid 'changes_made' field in LLM response");
+  }
+  if (typeof obj.confidence !== "number") {
+    throw new Error("Missing or invalid 'confidence' field in LLM response");
+  }
+
+  const confidence = Math.max(0.0, Math.min(1.0, obj.confidence));
+
+  return {
+    refined_body: obj.refined_body,
+    changes_made: obj.changes_made,
+    confidence,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Refinement function
+// ---------------------------------------------------------------------------
+
+/** Refine a body proposal based on validation feedback. */
+export async function refineBodyProposal(
+  proposal: BodyEvolutionProposal,
+  validationResult: BodyValidationResult,
+  agent: string,
+  modelFlag?: string,
+): Promise<BodyEvolutionProposal> {
+  const prompt = buildRefinementPrompt(
+    proposal.proposed_body,
+    validationResult,
+    proposal.skill_name,
+    validationResult.regressions,
+  );
+
+  const rawResponse = await callLlm(BODY_REFINER_SYSTEM, prompt, agent, modelFlag);
+  const { refined_body, changes_made, confidence } = parseRefinementResponse(rawResponse);
+
+  return {
+    ...proposal,
+    proposal_id: `${proposal.proposal_id}-refined-${Date.now()}`,
+    proposed_body: refined_body,
+    rationale: `${proposal.rationale}\n\nRefinement: ${changes_made}`,
+    confidence,
+    created_at: new Date().toISOString(),
+    status: "pending",
+  };
+}

package/cli/selftune/evolution/rollback.ts

@@ -11,6 +11,7 @@ import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from
 import { basename, dirname, join } from "node:path";
 import { parseArgs } from "node:util";
 
+import { updateContextAfterRollback } from "../memory/writer.js";
 import type { EvolutionAuditEntry } from "../types.js";
 import { appendAuditEntry, getLastDeployedProposal, readAuditTrail } from "./audit.js";
 import { replaceDescription } from "./deploy-proposal.js";
@@ -153,11 +154,19 @@ export async function rollback(options: RollbackOptions): Promise<RollbackResult
     };
     appendAuditEntry(auditEntry, logPath);
 
-    return {
+    const backupResult: RollbackResult = {
       rolledBack: true,
       restoredDescription: originalContent,
       reason: "Restored from backup file",
     };
+
+    try {
+      updateContextAfterRollback(skillName, backupResult);
+    } catch {
+      // Memory writes should never fail the main operation
+    }
+
+    return backupResult;
   }
 
   // Strategy 2: Restore from audit trail's created entry (description only)
@@ -177,11 +186,19 @@ export async function rollback(options: RollbackOptions): Promise<RollbackResult
     };
     appendAuditEntry(auditEntry, logPath);
 
-    return {
+    const auditResult: RollbackResult = {
       rolledBack: true,
       restoredDescription: originalFromAudit,
       reason: "Restored from audit trail",
     };
+
+    try {
+      updateContextAfterRollback(skillName, auditResult);
+    } catch {
+      // Memory writes should never fail the main operation
+    }
+
+    return auditResult;
   }
 
   // No restoration source available
@@ -206,10 +223,10 @@ export async function cliMain(): Promise<void> {
   });
 
   if (values.help) {
-    console.log(`selftune rollback — Rollback a skill to its pre-evolution state
+    console.log(`selftune evolve rollback — Rollback a skill to its pre-evolution state
 
 Usage:
-  selftune rollback --skill <name> --skill-path <path> [options]
+  selftune evolve rollback --skill <name> --skill-path <path> [options]
 
 Options:
   --skill             Skill name (required)