selftune 0.1.4 → 0.2.0

package/cli/selftune/evolution/propose-routing.ts

@@ -0,0 +1,166 @@
+/**
+ * propose-routing.ts
+ *
+ * Generates improved routing table proposals using LLM analysis of failure
+ * patterns. Targets the `## Workflow Routing` section of a SKILL.md file.
+ */
+
+import type { BodyEvolutionProposal, EvolutionTarget, FailurePattern } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+
+/** System prompt for the routing table proposer LLM. */
+export const ROUTING_PROPOSER_SYSTEM = `You are a workflow routing optimizer for an AI agent skill system.
+
+Your task is to analyze the current routing table and its failure patterns,
+then propose an improved routing table that would correctly route missed queries
+while preserving correct routing for existing queries.
+
+Rules:
+- The routing table must be a valid markdown table with | Trigger | Workflow | columns.
+- Each row maps a trigger pattern to the workflow it should activate.
+- Cover the semantic space of the missed queries without being too broad.
+- Maintain the original intent and scope of the skill routing.
+- Output ONLY valid JSON with exactly these fields:
+  - "proposed_routing" (string): the improved routing table in markdown format
+  - "rationale" (string): explanation of what changed and why
+  - "confidence" (number): 0.0-1.0 how confident you are this improves routing
+
+Do NOT include any text outside the JSON object.`;
+
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+
+/** Build the user prompt for routing table proposal. */
+export function buildRoutingProposalPrompt(
+  currentRouting: string,
+  fullSkillContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+): string {
+  const patternLines = failurePatterns.map((p) => {
+    const queries = p.missed_queries.map((q) => `    - "${q}"`).join("\n");
+    return `  Pattern ${p.pattern_id} (frequency: ${p.frequency}, type: ${p.invocation_type}):\n${queries}`;
+  });
+
+  const missedLines = missedQueries.map((q) => `  - "${q}"`).join("\n");
+
+  // Build failure feedback section if any patterns have feedback
+  const feedbackLines: string[] = [];
+  for (const p of failurePatterns) {
+    if (p.feedback && p.feedback.length > 0) {
+      for (const fb of p.feedback) {
+        feedbackLines.push(`  Query: "${fb.query}"`);
+        feedbackLines.push(`    Failure reason: ${fb.failure_reason}`);
+        feedbackLines.push(`    Improvement hint: ${fb.improvement_hint}`);
+      }
+    }
+  }
+  const feedbackSection =
+    feedbackLines.length > 0 ? `\n\nStructured Failure Analysis:\n${feedbackLines.join("\n")}` : "";
+
+  return `Skill Name: ${skillName}
+
+Current Routing Table:
+${currentRouting}
+
+Full Skill Content:
+${fullSkillContent}
+
+Failure Patterns:
+${patternLines.join("\n\n")}
+
+All Missed Queries:
+${missedLines}${feedbackSection}
+
+Propose an improved routing table for the "${skillName}" skill that would correctly route the missed queries listed above. Output ONLY a JSON object with "proposed_routing", "rationale", and "confidence" fields.`;
+}
+
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+
+/** Parse LLM response text into structured routing proposal data. */
+export function parseRoutingProposalResponse(raw: string): {
+  proposed_routing: string;
+  rationale: string;
+  confidence: number;
+} {
+  const cleaned = stripMarkdownFences(raw);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse LLM response as JSON: ${cleaned.slice(0, 200)}`);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new Error("LLM response is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  if (typeof obj.proposed_routing !== "string") {
+    throw new Error("Missing or invalid 'proposed_routing' field in LLM response");
+  }
+  if (typeof obj.rationale !== "string") {
+    throw new Error("Missing or invalid 'rationale' field in LLM response");
+  }
+  if (typeof obj.confidence !== "number") {
+    throw new Error("Missing or invalid 'confidence' field in LLM response");
+  }
+
+  const confidence = Math.max(0.0, Math.min(1.0, obj.confidence));
+
+  return {
+    proposed_routing: obj.proposed_routing,
+    rationale: obj.rationale,
+    confidence,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Proposal generator
+// ---------------------------------------------------------------------------
+
+/** Generate a routing table evolution proposal using LLM. */
+export async function generateRoutingProposal(
+  currentRouting: string,
+  fullSkillContent: string,
+  failurePatterns: FailurePattern[],
+  missedQueries: string[],
+  skillName: string,
+  skillPath: string,
+  agent: string,
+  modelFlag?: string,
+): Promise<BodyEvolutionProposal> {
+  const prompt = buildRoutingProposalPrompt(
+    currentRouting,
+    fullSkillContent,
+    failurePatterns,
+    missedQueries,
+    skillName,
+  );
+  const rawResponse = await callLlm(ROUTING_PROPOSER_SYSTEM, prompt, agent, modelFlag);
+  const { proposed_routing, rationale, confidence } = parseRoutingProposalResponse(rawResponse);
+
+  return {
+    proposal_id: `evo-routing-${skillName}-${Date.now()}`,
+    skill_name: skillName,
+    skill_path: skillPath,
+    original_body: currentRouting,
+    proposed_body: proposed_routing,
+    rationale,
+    target: "routing" as EvolutionTarget,
+    failure_patterns: failurePatterns.map((p) => p.pattern_id),
+    confidence,
+    created_at: new Date().toISOString(),
+    status: "pending",
+  };
+}

package/cli/selftune/evolution/refine-body.ts

@@ -0,0 +1,141 @@
+/**
+ * refine-body.ts
+ *
+ * Takes failure feedback from a validation pass and asks the teacher LLM
+ * to revise specific sections of a body proposal.
+ */
+
+import type { BodyEvolutionProposal, BodyValidationResult } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+
+// ---------------------------------------------------------------------------
+// System prompt
+// ---------------------------------------------------------------------------
+
+/** System prompt for the body refiner (teacher) LLM. */
+export const BODY_REFINER_SYSTEM = `You are an expert skill document refiner for an AI agent routing system.
+
+You are given a proposed SKILL.md body that failed one or more validation gates.
+Your task is to revise the body to address the specific failures while preserving
+the parts that passed validation.
+
+Rules:
+- Address each failure reason specifically.
+- Preserve structural elements: ## Workflow Routing table, ## sections.
+- Keep the routing table as a valid markdown table with | Trigger | Workflow | columns.
+- Do not make unnecessary changes to parts that passed validation.
+- Output ONLY valid JSON with exactly these fields:
+  - "refined_body" (string): the revised skill body (markdown, everything below the title)
+  - "changes_made" (string): summary of what was changed
+  - "confidence" (number): 0.0-1.0 how confident you are this addresses the failures
+
+Do NOT include any text outside the JSON object.`;
+
+// ---------------------------------------------------------------------------
+// Prompt builder
+// ---------------------------------------------------------------------------
+
+/** Build the refinement prompt from validation feedback. */
+export function buildRefinementPrompt(
+  proposedBody: string,
+  validationResult: BodyValidationResult,
+  skillName: string,
+  regressionQueries?: string[],
+): string {
+  const failedGates = validationResult.gate_results
+    .filter((g) => !g.passed)
+    .map((g) => `  - ${g.gate}: ${g.reason}`)
+    .join("\n");
+
+  const regressionSection =
+    regressionQueries && regressionQueries.length > 0
+      ? `\n\nRegression Queries (these worked before but broke after):\n${regressionQueries.map((q) => `  - "${q}"`).join("\n")}`
+      : "";
+
+  return `Skill Name: ${skillName}
+
+Current Proposed Body:
+${proposedBody}
+
+Failed Validation Gates:
+${failedGates}
+${regressionSection}
+
+Revise the proposed body to address the failed validation gates. Preserve what works, fix what doesn't. Output ONLY a JSON object with "refined_body", "changes_made", and "confidence" fields.`;
+}
+
+// ---------------------------------------------------------------------------
+// Response parser
+// ---------------------------------------------------------------------------
+
+/** Parse LLM response text into structured refinement data. */
+export function parseRefinementResponse(raw: string): {
+  refined_body: string;
+  changes_made: string;
+  confidence: number;
+} {
+  const cleaned = stripMarkdownFences(raw);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    throw new Error(`Failed to parse LLM response as JSON: ${cleaned.slice(0, 200)}`);
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new Error("LLM response is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  if (typeof obj.refined_body !== "string") {
+    throw new Error("Missing or invalid 'refined_body' field in LLM response");
+  }
+  if (typeof obj.changes_made !== "string") {
+    throw new Error("Missing or invalid 'changes_made' field in LLM response");
+  }
+  if (typeof obj.confidence !== "number") {
+    throw new Error("Missing or invalid 'confidence' field in LLM response");
+  }
+
+  const confidence = Math.max(0.0, Math.min(1.0, obj.confidence));
+
+  return {
+    refined_body: obj.refined_body,
+    changes_made: obj.changes_made,
+    confidence,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Refinement function
+// ---------------------------------------------------------------------------
+
+/** Refine a body proposal based on validation feedback. */
+export async function refineBodyProposal(
+  proposal: BodyEvolutionProposal,
+  validationResult: BodyValidationResult,
+  agent: string,
+  modelFlag?: string,
+): Promise<BodyEvolutionProposal> {
+  const prompt = buildRefinementPrompt(
+    proposal.proposed_body,
+    validationResult,
+    proposal.skill_name,
+    validationResult.regressions,
+  );
+
+  const rawResponse = await callLlm(BODY_REFINER_SYSTEM, prompt, agent, modelFlag);
+  const { refined_body, changes_made, confidence } = parseRefinementResponse(rawResponse);
+
+  return {
+    ...proposal,
+    proposal_id: `${proposal.proposal_id}-refined-${Date.now()}`,
+    proposed_body: refined_body,
+    rationale: `${proposal.rationale}\n\nRefinement: ${changes_made}`,
+    confidence,
+    created_at: new Date().toISOString(),
+    status: "pending",
+  };
+}

package/cli/selftune/evolution/rollback.ts

@@ -11,6 +11,7 @@ import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from
 import { basename, dirname, join } from "node:path";
 import { parseArgs } from "node:util";
 
+import { updateContextAfterRollback } from "../memory/writer.js";
 import type { EvolutionAuditEntry } from "../types.js";
 import { appendAuditEntry, getLastDeployedProposal, readAuditTrail } from "./audit.js";
 import { replaceDescription } from "./deploy-proposal.js";
@@ -153,11 +154,19 @@ export async function rollback(options: RollbackOptions): Promise<RollbackResult
     };
     appendAuditEntry(auditEntry, logPath);
 
-    return {
+    const backupResult: RollbackResult = {
       rolledBack: true,
       restoredDescription: originalContent,
       reason: "Restored from backup file",
     };
+
+    try {
+      updateContextAfterRollback(skillName, backupResult);
+    } catch {
+      // Memory writes should never fail the main operation
+    }
+
+    return backupResult;
   }
 
   // Strategy 2: Restore from audit trail's created entry (description only)
@@ -177,11 +186,19 @@ export async function rollback(options: RollbackOptions): Promise<RollbackResult
     };
     appendAuditEntry(auditEntry, logPath);
 
-    return {
+    const auditResult: RollbackResult = {
       rolledBack: true,
       restoredDescription: originalFromAudit,
       reason: "Restored from audit trail",
     };
+
+    try {
+      updateContextAfterRollback(skillName, auditResult);
+    } catch {
+      // Memory writes should never fail the main operation
+    }
+
+    return auditResult;
   }
 
   // No restoration source available

package/cli/selftune/evolution/validate-body.ts

@@ -0,0 +1,254 @@
+/**
+ * validate-body.ts
+ *
+ * 3-gate validation for full body evolution proposals:
+ *   Gate 1 (structural): Pure code — YAML frontmatter, # Title, ## Workflow Routing preserved
+ *   Gate 2 (trigger accuracy): Student model YES/NO per eval entry
+ *   Gate 3 (quality): Student model rates body clarity/completeness 0.0-1.0
+ */
+
+import type { BodyEvolutionProposal, BodyValidationResult, EvalEntry } from "../types.js";
+import { callLlm, stripMarkdownFences } from "../utils/llm-call.js";
+import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
+
+// ---------------------------------------------------------------------------
+// Gate 1: Structural validation (pure code, no LLM)
+// ---------------------------------------------------------------------------
+
+/**
+ * Check that a proposed body preserves required structural elements.
+ * Verifies:
+ *  - Contains a ## Workflow Routing section
+ *  - Routing table has valid markdown table syntax
+ *  - Body is non-empty
+ */
+export function validateBodyStructure(proposedBody: string): { valid: boolean; reason: string } {
+  if (!proposedBody || proposedBody.trim().length === 0) {
+    return { valid: false, reason: "Proposed body is empty" };
+  }
+
+  // Check for ## Workflow Routing section
+  if (!proposedBody.includes("## Workflow Routing")) {
+    return { valid: false, reason: "Missing required '## Workflow Routing' section" };
+  }
+
+  // Extract the routing section and check for table syntax
+  const routingIdx = proposedBody.indexOf("## Workflow Routing");
+  const afterRouting = proposedBody.slice(routingIdx + "## Workflow Routing".length);
+  // Find end of section (next ## heading or EOF)
+  const nextSectionMatch = afterRouting.match(/\n## /);
+  const routingContent = nextSectionMatch
+    ? afterRouting.slice(0, nextSectionMatch.index)
+    : afterRouting;
+
+  // Check for pipe-delimited table rows
+  const tableLines = routingContent
+    .split("\n")
+    .filter((l) => l.trim().startsWith("|") && l.trim().endsWith("|"));
+  if (tableLines.length < 2) {
+    return {
+      valid: false,
+      reason:
+        "Workflow Routing section lacks a valid markdown table (need header + separator + rows)",
+    };
+  }
+
+  return { valid: true, reason: "Structural validation passed" };
+}
+
+// ---------------------------------------------------------------------------
+// Gate 2: Trigger accuracy (student model YES/NO)
+// ---------------------------------------------------------------------------
+
+/**
+ * Run trigger checks on the eval set using the proposed body content.
+ * Returns before/after pass rates.
+ */
+export async function validateBodyTriggerAccuracy(
+  originalBody: string,
+  proposedBody: string,
+  evalSet: EvalEntry[],
+  agent: string,
+  modelFlag?: string,
+): Promise<{
+  before_pass_rate: number;
+  after_pass_rate: number;
+  improved: boolean;
+  regressions: string[];
+}> {
+  if (evalSet.length === 0) {
+    return { before_pass_rate: 0, after_pass_rate: 0, improved: false, regressions: [] };
+  }
+
+  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
+  let beforePassed = 0;
+  let afterPassed = 0;
+  const regressions: string[] = [];
+
+  for (const entry of evalSet) {
+    // Check with original body
+    const beforePrompt = buildTriggerCheckPrompt(originalBody, entry.query);
+    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
+    const beforeTriggered = parseTriggerResponse(beforeRaw);
+    const beforePass =
+      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
+
+    // Check with proposed body
+    const afterPrompt = buildTriggerCheckPrompt(proposedBody, entry.query);
+    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
+    const afterTriggered = parseTriggerResponse(afterRaw);
+    const afterPass =
+      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
+
+    if (beforePass) beforePassed++;
+    if (afterPass) afterPassed++;
+
+    // Track regressions
+    if (beforePass && !afterPass) {
+      regressions.push(entry.query);
+    }
+  }
+
+  const total = evalSet.length;
+  const beforePassRate = beforePassed / total;
+  const afterPassRate = afterPassed / total;
+
+  return {
+    before_pass_rate: beforePassRate,
+    after_pass_rate: afterPassRate,
+    improved: afterPassRate > beforePassRate,
+    regressions,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Gate 3: Quality assessment (student model 0.0-1.0)
+// ---------------------------------------------------------------------------
+
+/** System prompt for quality assessment. */
+const QUALITY_ASSESSMENT_SYSTEM = `You are a skill document quality assessor for an AI agent system.
+
+Rate the quality of the provided skill document body on these dimensions:
+- Clarity: Is the description clear and unambiguous?
+- Completeness: Does it cover the expected use cases?
+- Structure: Is it well-organized with proper sections?
+- Routing accuracy: Does the routing table seem comprehensive?
+
+Output ONLY valid JSON with exactly these fields:
+  - "score" (number): Overall quality score 0.0-1.0
+  - "reason" (string): Brief explanation of the score
+
+Do NOT include any text outside the JSON object.`;
+
+/** Assess the quality of a proposed body via student model. */
+export async function assessBodyQuality(
+  proposedBody: string,
+  skillName: string,
+  agent: string,
+  modelFlag?: string,
+): Promise<{ score: number; reason: string }> {
+  const userPrompt = `Skill Name: ${skillName}
+
+Proposed Skill Body:
+${proposedBody}
+
+Rate the quality of this skill document body. Output ONLY a JSON object with "score" (0.0-1.0) and "reason" fields.`;
+
+  const rawResponse = await callLlm(QUALITY_ASSESSMENT_SYSTEM, userPrompt, agent, modelFlag);
+  const cleaned = stripMarkdownFences(rawResponse);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch {
+    // If parsing fails, return a conservative default
+    return { score: 0.5, reason: "Failed to parse quality assessment response" };
+  }
+
+  if (typeof parsed !== "object" || parsed === null) {
+    return { score: 0.5, reason: "Quality assessment response is not a JSON object" };
+  }
+
+  const obj = parsed as Record<string, unknown>;
+  const score = typeof obj.score === "number" ? Math.max(0.0, Math.min(1.0, obj.score)) : 0.5;
+  const reason = typeof obj.reason === "string" ? obj.reason : "No reason provided";
+
+  return { score, reason };
+}
+
+// ---------------------------------------------------------------------------
+// Full 3-gate body validation
+// ---------------------------------------------------------------------------
+
+/** Minimum quality score to pass Gate 3. */
+const QUALITY_THRESHOLD = 0.6;
+
+/** Validate a body proposal through all 3 gates. */
+export async function validateBodyProposal(
+  proposal: BodyEvolutionProposal,
+  evalSet: EvalEntry[],
+  agent: string,
+  modelFlag?: string,
+  qualityThreshold = QUALITY_THRESHOLD,
+): Promise<BodyValidationResult> {
+  const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
+
+  // Gate 1: Structural validation (pure code)
+  const structural = validateBodyStructure(proposal.proposed_body);
+  gateResults.push({
+    gate: "structural",
+    passed: structural.valid,
+    reason: structural.reason,
+  });
+
+  if (!structural.valid) {
+    return {
+      proposal_id: proposal.proposal_id,
+      gates_passed: 0,
+      gates_total: 3,
+      gate_results: gateResults,
+      improved: false,
+      regressions: [],
+    };
+  }
+
+  // Gate 2: Trigger accuracy (student model)
+  const accuracy = await validateBodyTriggerAccuracy(
+    proposal.original_body,
+    proposal.proposed_body,
+    evalSet,
+    agent,
+    modelFlag,
+  );
+  gateResults.push({
+    gate: "trigger_accuracy",
+    passed: accuracy.improved,
+    reason: accuracy.improved
+      ? `Improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`
+      : `Not improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`,
+  });
+
+  // Gate 3: Quality assessment (student model)
+  const quality = await assessBodyQuality(
+    proposal.proposed_body,
+    proposal.skill_name,
+    agent,
+    modelFlag,
+  );
+  gateResults.push({
+    gate: "quality",
+    passed: quality.score >= qualityThreshold,
+    reason: `Quality score: ${quality.score.toFixed(2)} (threshold: ${qualityThreshold}) — ${quality.reason}`,
+  });
+
+  const gatesPassed = gateResults.filter((g) => g.passed).length;
+
+  return {
+    proposal_id: proposal.proposal_id,
+    gates_passed: gatesPassed,
+    gates_total: 3,
+    gate_results: gateResults,
+    improved: gatesPassed === 3,
+    regressions: accuracy.regressions,
+  };
+}