selftune 0.1.4 → 0.2.0

package/cli/selftune/types.ts

@@ -7,7 +7,7 @@
 // ---------------------------------------------------------------------------
 
 export interface SelftuneConfig {
-  agent_type: "claude_code" | "codex" | "opencode" | "unknown";
+  agent_type: "claude_code" | "codex" | "opencode" | "openclaw" | "unknown";
   cli_path: string;
   llm_mode: "agent";
   agent_cli: string | null;
@@ -75,15 +75,19 @@ export interface TranscriptMetrics {
 // Hook payloads (received via stdin from Claude Code)
 // ---------------------------------------------------------------------------
 
+// Shared base for pre/post tool-use hook payloads
+export interface BaseToolUsePayload {
+  tool_name: string;
+  tool_input: Record<string, unknown>;
+  session_id?: string;
+}
+
 export interface PromptSubmitPayload {
   user_prompt: string;
   session_id?: string;
 }
 
-export interface PostToolUsePayload {
-  tool_name: string;
-  tool_input: Record<string, unknown>;
-  session_id?: string;
+export interface PostToolUsePayload extends BaseToolUsePayload {
   transcript_path?: string;
 }
 
@@ -113,6 +117,8 @@ export interface GradingExpectation {
   text: string;
   passed: boolean;
   evidence: string;
+  score?: number; // 0.0-1.0 graduated confidence
+  source?: "pre-gate" | "llm"; // which grading path produced this
 }
 
 export interface GradingClaim {
@@ -127,6 +133,15 @@ export interface GradingSummary {
   failed: number;
   total: number;
   pass_rate: number;
+  mean_score?: number; // mean of all expectation scores
+  score_std_dev?: number; // standard deviation
+}
+
+export interface FailureFeedback {
+  query: string;
+  failure_reason: string;
+  improvement_hint: string;
+  invocation_type?: InvocationType;
 }
 
 /** Raw output from the LLM grader (before assembly into GradingResult). */
@@ -135,6 +150,7 @@ export interface GraderOutput {
   summary: GradingSummary;
   claims: GradingClaim[];
   eval_feedback: EvalFeedback;
+  failure_feedback?: FailureFeedback[];
 }
 
 export interface EvalFeedback {
@@ -152,6 +168,7 @@ export interface GradingResult {
   execution_metrics: ExecutionMetrics;
   claims: GradingClaim[];
   eval_feedback: EvalFeedback;
+  failure_feedback?: FailureFeedback[];
 }
 
 export interface ExecutionMetrics {
@@ -197,6 +214,7 @@ export interface FailurePattern {
   frequency: number;
   sample_sessions: string[];
   extracted_at: string;
+  feedback?: FailureFeedback[];
 }
 
 export interface EvolutionProposal {
@@ -226,6 +244,7 @@ export interface EvalPassRate {
 export interface EvolutionAuditEntry {
   timestamp: string;
   proposal_id: string;
+  skill_name?: string;
   action: "created" | "validated" | "deployed" | "rolled_back" | "rejected";
   details: string;
   eval_snapshot?: EvalPassRate;
@@ -239,6 +258,68 @@ export interface EvolutionConfig {
   dry_run: boolean;
 }
 
+// ---------------------------------------------------------------------------
+// Validation result base (self-contained for Pareto types)
+// ---------------------------------------------------------------------------
+
+/** Compact summary of an evolve run, used for CLI JSON output. */
+export interface EvolveResultSummary {
+  skill: string;
+  deployed: boolean;
+  reason: string;
+  before: number;
+  after: number;
+  net_change: number;
+  improved: boolean;
+  regressions: number;
+  new_passes: number;
+  confidence: number;
+  llm_calls: number;
+  elapsed_s: number;
+  proposal_id: string;
+  rationale: string;
+  version?: string;
+  dashboard_url: string;
+}
+
+export interface ValidationResultBase {
+  proposal_id: string;
+  before_pass_rate: number;
+  after_pass_rate: number;
+  improved: boolean;
+  regressions: EvalEntry[];
+  new_passes: EvalEntry[];
+  net_change: number;
+  by_invocation_type?: InvocationTypeScores;
+  per_entry_results?: Array<{ entry: EvalEntry; before_pass: boolean; after_pass: boolean }>;
+}
+
+// ---------------------------------------------------------------------------
+// Pareto types (multi-dimensional evolution selection)
+// ---------------------------------------------------------------------------
+
+export interface InvocationTypeScores {
+  explicit: { passed: number; total: number; pass_rate: number };
+  implicit: { passed: number; total: number; pass_rate: number };
+  contextual: { passed: number; total: number; pass_rate: number };
+  negative: { passed: number; total: number; pass_rate: number };
+}
+
+export interface ParetoCandidate {
+  proposal: EvolutionProposal;
+  validation: ValidationResultBase;
+  invocation_scores: InvocationTypeScores;
+  dominates_on: InvocationType[];
+  token_efficiency_score?: number;
+}
+
+export interface ParetoSelectionResult {
+  selected_proposal: EvolutionProposal;
+  frontier: ParetoCandidate[];
+  merge_applied: boolean;
+  merge_sources: string[];
+}
+
 // ---------------------------------------------------------------------------
 // Monitoring types (v0.4)
 // ---------------------------------------------------------------------------
@@ -253,3 +334,294 @@ export interface MonitoringSnapshot {
   regression_detected: boolean;
   baseline_pass_rate: number;
 }
+
+// ---------------------------------------------------------------------------
+// Activation rule types (v0.5 — auto-activate hooks)
+// ---------------------------------------------------------------------------
+
+export interface ActivationRule {
+  id: string;
+  description: string;
+  /** Evaluate whether this rule fires. Returns a suggestion string or null. */
+  evaluate: (ctx: ActivationContext) => string | null;
+}
+
+export interface ActivationContext {
+  session_id: string;
+  query_log_path: string;
+  telemetry_log_path: string;
+  evolution_audit_log_path: string;
+  selftune_dir: string;
+  settings_path: string;
+}
+
+export interface SessionState {
+  session_id: string;
+  suggestions_shown: string[]; // rule IDs already fired this session
+  updated_at: string;
+}
+
+// ---------------------------------------------------------------------------
+// PreToolUse hook payloads
+// ---------------------------------------------------------------------------
+
+export interface PreToolUsePayload extends BaseToolUsePayload {}
+
+// ---------------------------------------------------------------------------
+// Evolution memory types (session context persistence)
+// ---------------------------------------------------------------------------
+
+export interface EvolutionMemory {
+  context: MemoryContext;
+  plan: MemoryPlan;
+  decisions: DecisionRecord[];
+}
+
+export interface MemoryContext {
+  activeEvolutions: Array<{
+    skillName: string;
+    status: string;
+    description: string;
+  }>;
+  knownIssues: string[];
+  lastUpdated: string;
+}
+
+export interface MemoryPlan {
+  currentPriorities: string[];
+  strategy: string;
+  lastUpdated: string;
+}
+
+export interface DecisionRecord {
+  timestamp: string;
+  /** Imperative verb for markdown headings (e.g. "evolve", "rollback", "watch"). */
+  actionType: string;
+  skillName: string;
+  /** Past-tense result state used programmatically. */
+  action: "evolved" | "rolled-back" | "watched";
+  rationale: string;
+  result: string;
+}
+
+// ---------------------------------------------------------------------------
+// Contribution types (contribute command)
+// ---------------------------------------------------------------------------
+
+export interface ContributionQuery {
+  query: string;
+  invocation_type: InvocationType;
+  source: string;
+}
+
+export interface ContributionEvalEntry {
+  query: string;
+  should_trigger: boolean;
+  invocation_type?: InvocationType;
+}
+
+export interface ContributionGradingSummary {
+  total_sessions: number;
+  graded_sessions: number;
+  average_pass_rate: number;
+  expectation_count: number;
+}
+
+export interface ContributionEvolutionSummary {
+  total_proposals: number;
+  deployed_proposals: number;
+  rolled_back_proposals: number;
+  average_improvement: number;
+}
+
+export interface ContributionSessionMetrics {
+  total_sessions: number;
+  avg_assistant_turns: number;
+  avg_tool_calls: number;
+  avg_errors: number;
+  top_tools: Array<{ tool: string; count: number }>;
+}
+
+export interface ContributionBundle {
+  schema_version: "1.0" | "1.1" | "1.2";
+  skill_name?: string;
+  contributor_id: string;
+  created_at: string;
+  selftune_version: string;
+  agent_type: string;
+  sanitization_level: "conservative" | "aggressive";
+  positive_queries: ContributionQuery[];
+  eval_entries: ContributionEvalEntry[];
+  grading_summary: ContributionGradingSummary | null;
+  evolution_summary: ContributionEvolutionSummary | null;
+  session_metrics: ContributionSessionMetrics;
+  unmatched_queries?: Array<{ query: string; timestamp: string }>;
+  pending_proposals?: Array<{
+    proposal_id: string;
+    skill_name?: string;
+    action: string;
+    timestamp: string;
+    details: string;
+  }>;
+}
+
+// ---------------------------------------------------------------------------
+// Evolution target types (v0.6 — body + routing evolution)
+// ---------------------------------------------------------------------------
+
+/** Which part of a skill is being evolved. */
+export type EvolutionTarget = "description" | "routing" | "body";
+
+/** Parsed sections of a SKILL.md file. */
+export interface SkillSections {
+  frontmatter: string;
+  title: string;
+  description: string;
+  sections: Record<string, string>;
+}
+
+/** Proposal for evolving the full body of a SKILL.md. */
+export interface BodyEvolutionProposal {
+  proposal_id: string;
+  skill_name: string;
+  skill_path: string;
+  original_body: string;
+  proposed_body: string;
+  rationale: string;
+  target: EvolutionTarget;
+  failure_patterns: string[];
+  confidence: number;
+  created_at: string;
+  status: "pending" | "validated" | "deployed" | "rolled_back";
+}
+
+/** Closed union of gate names used in the validation pipeline. */
+export type ValidationGate = "structural" | "trigger_accuracy" | "quality";
+
+/** Result of validating a body evolution proposal. */
+export interface BodyValidationResult {
+  proposal_id: string;
+  gates_passed: number;
+  gates_total: number;
+  gate_results: Array<{ gate: ValidationGate; passed: boolean; reason: string }>;
+  improved: boolean;
+  regressions: string[];
+}
+
+/** Configuration for which LLM model a role should use. */
+export interface LlmRoleConfig {
+  role: string;
+  model: string;
+  temperature?: number;
+  max_tokens?: number;
+}
+
+/** Token usage metrics for a session or eval run. */
+export interface TokenUsageMetrics {
+  input_tokens: number;
+  output_tokens: number;
+  total_tokens: number;
+  estimated_cost_usd?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Baseline comparison types
+// ---------------------------------------------------------------------------
+
+/** Result of a no-skill baseline measurement. */
+export interface BaselineResult {
+  skill_name: string;
+  query: string;
+  with_skill: boolean;
+  triggered: boolean;
+  pass: boolean;
+  latency_ms?: number;
+  tokens?: TokenUsageMetrics;
+  measured_at: string;
+}
+
+// ---------------------------------------------------------------------------
+// Skill unit test types
+// ---------------------------------------------------------------------------
+
+/** Type of assertion for a skill unit test. */
+export type AssertionType =
+  | "contains"
+  | "not_contains"
+  | "regex"
+  | "json_path"
+  | "tool_called"
+  | "tool_not_called";
+
+/** A single assertion within a skill unit test. */
+export interface SkillAssertion {
+  type: AssertionType;
+  value: string;
+  description?: string;
+}
+
+/** A skill unit test case. */
+export interface SkillUnitTest {
+  id: string;
+  skill_name: string;
+  query: string;
+  assertions: SkillAssertion[];
+  timeout_ms?: number;
+  tags?: string[];
+}
+
+/** Result of running a single skill unit test. */
+export interface UnitTestResult {
+  test_id: string;
+  passed: boolean;
+  assertion_results: Array<{ assertion: SkillAssertion; passed: boolean; actual?: string }>;
+  duration_ms: number;
+  error?: string;
+}
+
+/** Aggregated result of a skill unit test suite. */
+export interface UnitTestSuiteResult {
+  skill_name: string;
+  total: number;
+  passed: number;
+  failed: number;
+  pass_rate: number;
+  results: UnitTestResult[];
+  run_at: string;
+}
+
+// ---------------------------------------------------------------------------
+// Composability types
+// ---------------------------------------------------------------------------
+
+/** A pair of skills that co-occur in sessions. */
+export interface CoOccurrencePair {
+  skill_a: string;
+  skill_b: string;
+  co_occurrence_count: number;
+  conflict_detected: boolean;
+  conflict_reason?: string;
+}
+
+/** Report on skill composability / conflicts. */
+export interface ComposabilityReport {
+  pairs: CoOccurrencePair[];
+  total_sessions_analyzed: number;
+  conflict_count: number;
+  generated_at: string;
+}
+
+// ---------------------------------------------------------------------------
+// SkillsBench types
+// ---------------------------------------------------------------------------
+
+/** A task from the SkillsBench benchmark suite. */
+export interface SkillsBenchTask {
+  task_id: string;
+  category: string;
+  query: string;
+  expected_skill?: string;
+  expected_tools?: string[];
+  difficulty: "easy" | "medium" | "hard";
+  tags?: string[];
+}

package/cli/selftune/utils/frontmatter.ts

@@ -0,0 +1,217 @@
+/**
+ * frontmatter.ts
+ *
+ * Line-based YAML frontmatter parser for SKILL.md files.
+ * Extracts name, description, and version without a YAML library.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface SkillFrontmatter {
+  name: string;
+  description: string;
+  version: string;
+  body: string;
+}
+
+// ---------------------------------------------------------------------------
+// Parser
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse YAML frontmatter from a SKILL.md file.
+ *
+ * Handles two description formats:
+ *   - Single-line:  `description: When the user wants to...`
+ *   - Folded scalar: `description: >\n  Multi-line text...`
+ *
+ * Handles two version locations:
+ *   - Top-level: `version: 1.0.0`
+ *   - Nested:    `metadata:\n  version: 1.0.0`
+ *
+ * Returns the full content as description if no frontmatter is found.
+ */
+export function parseFrontmatter(content: string): SkillFrontmatter {
+  const lines = content.split("\n");
+
+  // Check for opening delimiter
+  if (lines[0]?.trim() !== "---") {
+    return { name: "", description: content, version: "", body: content };
+  }
+
+  // Find closing delimiter
+  let endIdx = -1;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i].trim() === "---") {
+      endIdx = i;
+      break;
+    }
+  }
+
+  if (endIdx < 0) {
+    return { name: "", description: content, version: "", body: content };
+  }
+
+  const yamlLines = lines.slice(1, endIdx);
+  const body = lines
+    .slice(endIdx + 1)
+    .join("\n")
+    .replace(/^\n+/, "");
+
+  let name = "";
+  let description = "";
+  let version = "";
+  let inMetadata = false;
+
+  for (let i = 0; i < yamlLines.length; i++) {
+    const line = yamlLines[i];
+    const trimmed = line.trimEnd();
+
+    // Top-level `name:`
+    if (trimmed.startsWith("name:")) {
+      name = trimmed.slice("name:".length).trim();
+      inMetadata = false;
+      continue;
+    }
+
+    // Top-level `version:`
+    if (trimmed.startsWith("version:") && !trimmed.startsWith("  ")) {
+      version = trimmed.slice("version:".length).trim();
+      inMetadata = false;
+      continue;
+    }
+
+    // `metadata:` block start
+    if (trimmed === "metadata:" || trimmed.startsWith("metadata:")) {
+      inMetadata = true;
+      continue;
+    }
+
+    // Nested `version:` inside metadata
+    if (inMetadata && /^\s+version:/.test(trimmed)) {
+      version = trimmed.replace(/^\s+version:\s*/, "");
+      continue;
+    }
+
+    // Top-level `description:` — single-line or folded scalar
+    if (trimmed.startsWith("description:")) {
+      inMetadata = false;
+      const afterKey = trimmed.slice("description:".length).trim();
+
+      if (afterKey === ">" || afterKey === "|") {
+        // Folded/literal scalar: collect indented continuation lines
+        const descParts: string[] = [];
+        let j = i + 1;
+        while (j < yamlLines.length) {
+          const next = yamlLines[j];
+          // Continuation line must be indented (starts with whitespace)
+          if (next.length > 0 && /^\s/.test(next)) {
+            descParts.push(next.replace(/^\s+/, ""));
+          } else {
+            break;
+          }
+          j++;
+        }
+        description = descParts.join(" ").trim();
+        i = j - 1; // advance past consumed lines
+      } else {
+        // Single-line value
+        description = afterKey;
+      }
+      continue;
+    }
+
+    // Any other top-level key resets inMetadata
+    if (/^\S/.test(trimmed) && trimmed.includes(":")) {
+      inMetadata = false;
+    }
+  }
+
+  return { name, description, version, body };
+}
+
+// ---------------------------------------------------------------------------
+// Frontmatter description replacement
+// ---------------------------------------------------------------------------
+
+/**
+ * Replace the `description:` field in YAML frontmatter, preserving all other
+ * content. If the new description contains special YAML characters, it is
+ * written as a folded scalar (`description: >`).
+ *
+ * Returns the original content unchanged if no frontmatter is found.
+ */
+export function replaceFrontmatterDescription(content: string, newDescription: string): string {
+  const lines = content.split("\n");
+
+  if (lines[0]?.trim() !== "---") return content;
+
+  let endIdx = -1;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i].trim() === "---") {
+      endIdx = i;
+      break;
+    }
+  }
+  if (endIdx < 0) return content;
+
+  // Find and replace the description within frontmatter lines
+  const yamlLines = lines.slice(1, endIdx);
+  const newYamlLines: string[] = [];
+  let i = 0;
+  let replaced = false;
+
+  while (i < yamlLines.length) {
+    const trimmed = yamlLines[i].trimEnd();
+
+    if (trimmed.startsWith("description:")) {
+      replaced = true;
+      const afterKey = trimmed.slice("description:".length).trim();
+
+      // Skip continuation lines of folded/literal scalars
+      if (afterKey === ">" || afterKey === "|") {
+        i++;
+        while (i < yamlLines.length && yamlLines[i].length > 0 && /^\s/.test(yamlLines[i])) {
+          i++;
+        }
+      } else {
+        i++;
+      }
+
+      // Write new description — use folded scalar if it's long or has special chars
+      const needsFolded = newDescription.length > 120 || /[:#"'[\]{}|>]/.test(newDescription);
+      if (needsFolded) {
+        newYamlLines.push("description: >");
+        // Wrap at ~78 chars with 2-space indent
+        const words = newDescription.split(/\s+/);
+        let line = "  ";
+        for (const word of words) {
+          if (line.length + word.length + 1 > 80 && line.trim().length > 0) {
+            newYamlLines.push(line);
+            line = `  ${word}`;
+          } else {
+            line = line.trim().length === 0 ? `  ${word}` : `${line} ${word}`;
+          }
+        }
+        if (line.trim().length > 0) newYamlLines.push(line);
+      } else {
+        newYamlLines.push(`description: ${newDescription}`);
+      }
+      continue;
+    }
+
+    newYamlLines.push(yamlLines[i]);
+    i++;
+  }
+
+  // If description wasn't found in frontmatter, add it
+  if (!replaced) {
+    newYamlLines.push(`description: ${newDescription}`);
+  }
+
+  const before = lines[0]; // "---"
+  const after = lines.slice(endIdx); // "---" + body
+  return [before, ...newYamlLines, ...after].join("\n");
+}