selftune 0.1.4 → 0.2.1

package/cli/selftune/evolution/validate-routing.ts

@@ -0,0 +1,177 @@
+/**
+ * validate-routing.ts
+ *
+ * Validates a routing table evolution proposal by checking structural validity
+ * and running trigger accuracy checks against an eval set.
+ */
+
+import type { BodyEvolutionProposal, BodyValidationResult, EvalEntry } from "../types.js";
+import { callLlm } from "../utils/llm-call.js";
+import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
+
+// ---------------------------------------------------------------------------
+// Structural validation
+// ---------------------------------------------------------------------------
+
+/**
+ * Check that a routing table is valid markdown table syntax with
+ * `| Trigger | Workflow |` columns.
+ */
+export function validateRoutingStructure(routing: string): { valid: boolean; reason: string } {
+  const lines = routing
+    .trim()
+    .split("\n")
+    .filter((l) => l.trim().length > 0);
+
+  if (lines.length < 2) {
+    return { valid: false, reason: "Routing table must have at least a header and one data row" };
+  }
+
+  // Check header row contains Trigger and Workflow columns
+  const headerLine = lines[0].trim();
+  if (!headerLine.startsWith("|") || !headerLine.endsWith("|")) {
+    return {
+      valid: false,
+      reason: "Header row must be a markdown table row starting and ending with |",
+    };
+  }
+
+  const headerLower = headerLine.toLowerCase();
+  if (!headerLower.includes("trigger") || !headerLower.includes("workflow")) {
+    return { valid: false, reason: "Header must contain 'Trigger' and 'Workflow' columns" };
+  }
+
+  // Check separator row (line 2) has dashes
+  const separatorLine = lines[1].trim();
+  if (!separatorLine.includes("---")) {
+    return { valid: false, reason: "Second row must be a markdown table separator (contains ---)" };
+  }
+
+  // Check at least one data row
+  if (lines.length < 3) {
+    return { valid: false, reason: "Routing table must have at least one data row" };
+  }
+
+  // Check data rows are pipe-delimited
+  for (let i = 2; i < lines.length; i++) {
+    const row = lines[i].trim();
+    if (!row.startsWith("|") || !row.endsWith("|")) {
+      return { valid: false, reason: `Data row ${i - 1} is not a valid markdown table row` };
+    }
+  }
+
+  return { valid: true, reason: "Valid markdown routing table" };
+}
+
+// ---------------------------------------------------------------------------
+// Trigger accuracy validation
+// ---------------------------------------------------------------------------
+
+/**
+ * Run before/after trigger checks on the eval set using the routing content.
+ * Returns pass rates for comparison.
+ */
+export async function validateRoutingTriggerAccuracy(
+  originalRouting: string,
+  proposedRouting: string,
+  evalSet: EvalEntry[],
+  agent: string,
+  modelFlag?: string,
+): Promise<{ before_pass_rate: number; after_pass_rate: number; improved: boolean }> {
+  if (evalSet.length === 0) {
+    return { before_pass_rate: 0, after_pass_rate: 0, improved: false };
+  }
+
+  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
+  let beforePassed = 0;
+  let afterPassed = 0;
+
+  for (const entry of evalSet) {
+    // Check with original routing
+    const beforePrompt = buildTriggerCheckPrompt(originalRouting, entry.query);
+    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
+    const beforeTriggered = parseTriggerResponse(beforeRaw);
+    const beforePass =
+      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
+
+    // Check with proposed routing
+    const afterPrompt = buildTriggerCheckPrompt(proposedRouting, entry.query);
+    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
+    const afterTriggered = parseTriggerResponse(afterRaw);
+    const afterPass =
+      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
+
+    if (beforePass) beforePassed++;
+    if (afterPass) afterPassed++;
+  }
+
+  const total = evalSet.length;
+  const beforePassRate = beforePassed / total;
+  const afterPassRate = afterPassed / total;
+
+  return {
+    before_pass_rate: beforePassRate,
+    after_pass_rate: afterPassRate,
+    improved: afterPassRate > beforePassRate,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Full routing validation
+// ---------------------------------------------------------------------------
+
+/** Validate a routing table proposal: structural check + trigger accuracy. */
+export async function validateRoutingProposal(
+  proposal: BodyEvolutionProposal,
+  evalSet: EvalEntry[],
+  agent: string,
+  modelFlag?: string,
+): Promise<BodyValidationResult> {
+  const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
+
+  // Gate 1: Structural validation
+  const structural = validateRoutingStructure(proposal.proposed_body);
+  gateResults.push({
+    gate: "structural",
+    passed: structural.valid,
+    reason: structural.reason,
+  });
+
+  if (!structural.valid) {
+    return {
+      proposal_id: proposal.proposal_id,
+      gates_passed: 0,
+      gates_total: 2,
+      gate_results: gateResults,
+      improved: false,
+      regressions: [],
+    };
+  }
+
+  // Gate 2: Trigger accuracy
+  const accuracy = await validateRoutingTriggerAccuracy(
+    proposal.original_body,
+    proposal.proposed_body,
+    evalSet,
+    agent,
+    modelFlag,
+  );
+  gateResults.push({
+    gate: "trigger_accuracy",
+    passed: accuracy.improved,
+    reason: accuracy.improved
+      ? `Improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`
+      : `Not improved: ${(accuracy.before_pass_rate * 100).toFixed(1)}% -> ${(accuracy.after_pass_rate * 100).toFixed(1)}%`,
+  });
+
+  const gatesPassed = gateResults.filter((g) => g.passed).length;
+
+  return {
+    proposal_id: proposal.proposal_id,
+    gates_passed: gatesPassed,
+    gates_total: 2,
+    gate_results: gateResults,
+    improved: gatesPassed === 2,
+    regressions: [],
+  };
+}

package/cli/selftune/grading/auto-grade.ts

@@ -0,0 +1,200 @@
+#!/usr/bin/env bun
+/**
+ * auto-grade.ts
+ *
+ * Frictionless grading command that auto-finds the most recent real session
+ * for a skill, auto-derives expectations from SKILL.md, grades, and outputs results.
+ *
+ * Usage:
+ *   selftune auto-grade --skill <name> [--skill-path <path>] [--output <path>] [--agent <agent>]
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { parseArgs } from "node:util";
+
+import { AGENT_CANDIDATES, TELEMETRY_LOG } from "../constants.js";
+import type { GradingResult, SessionTelemetryRecord } from "../types.js";
+import { readJsonl } from "../utils/jsonl.js";
+import { detectAgent as _detectAgent } from "../utils/llm-call.js";
+import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
+import { readExcerpt } from "../utils/transcript.js";
+import {
+  buildDefaultGradingOutputPath,
+  deriveExpectationsFromSkill,
+  gradeSession,
+  resolveLatestSessionForSkill,
+  resolveSessionById,
+} from "./grade-session.js";
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string" },
+      "skill-path": { type: "string" },
+      "session-id": { type: "string" },
+      "telemetry-log": { type: "string", default: TELEMETRY_LOG },
+      output: { type: "string" },
+      agent: { type: "string" },
+      "show-transcript": { type: "boolean", default: false },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune auto-grade — Frictionless skill session grading
+
+Usage:
+  selftune auto-grade --skill <name> [options]
+
+Options:
+  --skill             Skill name (required)
+  --skill-path        Path to SKILL.md (auto-detected if omitted)
+  --session-id        Grade a specific session (auto-detects most recent if omitted)
+  --telemetry-log     Path to telemetry log (default: ~/.claude/session_telemetry_log.jsonl)
+  --output            Output path for grading JSON (default: ~/.selftune/grading/result-<session>.json)
+  --agent             Agent CLI to use (${AGENT_CANDIDATES.join(", ")})
+  --show-transcript   Print transcript excerpt before grading
+  -h, --help          Show this help message`);
+    process.exit(0);
+  }
+
+  const skill = values.skill;
+  if (!skill) {
+    console.error("[ERROR] --skill is required");
+    process.exit(1);
+  }
+
+  // --- Determine agent ---
+  let agent: string | null = null;
+  const validAgents = [...AGENT_CANDIDATES];
+  if (values.agent) {
+    if (!validAgents.includes(values.agent)) {
+      console.error(
+        `[ERROR] Invalid --agent '${values.agent}'. Expected one of: ${validAgents.join(", ")}`,
+      );
+      process.exit(1);
+    }
+    agent = values.agent;
+  } else {
+    agent = _detectAgent();
+  }
+
+  if (!agent) {
+    console.error(
+      `[ERROR] No supported agent CLI (${AGENT_CANDIDATES.join("/")}) found in PATH.\n` +
+        "Install one of the supported agent CLIs.",
+    );
+    process.exit(1);
+  }
+
+  console.error(`[INFO] Auto-grade via agent: ${agent}`);
+
+  // --- Auto-find session ---
+  const telemetryLog = values["telemetry-log"] ?? TELEMETRY_LOG;
+  const telRecords = readJsonl<SessionTelemetryRecord>(telemetryLog);
+  const skillUsageRecords = readEffectiveSkillUsageRecords();
+
+  let telemetry: SessionTelemetryRecord;
+  let sessionId: string;
+  let transcriptPath: string;
+
+  if (values["session-id"]) {
+    sessionId = values["session-id"];
+    const resolved = resolveSessionById(telRecords, sessionId);
+    if (!resolved) {
+      console.error(
+        `[ERROR] Session '${sessionId}' not found in telemetry or recoverable transcript data. ` +
+          "Check the session ID or omit --session-id to auto-select the latest matching session.",
+      );
+      process.exit(1);
+    }
+    telemetry = resolved.telemetry;
+    transcriptPath = resolved.transcriptPath;
+  } else {
+    const resolved = resolveLatestSessionForSkill(telRecords, skillUsageRecords, skill);
+    if (!resolved) {
+      console.error(
+        `[ERROR] No session found for skill '${skill}'. Run the skill first, or pass --session-id.`,
+      );
+      process.exit(1);
+    }
+    telemetry = resolved.telemetry;
+    sessionId = resolved.sessionId ?? "unknown";
+    transcriptPath = resolved.transcriptPath ?? "";
+    const note =
+      resolved.source === "telemetry" ? "" : ` (${resolved.source.replaceAll("_", " ")})`;
+    console.error(`[INFO] Found most recent '${skill}' session: ${sessionId}${note}`);
+  }
+
+  const transcriptExcerpt = transcriptPath ? readExcerpt(transcriptPath) : "(no transcript)";
+
+  if (values["show-transcript"]) {
+    console.log("=== TRANSCRIPT EXCERPT ===");
+    console.log(transcriptExcerpt);
+    console.log("==========================\n");
+  }
+
+  // --- Auto-derive expectations ---
+  const derived = deriveExpectationsFromSkill(skill, values["skill-path"]);
+  if (derived.derived) {
+    console.error(
+      `[INFO] Auto-derived ${derived.expectations.length} expectations from ${derived.source}`,
+    );
+  } else {
+    console.error(`[WARN] Using generic expectations (${derived.source})`);
+  }
+  const expectations = derived.expectations;
+
+  let result: GradingResult;
+  try {
+    result = await gradeSession({
+      expectations,
+      telemetry,
+      sessionId,
+      skillName: skill,
+      transcriptExcerpt,
+      transcriptPath,
+      agent,
+    });
+  } catch (err) {
+    console.error(`[ERROR] ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  }
+
+  const outputPath = values.output ?? buildDefaultGradingOutputPath(sessionId);
+  const outputDir = dirname(outputPath);
+  if (outputDir !== ".") {
+    mkdirSync(outputDir, { recursive: true });
+  }
+  writeFileSync(outputPath, JSON.stringify(result, null, 2), "utf-8");
+
+  // Print summary
+  const { summary } = result;
+  const rate = summary.pass_rate ?? 0;
+  const meanStr =
+    summary.mean_score != null ? ` | mean score: ${summary.mean_score.toFixed(2)}` : "";
+  console.log(
+    `\nResults: ${summary.passed}/${summary.total} passed (${Math.round(rate * 100)}%)${meanStr}`,
+  );
+  for (const exp of result.expectations ?? []) {
+    const icon = exp.passed ? "\u2713" : "\u2717";
+    const scoreStr = exp.score != null ? ` [${exp.score.toFixed(1)}]` : "";
+    const sourceStr = exp.source ? ` (${exp.source})` : "";
+    console.log(`  ${icon}${scoreStr}${sourceStr} ${String(exp.text ?? "").slice(0, 70)}`);
+    if (!exp.passed) {
+      console.log(`      -> ${String(exp.evidence ?? "").slice(0, 100)}`);
+    }
+  }
+
+  console.log(`\nWrote ${outputPath}`);
+}
+
+// Guard: only run when invoked directly
+if (import.meta.main) {
+  cliMain().catch((err) => {
+    console.error(`[FATAL] ${err}`);
+    process.exit(1);
+  });
+}