selftune 0.2.22 → 0.2.23

package/cli/selftune/evolution/validate-routing.ts

@@ -3,6 +3,9 @@
  *
  * Validates a routing table evolution proposal by checking structural validity
  * and running trigger accuracy checks against an eval set.
+ *
+ * Delegates replay-based and judge-based validation to dedicated engines
+ * (engines/replay-engine.ts and engines/judge-engine.ts).
  */
 
 import type {
@@ -10,28 +13,20 @@ import type {
   BodyValidationResult,
   EvalEntry,
   RoutingReplayEntryResult,
-  RoutingReplayFixture,
   ValidationMode,
 } from "../types.js";
-import { callLlm } from "../utils/llm-call.js";
-import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
-import { runHostReplayFixture } from "./validate-host-replay.js";
-
-export interface RoutingReplayRunnerInput {
-  routing: string;
-  evalSet: EvalEntry[];
-  agent: string;
-  fixture: RoutingReplayFixture;
-}
-
-export type RoutingReplayRunner = (
-  input: RoutingReplayRunnerInput,
-) => Promise<RoutingReplayEntryResult[]>;
-
-export interface RoutingValidationOptions {
-  replayFixture?: RoutingReplayFixture;
-  replayRunner?: RoutingReplayRunner;
-}
+import { runJudgeValidation } from "./engines/judge-engine.js";
+import {
+  runReplayValidation,
+  type ReplayRunner,
+  type ReplayRunnerInput,
+  type ReplayValidationOptions,
+} from "./engines/replay-engine.js";
+
+// Re-export engine types for backward compatibility
+export type { ReplayRunnerInput as RoutingReplayRunnerInput };
+export type { ReplayRunner as RoutingReplayRunner };
+export type { ReplayValidationOptions as RoutingValidationOptions };
 
 export interface RoutingTriggerAccuracyResult {
   before_pass_rate: number;
@@ -41,6 +36,7 @@ export interface RoutingTriggerAccuracyResult {
   validation_agent: string;
   validation_fixture_id?: string;
   per_entry_results?: RoutingReplayEntryResult[];
+  before_entry_results?: RoutingReplayEntryResult[];
 }
 
 // ---------------------------------------------------------------------------
@@ -104,6 +100,9 @@ export function validateRoutingStructure(routing: string): { valid: boolean; rea
 /**
  * Run before/after trigger checks on the eval set using the routing content.
  * Returns pass rates for comparison.
+ *
+ * Prefers replay-backed validation when a fixture is available,
+ * falls back to LLM judge otherwise.
  */
 export async function validateRoutingTriggerAccuracy(
   originalRouting: string,
@@ -111,7 +110,7 @@ export async function validateRoutingTriggerAccuracy(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
-  options: RoutingValidationOptions = {},
+  options: ReplayValidationOptions = {},
 ): Promise<RoutingTriggerAccuracyResult> {
   if (evalSet.length === 0) {
     return {
@@ -123,93 +122,34 @@ export async function validateRoutingTriggerAccuracy(
     };
   }
 
-  if (options.replayFixture && options.replayRunner) {
-    const beforeResults = await options.replayRunner({
-      routing: originalRouting,
-      evalSet,
-      agent,
-      fixture: options.replayFixture,
-    });
-    const afterResults = await options.replayRunner({
-      routing: proposedRouting,
-      evalSet,
-      agent,
-      fixture: options.replayFixture,
-    });
-    const beforePassed = beforeResults.filter((result) => result.passed).length;
-    const afterPassed = afterResults.filter((result) => result.passed).length;
-    const total = evalSet.length;
-
-    return {
-      before_pass_rate: beforePassed / total,
-      after_pass_rate: afterPassed / total,
-      improved: afterPassed > beforePassed,
-      validation_mode: "host_replay",
-      validation_agent: agent,
-      validation_fixture_id: options.replayFixture.fixture_id,
-      per_entry_results: afterResults,
-    };
-  }
-
-  if (options.replayFixture) {
-    const beforeResults = runHostReplayFixture({
-      routing: originalRouting,
-      evalSet,
-      fixture: options.replayFixture,
-    });
-    const afterResults = runHostReplayFixture({
-      routing: proposedRouting,
-      evalSet,
-      fixture: options.replayFixture,
-    });
-    const beforePassed = beforeResults.filter((result) => result.passed).length;
-    const afterPassed = afterResults.filter((result) => result.passed).length;
-    const total = evalSet.length;
-
-    return {
-      before_pass_rate: beforePassed / total,
-      after_pass_rate: afterPassed / total,
-      improved: afterPassed > beforePassed,
-      validation_mode: "host_replay",
-      validation_agent: agent,
-      validation_fixture_id: options.replayFixture.fixture_id,
-      per_entry_results: afterResults,
-    };
-  }
-
-  const systemPrompt = "You are an evaluation assistant. Answer only YES or NO.";
-  let beforePassed = 0;
-  let afterPassed = 0;
-
-  for (const entry of evalSet) {
-    // Check with original routing
-    const beforePrompt = buildTriggerCheckPrompt(originalRouting, entry.query);
-    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
-    const beforeTriggered = parseTriggerResponse(beforeRaw);
-    const beforePass =
-      (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
-
-    // Check with proposed routing
-    const afterPrompt = buildTriggerCheckPrompt(proposedRouting, entry.query);
-    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
-    const afterTriggered = parseTriggerResponse(afterRaw);
-    const afterPass =
-      (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
+  // Try replay-backed validation first
+  const replayResult = await runReplayValidation(
+    originalRouting,
+    proposedRouting,
+    evalSet,
+    agent,
+    options,
+  );
 
-    if (beforePass) beforePassed++;
-    if (afterPass) afterPassed++;
+  if (replayResult) {
+    return replayResult;
   }
 
-  const total = evalSet.length;
-  const beforePassRate = beforePassed / total;
-  const afterPassRate = afterPassed / total;
+  // Fall back to LLM judge
+  const judgeResult = await runJudgeValidation(
+    originalRouting,
+    proposedRouting,
+    evalSet,
+    agent,
+    modelFlag,
+  );
 
   return {
-    before_pass_rate: beforePassRate,
-    after_pass_rate: afterPassRate,
-    improved: afterPassRate > beforePassRate,
-    validation_mode: "llm_judge",
-    validation_agent: agent,
+    before_pass_rate: judgeResult.before_pass_rate,
+    after_pass_rate: judgeResult.after_pass_rate,
+    improved: judgeResult.improved,
+    validation_mode: judgeResult.validation_mode,
+    validation_agent: judgeResult.validation_agent,
   };
 }
 
@@ -223,7 +163,7 @@ export async function validateRoutingProposal(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
-  options: RoutingValidationOptions = {},
+  options: ReplayValidationOptions = {},
 ): Promise<BodyValidationResult> {
   const gateResults: Array<{ gate: string; passed: boolean; reason: string }> = [];
 
@@ -280,5 +220,6 @@ export async function validateRoutingProposal(
     before_pass_rate: accuracy.before_pass_rate,
     after_pass_rate: accuracy.after_pass_rate,
     per_entry_results: accuracy.per_entry_results,
+    before_entry_results: accuracy.before_entry_results,
   };
 }

package/cli/selftune/hooks/skill-eval.ts

@@ -25,7 +25,7 @@ import {
   getLatestPromptIdentity,
 } from "../normalization.js";
 import type { PostToolUsePayload, SkillUsageRecord } from "../types.js";
-import { classifySkillPath } from "../utils/skill-discovery.js";
+import { classifySkillPath, isTestFixturePath } from "../utils/skill-discovery.js";
 import { getLastUserMessage } from "../utils/transcript.js";
 
 /**
@@ -122,6 +122,7 @@ export async function processToolUse(
   const skillName = extractSkillName(filePath);
 
   if (skillName === null) return null;
+  if (isTestFixturePath(filePath)) return null;
 
   const transcriptPath = payload.transcript_path ?? "";
   const sessionId = payload.session_id ?? "unknown";

package/cli/selftune/hooks-shared/types.ts

@@ -83,6 +83,7 @@ export const PLATFORM_EVENT_MAP: Record<HookPlatform, Partial<Record<HookEventTy
     session_end: "TaskComplete",
   },
   pi: {
+    prompt_submit: "message",
     pre_tool_use: "tool_call",
     post_tool_use: "tool_result",
     session_end: "session_shutdown",

package/cli/selftune/index.ts

@@ -3,7 +3,7 @@
  * selftune CLI entry point.
  *
  * Usage:
- *   selftune ingest <agent>     — Ingest agent sessions (claude, codex, opencode, openclaw, wrap-codex)
+ *   selftune ingest <agent>     — Ingest agent sessions (claude, codex, opencode, openclaw, pi, wrap-codex)
  *   selftune grade [mode]       — Grade skill sessions (auto, baseline)
  *   selftune evolve [target]    — Evolve skill descriptions (body, rollback)
  *   selftune eval <action>      — Evaluation tools (generate, unit-test, import, composability, family-overlap)
@@ -28,11 +28,13 @@
  *   selftune export-canonical   — Export canonical telemetry for downstream ingestion
  *   selftune recover            — Recover SQLite from legacy/exported JSONL
  *   selftune telemetry          — Manage anonymous usage analytics (status, enable, disable)
+ *   selftune registry <sub>    — Team skill distribution (push, install, sync, status, rollback, history, list)
  *   selftune alpha <subcommand> — Alpha program management (upload)
  *   selftune hook <name>        — Run a hook by name (prompt-log, session-stop, etc.)
  *   selftune codex <subcommand> — Codex platform hooks (hook, install)
  *   selftune opencode <sub>     — OpenCode platform hooks (hook, install)
  *   selftune cline <subcommand> — Cline platform hooks (hook, install)
+ *   selftune pi <subcommand>    — Pi platform hooks (hook, install)
  */
 
 import { CLIError, handleCLIError } from "./utils/cli-error.js";
@@ -49,7 +51,7 @@ Usage:
   selftune <command> [options]
 
 Commands:
-  ingest <agent>     Ingest agent sessions (claude, codex, opencode, openclaw, wrap-codex)
+  ingest <agent>     Ingest agent sessions (claude, codex, opencode, openclaw, pi, wrap-codex)
   grade [mode]       Grade skill sessions (auto, baseline)
   evolve [target]    Evolve skill descriptions (body, rollback)
   eval <action>      Evaluation tools (generate, unit-test, import, composability, family-overlap)
@@ -73,19 +75,21 @@ Commands:
   export             Export SQLite data to JSONL snapshots
   export-canonical   Export canonical telemetry for downstream ingestion
   recover            Recover SQLite from legacy/exported JSONL
+  registry <sub>    Team skill distribution (push, install, sync, status, rollback, history, list)
   alpha <subcommand> Alpha program management (upload)
   telemetry          Manage anonymous usage analytics (status, enable, disable)
   hook <name>        Run a hook by name (prompt-log, session-stop, etc.)
   codex <sub>        Codex platform hooks (hook, install)
   opencode <sub>     OpenCode platform hooks (hook, install)
   cline <sub>        Cline platform hooks (hook, install)
+  pi <sub>           Pi platform hooks (hook, install)
 
 Run 'selftune <command> --help' for command-specific options.`);
   process.exit(0);
 }
 
 // Fast-path commands (real-time hooks) — skip analytics and auto-update to minimize latency
-const FAST_COMMANDS: ReadonlySet<string> = new Set(["hook", "codex", "opencode", "cline"]);
+const FAST_COMMANDS: ReadonlySet<string> = new Set(["hook", "codex", "opencode", "cline", "pi"]);
 
 // Track command usage (lazy import — skip for hooks and --help to avoid loading crypto/os)
 if (command && !FAST_COMMANDS.has(command) && command !== "--help" && command !== "-h") {
@@ -129,6 +133,7 @@ Agents:
   codex        Ingest Codex rollout logs (experimental)
   opencode     Ingest OpenCode sessions (experimental)
   openclaw     Ingest OpenClaw sessions (experimental)
+  pi           Ingest Pi sessions (experimental)
   wrap-codex   Wrap codex exec with real-time telemetry (experimental)
 
 Run 'selftune ingest <agent> --help' for agent-specific options.`);
@@ -157,6 +162,11 @@ Run 'selftune ingest <agent> --help' for agent-specific options.`);
         cliMain();
         break;
       }
+      case "pi": {
+        const { cliMain } = await import("./ingestors/pi-ingest.js");
+        cliMain();
+        break;
+      }
       case "wrap-codex": {
         const { cliMain } = await import("./ingestors/codex-wrapper.js");
         await cliMain();
@@ -620,6 +630,11 @@ Options:
     await cliMain();
     break;
   }
+  case "registry": {
+    const { cliMain } = await import("./registry/index.js");
+    await cliMain();
+    break;
+  }
   case "alpha": {
     const sub = process.argv[2];
     if (!sub || sub === "--help" || sub === "-h") {
@@ -828,9 +843,12 @@ Output:
 
   case "codex":
   case "opencode":
-  case "cline": {
+  case "cline":
+  case "pi": {
     const platform = command;
-    const displayName = { codex: "Codex", opencode: "OpenCode", cline: "Cline" }[platform];
+    const displayName = { codex: "Codex", opencode: "OpenCode", cline: "Cline", pi: "Pi" }[
+      platform
+    ];
     const sub = process.argv[2];
     if (!sub || sub === "--help" || sub === "-h") {
       console.log(`selftune ${platform} — ${displayName} platform hooks