selftune 0.2.22 → 0.2.24

package/cli/selftune/eval/baseline.ts

@@ -10,9 +10,10 @@
 
 import { parseArgs } from "node:util";
 
+import { writeGradingBaseline } from "../localdb/direct-write.js";
 import type { BaselineResult, EvalEntry } from "../types.js";
 import { CLIError, handleCLIError } from "../utils/cli-error.js";
-import { callLlm } from "../utils/llm-call.js";
+import { callLlm, detectLlmAgent } from "../utils/llm-call.js";
 import { buildTriggerCheckPrompt, parseTriggerResponse } from "../utils/trigger-check.js";
 
 // ---------------------------------------------------------------------------
@@ -208,7 +209,6 @@ Options:
   }
 
   // Detect agent
-  const { detectAgent } = await import("../utils/llm-call.js");
   const requestedAgent = values.agent;
   if (requestedAgent && !Bun.which(requestedAgent)) {
     throw new CLIError(
@@ -217,12 +217,12 @@ Options:
       "Install it or omit --agent to use auto-detection",
     );
   }
-  const agent = requestedAgent ?? detectAgent();
+  const agent = requestedAgent ?? detectLlmAgent();
   if (!agent) {
     throw new CLIError(
-      "No agent CLI (claude/codex/opencode) found in PATH",
+      "No agent CLI (claude/codex/opencode/pi) found in PATH",
       "AGENT_NOT_FOUND",
-      "Install Claude Code, Codex, or OpenCode",
+      "Install Claude Code, Codex, OpenCode, or Pi",
     );
   }
 
@@ -233,6 +233,22 @@ Options:
     agent,
   });
 
+  writeGradingBaseline({
+    skill_name: values.skill,
+    proposal_id: null,
+    measured_at: result.measured_at,
+    pass_rate: result.with_skill_pass_rate,
+    mean_score: null,
+    sample_size: evalSet.length,
+    grading_results_json: JSON.stringify({
+      baseline_pass_rate: result.baseline_pass_rate,
+      with_skill_pass_rate: result.with_skill_pass_rate,
+      lift: result.lift,
+      adds_value: result.adds_value,
+      per_entry: result.per_entry,
+    }),
+  });
+
   console.log(JSON.stringify(result, null, 2));
   process.exit(result.adds_value ? 0 : 1);
 }

package/cli/selftune/eval/execution-eval.ts

@@ -0,0 +1,170 @@
+/**
+ * execution-eval.ts
+ *
+ * Experimental execution eval harness — runs assertion-based evals
+ * in a staged skill workspace. Phase 2 of eval system gap closure.
+ *
+ * Behind experimental flag: must be explicitly opted into.
+ */
+
+import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ExecutionAssertion, ExecutionEvalEntry } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Result types
+// ---------------------------------------------------------------------------
+
+export interface ExecutionEvalResult {
+  entry: ExecutionEvalEntry;
+  passed: boolean;
+  assertion_results: AssertionResult[];
+  workspace_path?: string;
+  elapsed_ms: number;
+}
+
+export interface AssertionResult {
+  assertion: ExecutionAssertion;
+  passed: boolean;
+  actual?: string;
+  error?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Staged workspace management
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a staged workspace for execution eval isolation.
+ * Copies the skill into a temp directory so assertions don't affect the real skill.
+ */
+export function createStagedWorkspace(skillPath: string): string {
+  const workspace = mkdtempSync(join(tmpdir(), "selftune-exec-eval-"));
+  const skillDir = join(workspace, "skill");
+  mkdirSync(skillDir, { recursive: true });
+
+  if (existsSync(skillPath)) {
+    copyFileSync(skillPath, join(skillDir, "SKILL.md"));
+  }
+
+  return workspace;
+}
+
+/**
+ * Clean up a staged workspace.
+ */
+export function cleanupStagedWorkspace(workspacePath: string): void {
+  try {
+    rmSync(workspacePath, { recursive: true, force: true });
+  } catch {
+    // Best-effort cleanup — non-fatal
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Assertion runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a single execution assertion against a workspace.
+ */
+export function runAssertion(
+  assertion: ExecutionAssertion,
+  workspacePath: string,
+): AssertionResult {
+  try {
+    switch (assertion.type) {
+      case "file_exists": {
+        const target = join(workspacePath, assertion.target);
+        const exists = existsSync(target);
+        const passed = assertion.negated ? !exists : exists;
+        return { assertion, passed, actual: exists ? "exists" : "not found" };
+      }
+
+      case "file_contains": {
+        const target = join(workspacePath, assertion.target);
+        if (!existsSync(target)) {
+          return { assertion, passed: !!assertion.negated, actual: "file not found" };
+        }
+        const content = readFileSync(target, "utf-8");
+        const pattern = new RegExp(assertion.expected ?? "");
+        const matches = pattern.test(content);
+        const passed = assertion.negated ? !matches : matches;
+        return { assertion, passed, actual: matches ? "matched" : "no match" };
+      }
+
+      case "command_output":
+      case "skill_triggered":
+      case "custom": {
+        return {
+          assertion,
+          passed: false,
+          error: `Assertion type "${assertion.type}" not yet implemented`,
+        };
+      }
+
+      default:
+        return { assertion, passed: false, error: "Unknown assertion type" };
+    }
+  } catch (err) {
+    return { assertion, passed: false, error: String(err) };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Execution eval runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run all execution evals for a set of entries.
+ * Requires experimental opt-in (entries must have `experimental: true`).
+ */
+export async function runExecutionEvals(
+  entries: ExecutionEvalEntry[],
+  skillPath: string,
+  options?: { gateDeployment?: boolean },
+): Promise<{
+  results: ExecutionEvalResult[];
+  gate_passed: boolean;
+  summary: { total: number; passed: number; failed: number };
+}> {
+  const results: ExecutionEvalResult[] = [];
+
+  for (const entry of entries) {
+    const start = Date.now();
+    let workspace: string | undefined;
+
+    try {
+      if (entry.requires_workspace) {
+        workspace = createStagedWorkspace(skillPath);
+      }
+
+      const assertionResults = entry.assertions.map((a) => runAssertion(a, workspace ?? "."));
+
+      const passed = assertionResults.every((r) => r.passed);
+
+      results.push({
+        entry,
+        passed,
+        assertion_results: assertionResults,
+        workspace_path: workspace,
+        elapsed_ms: Date.now() - start,
+      });
+    } finally {
+      if (workspace) {
+        cleanupStagedWorkspace(workspace);
+      }
+    }
+  }
+
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.length - passed;
+  const gatePassed = !options?.gateDeployment || failed === 0;
+
+  return {
+    results,
+    gate_passed: gatePassed,
+    summary: { total: results.length, passed, failed },
+  };
+}

package/cli/selftune/eval/family-overlap.ts

@@ -438,7 +438,7 @@ function buildRefactorProposal(
     return {
       workflow_name: workflowName,
       source_skill: skillName,
-      suggested_path: `Workflows/${workflowName}.md`,
+      suggested_path: `workflows/${workflowName}.md`,
     };
   });
 
@@ -453,7 +453,7 @@ function buildRefactorProposal(
     migration_notes: [
       `Create a parent skill \`${parentSkillName}\` whose SKILL.md routes into internal workflows instead of exposing each family member as a primary top-level trigger surface.`,
       "Keep the existing sibling skills as thin compatibility aliases for at least one release cycle while usage shifts to the parent skill.",
-      "Move execution-specific instructions into internal Workflows/ or references/ files so the parent SKILL.md stays focused on routing and progressive disclosure.",
+      "Move execution-specific instructions into internal workflows/ or references/ files so the parent SKILL.md stays focused on routing and progressive disclosure.",
       "Use the compatibility aliases to measure whether trigger quality improves before removing the old skill entry points.",
     ],
   };