selftune 0.2.30 → 0.2.32

package/cli/selftune/create/replay.ts

@@ -0,0 +1,330 @@
+import { existsSync, readFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "../command-surface.js";
+import { parseSkillSections } from "../evolution/deploy-proposal.js";
+import {
+  buildRoutingReplayFixture,
+  resolveRuntimeReplayPlatform,
+  runHostRuntimeReplayFixture,
+  type RuntimeReplayInvoker,
+} from "../evolution/validate-host-replay.js";
+import { writeReplayEntryResultsToDb } from "../localdb/direct-write.js";
+import { getCanonicalEvalSetPath } from "../testing-readiness.js";
+import type {
+  EvalEntry,
+  ReplayStagingMode,
+  RoutingReplayEntryResult,
+  RuntimeReplayAggregateMetrics,
+} from "../types.js";
+import { isLlmBackedAgent, detectLlmAgent } from "../utils/llm-call.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
+import { readCreateSkillContext } from "./readiness.js";
+
+export type CreateReplayMode = ReplayStagingMode;
+
+export interface CreateReplayResult {
+  skill: string;
+  skill_path: string;
+  mode: CreateReplayMode;
+  agent: string;
+  proposal_id: string;
+  total: number;
+  passed: number;
+  failed: number;
+  pass_rate: number;
+  fixture_id: string;
+  results: RoutingReplayEntryResult[];
+  runtime_metrics: RuntimeReplayAggregateMetrics;
+}
+
+export interface RunCreateReplayOptions {
+  skillPath: string;
+  mode: CreateReplayMode;
+  agent?: string | null;
+  evalSetPath?: string;
+  includeTargetSkill?: boolean;
+  runtimeInvoker?: RuntimeReplayInvoker;
+}
+
+export function loadCreateEvalSet(skillName: string, explicitPath?: string): EvalEntry[] {
+  const path = explicitPath?.trim() || getCanonicalEvalSetPath(skillName);
+  if (!existsSync(path)) {
+    throw new CLIError(
+      `No canonical eval set found for "${skillName}" at ${path}.`,
+      "MISSING_DATA",
+      `Run selftune eval generate --skill ${skillName} --skill-path /path/to/${skillName}/SKILL.md --auto-synthetic`,
+    );
+  }
+
+  try {
+    const parsed = JSON.parse(readFileSync(path, "utf-8")) as unknown;
+    if (!Array.isArray(parsed)) {
+      throw new Error("expected a JSON array");
+    }
+    return parsed as EvalEntry[];
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new CLIError(
+      `Eval set at ${path} is invalid: ${message}`,
+      "INVALID_FLAG",
+      `Regenerate the eval set with selftune eval generate --skill ${skillName}`,
+    );
+  }
+}
+
+function resolveReplayAgent(requestedAgent?: string | null): string {
+  if (requestedAgent) {
+    if (!isLlmBackedAgent(requestedAgent)) {
+      throw new CLIError(
+        `Unsupported --agent value "${requestedAgent}".`,
+        "INVALID_FLAG",
+        "Use claude, codex, opencode, or pi.",
+      );
+    }
+    if (!Bun.which(requestedAgent)) {
+      throw new CLIError(
+        `Agent CLI '${requestedAgent}' not found in PATH`,
+        "AGENT_NOT_FOUND",
+        "Install it or omit --agent to use auto-detection",
+      );
+    }
+    return requestedAgent;
+  }
+
+  const detected = detectLlmAgent();
+  if (!detected) {
+    throw new CLIError(
+      "No supported runtime replay agent was found in PATH.",
+      "AGENT_NOT_FOUND",
+      "Install Claude Code, Codex, OpenCode, or Pi, or pass --agent explicitly.",
+    );
+  }
+  return detected;
+}
+
+function buildReplayContent(
+  skillContent: string,
+  mode: CreateReplayMode,
+): {
+  content: string;
+  contentTarget: "routing" | "body";
+} {
+  const parsed = parseSkillSections(skillContent);
+  if (mode === "routing") {
+    return {
+      content: parsed.sections["Workflow Routing"] ?? "",
+      contentTarget: "routing",
+    };
+  }
+
+  const bodyParts: string[] = [];
+  if (parsed.description.trim()) {
+    bodyParts.push(parsed.description.trim());
+  }
+  for (const [sectionName, sectionContent] of Object.entries(parsed.sections)) {
+    bodyParts.push(`## ${sectionName}`);
+    bodyParts.push("");
+    bodyParts.push(sectionContent.trim());
+    bodyParts.push("");
+  }
+
+  return {
+    content: bodyParts.join("\n").trim(),
+    contentTarget: "body",
+  };
+}
+
+function persistReplayResults(
+  proposalId: string,
+  skillName: string,
+  mode: CreateReplayMode,
+  results: RoutingReplayEntryResult[],
+): void {
+  writeReplayEntryResultsToDb(
+    results.map((result) => ({
+      proposal_id: proposalId,
+      skill_name: skillName,
+      validation_mode: "host_replay",
+      phase: `current_${mode}`,
+      query: result.query,
+      should_trigger: result.should_trigger,
+      triggered: result.triggered,
+      passed: result.passed,
+      evidence: result.evidence,
+    })),
+  );
+}
+
+function sumKnownMetric(values: Array<number | null | undefined>): {
+  total: number | null;
+  count: number;
+} {
+  let total = 0;
+  let count = 0;
+  for (const value of values) {
+    if (typeof value !== "number" || !Number.isFinite(value)) continue;
+    total += value;
+    count += 1;
+  }
+  return {
+    total: count > 0 ? total : null,
+    count,
+  };
+}
+
+export function summarizeReplayRuntimeMetrics(
+  results: RoutingReplayEntryResult[],
+): RuntimeReplayAggregateMetrics {
+  const evalRuns = results.length;
+  const totalDurationMs = results.reduce(
+    (sum, result) => sum + (result.runtime_metrics?.duration_ms ?? 0),
+    0,
+  );
+  const inputTokens = sumKnownMetric(results.map((result) => result.runtime_metrics?.input_tokens));
+  const outputTokens = sumKnownMetric(
+    results.map((result) => result.runtime_metrics?.output_tokens),
+  );
+  const cacheCreationTokens = sumKnownMetric(
+    results.map((result) => result.runtime_metrics?.cache_creation_input_tokens),
+  );
+  const cacheReadTokens = sumKnownMetric(
+    results.map((result) => result.runtime_metrics?.cache_read_input_tokens),
+  );
+  const totalCost = sumKnownMetric(results.map((result) => result.runtime_metrics?.total_cost_usd));
+  const totalTurns = sumKnownMetric(results.map((result) => result.runtime_metrics?.num_turns));
+  const usageObservations = results.filter((result) => {
+    const metrics = result.runtime_metrics;
+    return Boolean(
+      metrics &&
+      (metrics.input_tokens != null ||
+        metrics.output_tokens != null ||
+        metrics.total_cost_usd != null ||
+        metrics.num_turns != null),
+    );
+  }).length;
+
+  return {
+    eval_runs: evalRuns,
+    usage_observations: usageObservations,
+    total_duration_ms: totalDurationMs,
+    avg_duration_ms: evalRuns > 0 ? totalDurationMs / evalRuns : 0,
+    total_input_tokens: inputTokens.total,
+    total_output_tokens: outputTokens.total,
+    total_cache_creation_input_tokens: cacheCreationTokens.total,
+    total_cache_read_input_tokens: cacheReadTokens.total,
+    total_cost_usd: totalCost.total,
+    total_turns: totalTurns.total,
+  };
+}
+
+export async function runCreateReplay(
+  options: RunCreateReplayOptions,
+): Promise<CreateReplayResult> {
+  const context = readCreateSkillContext(options.skillPath);
+  const agent = resolveReplayAgent(options.agent);
+  const platform = resolveRuntimeReplayPlatform(agent);
+  if (!platform) {
+    throw new CLIError(
+      `Runtime replay is unavailable for agent "${agent}".`,
+      "REPLAY_UNAVAILABLE",
+      "Use claude, codex, or opencode for create replay.",
+    );
+  }
+
+  const evalSet = loadCreateEvalSet(context.skill_name, options.evalSetPath);
+  const { content, contentTarget } = buildReplayContent(context.skill_content, options.mode);
+  const fixture = buildRoutingReplayFixture({
+    skillName: context.skill_name,
+    skillPath: context.skill_path,
+    platform,
+    stagingMode: options.mode,
+  });
+
+  const results = await runHostRuntimeReplayFixture({
+    routing: content,
+    evalSet,
+    fixture,
+    contentTarget,
+    includeTargetSkill: options.includeTargetSkill,
+    runtimeInvoker: options.runtimeInvoker,
+  });
+
+  const passed = results.filter((result) => result.passed).length;
+  const total = results.length;
+  const proposalId = `create-replay-${context.skill_name}-${options.mode}-${Date.now()}`;
+  persistReplayResults(proposalId, context.skill_name, options.mode, results);
+  const runtimeMetrics = summarizeReplayRuntimeMetrics(results);
+
+  return {
+    skill: context.skill_name,
+    skill_path: context.skill_path,
+    mode: options.mode,
+    agent,
+    proposal_id: proposalId,
+    total,
+    passed,
+    failed: total - passed,
+    pass_rate: total > 0 ? passed / total : 0,
+    fixture_id: fixture.fixture_id,
+    results,
+    runtime_metrics: runtimeMetrics,
+  };
+}
+
+function formatReplayResult(result: CreateReplayResult): string {
+  return [
+    `Skill: ${result.skill}`,
+    `Mode: ${result.mode}`,
+    `Agent: ${result.agent}`,
+    `Pass rate: ${(result.pass_rate * 100).toFixed(1)}% (${result.passed}/${result.total})`,
+    `Replay record: ${result.proposal_id}`,
+  ].join("\n");
+}
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "skill-path": { type: "string" },
+      mode: { type: "string", default: "routing" },
+      agent: { type: "string" },
+      "eval-set": { type: "string" },
+      json: { type: "boolean", default: false },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(renderCommandHelp(PUBLIC_COMMAND_SURFACES.createReplay));
+    process.exit(0);
+  }
+
+  const mode = values.mode;
+  if (mode !== "routing" && mode !== "package") {
+    throw new CLIError(
+      `Unsupported --mode value "${mode}".`,
+      "INVALID_FLAG",
+      "Use --mode routing or --mode package.",
+    );
+  }
+
+  const result = await runCreateReplay({
+    skillPath: values["skill-path"] ?? "",
+    mode,
+    agent: values.agent,
+    evalSetPath: values["eval-set"],
+  });
+
+  if (values.json || !process.stdout.isTTY) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log(formatReplayResult(result));
+  }
+
+  process.exit(result.failed === 0 ? 0 : 1);
+}
+
+if (import.meta.main) {
+  cliMain().catch(handleCLIError);
+}

package/cli/selftune/create/report.ts

@@ -0,0 +1,74 @@
+import { parseArgs } from "node:util";
+
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "../command-surface.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
+import {
+  formatCreatePackageBenchmarkReport,
+  runCreatePackageEvaluation,
+  type CreatePackageEvaluationDeps,
+  type CreatePackageEvaluationResult,
+} from "./package-evaluator.js";
+
+export interface RunCreateReportOptions {
+  skillPath: string;
+  agent?: string;
+  evalSetPath?: string;
+}
+
+export async function runCreateReport(
+  options: RunCreateReportOptions,
+  deps: CreatePackageEvaluationDeps = {},
+): Promise<CreatePackageEvaluationResult> {
+  if (!options.skillPath.trim()) {
+    throw new CLIError(
+      "--skill-path <path> is required.",
+      "MISSING_FLAG",
+      "selftune create report --skill-path <path>",
+    );
+  }
+
+  return runCreatePackageEvaluation(
+    {
+      skillPath: options.skillPath,
+      agent: options.agent,
+      evalSetPath: options.evalSetPath,
+    },
+    deps,
+  );
+}
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "skill-path": { type: "string" },
+      agent: { type: "string" },
+      "eval-set": { type: "string" },
+      json: { type: "boolean", default: false },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(renderCommandHelp(PUBLIC_COMMAND_SURFACES.createReport));
+    process.exit(0);
+  }
+
+  const result = await runCreateReport({
+    skillPath: values["skill-path"] ?? "",
+    agent: values.agent,
+    evalSetPath: values["eval-set"],
+  });
+
+  if (values.json || !process.stdout.isTTY) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log(formatCreatePackageBenchmarkReport(result));
+  }
+
+  process.exit(result.summary.evaluation_passed ? 0 : 1);
+}
+
+if (import.meta.main) {
+  cliMain().catch(handleCLIError);
+}

package/cli/selftune/create/scaffold.ts

@@ -0,0 +1,121 @@
+import { existsSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "../command-surface.js";
+import { getDb } from "../localdb/db.js";
+import { querySessionTelemetry, querySkillUsageRecords } from "../localdb/queries.js";
+import type {
+  DiscoveredWorkflow,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+  WorkflowDiscoveryReport,
+} from "../types.js";
+import { CLIError, handleCLIError } from "../utils/cli-error.js";
+import { discoverWorkflows } from "../workflows/discover.js";
+import { buildWorkflowSkillDraft } from "../workflows/skill-scaffold.js";
+import { writeCreateSkillDraft } from "./init.js";
+
+function resolveWorkflowSelection(
+  report: WorkflowDiscoveryReport,
+  selection: string | undefined,
+): DiscoveredWorkflow {
+  if (!selection) {
+    throw new CLIError(
+      "--from-workflow <id|index> is required",
+      "MISSING_FLAG",
+      "selftune create scaffold --from-workflow <id|index>",
+    );
+  }
+
+  let workflow = report.workflows.find((candidate) => candidate.workflow_id === selection);
+  if (!workflow) {
+    const index = Number.parseInt(selection, 10);
+    if (!Number.isNaN(index) && index >= 1 && index <= report.workflows.length) {
+      workflow = report.workflows[index - 1];
+    }
+  }
+
+  if (!workflow) {
+    throw new CLIError(
+      `No workflow found matching "${selection}".`,
+      "INVALID_FLAG",
+      "Run 'selftune workflows' to inspect discovered workflows first.",
+    );
+  }
+
+  return workflow;
+}
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "from-workflow": { type: "string" },
+      "output-dir": { type: "string" },
+      "skill-name": { type: "string" },
+      description: { type: "string" },
+      write: { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+      json: { type: "boolean", default: false },
+      "min-occurrences": { type: "string" },
+      skill: { type: "string" },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(renderCommandHelp(PUBLIC_COMMAND_SURFACES.createScaffold));
+    process.exit(0);
+  }
+
+  const minOccurrences = values["min-occurrences"]
+    ? Number.parseInt(values["min-occurrences"], 10)
+    : undefined;
+  if (minOccurrences !== undefined && (Number.isNaN(minOccurrences) || minOccurrences < 0)) {
+    throw new CLIError("--min-occurrences must be a non-negative integer.", "INVALID_FLAG");
+  }
+
+  const db = getDb();
+  const telemetry = querySessionTelemetry(db) as SessionTelemetryRecord[];
+  const usage = querySkillUsageRecords(db) as SkillUsageRecord[];
+  const report = discoverWorkflows(telemetry, usage, {
+    minOccurrences,
+    skill: values.skill,
+  });
+  const workflow = resolveWorkflowSelection(report, values["from-workflow"]);
+  const draft = buildWorkflowSkillDraft(workflow, {
+    outputDir: values["output-dir"],
+    skillName: values["skill-name"],
+    description: values.description,
+    generatedBy: "selftune create scaffold",
+  });
+
+  if (values.write) {
+    const result = writeCreateSkillDraft(draft, { force: values.force });
+    if (values.json || !process.stdout.isTTY) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    console.log(
+      `Scaffolded skill package "${draft.skill_name}" to ${draft.skill_dir}${result.overwritten ? " (overwritten)" : ""}`,
+    );
+    return;
+  }
+
+  if (values.json || !process.stdout.isTTY) {
+    console.log(JSON.stringify({ ...draft, written: false }, null, 2));
+    return;
+  }
+
+  console.log(draft.content);
+  if (existsSync(draft.skill_dir)) {
+    console.log("");
+    console.log(
+      `[WARN] ${draft.skill_dir} already exists. Re-run with --write --force to overwrite.`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  cliMain().catch(handleCLIError);
+}

package/cli/selftune/create/skills-ref-adapter.ts

@@ -0,0 +1,177 @@
+import type { AgentSkillValidationIssue, AgentSkillValidationResult } from "../types.js";
+
+interface ValidatorCommand {
+  command: string;
+  argv: string[];
+}
+
+export interface ValidateAgentSkillDeps {
+  which?: (command: string) => string | null;
+  spawnSync?: typeof Bun.spawnSync;
+}
+
+const VALIDATOR_COMMANDS: readonly ValidatorCommand[] = [
+  {
+    command: "uvx --from skills-ref agentskills validate",
+    argv: ["uvx", "--from", "skills-ref", "agentskills", "validate"],
+  },
+  {
+    command: "uvx skills-ref validate",
+    argv: ["uvx", "skills-ref", "validate"],
+  },
+  {
+    command: "npx skills-ref validate",
+    argv: ["npx", "skills-ref", "validate"],
+  },
+] as const;
+
+function classifyIssueLevel(line: string): "error" | "warning" {
+  return /\bwarn(?:ing)?\b/i.test(line) ? "warning" : "error";
+}
+
+function normalizeIssues(
+  stdout: string,
+  stderr: string,
+  exitCode: number | null,
+): AgentSkillValidationIssue[] {
+  const merged = `${stderr}\n${stdout}`.trim();
+  if (!merged) {
+    return exitCode === 0
+      ? []
+      : [
+          {
+            level: "error",
+            code: "validation_failed",
+            message: `skills-ref exited with code ${exitCode ?? "unknown"}.`,
+          },
+        ];
+  }
+
+  const seen = new Set<string>();
+  const lines = merged
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .filter((line) => {
+      if (seen.has(line)) return false;
+      seen.add(line);
+      return true;
+    });
+
+  return lines.map((line, index) => ({
+    level: classifyIssueLevel(line),
+    code: `skills_ref_${index + 1}`,
+    message: line,
+  }));
+}
+
+function readSpawnText(output: unknown): string {
+  if (typeof output === "string") return output;
+  if (output == null) return "";
+  return Buffer.from(output as ArrayBufferLike).toString("utf-8");
+}
+
+function isValidatorInvocationFailure(
+  stdout: string,
+  stderr: string,
+  exitCode: number | null,
+): boolean {
+  if (exitCode === 0) return false;
+  const merged = `${stderr}\n${stdout}`.trim();
+  if (!merged) return true;
+
+  return [
+    /No such option/i,
+    /unknown command/i,
+    /unknown argument/i,
+    /unrecognized (argument|option)/i,
+    /usage:\s*(uvx|npx|skills-ref|agentskills)\b/i,
+    /command not found/i,
+    /not found in PATH/i,
+    /No such file or directory/i,
+    /Unable to locate executable/i,
+    /The executable [`'"]?agentskills[`'"]? was not found/i,
+    /No package .*skills-ref/i,
+    /failed to resolve/i,
+  ].some((pattern) => pattern.test(merged));
+}
+
+export async function validateAgentSkill(
+  skillDir: string,
+  deps: ValidateAgentSkillDeps = {},
+): Promise<AgentSkillValidationResult> {
+  const which = deps.which ?? ((command: string) => Bun.which(command));
+  const spawnSync = deps.spawnSync ?? Bun.spawnSync;
+
+  const candidates = VALIDATOR_COMMANDS.filter((option) => which(option.argv[0]) != null);
+  if (candidates.length === 0) {
+    return {
+      ok: false,
+      issues: [
+        {
+          level: "error",
+          code: "validator_unavailable",
+          message:
+            "No Agent Skills validator was found. Install uv/uvx or use npx so selftune can run skills-ref validate.",
+        },
+      ],
+      raw_stdout: "",
+      raw_stderr: "",
+      exit_code: null,
+      validator: "skills-ref",
+      command: null,
+    };
+  }
+
+  let lastFailure: AgentSkillValidationResult | null = null;
+
+  for (const candidate of candidates) {
+    const result = spawnSync([...candidate.argv, skillDir], {
+      cwd: skillDir,
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+
+    const stdout = readSpawnText(result.stdout);
+    const stderr = readSpawnText(result.stderr);
+    const exitCode = result.exitCode;
+    const issues = normalizeIssues(stdout, stderr, exitCode);
+    const response: AgentSkillValidationResult = {
+      ok: exitCode === 0,
+      issues: exitCode === 0 ? issues.filter((issue) => issue.level === "warning") : issues,
+      raw_stdout: stdout,
+      raw_stderr: stderr,
+      exit_code: exitCode,
+      validator: "skills-ref",
+      command: `${candidate.command} ${skillDir}`,
+    };
+
+    if (exitCode === 0) {
+      return response;
+    }
+
+    lastFailure = response;
+    if (!isValidatorInvocationFailure(stdout, stderr, exitCode)) {
+      return response;
+    }
+  }
+
+  return (
+    lastFailure ?? {
+      ok: false,
+      issues: [
+        {
+          level: "error",
+          code: "validation_failed",
+          message: "skills-ref validation failed for an unknown reason.",
+        },
+      ],
+      raw_stdout: "",
+      raw_stderr: "",
+      exit_code: null,
+      validator: "skills-ref",
+      command: null,
+    }
+  );
+}