selftune 0.2.31 → 0.2.32

package/cli/selftune/verify.ts

@@ -0,0 +1,205 @@
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+
+import type { CreatePackageEvaluationResult } from "./create/package-evaluator.js";
+import { formatCreatePackageBenchmarkReport } from "./create/package-evaluator.js";
+import { runCreateReport } from "./create/report.js";
+import { computeCreateCheckResult, formatCreateCheckResult } from "./create/readiness.js";
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "./command-surface.js";
+import type { CreateCheckResult } from "./types.js";
+import { handleCLIError } from "./utils/cli-error.js";
+
+export interface VerifyResult {
+  skill: string;
+  skill_path: string;
+  readiness_state: CreateCheckResult["state"];
+  verified: boolean;
+  next_command: string | null;
+  readiness: CreateCheckResult;
+  report: CreatePackageEvaluationResult | null;
+}
+
+export interface RunVerifyOptions {
+  skillPath: string;
+  agent?: string;
+  evalSetPath?: string;
+  autoFix?: boolean;
+}
+
+export interface RunVerifyDeps {
+  computeCreateCheckResult?: typeof computeCreateCheckResult;
+  runCreateReport?: typeof runCreateReport;
+  runSelftuneSubCommand?: (command: string[]) => {
+    exitCode: number | null;
+    stdout: string;
+    stderr: string;
+  };
+}
+
+const MAX_AUTO_FIX_ITERATIONS = 4;
+
+function runSelftuneSubCommand(command: string[]): {
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+} {
+  const indexPath = join(import.meta.dir, "index.ts");
+  const result = Bun.spawnSync(["bun", "run", indexPath, ...command], {
+    stdout: "pipe",
+    stderr: "pipe",
+    env: process.env,
+  });
+  return {
+    exitCode: result.exitCode,
+    stdout: Buffer.from(result.stdout).toString("utf-8"),
+    stderr: Buffer.from(result.stderr).toString("utf-8"),
+  };
+}
+
+function withPublishRecommendation(
+  result: CreatePackageEvaluationResult,
+): CreatePackageEvaluationResult {
+  if (!result.summary.evaluation_passed) return result;
+  return {
+    ...result,
+    summary: {
+      ...result.summary,
+      next_command: `selftune publish --skill-path ${result.summary.skill_path}`,
+    },
+  };
+}
+
+function buildAutoFixCommand(
+  readiness: CreateCheckResult,
+  options: Pick<RunVerifyOptions, "evalSetPath">,
+): string[] | null {
+  switch (readiness.state) {
+    case "needs_evals":
+      return [
+        "eval",
+        "generate",
+        "--skill",
+        readiness.skill,
+        "--skill-path",
+        readiness.skill_path,
+        "--auto-synthetic",
+      ];
+    case "needs_unit_tests":
+      return [
+        "eval",
+        "unit-test",
+        "--skill",
+        readiness.skill,
+        "--generate",
+        ...(options.evalSetPath ? ["--eval-set", options.evalSetPath] : []),
+        "--skill-path",
+        readiness.skill_path,
+      ];
+    case "needs_routing_replay":
+      return ["create", "replay", "--skill-path", readiness.skill_path];
+    case "needs_baseline":
+      return ["create", "baseline", "--skill-path", readiness.skill_path];
+    default:
+      return null;
+  }
+}
+
+export async function runVerify(
+  options: RunVerifyOptions,
+  deps: RunVerifyDeps = {},
+): Promise<VerifyResult> {
+  const computeReadiness = deps.computeCreateCheckResult ?? computeCreateCheckResult;
+  const buildReport = deps.runCreateReport ?? runCreateReport;
+  const execSubCommand = deps.runSelftuneSubCommand ?? runSelftuneSubCommand;
+  let readiness = await computeReadiness(options.skillPath);
+  const autoFix = options.autoFix !== false;
+
+  if (!readiness.ok && autoFix) {
+    for (let i = 0; i < MAX_AUTO_FIX_ITERATIONS; i++) {
+      const command = buildAutoFixCommand(readiness, options);
+      if (!command) break;
+
+      process.stderr.write(`[verify] Auto-fixing: selftune ${command.join(" ")}\n`);
+
+      const result = execSubCommand(command);
+      if (result.exitCode !== 0) {
+        process.stderr.write(`[verify] Sub-command exited ${result.exitCode}, stopping auto-fix\n`);
+        break;
+      }
+
+      // eslint-disable-next-line no-await-in-loop -- each remediation step changes the next readiness state
+      readiness = await computeReadiness(options.skillPath);
+      if (readiness.ok) break;
+    }
+  }
+
+  if (!readiness.ok) {
+    return {
+      skill: readiness.skill,
+      skill_path: readiness.skill_path,
+      readiness_state: readiness.state,
+      verified: false,
+      next_command: readiness.next_command,
+      readiness,
+      report: null,
+    };
+  }
+
+  const report = withPublishRecommendation(
+    await buildReport({
+      skillPath: readiness.skill_path,
+      agent: options.agent,
+      evalSetPath: options.evalSetPath,
+    }),
+  );
+
+  return {
+    skill: readiness.skill,
+    skill_path: readiness.skill_path,
+    readiness_state: readiness.state,
+    verified: report.summary.evaluation_passed,
+    next_command: report.summary.next_command,
+    readiness,
+    report,
+  };
+}
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      "skill-path": { type: "string" },
+      agent: { type: "string" },
+      "eval-set": { type: "string" },
+      "no-auto-fix": { type: "boolean", default: false },
+      json: { type: "boolean", default: false },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(renderCommandHelp(PUBLIC_COMMAND_SURFACES.verify));
+    process.exit(0);
+  }
+
+  const result = await runVerify({
+    skillPath: values["skill-path"] ?? "",
+    agent: values.agent,
+    evalSetPath: values["eval-set"],
+    autoFix: !values["no-auto-fix"],
+  });
+
+  if (values.json || !process.stdout.isTTY) {
+    console.log(JSON.stringify(result, null, 2));
+  } else if (result.report) {
+    console.log(formatCreatePackageBenchmarkReport(result.report));
+  } else {
+    console.log(formatCreateCheckResult(result.readiness));
+  }
+
+  process.exit(result.verified ? 0 : 1);
+}
+
+if (import.meta.main) {
+  cliMain().catch(handleCLIError);
+}

package/cli/selftune/workflows/proposals.ts

@@ -129,7 +129,7 @@ export function discoverWorkflowSkillProposals(
 
     const summary = buildWorkflowProposalSummary(workflow, draft);
     const currentValue = `No dedicated workflow skill exists for ${workflow.skills.join(" -> ")}.`;
-    const proposedValue = `Create ${draft.skill_name} at ${draft.skill_path}`;
+    const proposedValue = `Create package ${draft.skill_name} at ${draft.skill_dir}`;
     const queryClause = workflow.representative_query.trim()
       ? ` Common trigger: "${workflow.representative_query.trim()}".`
       : "";

package/cli/selftune/workflows/skill-scaffold.ts

@@ -1,23 +1,20 @@
 /**
  * skill-scaffold.ts
  *
- * Builds draft workflow skills from repeated telemetry-discovered workflows.
- * The draft is preview-first by default so agents can review the scaffold before
- * writing it into a local skill registry.
+ * Builds draft workflow skill packages from repeated telemetry-discovered
+ * workflows. The draft is preview-first by default so agents can review the
+ * scaffold before writing it into a local skill registry.
  */
 
 import { join } from "node:path";
 
+import type { CreateSkillDraft } from "../create/templates.js";
+import { buildCreateSkillDraft } from "../create/templates.js";
 import type { DiscoveredWorkflow } from "../types.js";
 import { findGitRepositoryRoot } from "../utils/skill-discovery.js";
 
-export interface WorkflowSkillDraft {
+export interface WorkflowSkillDraft extends CreateSkillDraft {
   title: string;
-  skill_name: string;
-  description: string;
-  output_dir: string;
-  skill_dir: string;
-  skill_path: string;
   content: string;
   source_workflow: {
     workflow_id: string;
@@ -33,6 +30,7 @@ export interface WorkflowSkillDraftOptions {
   skillName?: string;
   description?: string;
   cwd?: string;
+  generatedBy?: string;
 }
 
 const STOPWORDS = new Set([
@@ -85,6 +83,11 @@ function deriveBaseLabel(workflow: DiscoveredWorkflow): string {
   return `${workflow.skills.join(" ")} workflow`;
 }
 
+export function getDefaultWorkflowSkillOutputDir(cwd: string = process.cwd()): string {
+  const repoRoot = findGitRepositoryRoot(cwd);
+  return join(repoRoot ?? cwd, ".agents", "skills");
+}
+
 function formatList(items: string[]): string {
   if (items.length === 0) return "";
   if (items.length === 1) return items[0];
@@ -92,30 +95,6 @@ function formatList(items: string[]): string {
   return `${items.slice(0, -1).join(", ")}, and ${items[items.length - 1]}`;
 }
 
-function wrapFoldedScalar(value: string, width = 78): string[] {
-  const words = value.split(/\s+/).filter(Boolean);
-  const lines: string[] = [];
-  let current = "";
-
-  for (const word of words) {
-    const candidate = current.length === 0 ? word : `${current} ${word}`;
-    if (candidate.length > width && current.length > 0) {
-      lines.push(`  ${current}`);
-      current = word;
-    } else {
-      current = candidate;
-    }
-  }
-
-  if (current.length > 0) lines.push(`  ${current}`);
-  return lines.length > 0 ? lines : ["  "];
-}
-
-export function getDefaultWorkflowSkillOutputDir(cwd: string = process.cwd()): string {
-  const repoRoot = findGitRepositoryRoot(cwd);
-  return join(repoRoot ?? cwd, ".agents", "skills");
-}
-
 export function buildWorkflowSkillDescription(
   workflow: DiscoveredWorkflow,
   override?: string,
@@ -131,16 +110,15 @@ export function buildWorkflowSkillDescription(
   return `Use when the task consistently needs ${chain} in sequence.`;
 }
 
-export function buildWorkflowSkillContent(
+function buildWorkflowSkillContent(
   workflow: DiscoveredWorkflow,
   title: string,
   skillName: string,
   description: string,
+  generatedBy: string,
 ): string {
-  const workflowName = title.endsWith("Workflow") ? title : `${title} Workflow`;
   const chain = workflow.skills.join(" → ");
   const query = workflow.representative_query.trim();
-  const foldedDescription = wrapFoldedScalar(description).join("\n");
 
   const whenToUseLines =
     query.length > 0
@@ -150,49 +128,121 @@ export function buildWorkflowSkillContent(
         ]
       : [`- The request repeatedly needs this skill chain: ${chain}`];
 
-  const executionPlanLines = workflow.skills.map(
-    (skill, index) =>
-      `${index + 1}. Invoke \`${skill}\` in its established role for this workflow.`,
-  );
-
   return `---
 name: ${skillName}
 description: >
-${foldedDescription}
+  ${description}
 metadata:
   author: selftune-autogen
   version: 0.1.0
-  category: developer-tools
-  generated_by: selftune workflows scaffold
+  category: custom
+  generated_by: ${generatedBy}
   source_workflow_id: ${workflow.workflow_id}
 ---
 
 # ${title}
 
-This draft skill was scaffolded by selftune from repeated workflow telemetry.
-Review the routing language and execution notes before broad distribution.
+This draft skill package was scaffolded by selftune from repeated workflow
+telemetry. Review the routing language and package contents before broad
+distribution.
 
 ## When to Use
 
 ${whenToUseLines.join("\n")}
 
-## Execution Plan
+## Workflow Routing
 
-${executionPlanLines.join("\n")}
+| Trigger | Workflow | File |
+| --- | --- | --- |
+| Default execution path | Default | workflows/default.md |
 
-## Workflows
+## Package Resources
 
-### ${workflowName}
-- **Skills:** ${chain}
-${query.length > 0 ? `- **Trigger:** ${query}\n` : ""}- **Source:** Discovered from ${workflow.occurrence_count} sessions (synergy: ${workflow.synergy_score.toFixed(2)})
+- \`workflows/default.md\` contains the ordered execution steps for the workflow.
+- \`references/overview.md\` preserves the observed provenance and trigger
+  evidence from telemetry.
+- \`selftune.create.json\` records the package-level readiness hints for future
+  validation.
 
 ## Notes
 
-- This is a proposal scaffold, not a silently published marketplace skill.
+- This is a proposal scaffold, not a silently published skill.
+- Source workflow: ${chain}
 - Add tighter scope boundaries and richer examples before publishing.
 `;
 }
 
+function buildWorkflowDefaultContent(workflow: DiscoveredWorkflow, title: string): string {
+  const query = workflow.representative_query.trim();
+  const chain = workflow.skills.join(" → ");
+  const executionPlanLines = workflow.skills.map(
+    (skill, index) =>
+      `${index + 1}. Invoke \`${skill}\` in its established role for this workflow.`,
+  );
+
+  return `# ${title} Default Workflow
+
+## When to Use
+
+Use this after \`../SKILL.md\` has already matched the request.${query.length > 0 ? ` The representative trigger was "${query}".` : ""}
+
+## Goal
+
+Coordinate this repeated multi-skill chain without making the top-level router
+carry all of the execution detail: ${chain}.
+
+## Steps
+
+${executionPlanLines.join("\n")}
+
+## Notes
+
+- Provenance lives in \`../references/overview.md\`.
+- Replace these placeholders with concrete execution mechanics before shipping.
+`;
+}
+
+function buildWorkflowReferenceContent(workflow: DiscoveredWorkflow, title: string): string {
+  const query = workflow.representative_query.trim();
+  const chain = workflow.skills.join(" → ");
+
+  return `# ${title} Overview
+
+This package was scaffolded from observed workflow telemetry.
+
+## Provenance
+
+- Workflow ID: ${workflow.workflow_id}
+- Skills: ${chain}
+- Observed sessions: ${workflow.occurrence_count}
+- Synergy score: ${workflow.synergy_score.toFixed(2)}
+- Sequence consistency: ${Math.round(workflow.sequence_consistency * 100)}%
+- Completion rate: ${Math.round(workflow.completion_rate * 100)}%
+${query.length > 0 ? `- Representative trigger: ${query}` : "- Representative trigger: not captured"}
+
+## Authoring Notes
+
+- Keep the router lean in \`../SKILL.md\`.
+- Move detailed steps into \`../workflows/default.md\`.
+- Expand this file with examples, vocabulary, and edge cases as the skill matures.
+`;
+}
+
+function replaceDraftFile(
+  draft: CreateSkillDraft,
+  relativePath: string,
+  content: string,
+): CreateSkillDraft["files"] {
+  return draft.files.map((file) =>
+    file.relative_path === relativePath
+      ? {
+          ...file,
+          content,
+        }
+      : file,
+  );
+}
+
 export function buildWorkflowSkillDraft(
   workflow: DiscoveredWorkflow,
   options: WorkflowSkillDraftOptions = {},
@@ -201,18 +251,38 @@ export function buildWorkflowSkillDraft(
   const skillName = slugifyWorkflowSkillName(baseLabel);
   const title = titleCase(baseLabel) || titleCase(`${workflow.skills.join(" ")} workflow`);
   const description = buildWorkflowSkillDescription(workflow, options.description);
-  const outputDir = options.outputDir?.trim() || getDefaultWorkflowSkillOutputDir(options.cwd);
-  const skillDir = join(outputDir, skillName);
-  const skillPath = join(skillDir, "SKILL.md");
+  const generatedBy = options.generatedBy?.trim() || "selftune workflows scaffold";
+  const baseDraft = buildCreateSkillDraft({
+    name: title,
+    description,
+    outputDir: options.outputDir,
+    cwd: options.cwd,
+  });
+  const skillContent = buildWorkflowSkillContent(
+    workflow,
+    title,
+    skillName,
+    description,
+    generatedBy,
+  );
+  const workflowContent = buildWorkflowDefaultContent(workflow, title);
+  const referenceContent = buildWorkflowReferenceContent(workflow, title);
+  const files = replaceDraftFile(baseDraft, "SKILL.md", skillContent).map((file) => {
+    if (file.relative_path === "workflows/default.md") {
+      return { ...file, content: workflowContent };
+    }
+    if (file.relative_path === "references/overview.md") {
+      return { ...file, content: referenceContent };
+    }
+    return file;
+  });
 
-  return {
+  const draft: WorkflowSkillDraft = {
+    ...baseDraft,
     title,
     skill_name: skillName,
-    description,
-    output_dir: outputDir,
-    skill_dir: skillDir,
-    skill_path: skillPath,
-    content: buildWorkflowSkillContent(workflow, title, skillName, description),
+    files,
+    content: "",
     source_workflow: {
       workflow_id: workflow.workflow_id,
       skills: workflow.skills,
@@ -221,6 +291,9 @@ export function buildWorkflowSkillDraft(
       representative_query: workflow.representative_query,
     },
   };
+
+  draft.content = formatWorkflowSkillDraft(draft);
+  return draft;
 }
 
 export function formatWorkflowSkillDraft(draft: WorkflowSkillDraft): string {
@@ -236,6 +309,11 @@ export function formatWorkflowSkillDraft(draft: WorkflowSkillDraft): string {
     lines.push(`Representative query: "${draft.source_workflow.representative_query.trim()}"`);
   }
 
-  lines.push("", draft.content.trimEnd());
+  lines.push("");
+  for (const file of draft.files) {
+    lines.push(`=== ${file.relative_path} ===`, file.content.trimEnd(), "");
+  }
+
+  lines.push("Empty directories:", "  - scripts/", "  - assets/");
   return lines.join("\n");
 }

package/cli/selftune/workflows/workflows.ts

@@ -8,9 +8,10 @@
  *  - cliMain()          (reads logs, discovers workflows, prints output or saves)
  */
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { parseArgs } from "node:util";
 
+import { writeCreateSkillDraft } from "../create/init.js";
 import { getDb } from "../localdb/db.js";
 import { querySessionTelemetry, querySkillUsageRecords } from "../localdb/queries.js";
 import type {
@@ -242,13 +243,12 @@ Options:
         );
       }
 
-      mkdirSync(draft.skill_dir, { recursive: true });
-      writeFileSync(draft.skill_path, draft.content, "utf-8");
+      writeCreateSkillDraft(draft, { force: values.force });
 
       if (values.json || !process.stdout.isTTY) {
         console.log(JSON.stringify({ ...draft, written: true }, null, 2));
       } else {
-        console.log(`Scaffolded skill "${draft.skill_name}" to ${draft.skill_path}`);
+        console.log(`Scaffolded skill package "${draft.skill_name}" to ${draft.skill_dir}`);
       }
       return;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "selftune",
-  "version": "0.2.31",
+  "version": "0.2.32",
   "description": "Skill-level observability and self-improvement for AI agents — monitors skill routing, detects missed triggers, and evolves descriptions automatically",
   "keywords": [
     "agent",