selftune 0.2.31 → 0.2.32

package/cli/selftune/types.ts

@@ -745,6 +745,7 @@ export interface BodyEvolutionProposal {
 export type ValidationGate = "structural" | "trigger_accuracy" | "quality";
 
 export type ValidationMode = "structural_guard" | "host_replay" | "llm_judge";
+export type ReplayStagingMode = "routing" | "package";
 
 export interface RoutingReplayFixture {
   fixture_id: string;
@@ -753,6 +754,7 @@ export interface RoutingReplayFixture {
   target_skill_path: string;
   competing_skill_paths: string[];
   workspace_root?: string;
+  skill_staging_mode?: ReplayStagingMode;
 }
 
 export interface RoutingReplayEntryResult {
@@ -761,6 +763,30 @@ export interface RoutingReplayEntryResult {
   triggered: boolean;
   passed: boolean;
   evidence?: string;
+  runtime_metrics?: RuntimeReplayEntryMetrics;
+}
+
+export interface RuntimeReplayEntryMetrics {
+  input_tokens: number | null;
+  output_tokens: number | null;
+  cache_creation_input_tokens: number | null;
+  cache_read_input_tokens: number | null;
+  total_cost_usd: number | null;
+  duration_ms: number | null;
+  num_turns: number | null;
+}
+
+export interface RuntimeReplayAggregateMetrics {
+  eval_runs: number;
+  usage_observations: number;
+  total_duration_ms: number;
+  avg_duration_ms: number;
+  total_input_tokens: number | null;
+  total_output_tokens: number | null;
+  total_cache_creation_input_tokens: number | null;
+  total_cache_read_input_tokens: number | null;
+  total_cost_usd: number | null;
+  total_turns: number | null;
 }
 
 /** Result of validating a body evolution proposal. */
@@ -808,11 +834,193 @@ export interface BaselineResult {
   with_skill: boolean;
   triggered: boolean;
   pass: boolean;
+  evidence?: string;
   latency_ms?: number;
   tokens?: TokenUsageMetrics;
   measured_at: string;
 }
 
+export type CreatePackageEvaluationStatus = "passed" | "replay_failed" | "baseline_failed";
+
+export interface CreatePackageReplaySummary {
+  mode: ReplayStagingMode;
+  validation_mode: "host_replay";
+  agent: string;
+  proposal_id: string;
+  fixture_id: string;
+  total: number;
+  passed: number;
+  failed: number;
+  pass_rate: number;
+  runtime_metrics?: RuntimeReplayAggregateMetrics;
+}
+
+export interface CreatePackageBaselineSummary {
+  mode: ReplayStagingMode;
+  baseline_pass_rate: number;
+  with_skill_pass_rate: number;
+  lift: number;
+  adds_value: boolean;
+  measured_at: string;
+  sample_size?: number;
+  runtime_metrics?: {
+    with_skill: RuntimeReplayAggregateMetrics;
+    without_skill: RuntimeReplayAggregateMetrics;
+  };
+}
+
+export interface CreatePackageEvaluationEvidenceSample {
+  query: string;
+  evidence: string | null;
+}
+
+export interface CreatePackageEvaluationEvidenceSummary {
+  replay_failures: number;
+  baseline_wins: number;
+  baseline_regressions: number;
+  replay_failure_samples: CreatePackageEvaluationEvidenceSample[];
+  baseline_win_samples: CreatePackageEvaluationEvidenceSample[];
+  baseline_regression_samples: CreatePackageEvaluationEvidenceSample[];
+}
+
+export interface CreatePackageEvaluationEfficiencySummary {
+  with_skill: RuntimeReplayAggregateMetrics;
+  without_skill: RuntimeReplayAggregateMetrics;
+}
+
+export interface CreatePackageEvaluationWatchEfficiencyRegressionSummary {
+  sample_size: number;
+  baseline_avg_duration_ms: number | null;
+  observed_avg_duration_ms: number | null;
+  duration_delta_ratio: number | null;
+  baseline_avg_input_tokens: number | null;
+  observed_avg_input_tokens: number | null;
+  input_tokens_delta_ratio: number | null;
+  baseline_avg_output_tokens: number | null;
+  observed_avg_output_tokens: number | null;
+  output_tokens_delta_ratio: number | null;
+  baseline_avg_turns: number | null;
+  observed_avg_turns: number | null;
+  turns_delta_ratio: number | null;
+}
+
+export interface CreatePackageEvaluationWatchSummary {
+  snapshot: MonitoringSnapshot;
+  alert: string | null;
+  rolled_back: boolean;
+  recommendation: string;
+  recommended_command: string | null;
+  grade_alert: string | null;
+  grade_regression: { before: number; after: number; delta: number } | null;
+  efficiency_alert?: string | null;
+  efficiency_regression?: CreatePackageEvaluationWatchEfficiencyRegressionSummary | null;
+}
+
+export interface CreatePackageEvaluationGradingBaselineSummary {
+  proposal_id: string | null;
+  measured_at: string;
+  pass_rate: number;
+  mean_score: number | null;
+  sample_size: number;
+}
+
+export interface CreatePackageEvaluationGradingRecentSummary {
+  sample_size: number;
+  average_pass_rate: number | null;
+  average_mean_score: number | null;
+  newest_graded_at: string | null;
+  oldest_graded_at: string | null;
+}
+
+export interface CreatePackageEvaluationGradingSummary {
+  baseline: CreatePackageEvaluationGradingBaselineSummary | null;
+  recent: CreatePackageEvaluationGradingRecentSummary | null;
+  pass_rate_delta: number | null;
+  mean_score_delta: number | null;
+  regressed: boolean | null;
+}
+
+export interface CreatePackageEvaluationUnitTestFailureSummary {
+  test_id: string;
+  error: string | null;
+  failed_assertions: string[];
+}
+
+export interface CreatePackageEvaluationUnitTestSummary {
+  total: number;
+  passed: number;
+  failed: number;
+  pass_rate: number;
+  run_at: string;
+  failing_tests: CreatePackageEvaluationUnitTestFailureSummary[];
+}
+
+export interface CreatePackageBodySummary {
+  structural_valid: boolean;
+  structural_reason: string;
+  quality_score: number | null;
+  quality_reason: string | null;
+  quality_threshold: number;
+  quality_passed: boolean | null;
+  valid: boolean;
+}
+
+export type CreatePackageEvaluationSource = "fresh" | "artifact_cache" | "candidate_cache";
+export type CreatePackageCandidateAcceptanceDecision = "root" | "accepted" | "rejected";
+
+export interface CreatePackageCandidateAcceptanceSummary {
+  decision: CreatePackageCandidateAcceptanceDecision;
+  compared_to_candidate_id: string | null;
+  decided_at: string;
+  rationale: string;
+  replay_pass_rate_delta: number | null;
+  routing_pass_rate_delta: number | null;
+  baseline_lift_delta: number | null;
+  body_quality_delta: number | null;
+  unit_test_pass_rate_delta: number | null;
+}
+
+export interface CreatePackageEvaluationSummary {
+  skill_name: string;
+  skill_path: string;
+  mode: ReplayStagingMode;
+  package_fingerprint?: string;
+  candidate_id?: string;
+  parent_candidate_id?: string | null;
+  candidate_generation?: number | null;
+  evaluation_source?: CreatePackageEvaluationSource;
+  status: CreatePackageEvaluationStatus;
+  evaluation_passed: boolean;
+  next_command: string | null;
+  replay: CreatePackageReplaySummary;
+  routing?: CreatePackageReplaySummary;
+  baseline: CreatePackageBaselineSummary;
+  evidence?: CreatePackageEvaluationEvidenceSummary;
+  efficiency?: CreatePackageEvaluationEfficiencySummary;
+  grading?: CreatePackageEvaluationGradingSummary;
+  body?: CreatePackageBodySummary;
+  unit_tests?: CreatePackageEvaluationUnitTestSummary;
+  watch?: CreatePackageEvaluationWatchSummary;
+  candidate_acceptance?: CreatePackageCandidateAcceptanceSummary;
+}
+
+export interface CreatePackageCandidateRecord {
+  candidate_id: string;
+  skill_name: string;
+  skill_path: string;
+  package_fingerprint: string;
+  parent_candidate_id: string | null;
+  candidate_generation: number;
+  evaluation_count: number;
+  first_evaluated_at: string;
+  last_evaluated_at: string;
+  latest_status: CreatePackageEvaluationStatus;
+  latest_evaluation_source: CreatePackageEvaluationSource | null;
+  latest_acceptance_decision: CreatePackageCandidateAcceptanceDecision | null;
+  artifact_path: string | null;
+  summary: CreatePackageEvaluationSummary;
+}
+
 // ---------------------------------------------------------------------------
 // Skill unit test types
 // ---------------------------------------------------------------------------
@@ -863,6 +1071,78 @@ export interface UnitTestSuiteResult {
   run_at: string;
 }
 
+export interface AgentSkillValidationIssue {
+  level: "error" | "warning";
+  code: string;
+  message: string;
+  path?: string;
+}
+
+export interface AgentSkillValidationResult {
+  ok: boolean;
+  issues: AgentSkillValidationIssue[];
+  raw_stdout: string;
+  raw_stderr: string;
+  exit_code: number | null;
+  validator: "skills-ref";
+  command: string | null;
+}
+
+export type CreateCheckState =
+  | "blocked_spec_validation"
+  | "needs_spec_validation"
+  | "needs_package_resources"
+  | "needs_evals"
+  | "needs_unit_tests"
+  | "needs_routing_replay"
+  | "needs_baseline"
+  | "ready_to_publish";
+
+export interface CreateCheckChecks {
+  skill_md: boolean;
+  frontmatter_present: boolean;
+  skill_name_matches_dir: boolean;
+  description_present: boolean;
+  description_within_budget: boolean;
+  skill_md_within_line_budget: boolean;
+  manifest_present: boolean;
+  workflow_entry: boolean;
+  references_present: boolean;
+  scripts_present: boolean;
+  assets_present: boolean;
+  evals_present: boolean;
+  unit_tests_present: boolean;
+  routing_replay_ready: boolean;
+  routing_replay_recorded: boolean;
+  package_replay_ready: boolean;
+  baseline_present: boolean;
+}
+
+export interface CreateCheckReadiness {
+  ok: boolean;
+  state: CreateCheckState;
+  summary: string;
+  next_command: string | null;
+  checks: CreateCheckChecks;
+  skill_name: string;
+  skill_dir: string;
+  skill_path: string;
+  entry_workflow: string;
+  manifest_present: boolean;
+  description_quality: DescriptionQualityScore;
+}
+
+export interface CreateCheckResult {
+  skill: string;
+  skill_dir: string;
+  skill_path: string;
+  ok: boolean;
+  state: CreateCheckState;
+  next_command: string | null;
+  spec_validation: AgentSkillValidationResult;
+  readiness: CreateCheckReadiness;
+}
+
 // ---------------------------------------------------------------------------
 // Composability types
 // ---------------------------------------------------------------------------
@@ -1034,3 +1314,39 @@ export interface WorkflowDiscoveryReport {
   total_sessions_analyzed: number;
   generated_at: string;
 }
+
+// ---------------------------------------------------------------------------
+// Package search types (bounded package evolution)
+// ---------------------------------------------------------------------------
+
+/** Provenance trail for a package search run. */
+export interface PackageSearchProvenance {
+  frontier_size: number;
+  parent_selection_method: string;
+  candidate_fingerprints: string[];
+  surface_plan?: {
+    routing_count: number;
+    body_count: number;
+    weakness_source: string;
+    routing_weakness: number | null;
+    body_weakness: number | null;
+  };
+  evaluation_summaries: Array<{
+    candidate_id: string;
+    decision: string;
+    rationale: string;
+  }>;
+}
+
+/** Result of a bounded package search run. */
+export interface PackageSearchRunResult {
+  search_id: string;
+  skill_name: string;
+  parent_candidate_id: string | null;
+  candidates_evaluated: number;
+  winner_candidate_id: string | null;
+  winner_rationale: string | null;
+  started_at: string;
+  completed_at: string;
+  provenance: PackageSearchProvenance;
+}

package/cli/selftune/utils/eval-readiness.ts

@@ -0,0 +1 @@
+export const MIN_LOG_READY_POSITIVES = 3;

package/cli/selftune/utils/json-output.ts

@@ -0,0 +1,11 @@
+export function extractJsonObject(text: string): Record<string, unknown> | null {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null;
+
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
+}

package/cli/selftune/utils/lifecycle-surface.ts

@@ -0,0 +1,48 @@
+export function normalizeLifecycleCommand(command: string | null | undefined): string | null {
+  if (!command) return null;
+
+  let normalized = command;
+  normalized = normalized.replace(/\bselftune create replay\b/g, "selftune verify");
+  normalized = normalized.replace(/\bselftune create baseline\b/g, "selftune verify");
+  normalized = normalized.replace(/\bselftune create check\b/g, "selftune verify");
+  normalized = normalized.replace(/\bselftune create publish\b/g, "selftune publish");
+  normalized = normalized.replace(/\bselftune evolve[- ]body\b/g, "selftune improve --scope body");
+  normalized = normalized.replace(/\bselftune evolve\b/g, "selftune improve");
+  normalized = normalized.replace(/\bselftune search-run\b/g, "selftune improve --scope package");
+  normalized = normalized.replace(/\bselftune orchestrate\b/g, "selftune run");
+  normalized = normalized.replace(/\s+--watch(?=\s|$)/g, "");
+  normalized = normalized.replace(/\s{2,}/g, " ").trim();
+
+  return normalized;
+}
+
+export function normalizeLifecycleText(text: string | null | undefined): string {
+  if (!text) return "";
+
+  return text
+    .replace(/\bRun create replay\b/g, "Run verify")
+    .replace(/\brun create replay\b/g, "run verify")
+    .replace(/\bcreate replay\b/g, "verify")
+    .replace(/\bCreate replay\b/g, "Verify")
+    .replace(/\bRun create baseline\b/g, "Run verify")
+    .replace(/\brun create baseline\b/g, "run verify")
+    .replace(/\bcreate baseline\b/g, "verify")
+    .replace(/\bCreate baseline\b/g, "Verify")
+    .replace(/\bRun create check\b/g, "Run verify")
+    .replace(/\brun create check\b/g, "run verify")
+    .replace(/\bcreate check\b/g, "verify")
+    .replace(/\bCreate check\b/g, "Verify")
+    .replace(/\bRun create publish\b/g, "Run publish")
+    .replace(/\brun create publish\b/g, "run publish")
+    .replace(/\bcreate publish\b/g, "publish")
+    .replace(/\bCreate publish\b/g, "Publish")
+    .replace(/\bevolve body\b/g, "improve --scope body")
+    .replace(/\bEvolve body\b/g, "Improve --scope body")
+    .replace(/\bevolve\b/g, "improve")
+    .replace(/\bEvolve\b/g, "Improve")
+    .replace(/\bsearch-run\b/g, "improve --scope package")
+    .replace(/\bSearch-run\b/g, "Improve --scope package")
+    .replace(/\bselftune orchestrate\b/g, "selftune run")
+    .replace(/\bOrchestrate\b/g, "Run")
+    .replace(/\borchestrate\b/g, "run");
+}

package/cli/selftune/utils/query-filter.ts

@@ -53,6 +53,42 @@ const LEADING_WRAPPED_QUERY_TAGS = [
   "local-command-stdout",
   "local-command-stderr",
   "command-name",
+  "command-message",
+  "command-args",
+] as const;
+
+const SKILL_MAINTENANCE_VERBS = [
+  "grade",
+  "review",
+  "audit",
+  "inspect",
+  "analyze",
+  "analyse",
+  "understand",
+  "explain",
+  "find",
+  "locate",
+  "update",
+  "fix",
+  "repair",
+  "improve",
+  "debug",
+  "document",
+  "publish api",
+] as const;
+
+const SKILL_MAINTENANCE_NOUNS = [
+  "skill",
+  "skills",
+  "readme",
+  "docs",
+  "documentation",
+  "workflow",
+  "workflows",
+  "reference",
+  "references",
+  "files",
+  "format",
 ] as const;
 
 function stripLeadingWrappedQueryText(query: string): string {
@@ -81,7 +117,7 @@ export function extractActionableQueryText(query: string): string | null {
   const trimmed = query.trim();
   if (!trimmed || trimmed === "-" || trimmed === "(query not found)") return null;
 
-  const candidate = stripLeadingWrappedQueryText(trimmed) || trimmed;
+  const candidate = stripLeadingWrappedQueryText(trimmed);
   if (!candidate || candidate === "-" || candidate === "(query not found)") return null;
 
   const isBlocked =
@@ -92,6 +128,51 @@ export function extractActionableQueryText(query: string): string | null {
   return isBlocked ? null : candidate;
 }
 
+function normalizeSkillNameVariants(skillName: string): string[] {
+  const trimmed = skillName.trim();
+  if (!trimmed) return [];
+
+  const variants = new Set<string>();
+  const lower = trimmed.toLowerCase();
+  variants.add(lower);
+  variants.add(lower.replace(/[-_]+/g, " "));
+  variants.add(lower.replace(/[-_\s]+/g, ""));
+  variants.add(
+    trimmed
+      .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+      .replace(/[-_]+/g, " ")
+      .toLowerCase(),
+  );
+
+  return [...variants].filter(Boolean);
+}
+
+export function isLikelySkillMaintenanceQuery(query: string, skillName?: string): boolean {
+  const candidate = extractActionableQueryText(query);
+  if (!candidate) return false;
+
+  const lowered = candidate.toLowerCase().replace(/\s+/g, " ").trim();
+  const mentionsMaintenanceVerb = SKILL_MAINTENANCE_VERBS.some((verb) => lowered.includes(verb));
+  const mentionsMaintenanceNoun = SKILL_MAINTENANCE_NOUNS.some((noun) => lowered.includes(noun));
+  const mentionsHowItWorks = /\bhow\b[\s\S]{0,80}\bworks?\b/.test(lowered);
+  const mentionsSkillName = skillName
+    ? normalizeSkillNameVariants(skillName).some(
+        (variant) => variant.length > 0 && lowered.includes(variant),
+      )
+    : false;
+
+  if (mentionsHowItWorks && mentionsSkillName) return true;
+  if (mentionsMaintenanceVerb && mentionsMaintenanceNoun) return true;
+  if (mentionsMaintenanceVerb && mentionsSkillName) return true;
+  return false;
+}
+
+export function extractPositiveEvalQueryText(query: string, skillName?: string): string | null {
+  const candidate = extractActionableQueryText(query);
+  if (!candidate) return null;
+  return isLikelySkillMaintenanceQuery(candidate, skillName) ? null : candidate;
+}
+
 export function isActionableQueryText(query: string): boolean {
   return extractActionableQueryText(query) !== null;
 }

package/cli/selftune/utils/tui.ts

@@ -26,13 +26,96 @@ function createNoopTUI(): EvolveTUI {
   return { step() {}, done() {}, fail() {}, finish() {}, destroy() {} };
 }
 
+function createPlainTextTUI(opts: { skillName: string; model: string }): EvolveTUI {
+  const write = (s: string) => process.stderr.write(s);
+  let stepStartTime = Date.now();
+  let currentLabel = "";
+  let hasActiveStep = false;
+  let destroyed = false;
+
+  const checkMark = process.env.NO_COLOR ? "+" : "\u2713";
+  const crossMark = process.env.NO_COLOR ? "x" : "\u2717";
+
+  write(`\n  selftune evolve \u2500\u2500 ${opts.skillName} \u2500\u2500 ${opts.model}\n\n`);
+
+  function formatTime(ms: number): string {
+    return `${(ms / 1000).toFixed(1)}s`;
+  }
+
+  function writeStartedLine(label: string): void {
+    write(`  -> ${label}\n`);
+  }
+
+  function writeCompletedLine(marker: string, label: string, elapsed: number): void {
+    const time = formatTime(elapsed);
+    const padding = Math.max(1, 48 - label.length);
+    write(`  ${marker} ${label}${" ".repeat(padding)}${time}\n`);
+  }
+
+  function completeCurrentStep(marker: string, label: string): void {
+    const elapsed = Date.now() - stepStartTime;
+    hasActiveStep = false;
+    writeCompletedLine(marker, label, elapsed);
+  }
+
+  return {
+    step(label: string): void {
+      if (destroyed) return;
+      if (hasActiveStep) {
+        completeCurrentStep(checkMark, currentLabel);
+      }
+      currentLabel = label;
+      stepStartTime = Date.now();
+      hasActiveStep = true;
+      writeStartedLine(label);
+    },
+
+    done(label: string): void {
+      if (destroyed) return;
+      if (hasActiveStep) {
+        completeCurrentStep(checkMark, label);
+      } else {
+        writeCompletedLine(checkMark, label, 0);
+      }
+      currentLabel = "";
+    },
+
+    fail(label: string): void {
+      if (destroyed) return;
+      if (hasActiveStep) {
+        completeCurrentStep(crossMark, label);
+      } else {
+        writeCompletedLine(crossMark, label, 0);
+      }
+      currentLabel = "";
+    },
+
+    finish(summary: string): void {
+      if (destroyed) return;
+      if (hasActiveStep) {
+        completeCurrentStep(checkMark, currentLabel);
+      }
+      write(`\n  ${summary}\n`);
+      destroyed = true;
+    },
+
+    destroy(): void {
+      destroyed = true;
+      hasActiveStep = false;
+      currentLabel = "";
+    },
+  };
+}
+
 export function createEvolveTUI(opts: { skillName: string; model: string }): EvolveTUI {
   const noColor = !!process.env.NO_COLOR;
   const isTTY = !!process.stderr.isTTY;
+  const isTestEnvironment = process.env.BUN_ENV?.includes("test");
 
-  // If not a TTY, return no-op to avoid ANSI noise in pipes/tests
+  // Non-interactive agent runs still need durable progress lines. Keep tests
+  // silent by default unless explicitly forced.
   if (!isTTY && !process.env.SELFTUNE_TUI_FORCE) {
-    return createNoopTUI();
+    return isTestEnvironment ? createNoopTUI() : createPlainTextTUI(opts);
   }
 
   const write = (s: string) => process.stderr.write(s);