selftune 0.2.23 → 0.2.24

package/cli/selftune/localdb/schema.ts

@@ -231,6 +231,20 @@ CREATE TABLE IF NOT EXISTS grading_results (
   execution_metrics_json  TEXT
 )`;
 
+// -- Grading baselines table (pre/post deploy grade snapshots) ---------------
+
+export const CREATE_GRADING_BASELINES = `
+CREATE TABLE IF NOT EXISTS grading_baselines (
+  id                    INTEGER PRIMARY KEY AUTOINCREMENT,
+  skill_name            TEXT NOT NULL,
+  proposal_id           TEXT,
+  measured_at           TEXT NOT NULL,
+  pass_rate             REAL NOT NULL,
+  mean_score            REAL,
+  sample_size           INTEGER NOT NULL,
+  grading_results_json  TEXT
+)`;
+
 // -- Improvement signal table (from signal_log.jsonl) ------------------------
 
 export const CREATE_IMPROVEMENT_SIGNALS = `
@@ -369,6 +383,11 @@ export const CREATE_INDEXES = [
   `CREATE INDEX IF NOT EXISTS idx_grading_skill ON grading_results(skill_name)`,
   `CREATE INDEX IF NOT EXISTS idx_grading_ts ON grading_results(graded_at)`,
   `CREATE UNIQUE INDEX IF NOT EXISTS idx_grading_dedup ON grading_results(session_id, skill_name, graded_at)`,
+  // -- Grading baseline indexes ------------------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_grading_bl_skill ON grading_baselines(skill_name)`,
+  `CREATE INDEX IF NOT EXISTS idx_grading_bl_proposal ON grading_baselines(proposal_id)`,
+  `CREATE INDEX IF NOT EXISTS idx_grading_bl_ts ON grading_baselines(measured_at)`,
+  `CREATE INDEX IF NOT EXISTS idx_grading_bl_skill_proposal ON grading_baselines(skill_name, proposal_id, measured_at)`,
   // -- Improvement signal indexes ---------------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_signals_session ON improvement_signals(session_id)`,
   `CREATE INDEX IF NOT EXISTS idx_signals_consumed ON improvement_signals(consumed)`,
@@ -389,6 +408,7 @@ export const CREATE_INDEXES = [
   `CREATE INDEX IF NOT EXISTS idx_replay_entry_proposal ON replay_entry_results(proposal_id)`,
   `CREATE INDEX IF NOT EXISTS idx_replay_entry_skill ON replay_entry_results(skill_name)`,
   `CREATE INDEX IF NOT EXISTS idx_replay_entry_passed ON replay_entry_results(passed)`,
+  `CREATE INDEX IF NOT EXISTS idx_replay_entry_proposal_phase ON replay_entry_results(proposal_id, phase)`,
   // -- Commit tracking indexes ------------------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_commit_sha ON commit_tracking(commit_sha)`,
   `CREATE INDEX IF NOT EXISTS idx_commit_session ON commit_tracking(session_id)`,
@@ -485,6 +505,7 @@ export const ALL_DDL = [
   CREATE_ORCHESTRATE_RUNS,
   CREATE_QUERIES,
   CREATE_GRADING_RESULTS,
+  CREATE_GRADING_BASELINES,
   CREATE_IMPROVEMENT_SIGNALS,
   CREATE_UPLOAD_QUEUE,
   CREATE_CREATOR_CONTRIBUTION_STAGING,

package/cli/selftune/monitoring/watch.ts

@@ -8,12 +8,15 @@
 
 import { parseArgs } from "node:util";
 
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "../command-surface.js";
 import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
 import { classifyInvocation } from "../eval/hooks-to-evals.js";
 import { getLastDeployedProposal } from "../evolution/audit.js";
 import { getDb } from "../localdb/db.js";
 import {
+  queryGradingBaseline,
   queryQueryLog,
+  queryRecentGradingResults,
   querySessionTelemetry,
   querySkillUsageRecords,
 } from "../localdb/queries.js";
@@ -42,6 +45,10 @@ export interface WatchOptions {
   windowSessions: number;
   regressionThreshold: number;
   autoRollback: boolean;
+  /** Grade regression threshold (default 0.15). */
+  gradeRegressionThreshold?: number;
+  /** Enable grade-based regression watch (default true). */
+  enableGradeWatch?: boolean;
   /** Injected log paths for testing (override defaults). */
   _telemetryLogPath?: string;
   _skillLogPath?: string;
@@ -65,6 +72,8 @@ export interface WatchResult {
   rolledBack: boolean;
   recommendation: string;
   sync_result?: SyncResult;
+  gradeAlert?: string | null;
+  gradeRegression?: { before: number; after: number; delta: number } | null;
 }
 
 // ---------------------------------------------------------------------------
@@ -73,6 +82,7 @@ export interface WatchResult {
 
 const DEFAULT_BASELINE_PASS_RATE = 0.5;
 const DEFAULT_REGRESSION_THRESHOLD = 0.1;
+const DEFAULT_GRADE_REGRESSION_THRESHOLD = 0.15;
 export const MIN_MONITORING_SKILL_CHECKS = 3;
 
 // ---------------------------------------------------------------------------
@@ -190,6 +200,8 @@ export async function watch(options: WatchOptions): Promise<WatchResult> {
     skillPath,
     windowSessions = 20,
     regressionThreshold = DEFAULT_REGRESSION_THRESHOLD,
+    gradeRegressionThreshold = DEFAULT_GRADE_REGRESSION_THRESHOLD,
+    enableGradeWatch = true,
     autoRollback = false,
     _telemetryLogPath = TELEMETRY_LOG,
     _skillLogPath = SKILL_LOG,
@@ -235,26 +247,71 @@ export async function watch(options: WatchOptions): Promise<WatchResult> {
     regressionThreshold,
   );
 
-  // 4. Build alert and recommendation
-  let alert: string | null = null;
+  // 4. Build trigger alert. Grade alerts are added below before rollback
+  // decisions so either signal can drive automated rollback.
+  let triggerAlert: string | null = null;
   let rolledBack = false;
-  let recommendation: string;
 
   if (snapshot.regression_detected) {
-    alert = `regression detected for "${skillName}": pass_rate=${snapshot.pass_rate.toFixed(2)} below baseline=${baselinePassRate.toFixed(2)} minus threshold=${regressionThreshold.toFixed(2)}`;
-
-    // 5. Auto-rollback if enabled
-    if (autoRollback) {
-      const rollbackFn = _rollbackFn ?? (await loadRollbackFn());
-      const proposalId = lastDeployed?.proposal_id;
-      const rollbackResult = await rollbackFn({
-        skillName,
-        skillPath,
-        proposalId,
-      });
-      rolledBack = rollbackResult.rolledBack;
+    triggerAlert = `regression detected for "${skillName}": pass_rate=${snapshot.pass_rate.toFixed(2)} below baseline=${baselinePassRate.toFixed(2)} minus threshold=${regressionThreshold.toFixed(2)}`;
+  }
+
+  // 5. Grade regression detection (fail-open)
+  let gradeAlert: string | null = null;
+  let gradeRegression: { before: number; after: number; delta: number } | null = null;
+
+  if (enableGradeWatch) {
+    try {
+      const baseline = queryGradingBaseline(db, skillName, lastDeployed?.proposal_id);
+      const recentResults = queryRecentGradingResults(db, skillName, 10);
+
+      if (baseline && recentResults.length > 0) {
+        // Compute the average pass rate from recent grading results
+        const validResults = recentResults.filter((r) => r.pass_rate != null);
+        if (validResults.length > 0) {
+          const recentAvgPassRate =
+            validResults.reduce((sum, r) => sum + (r.pass_rate ?? 0), 0) / validResults.length;
+          const baselinePassRateGrade = baseline.pass_rate;
+          const delta = baselinePassRateGrade - recentAvgPassRate;
+
+          if (delta > gradeRegressionThreshold) {
+            gradeAlert = `grade regression detected for "${skillName}": baseline_grade_pass_rate=${baselinePassRateGrade.toFixed(2)}, recent_avg=${recentAvgPassRate.toFixed(2)}, delta=${delta.toFixed(2)} exceeds threshold=${gradeRegressionThreshold.toFixed(2)}`;
+            gradeRegression = {
+              before: baselinePassRateGrade,
+              after: recentAvgPassRate,
+              delta,
+            };
+          }
+        }
+      }
+    } catch (err) {
+      // Fail-open: grade watch should never block trigger monitoring
+      console.error(
+        JSON.stringify({
+          level: "debug",
+          code: "grade_watch_failed",
+          message: `Grade watch failed for "${skillName}": ${err instanceof Error ? err.message : String(err)}`,
+        }),
+      );
     }
+  }
 
+  const alerts = [triggerAlert, gradeAlert].filter((value): value is string => Boolean(value));
+  const alert = alerts.length > 0 ? alerts.join("\n") : null;
+
+  if (alert && autoRollback) {
+    const rollbackFn = _rollbackFn ?? (await loadRollbackFn());
+    const proposalId = lastDeployed?.proposal_id;
+    const rollbackResult = await rollbackFn({
+      skillName,
+      skillPath,
+      proposalId,
+    });
+    rolledBack = rollbackResult.rolledBack;
+  }
+
+  let recommendation: string;
+  if (alert) {
     recommendation = rolledBack
       ? `Rolled back "${skillName}" to previous version. Monitor to confirm recovery.`
       : `Consider running: selftune rollback --skill "${skillName}" --skill-path "${skillPath}"`;
@@ -285,6 +342,8 @@ export async function watch(options: WatchOptions): Promise<WatchResult> {
     alert,
     rolledBack,
     recommendation,
+    gradeAlert,
+    gradeRegression,
     ...(syncResult ? { sync_result: syncResult } : {}),
   };
 }
@@ -329,6 +388,8 @@ export async function cliMain(): Promise<void> {
       window: { type: "string", default: "20" },
       threshold: { type: "string", default: "0.1" },
       "auto-rollback": { type: "boolean", default: false },
+      "grade-threshold": { type: "string", default: "0.15" },
+      "no-grade-watch": { type: "boolean", default: false },
       "sync-first": { type: "boolean", default: false },
       "sync-force": { type: "boolean", default: false },
       help: { type: "boolean", default: false },
@@ -337,20 +398,7 @@ export async function cliMain(): Promise<void> {
   });
 
   if (values.help) {
-    console.log(`selftune watch — Monitor post-deploy skill health
-
-Usage:
-  selftune watch --skill <name> --skill-path <path> [options]
-
-Options:
-  --skill             Skill name (required)
-  --skill-path        Path to SKILL.md (required)
-  --window            Number of recent sessions to consider (default: 20)
-  --threshold         Regression threshold below baseline (default: 0.1)
-  --auto-rollback     Automatically rollback on regression detection
-  --sync-first        Refresh source-truth telemetry before reading watch inputs
-  --sync-force        Force a full rescan during --sync-first
-  --help              Show this help message`);
+    console.log(renderCommandHelp(PUBLIC_COMMAND_SURFACES.watch));
     process.exit(0);
   }
 
@@ -403,11 +451,30 @@ Options:
     );
   }
 
+  const rawGradeThreshold = values["grade-threshold"] ?? "0.15";
+  if (!/^\d+(\.\d+)?$/.test(rawGradeThreshold)) {
+    throw new CLIError(
+      "--grade-threshold must be a finite number between 0 and 1.",
+      "INVALID_FLAG",
+      "selftune watch --grade-threshold 0.15",
+    );
+  }
+  const gradeRegressionThreshold = Number.parseFloat(rawGradeThreshold);
+  if (gradeRegressionThreshold < 0 || gradeRegressionThreshold > 1) {
+    throw new CLIError(
+      "--grade-threshold must be a finite number between 0 and 1.",
+      "INVALID_FLAG",
+      "selftune watch --grade-threshold 0.15",
+    );
+  }
+
   const result = await watch({
     skillName: values.skill,
     skillPath: values["skill-path"],
     windowSessions,
     regressionThreshold,
+    gradeRegressionThreshold,
+    enableGradeWatch: !(values["no-grade-watch"] ?? false),
     autoRollback: values["auto-rollback"] ?? false,
     syncFirst: values["sync-first"] ?? false,
     syncForce: values["sync-force"] ?? false,

package/cli/selftune/normalization.ts

@@ -694,6 +694,7 @@ export function buildCanonicalSkillInvocation(
 }
 
 export interface BuildExecutionFactInput extends CanonicalBaseInput {
+  execution_fact_id?: string;
   occurred_at: string;
   prompt_id?: string;
   tool_calls_json: Record<string, number>;
@@ -716,6 +717,8 @@ export function buildCanonicalExecutionFact(
   const record: CanonicalExecutionFactRecord = {
     ...base,
     record_kind: "execution_fact",
+    execution_fact_id:
+      input.execution_fact_id ?? `${input.session_id}:${input.occurred_at}:execution_fact`,
     occurred_at: input.occurred_at,
     tool_calls_json: input.tool_calls_json,
     total_tool_calls: input.total_tool_calls,

package/cli/selftune/observability.ts

@@ -14,6 +14,7 @@ import { join } from "node:path";
 
 import { getAlphaGuidance } from "./agent-guidance.js";
 import { getAlphaLinkState, readAlphaIdentity } from "./alpha-identity.js";
+import { getSelftuneUpdateHint } from "./auto-update.js";
 import { LOG_DIR, REQUIRED_FIELDS, SELFTUNE_CONFIG_PATH } from "./constants.js";
 import { DB_PATH, getDb } from "./localdb/db.js";
 import type {
@@ -318,12 +319,13 @@ export async function checkVersionHealth(): Promise<HealthCheck[]> {
         if (cmp >= 0) {
           check.message = `v${currentVersion} (latest)`;
         } else {
+          const updateCommand = getSelftuneUpdateHint("latest");
           check.status = "warn";
-          check.message = `v${currentVersion} installed, v${latestVersion} available. Run: npx skills add selftune-dev/selftune`;
+          check.message = `v${currentVersion} installed, v${latestVersion} available. Run: ${updateCommand}`;
           check.guidance = {
             code: "version_update_available",
             message: "A newer selftune release is available.",
-            next_command: "npx skills add selftune-dev/selftune",
+            next_command: updateCommand,
             suggested_commands: ["selftune doctor"],
             blocking: false,
           };

package/cli/selftune/orchestrate/cli.ts

@@ -0,0 +1,161 @@
+import { parseArgs } from "node:util";
+
+import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "../command-surface.js";
+import type { OrchestrateOptions, OrchestrateResult } from "../orchestrate.js";
+import { CLIError } from "../utils/cli-error.js";
+
+export interface ParsedOrchestrateCliArgs {
+  showHelp: boolean;
+  warnings: string[];
+  loop: boolean;
+  loopIntervalSeconds: number;
+  runOptions: OrchestrateOptions;
+}
+
+function parsePositiveIntegerFlag(value: string, message: string, command: string): number {
+  if (!/^\d+$/.test(value) || Number(value) < 1) {
+    throw new CLIError(message, "INVALID_FLAG", command);
+  }
+  return Number(value);
+}
+
+function parseNonNegativeIntegerFlag(value: string, message: string, command: string): number {
+  if (!/^\d+$/.test(value)) {
+    throw new CLIError(message, "INVALID_FLAG", command);
+  }
+  return Number(value);
+}
+
+export function renderOrchestrateHelp(): string {
+  return renderCommandHelp(PUBLIC_COMMAND_SURFACES.orchestrate);
+}
+
+export function parseOrchestrateCliArgs(
+  argv: string[] = process.argv.slice(2),
+): ParsedOrchestrateCliArgs {
+  const { values } = parseArgs({
+    args: argv,
+    options: {
+      "dry-run": { type: "boolean", default: false },
+      "review-required": { type: "boolean", default: false },
+      "auto-approve": { type: "boolean", default: false },
+      skill: { type: "string" },
+      "max-skills": { type: "string", default: "5" },
+      "recent-window": { type: "string", default: "48" },
+      "sync-force": { type: "boolean", default: false },
+      "max-auto-grade": { type: "string", default: "5" },
+      loop: { type: "boolean", default: false },
+      "loop-interval": { type: "string", default: "3600" },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    return {
+      showHelp: true,
+      warnings: [],
+      loop: false,
+      loopIntervalSeconds: 3600,
+      runOptions: {
+        dryRun: false,
+        approvalMode: "auto",
+        maxSkills: 5,
+        recentWindowHours: 48,
+        syncForce: false,
+        maxAutoGrade: 5,
+      },
+    };
+  }
+
+  const loop = values.loop ?? false;
+  const maxSkills = parsePositiveIntegerFlag(
+    values["max-skills"] ?? "5",
+    "--max-skills must be a positive integer",
+    "selftune orchestrate --max-skills 5",
+  );
+  const recentWindowHours = parsePositiveIntegerFlag(
+    values["recent-window"] ?? "48",
+    "--recent-window must be a positive integer",
+    "selftune orchestrate --recent-window 48",
+  );
+  const maxAutoGrade = parseNonNegativeIntegerFlag(
+    values["max-auto-grade"] ?? "5",
+    "--max-auto-grade must be a non-negative integer",
+    "selftune orchestrate --max-auto-grade 5",
+  );
+
+  const loopIntervalRaw = values["loop-interval"] ?? "3600";
+  if (!/^\d+$/.test(loopIntervalRaw) || (loop && Number(loopIntervalRaw) < 60)) {
+    throw new CLIError(
+      "--loop-interval must be an integer >= 60 (seconds)",
+      "INVALID_FLAG",
+      "selftune orchestrate --loop --loop-interval 3600",
+    );
+  }
+
+  const warnings: string[] = [];
+  if (values["auto-approve"]) {
+    warnings.push(
+      "[orchestrate] --auto-approve is deprecated; autonomous mode is now the default.",
+    );
+  }
+
+  return {
+    showHelp: false,
+    warnings,
+    loop,
+    loopIntervalSeconds: Number(loopIntervalRaw),
+    runOptions: {
+      dryRun: values["dry-run"] ?? false,
+      approvalMode: values["review-required"] ? "review" : "auto",
+      skillFilter: values.skill,
+      maxSkills,
+      recentWindowHours,
+      syncForce: values["sync-force"] ?? false,
+      maxAutoGrade,
+    },
+  };
+}
+
+export function buildOrchestrateJsonOutput(result: OrchestrateResult) {
+  return {
+    ...result.summary,
+    ...(result.uploadSummary ? { upload: result.uploadSummary } : {}),
+    workflow_proposals: result.workflowProposals.map((proposal) => ({
+      proposal_id: proposal.proposal_id,
+      source_skill_name: proposal.source_skill_name,
+      workflow_id: proposal.workflow.workflow_id,
+      generated_skill_name: proposal.draft.skill_name,
+      output_path: proposal.draft.skill_path,
+      confidence: proposal.confidence,
+      reason: proposal.rationale,
+    })),
+    decisions: result.candidates.map((candidate) => ({
+      skill: candidate.skill,
+      action: candidate.action,
+      reason: candidate.reason,
+      ...(candidate.evolveResult
+        ? {
+            deployed: candidate.evolveResult.deployed,
+            evolveReason: candidate.evolveResult.reason,
+            validation: candidate.evolveResult.validation
+              ? {
+                  before: candidate.evolveResult.validation.before_pass_rate,
+                  after: candidate.evolveResult.validation.after_pass_rate,
+                  improved: candidate.evolveResult.validation.improved,
+                }
+              : null,
+          }
+        : {}),
+      ...(candidate.watchResult
+        ? {
+            alert: candidate.watchResult.alert,
+            rolledBack: candidate.watchResult.rolledBack,
+            passRate: candidate.watchResult.snapshot?.pass_rate ?? null,
+            recommendation: candidate.watchResult.recommendation,
+          }
+        : {}),
+    })),
+  };
+}