selftune 0.2.29 → 0.2.31

package/cli/selftune/dashboard-action-events.ts

@@ -0,0 +1,117 @@
+import { appendFileSync, existsSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { DASHBOARD_ACTION_STREAM_LOG } from "./constants.js";
+import type {
+  DashboardActionEvent,
+  DashboardActionMetrics,
+  DashboardActionName,
+  DashboardActionProgress,
+} from "./dashboard-contract.js";
+
+const ACTION_EVENT_ID_ENV = "SELFTUNE_DASHBOARD_ACTION_EVENT_ID";
+const ACTION_NAME_ENV = "SELFTUNE_DASHBOARD_ACTION_NAME";
+const ACTION_SKILL_NAME_ENV = "SELFTUNE_DASHBOARD_ACTION_SKILL_NAME";
+const ACTION_SKILL_PATH_ENV = "SELFTUNE_DASHBOARD_ACTION_SKILL_PATH";
+
+export interface DashboardActionContext {
+  eventId: string;
+  action: DashboardActionName;
+  skillName: string | null;
+  skillPath: string | null;
+}
+
+let currentContext: DashboardActionContext | null = null;
+
+function appendDashboardActionEvent(event: DashboardActionEvent): void {
+  try {
+    const path = process.env.SELFTUNE_DASHBOARD_ACTION_STREAM_LOG || DASHBOARD_ACTION_STREAM_LOG;
+    const parent = dirname(path);
+    if (!existsSync(parent)) {
+      mkdirSync(parent, { recursive: true });
+    }
+    appendFileSync(path, `${JSON.stringify(event)}\n`, "utf-8");
+  } catch {
+    // fail-open: dashboard instrumentation must never block the real CLI
+  }
+}
+
+function isDashboardActionName(value: string | undefined): value is DashboardActionName {
+  return (
+    value === "generate-evals" ||
+    value === "generate-unit-tests" ||
+    value === "replay-dry-run" ||
+    value === "measure-baseline" ||
+    value === "deploy-candidate" ||
+    value === "watch" ||
+    value === "orchestrate" ||
+    value === "rollback"
+  );
+}
+
+function readContextFromEnv(): DashboardActionContext | null {
+  const eventId = process.env[ACTION_EVENT_ID_ENV];
+  const action = process.env[ACTION_NAME_ENV];
+  if (!eventId || !action || !isDashboardActionName(action)) return null;
+
+  return {
+    eventId,
+    action,
+    skillName: process.env[ACTION_SKILL_NAME_ENV] ?? null,
+    skillPath: process.env[ACTION_SKILL_PATH_ENV] ?? null,
+  };
+}
+
+export function setCurrentDashboardActionContext(context: DashboardActionContext | null): void {
+  currentContext = context;
+}
+
+export function getCurrentDashboardActionContext(): DashboardActionContext | null {
+  return currentContext ?? readContextFromEnv();
+}
+
+export function dashboardActionContextEnv(
+  context: DashboardActionContext | null,
+): Record<string, string> {
+  if (!context) return {};
+
+  const env: Record<string, string> = {
+    [ACTION_EVENT_ID_ENV]: context.eventId,
+    [ACTION_NAME_ENV]: context.action,
+  };
+  if (context.skillName) env[ACTION_SKILL_NAME_ENV] = context.skillName;
+  if (context.skillPath) env[ACTION_SKILL_PATH_ENV] = context.skillPath;
+  return env;
+}
+
+export function emitDashboardActionMetrics(metrics: DashboardActionMetrics): void {
+  const context = getCurrentDashboardActionContext();
+  if (!context) return;
+
+  appendDashboardActionEvent({
+    event_id: context.eventId,
+    action: context.action,
+    stage: "metrics",
+    skill_name: context.skillName,
+    skill_path: context.skillPath,
+    ts: Date.now(),
+    metrics,
+  });
+}
+
+export function emitDashboardActionProgress(progress: DashboardActionProgress): void {
+  const context = getCurrentDashboardActionContext();
+  if (!context) return;
+
+  appendDashboardActionEvent({
+    event_id: context.eventId,
+    action: context.action,
+    stage: "progress",
+    skill_name: context.skillName,
+    skill_path: context.skillPath,
+    ts: Date.now(),
+    progress,
+  });
+}
+
+export { appendDashboardActionEvent };

package/cli/selftune/dashboard-action-instrumentation.ts

@@ -0,0 +1,103 @@
+import {
+  emitDashboardActionMetrics,
+  emitDashboardActionProgress,
+} from "./dashboard-action-events.js";
+import type {
+  DashboardActionMetrics,
+  DashboardActionProgress,
+  DashboardActionProgressUnit,
+} from "./dashboard-contract.js";
+import type { LlmCallObserver, LlmCallLifecycleEvent } from "./utils/llm-call.js";
+
+export interface DashboardStepProgressOptions {
+  current: number;
+  total: number;
+  status: DashboardActionProgress["status"];
+  unit?: DashboardActionProgressUnit;
+  phase?: string | null;
+  label?: string | null;
+  query?: string | null;
+  passed?: boolean | null;
+  evidence?: string | null;
+}
+
+export interface DashboardLlmObserverOptions {
+  current: number;
+  total: number;
+  phase: string;
+  label: string;
+}
+
+function buildRuntimeMetrics(event: LlmCallLifecycleEvent): DashboardActionMetrics {
+  return {
+    platform: event.platform,
+    model: event.model,
+    session_id: null,
+    input_tokens: null,
+    output_tokens: null,
+    cache_creation_input_tokens: null,
+    cache_read_input_tokens: null,
+    total_cost_usd: null,
+    duration_ms: event.durationMs,
+    num_turns: null,
+  };
+}
+
+function describeInvocation(event: LlmCallLifecycleEvent): string {
+  const parts = [event.platform, event.model].filter(Boolean);
+  return parts.length > 0 ? parts.join(" · ") : "runtime invoked";
+}
+
+function describeCompletion(event: LlmCallLifecycleEvent): string {
+  const durationText =
+    event.durationMs != null ? `${(event.durationMs / 1000).toFixed(1)}s` : "completed";
+  if (event.success === false && event.error) {
+    return event.error;
+  }
+  return `${describeInvocation(event)} · ${durationText}`;
+}
+
+export function emitDashboardStepProgress(options: DashboardStepProgressOptions): void {
+  emitDashboardActionProgress({
+    current: options.current,
+    total: options.total,
+    status: options.status,
+    unit: options.unit ?? "step",
+    phase: options.phase ?? null,
+    label: options.label ?? null,
+    query: options.query ?? null,
+    passed: options.passed ?? null,
+    evidence: options.evidence ?? null,
+  });
+}
+
+export function createDashboardLlmObserver(options: DashboardLlmObserverOptions): LlmCallObserver {
+  return {
+    onStart(event) {
+      emitDashboardActionMetrics(buildRuntimeMetrics(event));
+      emitDashboardStepProgress({
+        current: options.current,
+        total: options.total,
+        status: "started",
+        unit: "llm_call",
+        phase: options.phase,
+        label: options.label,
+        passed: null,
+        evidence: describeInvocation(event),
+      });
+    },
+    onFinish(event) {
+      emitDashboardActionMetrics(buildRuntimeMetrics(event));
+      emitDashboardStepProgress({
+        current: options.current,
+        total: options.total,
+        status: "finished",
+        unit: "llm_call",
+        phase: options.phase,
+        label: options.label,
+        passed: event.success,
+        evidence: describeCompletion(event),
+      });
+    },
+  };
+}

package/cli/selftune/dashboard-action-result.ts

@@ -0,0 +1,90 @@
+import type { DashboardActionName, DashboardActionResultSummary } from "./dashboard-contract.js";
+
+export interface DashboardActionOutcomeInput {
+  action: DashboardActionName;
+  stdout: string;
+  stderr: string | null;
+  exitCode: number | null;
+}
+
+export interface DashboardActionOutcome {
+  success: boolean;
+  error: string | null;
+  summary: DashboardActionResultSummary | null;
+}
+
+function extractJsonObject(stdout: string): Record<string, unknown> | null {
+  const trimmed = stdout.trim();
+  if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null;
+
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    return parsed && typeof parsed === "object" ? (parsed as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
+}
+
+function readBoolean(value: unknown): boolean | null {
+  return typeof value === "boolean" ? value : null;
+}
+
+function readNumber(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function readString(value: unknown): string | null {
+  return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+export function extractDashboardActionSummary(
+  action: DashboardActionName,
+  stdout: string,
+): DashboardActionResultSummary | null {
+  if (action !== "replay-dry-run") return null;
+
+  const parsed = extractJsonObject(stdout);
+  if (!parsed) return null;
+
+  return {
+    reason: readString(parsed["reason"]),
+    improved: readBoolean(parsed["improved"]),
+    deployed: readBoolean(parsed["deployed"]),
+    before_pass_rate: readNumber(parsed["before_pass_rate"]) ?? readNumber(parsed["before"]),
+    after_pass_rate: readNumber(parsed["after_pass_rate"]) ?? readNumber(parsed["after"]),
+    net_change: readNumber(parsed["net_change"]),
+    validation_mode: readString(parsed["validation_mode"]),
+  };
+}
+
+function isSuccessfulReplayDryRun(summary: DashboardActionResultSummary | null): boolean {
+  if (!summary) return false;
+
+  return (
+    summary.reason === "Dry run - proposal validated but not deployed" &&
+    summary.improved === true &&
+    summary.deployed === false
+  );
+}
+
+export function resolveDashboardActionOutcome(
+  input: DashboardActionOutcomeInput,
+): DashboardActionOutcome {
+  const summary = extractDashboardActionSummary(input.action, input.stdout);
+
+  if (input.exitCode === 0) {
+    return { success: true, error: null, summary };
+  }
+
+  if (input.action === "replay-dry-run" && isSuccessfulReplayDryRun(summary)) {
+    return { success: true, error: null, summary };
+  }
+
+  return {
+    success: false,
+    summary,
+    error:
+      input.stderr ||
+      (input.exitCode == null ? "Unknown action failure" : `Exit code ${input.exitCode}`),
+  };
+}

package/cli/selftune/dashboard-action-stream.ts

@@ -0,0 +1,252 @@
+import { randomUUID } from "node:crypto";
+import {
+  appendDashboardActionEvent,
+  setCurrentDashboardActionContext,
+} from "./dashboard-action-events.js";
+import { resolveDashboardActionOutcome } from "./dashboard-action-result.js";
+import type { DashboardActionName } from "./dashboard-contract.js";
+
+const STREAM_DISABLE_ENV = "SELFTUNE_DASHBOARD_STREAM_DISABLE";
+
+function readFlagValue(argv: string[], flag: string): string | null {
+  const index = argv.indexOf(flag);
+  if (index === -1) return null;
+  const value = argv[index + 1];
+  if (!value || value.startsWith("--")) return null;
+  return value;
+}
+
+function hasFlag(argv: string[], flag: string): boolean {
+  return argv.includes(flag);
+}
+
+function detectDashboardAction(argv: string[]): {
+  action: DashboardActionName;
+  skillName: string | null;
+  skillPath: string | null;
+} | null {
+  if (argv.includes("--help") || argv.includes("-h")) {
+    return null;
+  }
+
+  const [command, subcommand] = argv;
+
+  if (command === "eval" && subcommand === "generate") {
+    return {
+      action: "generate-evals",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  if (command === "eval" && subcommand === "unit-test" && hasFlag(argv, "--generate")) {
+    return {
+      action: "generate-unit-tests",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  if (command === "grade" && subcommand === "baseline") {
+    return {
+      action: "measure-baseline",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  if (command === "watch") {
+    return {
+      action: "watch",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  if (command === "orchestrate") {
+    return {
+      action: "orchestrate",
+      skillName: null,
+      skillPath: null,
+    };
+  }
+
+  if (command === "evolve" && subcommand === "rollback") {
+    return {
+      action: "rollback",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  if (command === "evolve" && (!subcommand || subcommand.startsWith("--"))) {
+    return {
+      action: hasFlag(argv, "--dry-run") ? "replay-dry-run" : "deploy-candidate",
+      skillName: readFlagValue(argv, "--skill"),
+      skillPath: readFlagValue(argv, "--skill-path"),
+    };
+  }
+
+  return null;
+}
+
+function normalizeChunk(chunk: unknown): string {
+  if (typeof chunk === "string") return chunk;
+  if (Buffer.isBuffer(chunk)) return chunk.toString("utf-8");
+  return String(chunk);
+}
+
+export interface DashboardActionStreamSession {
+  eventId: string;
+  finish: (exitCode?: number | null) => void;
+}
+
+export function startDashboardActionStream(argv: string[]): DashboardActionStreamSession | null {
+  if (process.env[STREAM_DISABLE_ENV] === "1") return null;
+
+  const detected = detectDashboardAction(argv);
+  if (!detected) return null;
+
+  const eventId = randomUUID();
+  let finished = false;
+  let lastError: string | null = null;
+  let suppressChunkCapture = false;
+  let stdoutBuffer = "";
+  let stderrBuffer = "";
+
+  appendDashboardActionEvent({
+    event_id: eventId,
+    action: detected.action,
+    stage: "started",
+    skill_name: detected.skillName,
+    skill_path: detected.skillPath,
+    ts: Date.now(),
+  });
+
+  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  const originalConsoleLog = console.log.bind(console);
+  const originalConsoleInfo = console.info.bind(console);
+  const originalConsoleWarn = console.warn.bind(console);
+  const originalConsoleError = console.error.bind(console);
+  setCurrentDashboardActionContext({
+    eventId,
+    action: detected.action,
+    skillName: detected.skillName,
+    skillPath: detected.skillPath,
+  });
+
+  process.stdout.write = ((chunk: unknown, ...args: unknown[]) => {
+    if (!suppressChunkCapture) {
+      stdoutBuffer += normalizeChunk(chunk);
+      appendDashboardActionEvent({
+        event_id: eventId,
+        action: detected.action,
+        stage: "stdout",
+        skill_name: detected.skillName,
+        skill_path: detected.skillPath,
+        ts: Date.now(),
+        chunk: normalizeChunk(chunk),
+      });
+    }
+    return originalStdoutWrite(chunk as never, ...(args as []));
+  }) as typeof process.stdout.write;
+
+  process.stderr.write = ((chunk: unknown, ...args: unknown[]) => {
+    const normalized = normalizeChunk(chunk);
+    if (!suppressChunkCapture) {
+      stderrBuffer += normalized;
+      if (normalized.trim()) {
+        lastError = normalized.trim();
+      }
+      appendDashboardActionEvent({
+        event_id: eventId,
+        action: detected.action,
+        stage: "stderr",
+        skill_name: detected.skillName,
+        skill_path: detected.skillPath,
+        ts: Date.now(),
+        chunk: normalized,
+      });
+    }
+    return originalStderrWrite(chunk as never, ...(args as []));
+  }) as typeof process.stderr.write;
+
+  function wrapConsole(
+    stage: "stdout" | "stderr",
+    originalMethod: typeof console.log,
+  ): typeof console.log {
+    return (...args: unknown[]) => {
+      const message = args
+        .map((arg) => (typeof arg === "string" ? arg : Bun.inspect(arg)))
+        .join(" ");
+      if (message.trim()) {
+        if (stage === "stderr") {
+          stderrBuffer += `${message}\n`;
+          lastError = message.trim();
+        } else {
+          stdoutBuffer += `${message}\n`;
+        }
+        appendDashboardActionEvent({
+          event_id: eventId,
+          action: detected.action,
+          stage,
+          skill_name: detected.skillName,
+          skill_path: detected.skillPath,
+          ts: Date.now(),
+          chunk: `${message}\n`,
+        });
+      }
+      suppressChunkCapture = true;
+      try {
+        originalMethod(...args);
+      } finally {
+        suppressChunkCapture = false;
+      }
+    };
+  }
+
+  console.log = wrapConsole("stdout", originalConsoleLog);
+  console.info = wrapConsole("stdout", originalConsoleInfo);
+  console.warn = wrapConsole("stderr", originalConsoleWarn);
+  console.error = wrapConsole("stderr", originalConsoleError);
+
+  const exitListener = (code: number) => {
+    finish(code);
+  };
+
+  function finish(exitCode?: number | null): void {
+    if (finished) return;
+    finished = true;
+    process.removeListener("exit", exitListener);
+    process.stdout.write = originalStdoutWrite;
+    process.stderr.write = originalStderrWrite;
+    console.log = originalConsoleLog;
+    console.info = originalConsoleInfo;
+    console.warn = originalConsoleWarn;
+    console.error = originalConsoleError;
+    setCurrentDashboardActionContext(null);
+    const outcome = resolveDashboardActionOutcome({
+      action: detected.action,
+      stdout: stdoutBuffer,
+      stderr: stderrBuffer || lastError,
+      exitCode: exitCode ?? 0,
+    });
+    appendDashboardActionEvent({
+      event_id: eventId,
+      action: detected.action,
+      stage: "finished",
+      skill_name: detected.skillName,
+      skill_path: detected.skillPath,
+      ts: Date.now(),
+      success: outcome.success,
+      exit_code: exitCode ?? 0,
+      error: outcome.error,
+      summary: outcome.summary,
+    });
+  }
+
+  process.once("exit", exitListener);
+
+  return { eventId, finish };
+}

package/cli/selftune/dashboard-contract.ts

@@ -353,6 +353,77 @@ export interface SkillTestingReadiness {
   latest_evolution_at: string | null;
 }
 
+export type DashboardActionName =
+  | "generate-evals"
+  | "generate-unit-tests"
+  | "replay-dry-run"
+  | "measure-baseline"
+  | "deploy-candidate"
+  | "watch"
+  | "orchestrate"
+  | "rollback";
+
+export type DashboardActionEventStage =
+  | "started"
+  | "progress"
+  | "stdout"
+  | "stderr"
+  | "metrics"
+  | "finished";
+
+export interface DashboardActionResultSummary {
+  reason: string | null;
+  improved: boolean | null;
+  deployed: boolean | null;
+  before_pass_rate: number | null;
+  after_pass_rate: number | null;
+  net_change: number | null;
+  validation_mode: string | null;
+}
+
+export interface DashboardActionMetrics {
+  platform: string | null;
+  model: string | null;
+  session_id: string | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  cache_creation_input_tokens: number | null;
+  cache_read_input_tokens: number | null;
+  total_cost_usd: number | null;
+  duration_ms: number | null;
+  num_turns: number | null;
+}
+
+export type DashboardActionProgressUnit = "eval" | "llm_call" | "step";
+
+export interface DashboardActionProgress {
+  current: number;
+  total: number;
+  status: "started" | "finished";
+  unit?: DashboardActionProgressUnit | null;
+  phase?: string | null;
+  label?: string | null;
+  query: string | null;
+  passed: boolean | null;
+  evidence: string | null;
+}
+
+export interface DashboardActionEvent {
+  event_id: string;
+  action: DashboardActionName;
+  stage: DashboardActionEventStage;
+  skill_name: string | null;
+  skill_path: string | null;
+  ts: number;
+  chunk?: string;
+  success?: boolean;
+  exit_code?: number | null;
+  error?: string | null;
+  summary?: DashboardActionResultSummary | null;
+  metrics?: DashboardActionMetrics | null;
+  progress?: DashboardActionProgress | null;
+}
+
 export interface CreatorTestingOverview {
   summary: string;
   counts: {
@@ -452,6 +523,10 @@ export interface HealthResponse {
   ok: boolean;
   service: string;
   version: string;
+  latest_version: string | null;
+  update_available: boolean;
+  auto_update_supported: boolean;
+  update_hint: string | null;
   pid: number;
   spa: boolean;
   spa_mode?: "dist" | "proxy" | "missing";
@@ -513,7 +588,12 @@ export interface CommitRecord {
 export interface CommitSummary {
   total_commits: number;
   unique_branches: number;
-  recent_commits: Array<{ sha: string; title: string; branch: string; timestamp: string }>;
+  recent_commits: Array<{
+    sha: string;
+    title: string;
+    branch: string;
+    timestamp: string;
+  }>;
 }
 
 // -- Trust-oriented types for skill report ------------------------------------