selftune 0.2.29 → 0.2.31

package/cli/selftune/evolution/validate-host-replay.ts

@@ -12,7 +12,12 @@ import {
 import { tmpdir } from "node:os";
 import { basename, dirname, isAbsolute, join } from "node:path";
 
+import {
+  emitDashboardActionMetrics,
+  emitDashboardActionProgress,
+} from "../dashboard-action-events.js";
 import type { EvalEntry, RoutingReplayEntryResult, RoutingReplayFixture } from "../types.js";
+import type { DashboardActionMetrics } from "../dashboard-contract.js";
 import { parseFrontmatter } from "../utils/frontmatter.js";
 import {
   containsWholeSkillMention,
@@ -95,6 +100,14 @@ function resolveObservedReplayPath(path: string, workspaceRoot: string): string
   return resolveReplayPath(isAbsolute(path) ? path : join(workspaceRoot, path));
 }
 
+function truncateReplayText(value: string | null | undefined, maxLength: number): string | null {
+  if (typeof value !== "string") return null;
+  const normalized = value.replace(/\s+/g, " ").trim();
+  if (!normalized) return null;
+  if (normalized.length <= maxLength) return normalized;
+  return `${normalized.slice(0, maxLength - 1)}…`;
+}
+
 function listCompetingSkillPaths(targetSkillPath: string): string[] {
   const normalizedTargetPath = resolveReplayPath(targetSkillPath);
   const targetSkillDir = dirname(normalizedTargetPath);
@@ -332,6 +345,128 @@ function normalizeReplayEventType(value: unknown): string {
   return typeof value === "string" ? value.replace(/[._]/g, "-").trim().toLowerCase() : "";
 }
 
+function readObject(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null ? (value as Record<string, unknown>) : null;
+}
+
+function readNumber(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function readString(value: unknown): string | null {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+
+function normalizeClaudeModel(value: string | null): string | null {
+  return value ? value.replace(/\[[^\]]+\]$/, "") : null;
+}
+
+function firstModelUsageKey(value: unknown): string | null {
+  const modelUsage = readObject(value);
+  if (!modelUsage) return null;
+  const firstKey = Object.keys(modelUsage)[0];
+  return normalizeClaudeModel(firstKey ?? null);
+}
+
+export function extractClaudeRuntimeReplayMetrics(line: string): DashboardActionMetrics | null {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = JSON.parse(trimmed) as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+
+  const eventType = readString(parsed.type);
+  const sessionId = readString(parsed.session_id);
+
+  if (eventType === "system" && readString(parsed.subtype) === "init") {
+    return {
+      platform: "claude_code",
+      model: normalizeClaudeModel(readString(parsed.model)),
+      session_id: sessionId,
+      input_tokens: null,
+      output_tokens: null,
+      cache_creation_input_tokens: null,
+      cache_read_input_tokens: null,
+      total_cost_usd: null,
+      duration_ms: null,
+      num_turns: null,
+    };
+  }
+
+  if (eventType === "assistant") {
+    const message = readObject(parsed.message);
+    const usage = readObject(message?.usage);
+    return {
+      platform: "claude_code",
+      model: normalizeClaudeModel(readString(message?.model)),
+      session_id: sessionId,
+      input_tokens: readNumber(usage?.input_tokens),
+      output_tokens: readNumber(usage?.output_tokens),
+      cache_creation_input_tokens: readNumber(usage?.cache_creation_input_tokens),
+      cache_read_input_tokens: readNumber(usage?.cache_read_input_tokens),
+      total_cost_usd: null,
+      duration_ms: null,
+      num_turns: null,
+    };
+  }
+
+  if (eventType === "result") {
+    const usage = readObject(parsed.usage);
+    return {
+      platform: "claude_code",
+      model: firstModelUsageKey(parsed.modelUsage),
+      session_id: sessionId,
+      input_tokens: readNumber(usage?.input_tokens),
+      output_tokens: readNumber(usage?.output_tokens),
+      cache_creation_input_tokens: readNumber(usage?.cache_creation_input_tokens),
+      cache_read_input_tokens: readNumber(usage?.cache_read_input_tokens),
+      total_cost_usd: readNumber(parsed.total_cost_usd),
+      duration_ms: readNumber(parsed.duration_ms),
+      num_turns: readNumber(parsed.num_turns),
+    };
+  }
+
+  return null;
+}
+
+async function readStreamText(
+  stream: ReadableStream<Uint8Array> | null | undefined,
+  onLine?: (line: string) => void,
+): Promise<string> {
+  if (!stream) return "";
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+  let output = "";
+  let buffered = "";
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    const chunk = decoder.decode(value, { stream: true });
+    if (!chunk) continue;
+    output += chunk;
+    buffered += chunk;
+    const lines = buffered.split("\n");
+    buffered = lines.pop() ?? "";
+    for (const line of lines) {
+      onLine?.(line);
+    }
+  }
+
+  const tail = decoder.decode();
+  if (tail) {
+    output += tail;
+    buffered += tail;
+  }
+  if (buffered) onLine?.(buffered);
+
+  return output;
+}
+
 export function parseCodexRuntimeReplayOutput(
   rawOutput: string,
   knownSkillNames: Set<string>,
@@ -591,7 +726,10 @@ async function invokeClaudeRuntimeReplay(
   const timeout = setTimeout(() => proc.kill(), CLAUDE_RUNTIME_REPLAY_TIMEOUT_MS);
 
   const [stdoutText, stderrText, exitCode] = await Promise.all([
-    new Response(proc.stdout).text(),
+    readStreamText(proc.stdout, (line) => {
+      const metrics = extractClaudeRuntimeReplayMetrics(line);
+      if (metrics) emitDashboardActionMetrics(metrics);
+    }),
     new Response(proc.stderr).text(),
     proc.exited,
   ]);
@@ -1032,20 +1170,59 @@ export async function runHostRuntimeReplayFixture(options: {
       options.contentTarget ?? "routing",
     );
     const results: RoutingReplayEntryResult[] = [];
-
-    for (const entry of options.evalSet) {
-      const observation = await invokeRuntime({
-        query: entry.query,
-        platform: options.fixture.platform,
-        workspaceRoot: workspace.rootDir,
-        skillRegistryDir: workspace.skillRegistryDir,
-        targetSkillName: options.fixture.target_skill_name,
-        targetSkillPath: workspace.targetSkillPath,
-        competingSkillPaths: workspace.competingSkillPaths,
+    const total = options.evalSet.length;
+
+    for (const [index, entry] of options.evalSet.entries()) {
+      const current = index + 1;
+      const querySnippet = truncateReplayText(entry.query, 120);
+
+      emitDashboardActionProgress({
+        current,
+        total,
+        status: "started",
+        query: querySnippet,
+        passed: null,
+        evidence: null,
       });
-      results.push(
-        evaluateRuntimeReplayObservation(entry, options.fixture, observation, workspace),
-      );
+
+      try {
+        const observation = await invokeRuntime({
+          query: entry.query,
+          platform: options.fixture.platform,
+          workspaceRoot: workspace.rootDir,
+          skillRegistryDir: workspace.skillRegistryDir,
+          targetSkillName: options.fixture.target_skill_name,
+          targetSkillPath: workspace.targetSkillPath,
+          competingSkillPaths: workspace.competingSkillPaths,
+        });
+        const result = evaluateRuntimeReplayObservation(
+          entry,
+          options.fixture,
+          observation,
+          workspace,
+        );
+        results.push(result);
+
+        emitDashboardActionProgress({
+          current,
+          total,
+          status: "finished",
+          query: querySnippet,
+          passed: result.passed,
+          evidence: truncateReplayText(result.evidence, 180),
+        });
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        emitDashboardActionProgress({
+          current,
+          total,
+          status: "finished",
+          query: querySnippet,
+          passed: false,
+          evidence: truncateReplayText(message, 180),
+        });
+        throw error;
+      }
     }
 
     return results;

package/cli/selftune/index.ts

@@ -43,6 +43,7 @@ import { PUBLIC_COMMAND_SURFACES, renderCommandHelp } from "./command-surface.js
 process.on("uncaughtException", handleCLIError);
 process.on("unhandledRejection", handleCLIError);
 
+const originalArgv = process.argv.slice(2);
 const command = process.argv[2];
 
 if (command === "--help" || command === "-h") {
@@ -112,6 +113,9 @@ if (!command) {
   process.exit(0);
 }
 
+const { startDashboardActionStream } = await import("./dashboard-action-stream.js");
+startDashboardActionStream(originalArgv);
+
 // Route to the appropriate subcommand module.
 // We use dynamic imports so only the needed module is loaded.
 // Each module exports a cliMain() function that the router calls explicitly,

package/cli/selftune/ingestors/opencode-ingest.ts

@@ -179,6 +179,77 @@ function normalizeContent(rawContent: unknown): Array<Record<string, unknown>> {
   return [];
 }
 
+function normalizeTimestampMs(rawValue: unknown): number {
+  if (typeof rawValue !== "number" || !Number.isFinite(rawValue)) {
+    return Date.now();
+  }
+  if (rawValue > 1e12) return rawValue;
+  if (rawValue > 1e9) return rawValue * 1000;
+  return rawValue;
+}
+
+function getTableColumns(db: Database, tableName: string): Set<string> {
+  const safeTableName = assertSafeIdentifier(tableName);
+  const rows = db.query(`PRAGMA table_info(${safeTableName})`).all() as Array<{
+    name: string;
+  }>;
+  return new Set(rows.map((row) => row.name));
+}
+
+function pickColumn(columns: Set<string>, candidates: string[]): string | null {
+  for (const candidate of candidates) {
+    if (columns.has(candidate)) return candidate;
+  }
+  return null;
+}
+
+function parseMessagePayload(rawValue: unknown): Record<string, unknown> | null {
+  if (typeof rawValue === "string") {
+    try {
+      const parsed = JSON.parse(rawValue) as unknown;
+      return typeof parsed === "object" && parsed !== null
+        ? (parsed as Record<string, unknown>)
+        : null;
+    } catch {
+      return null;
+    }
+  }
+  return typeof rawValue === "object" && rawValue !== null
+    ? (rawValue as Record<string, unknown>)
+    : null;
+}
+
+function extractMessageRole(
+  row: Record<string, unknown>,
+  payload: Record<string, unknown> | null,
+): string {
+  const rowRole = row.role;
+  if (typeof rowRole === "string" && rowRole.trim()) return rowRole;
+  const payloadRole = payload?.role;
+  return typeof payloadRole === "string" ? payloadRole : "";
+}
+
+function extractMessageBlocks(
+  row: Record<string, unknown>,
+  payload: Record<string, unknown> | null,
+): Array<Record<string, unknown>> {
+  const directBlocks = normalizeContent(row.content);
+  if (directBlocks.length > 0) return directBlocks;
+
+  const payloadBlocks = normalizeContent(payload?.content);
+  if (payloadBlocks.length > 0) return payloadBlocks;
+
+  const summary = payload?.summary;
+  if (typeof summary === "object" && summary !== null) {
+    const title = (summary as Record<string, unknown>).title;
+    if (typeof title === "string" && title.trim()) {
+      return [{ type: "text", text: title.trim() }];
+    }
+  }
+
+  return [];
+}
+
 /**
  * Read OpenCode sessions from SQLite database.
  */
@@ -207,19 +278,40 @@ export function readSessionsFromSqlite(
 
   const safeSessionsTable = assertSafeIdentifier(sessionsTable);
   const safeMessagesTable = assertSafeIdentifier(messagesTable);
+  const sessionColumns = getTableColumns(db, safeSessionsTable);
+  const messageColumns = getTableColumns(db, safeMessagesTable);
+  const sessionTimeColumn = pickColumn(sessionColumns, [
+    "created",
+    "time_created",
+    "createdAt",
+    "timeCreated",
+    "updated",
+    "time_updated",
+  ]);
+  const messageTimeColumn = pickColumn(messageColumns, [
+    "created",
+    "time_created",
+    "createdAt",
+    "timeCreated",
+    "updated",
+    "time_updated",
+  ]);
 
   // Get sessions
   let whereClause = "";
   const queryParams: number[] = [];
-  if (sinceTs) {
-    whereClause = "WHERE created > ?";
+  if (sinceTs && sessionTimeColumn) {
+    whereClause = `WHERE ${assertSafeIdentifier(sessionTimeColumn)} > ?`;
     queryParams.push(Math.floor(sinceTs * 1000));
   }
+  const orderBySessionColumn = sessionTimeColumn ? assertSafeIdentifier(sessionTimeColumn) : "id";
 
   let sessionRows: Array<Record<string, unknown>>;
   try {
     sessionRows = db
-      .query(`SELECT * FROM ${safeSessionsTable} ${whereClause} ORDER BY created ASC`)
+      .query(
+        `SELECT * FROM ${safeSessionsTable} ${whereClause} ORDER BY ${orderBySessionColumn} ASC`,
+      )
       .all(...queryParams) as Array<Record<string, unknown>>;
   } catch (e) {
     console.warn(`[WARN] Could not query sessions: ${e}`);
@@ -231,14 +323,19 @@ export function readSessionsFromSqlite(
 
   for (const sessionRow of sessionRows) {
     const sessionId = String(sessionRow.id);
-    const createdMs = sessionRow.created as number;
+    const createdMs = normalizeTimestampMs(
+      sessionTimeColumn ? sessionRow[sessionTimeColumn] : Date.now(),
+    );
     const timestamp = new Date(createdMs).toISOString();
 
     // Get messages for this session
     let msgRows: Array<Record<string, unknown>>;
     try {
+      const orderByMessageColumn = messageTimeColumn
+        ? ` ORDER BY ${assertSafeIdentifier(messageTimeColumn)} ASC`
+        : "";
       msgRows = db
-        .query(`SELECT * FROM ${safeMessagesTable} WHERE session_id = ? ORDER BY created ASC`)
+        .query(`SELECT * FROM ${safeMessagesTable} WHERE session_id = ?${orderByMessageColumn}`)
         .all(String(sessionRow.id)) as Array<Record<string, unknown>>;
     } catch {
       continue;
@@ -250,6 +347,7 @@ export function readSessionsFromSqlite(
     const skillDetections = new Map<string, TriggeredSkillDetection>();
     let errors = 0;
     let assistantTurns = 0;
+    let cwd = typeof sessionRow.directory === "string" ? sessionRow.directory : "";
 
     const noteSkillDetection = (skillName: string, hasSkillMdRead: boolean): void => {
       const normalizedSkillName = skillName.trim();
@@ -266,8 +364,16 @@ export function readSessionsFromSqlite(
     };
 
     for (const msg of msgRows) {
-      const role = (msg.role as string) ?? "";
-      const blocks = normalizeContent(msg.content ?? "[]");
+      const payload = parseMessagePayload(msg.data);
+      const role = extractMessageRole(msg, payload);
+      const blocks = extractMessageBlocks(msg, payload);
+      const payloadPath =
+        payload && typeof payload.path === "object" && payload.path !== null
+          ? (payload.path as Record<string, unknown>)
+          : null;
+      if (!cwd && payloadPath && typeof payloadPath.cwd === "string") {
+        cwd = payloadPath.cwd;
+      }
 
       if (role === "user") {
         if (!firstUserQuery) {
@@ -291,6 +397,9 @@ export function readSessionsFromSqlite(
         }
       } else if (role === "assistant") {
         assistantTurns += 1;
+        if (payload?.error) {
+          errors += 1;
+        }
         for (const block of blocks) {
           const blockType = (block.type as string) ?? "";
 
@@ -350,7 +459,7 @@ export function readSessionsFromSqlite(
       session_id: sessionId,
       source: "opencode",
       transcript_path: dbPath,
-      cwd: "",
+      cwd,
       last_user_query: firstUserQuery,
       query: firstUserQuery,
       tool_calls: toolCalls,

package/cli/selftune/localdb/schema.ts

@@ -245,6 +245,33 @@ CREATE TABLE IF NOT EXISTS grading_baselines (
   grading_results_json  TEXT
 )`;
 
+// -- Creator-loop artifact tables --------------------------------------------
+
+export const CREATE_CANONICAL_EVAL_SETS = `
+CREATE TABLE IF NOT EXISTS canonical_eval_sets (
+  skill_name    TEXT PRIMARY KEY,
+  stored_at     TEXT NOT NULL,
+  eval_set_json TEXT NOT NULL
+)`;
+
+export const CREATE_UNIT_TEST_FILES = `
+CREATE TABLE IF NOT EXISTS unit_test_files (
+  skill_name  TEXT PRIMARY KEY,
+  stored_at   TEXT NOT NULL,
+  tests_json  TEXT NOT NULL
+)`;
+
+export const CREATE_UNIT_TEST_RUN_RESULTS = `
+CREATE TABLE IF NOT EXISTS unit_test_run_results (
+  skill_name   TEXT PRIMARY KEY,
+  run_at       TEXT NOT NULL,
+  total        INTEGER NOT NULL,
+  passed       INTEGER NOT NULL,
+  failed       INTEGER NOT NULL,
+  pass_rate    REAL NOT NULL,
+  result_json  TEXT NOT NULL
+)`;
+
 // -- Improvement signal table (from signal_log.jsonl) ------------------------
 
 export const CREATE_IMPROVEMENT_SIGNALS = `
@@ -388,6 +415,10 @@ export const CREATE_INDEXES = [
   `CREATE INDEX IF NOT EXISTS idx_grading_bl_proposal ON grading_baselines(proposal_id)`,
   `CREATE INDEX IF NOT EXISTS idx_grading_bl_ts ON grading_baselines(measured_at)`,
   `CREATE INDEX IF NOT EXISTS idx_grading_bl_skill_proposal ON grading_baselines(skill_name, proposal_id, measured_at)`,
+  // -- Creator-loop artifact indexes -----------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_canonical_eval_sets_stored_at ON canonical_eval_sets(stored_at)`,
+  `CREATE INDEX IF NOT EXISTS idx_unit_test_files_stored_at ON unit_test_files(stored_at)`,
+  `CREATE INDEX IF NOT EXISTS idx_unit_test_run_results_run_at ON unit_test_run_results(run_at)`,
   // -- Improvement signal indexes ---------------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_signals_session ON improvement_signals(session_id)`,
   `CREATE INDEX IF NOT EXISTS idx_signals_consumed ON improvement_signals(consumed)`,
@@ -506,6 +537,9 @@ export const ALL_DDL = [
   CREATE_QUERIES,
   CREATE_GRADING_RESULTS,
   CREATE_GRADING_BASELINES,
+  CREATE_CANONICAL_EVAL_SETS,
+  CREATE_UNIT_TEST_FILES,
+  CREATE_UNIT_TEST_RUN_RESULTS,
   CREATE_IMPROVEMENT_SIGNALS,
   CREATE_UPLOAD_QUEUE,
   CREATE_CREATOR_CONTRIBUTION_STAGING,