selftune 0.1.4 → 0.2.1

package/cli/selftune/utils/transcript.ts

@@ -2,9 +2,11 @@
  * Transcript parsing utilities shared by hooks and grading.
  */
 
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname } from "node:path";
-import type { TranscriptMetrics } from "../types.js";
+import { CLAUDE_CODE_PROJECTS_DIR } from "../constants.js";
+import type { SessionTelemetryRecord, TranscriptMetrics } from "../types.js";
+import { isActionableQueryText } from "./query-filter.js";
 
 /**
  * Parse a Claude Code transcript JSONL and extract process metrics.
@@ -23,9 +25,14 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
   const toolCalls: Record<string, number> = {};
   const bashCommands: string[] = [];
   const skillsTriggered: string[] = [];
+  const skillsInvoked: string[] = [];
   let errors = 0;
   let assistantTurns = 0;
   let lastUserQuery = "";
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let firstTimestamp: string | null = null;
+  let lastTimestamp: string | null = null;
 
   for (const raw of lines) {
     const line = raw.trim();
@@ -38,6 +45,22 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
       continue;
     }
 
+    // Track timestamps for duration calculation
+    const ts = entry.timestamp as string | undefined;
+    if (ts) {
+      if (!firstTimestamp) firstTimestamp = ts;
+      lastTimestamp = ts;
+    }
+
+    // Accumulate token usage from usage objects
+    const usage = (entry.usage ?? (entry.message as Record<string, unknown>)?.usage) as
+      | Record<string, unknown>
+      | undefined;
+    if (usage && typeof usage === "object") {
+      if (typeof usage.input_tokens === "number") inputTokens += usage.input_tokens;
+      if (typeof usage.output_tokens === "number") outputTokens += usage.output_tokens;
+    }
+
     // Normalise: unwrap nested message if present
     const msg = (entry.message as Record<string, unknown>) ?? entry;
     const role = (msg.role as string) ?? (entry.role as string) ?? "";
@@ -45,19 +68,8 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
 
     // Track last user query
     if (role === "user") {
-      if (typeof content === "string" && content.trim()) {
-        lastUserQuery = content.trim();
-      } else if (Array.isArray(content)) {
-        const texts = content
-          .filter(
-            (p): p is Record<string, unknown> =>
-              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
-          )
-          .map((p) => (p.text as string) ?? "")
-          .filter(Boolean);
-        const text = texts.join(" ").trim();
-        if (text) lastUserQuery = text;
-      }
+      const text = extractActionableUserText(content);
+      if (text) lastUserQuery = text;
     }
 
     // Count assistant turns and parse tool use
@@ -72,7 +84,7 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
           toolCalls[toolName] = (toolCalls[toolName] ?? 0) + 1;
           const inp = (b.input as Record<string, unknown>) ?? {};
 
-          // Track SKILL.md reads
+          // Track SKILL.md reads (may be browsing — kept for backwards compat)
           const filePath = (inp.file_path as string) ?? "";
           if (basename(filePath).toUpperCase() === "SKILL.MD") {
             const skillName = basename(dirname(filePath));
@@ -81,6 +93,14 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
             }
           }
 
+          // Track actual Skill tool invocations (high-confidence signal)
+          if (toolName === "Skill") {
+            const skillArg = (inp.skill as string) ?? (inp.name as string) ?? "";
+            if (skillArg && !skillsInvoked.includes(skillArg)) {
+              skillsInvoked.push(skillArg);
+            }
+          }
+
           // Track bash commands
           if (toolName === "Bash") {
             const cmd = ((inp.command as string) ?? "").trim();
@@ -110,15 +130,167 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     }
   }
 
+  // Compute duration from first to last timestamp
+  let durationMs: number | undefined;
+  if (firstTimestamp && lastTimestamp && firstTimestamp !== lastTimestamp) {
+    const start = new Date(firstTimestamp).getTime();
+    const end = new Date(lastTimestamp).getTime();
+    if (!Number.isNaN(start) && !Number.isNaN(end) && end > start) {
+      durationMs = end - start;
+    }
+  }
+
   return {
     tool_calls: toolCalls,
     total_tool_calls: Object.values(toolCalls).reduce((a, b) => a + b, 0),
     bash_commands: bashCommands,
     skills_triggered: skillsTriggered,
+    skills_invoked: skillsInvoked,
     assistant_turns: assistantTurns,
     errors_encountered: errors,
     transcript_chars: totalChars,
     last_user_query: lastUserQuery,
+    ...(inputTokens > 0 ? { input_tokens: inputTokens } : {}),
+    ...(outputTokens > 0 ? { output_tokens: outputTokens } : {}),
+    ...(durationMs !== undefined ? { duration_ms: durationMs } : {}),
+  };
+}
+
+/**
+ * Extract actionable user queries from a Claude transcript.
+ */
+export function extractActionableUserQueries(
+  transcriptPath: string,
+): Array<{ query: string; timestamp: string }> {
+  if (!existsSync(transcriptPath)) return [];
+
+  let content: string;
+  try {
+    content = readFileSync(transcriptPath, "utf-8");
+  } catch {
+    return [];
+  }
+
+  const results: Array<{ query: string; timestamp: string }> = [];
+
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    const msg = (entry.message as Record<string, unknown>) ?? entry;
+    const role = (msg.role as string) ?? (entry.role as string) ?? "";
+    if (role !== "user") continue;
+
+    const text = extractActionableUserText(msg.content ?? entry.content ?? "");
+    if (!text || text.length < 4) continue;
+
+    const timestamp = (entry.timestamp as string) ?? (msg.timestamp as string) ?? "";
+    results.push({ query: text, timestamp });
+  }
+
+  return results;
+}
+
+/**
+ * Recursively find Claude transcript JSONL files under a projects directory.
+ */
+export function findTranscriptFiles(projectsDir: string, since?: Date): string[] {
+  if (!existsSync(projectsDir)) return [];
+
+  const files: string[] = [];
+
+  const walk = (dir: string): void => {
+    let entries: string[];
+    try {
+      entries = readdirSync(dir).sort();
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const entryPath = `${dir}/${entry}`;
+      try {
+        const stats = statSync(entryPath);
+
+        if (stats.isDirectory()) {
+          walk(entryPath);
+          continue;
+        }
+
+        if (!stats.isFile() || !entry.endsWith(".jsonl")) continue;
+        if (since && stats.mtime < since) continue;
+
+        files.push(entryPath);
+      } catch {
+        // Ignore unreadable files and keep scanning.
+      }
+    }
+  };
+
+  walk(projectsDir);
+  return files.sort();
+}
+
+/**
+ * Find a Claude transcript path by session ID.
+ */
+export function findTranscriptPathForSession(
+  sessionId: string,
+  projectsDir: string = CLAUDE_CODE_PROJECTS_DIR,
+): string | null {
+  const filename = `${sessionId}.jsonl`;
+  for (const transcriptPath of findTranscriptFiles(projectsDir)) {
+    if (basename(transcriptPath) === filename) return transcriptPath;
+  }
+  return null;
+}
+
+/**
+ * Build a SessionTelemetryRecord directly from a transcript file.
+ */
+export function buildTelemetryFromTranscript(
+  sessionId: string,
+  transcriptPath: string,
+  source = "claude_code_transcript_fallback",
+): SessionTelemetryRecord | null {
+  if (!existsSync(transcriptPath)) return null;
+
+  const metrics = parseTranscript(transcriptPath);
+  const userQueries = extractActionableUserQueries(transcriptPath);
+
+  let timestamp = userQueries[0]?.timestamp ?? "";
+  if (!timestamp) {
+    try {
+      timestamp = statSync(transcriptPath).mtime.toISOString();
+    } catch {
+      timestamp = new Date().toISOString();
+    }
+  }
+
+  return {
+    timestamp,
+    session_id: sessionId,
+    cwd: "",
+    transcript_path: transcriptPath,
+    tool_calls: metrics.tool_calls,
+    total_tool_calls: metrics.total_tool_calls,
+    bash_commands: metrics.bash_commands,
+    skills_triggered: metrics.skills_triggered,
+    skills_invoked: metrics.skills_invoked,
+    assistant_turns: metrics.assistant_turns,
+    errors_encountered: metrics.errors_encountered,
+    transcript_chars: metrics.transcript_chars,
+    last_user_query: metrics.last_user_query,
+    source,
+    input_tokens: metrics.input_tokens,
+    output_tokens: metrics.output_tokens,
   };
 }
 
@@ -142,14 +314,14 @@ export function getLastUserMessage(transcriptPath: string): string | null {
 
       // Format 1: top-level role field
       if (entry.role === "user") {
-        const text = extractUserText(entry.content);
+        const text = extractActionableUserText(entry.content);
         if (text) return text;
       }
 
       // Format 2: nested message object
       const msg = entry.message as Record<string, unknown> | undefined;
       if (msg && typeof msg === "object" && msg.role === "user") {
-        const text = extractUserText(msg.content);
+        const text = extractActionableUserText(msg.content);
         if (text) return text;
       }
     }
@@ -160,6 +332,40 @@ export function getLastUserMessage(transcriptPath: string): string | null {
   return null;
 }
 
+function extractTextParts(content: unknown): string {
+  if (!Array.isArray(content)) return "";
+
+  return content
+    .filter(
+      (part): part is Record<string, unknown> =>
+        typeof part === "object" &&
+        part !== null &&
+        (part as Record<string, unknown>).type === "text",
+    )
+    .map((part) => (part.text as string) ?? "")
+    .filter(Boolean)
+    .join(" ")
+    .trim();
+}
+
+function summarizeCodexFunctionArguments(argumentsText: unknown): string {
+  if (typeof argumentsText !== "string" || !argumentsText.trim()) return "";
+
+  try {
+    const parsed = JSON.parse(argumentsText) as Record<string, unknown>;
+    return (
+      (typeof parsed.cmd === "string" && parsed.cmd.trim()) ||
+      (typeof parsed.command === "string" && parsed.command.trim()) ||
+      (typeof parsed.file_path === "string" && parsed.file_path.trim()) ||
+      (typeof parsed.path === "string" && parsed.path.trim()) ||
+      (typeof parsed.query === "string" && parsed.query.trim()) ||
+      argumentsText.trim()
+    ).slice(0, 200);
+  } catch {
+    return argumentsText.trim().slice(0, 200);
+  }
+}
+
 /**
  * Parse a transcript into a human-readable excerpt for the grader.
  */
@@ -184,19 +390,13 @@ export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
     const msg = (entry.message as Record<string, unknown>) ?? entry;
     const role = (msg.role as string) ?? (entry.role as string) ?? "";
     const entryContent = msg.content ?? entry.content ?? "";
+    const eventType = (entry.type as string) ?? "";
 
     if (role === "user") {
       if (typeof entryContent === "string") {
         readable.push(`[USER] ${entryContent.slice(0, 200)}`);
       } else if (Array.isArray(entryContent)) {
-        const texts = entryContent
-          .filter(
-            (p): p is Record<string, unknown> =>
-              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
-          )
-          .map((p) => (p.text as string) ?? "")
-          .filter(Boolean);
-        const text = texts.join(" ").trim().slice(0, 200);
+        const text = extractTextParts(entryContent).slice(0, 200);
         if (text) readable.push(`[USER] ${text}`);
       }
     } else if (role === "assistant") {
@@ -218,6 +418,45 @@ export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
           }
         }
       }
+    } else if (eventType === "event_msg") {
+      const payload = (entry.payload as Record<string, unknown>) ?? {};
+      if (payload.type === "user_message") {
+        const text = extractActionableUserText(payload.message)?.slice(0, 200) ?? "";
+        if (text) readable.push(`[USER] ${text}`);
+      }
+    } else if (eventType === "turn.completed") {
+      const text = extractActionableUserText(entry.user_message)?.slice(0, 200) ?? "";
+      if (text) readable.push(`[USER] ${text}`);
+    } else if (eventType === "response_item") {
+      const payload = (entry.payload as Record<string, unknown>) ?? {};
+      const itemType = (payload.type as string) ?? "";
+
+      if (itemType === "function_call") {
+        const name = (payload.name as string) ?? "function_call";
+        const detail = summarizeCodexFunctionArguments(payload.arguments);
+        if (detail) readable.push(`[TOOL:${name}] ${detail}`);
+      } else if (itemType === "agent_reasoning") {
+        const text = ((payload.text as string) ?? "").trim().slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      } else if (itemType === "message" && (payload.role as string) === "assistant") {
+        const text = extractTextParts(payload.content).slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      }
+    } else if (
+      eventType === "item.completed" ||
+      eventType === "item.started" ||
+      eventType === "item.updated"
+    ) {
+      const item = (entry.item as Record<string, unknown>) ?? {};
+      const itemType = (item.item_type as string) ?? (item.type as string) ?? "";
+
+      if (itemType === "command_execution") {
+        const command = ((item.command as string) ?? "").trim().slice(0, 200);
+        if (command) readable.push(`[TOOL:command_execution] ${command}`);
+      } else {
+        const text = ((item.text as string) ?? "").trim().slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      }
     }
   }
 
@@ -228,12 +467,48 @@ export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
   return `${full.slice(0, head)}\n\n... [truncated] ...\n\n${full.slice(-tail)}`;
 }
 
+/**
+ * Extract token usage from a transcript JSONL by summing usage fields.
+ *
+ * Scans for entries with a `usage` object containing `input_tokens` and
+ * `output_tokens` (the format Claude Code transcripts use).
+ */
+export function extractTokenUsage(transcriptPath: string): { input: number; output: number } {
+  if (!existsSync(transcriptPath)) return { input: 0, output: 0 };
+
+  const content = readFileSync(transcriptPath, "utf-8");
+  const lines = content.split("\n");
+  let input = 0;
+  let output = 0;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    const usage = entry.usage as Record<string, unknown> | undefined;
+    if (usage && typeof usage === "object") {
+      if (typeof usage.input_tokens === "number") input += usage.input_tokens;
+      if (typeof usage.output_tokens === "number") output += usage.output_tokens;
+    }
+  }
+
+  return { input, output };
+}
+
 function emptyMetrics(): TranscriptMetrics {
   return {
     tool_calls: {},
     total_tool_calls: 0,
     bash_commands: [],
     skills_triggered: [],
+    skills_invoked: [],
     assistant_turns: 0,
     errors_encountered: 0,
     transcript_chars: 0,
@@ -258,3 +533,9 @@ function extractUserText(content: unknown): string | null {
   }
   return null;
 }
+
+function extractActionableUserText(content: unknown): string | null {
+  const text = extractUserText(content);
+  if (!text) return null;
+  return isActionableQueryText(text) ? text : null;
+}

package/cli/selftune/utils/trigger-check.ts

@@ -0,0 +1,89 @@
+/**
+ * Shared trigger-check utilities.
+ *
+ * Extracted from validate-proposal.ts so other modules (e.g. body validation,
+ * routing validation) can reuse the same prompt-building and response-parsing
+ * logic without depending on the evolution layer.
+ */
+
+// ---------------------------------------------------------------------------
+// Prompt building
+// ---------------------------------------------------------------------------
+
+/** Build the trigger check prompt for the LLM. */
+export function buildTriggerCheckPrompt(description: string, query: string): string {
+  return [
+    "Given this skill description, would the following user query trigger this skill?",
+    "Respond YES or NO only.",
+    "",
+    "Skill description:",
+    description,
+    "",
+    "User query:",
+    query,
+  ].join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Response parsing
+// ---------------------------------------------------------------------------
+
+/** Parse YES/NO from LLM response. */
+export function parseTriggerResponse(response: string): boolean {
+  const normalized = response.trim().toUpperCase();
+  if (normalized.startsWith("YES")) return true;
+  if (normalized.startsWith("NO")) return false;
+  return false; // conservative default
+}
+
+// ---------------------------------------------------------------------------
+// Batch prompt building
+// ---------------------------------------------------------------------------
+
+/** Build a batch trigger check prompt for multiple queries at once. */
+export function buildBatchTriggerCheckPrompt(description: string, queries: string[]): string {
+  const numbered = queries.map((q, i) => `${i + 1}. "${q}"`).join("\n");
+  return [
+    "Given this skill description, would each query trigger this skill?",
+    "Respond with the query number followed by YES or NO, one per line.",
+    "",
+    "Skill description:",
+    description,
+    "",
+    "Queries:",
+    numbered,
+  ].join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Batch response parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a batch YES/NO response. Returns a boolean array aligned to the
+ * original query order. Defaults to false for unparseable or missing lines.
+ */
+export function parseBatchTriggerResponse(response: string, queryCount: number): boolean[] {
+  const results: boolean[] = new Array(queryCount).fill(false);
+  const lines = response.trim().split("\n");
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    // Try to extract a number prefix: "1. YES", "1: YES", "1 YES", "1) YES"
+    const match = trimmed.match(/^(\d+)[.):\s]+\s*(.*)/);
+    if (!match) continue;
+
+    const idx = parseInt(match[1], 10) - 1; // 1-based to 0-based
+    if (idx < 0 || idx >= queryCount) continue;
+
+    const answer = match[2].trim().toUpperCase();
+    if (answer.startsWith("YES")) {
+      results[idx] = true;
+    }
+    // NO or anything else stays false (the default)
+  }
+
+  return results;
+}

package/cli/selftune/utils/tui.ts

@@ -0,0 +1,156 @@
+/**
+ * tui.ts
+ *
+ * Zero-dependency TUI primitives for the selftune evolve pipeline.
+ * Uses raw ANSI escape codes for spinners, timers, and step progression.
+ * All output goes to stderr to keep stdout clean for JSON results.
+ */
+
+const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+const TICK_MS = 80;
+
+export interface EvolveTUI {
+  /** Start a new step with a spinner. Completes the previous step (if any) with checkmark. */
+  step(label: string): void;
+  /** Complete the current step with checkmark and a custom label. */
+  done(label: string): void;
+  /** Complete the current step as failed with cross mark and a custom label. */
+  fail(label: string): void;
+  /** Stop all timers and print a summary line. */
+  finish(summary: string): void;
+  /** Clean up timers without printing. For error paths. */
+  destroy(): void;
+}
+
+function createNoopTUI(): EvolveTUI {
+  return { step() {}, done() {}, fail() {}, finish() {}, destroy() {} };
+}
+
+export function createEvolveTUI(opts: { skillName: string; model: string }): EvolveTUI {
+  const noColor = !!process.env.NO_COLOR;
+  const isTTY = !!process.stderr.isTTY;
+
+  // If not a TTY, return no-op to avoid ANSI noise in pipes/tests
+  if (!isTTY && !process.env.SELFTUNE_TUI_FORCE) {
+    return createNoopTUI();
+  }
+
+  const write = (s: string) => process.stderr.write(s);
+
+  let spinnerFrame = 0;
+  let stepStartTime = Date.now();
+  let currentLabel = "";
+  let hasActiveSpinner = false;
+  let intervalId: ReturnType<typeof setInterval> | null = null;
+  let destroyed = false;
+
+  const checkMark = noColor ? "+" : "\u2713";
+  const crossMark = noColor ? "x" : "\u2717";
+
+  // Print header
+  write(`\n  selftune evolve \u2500\u2500 ${opts.skillName} \u2500\u2500 ${opts.model}\n\n`);
+
+  function formatTime(ms: number): string {
+    return `${(ms / 1000).toFixed(1)}s`;
+  }
+
+  function clearSpinnerLine(): void {
+    if (hasActiveSpinner) {
+      write("\x1b[A\x1b[2K");
+    }
+  }
+
+  function writeSpinnerLine(): void {
+    const frame = noColor ? ">" : SPINNER_FRAMES[spinnerFrame % SPINNER_FRAMES.length];
+    const elapsed = formatTime(Date.now() - stepStartTime);
+    const padding = Math.max(1, 48 - currentLabel.length);
+    write(`  ${frame} ${currentLabel}${" ".repeat(padding)}${elapsed}\n`);
+    hasActiveSpinner = true;
+  }
+
+  function startSpinner(label: string): void {
+    currentLabel = label;
+    stepStartTime = Date.now();
+    spinnerFrame = 0;
+    writeSpinnerLine();
+    intervalId = setInterval(() => {
+      spinnerFrame++;
+      clearSpinnerLine();
+      writeSpinnerLine();
+    }, TICK_MS);
+  }
+
+  function stopSpinner(): void {
+    if (intervalId !== null) {
+      clearInterval(intervalId);
+      intervalId = null;
+    }
+  }
+
+  function writeCompletedLine(marker: string, label: string, elapsed: number): void {
+    const time = formatTime(elapsed);
+    const padding = Math.max(1, 48 - label.length);
+    write(`  ${marker} ${label}${" ".repeat(padding)}${time}\n`);
+  }
+
+  function completeCurrentStep(marker: string, label: string): void {
+    const elapsed = Date.now() - stepStartTime;
+    stopSpinner();
+    clearSpinnerLine();
+    hasActiveSpinner = false;
+    writeCompletedLine(marker, label, elapsed);
+  }
+
+  return {
+    step(label: string): void {
+      if (destroyed) return;
+      // Complete previous step if there was one
+      if (hasActiveSpinner) {
+        completeCurrentStep(checkMark, currentLabel);
+      }
+      startSpinner(label);
+    },
+
+    done(label: string): void {
+      if (destroyed) return;
+      if (hasActiveSpinner) {
+        // Complete active spinner with custom label
+        completeCurrentStep(checkMark, label);
+      } else {
+        // No active spinner — instant step
+        writeCompletedLine(checkMark, label, 0);
+      }
+      currentLabel = "";
+    },
+
+    fail(label: string): void {
+      if (destroyed) return;
+      if (hasActiveSpinner) {
+        completeCurrentStep(crossMark, label);
+      } else {
+        writeCompletedLine(crossMark, label, 0);
+      }
+      currentLabel = "";
+    },
+
+    finish(summary: string): void {
+      if (destroyed) return;
+      if (hasActiveSpinner) {
+        completeCurrentStep(checkMark, currentLabel);
+      }
+      stopSpinner();
+      write(`\n  ${summary}\n`);
+      destroyed = true;
+    },
+
+    destroy(): void {
+      if (destroyed) return;
+      stopSpinner();
+      if (hasActiveSpinner) {
+        clearSpinnerLine();
+        hasActiveSpinner = false;
+      }
+      destroyed = true;
+    },
+  };
+}