selftune 0.2.0 → 0.2.2

package/cli/selftune/utils/transcript.ts

@@ -2,9 +2,11 @@
  * Transcript parsing utilities shared by hooks and grading.
  */
 
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname } from "node:path";
-import type { TranscriptMetrics } from "../types.js";
+import { CLAUDE_CODE_PROJECTS_DIR } from "../constants.js";
+import type { SessionTelemetryRecord, TranscriptMetrics } from "../types.js";
+import { isActionableQueryText } from "./query-filter.js";
 
 /**
  * Parse a Claude Code transcript JSONL and extract process metrics.
@@ -23,9 +25,14 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
   const toolCalls: Record<string, number> = {};
   const bashCommands: string[] = [];
   const skillsTriggered: string[] = [];
+  const skillsInvoked: string[] = [];
   let errors = 0;
   let assistantTurns = 0;
   let lastUserQuery = "";
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let firstTimestamp: string | null = null;
+  let lastTimestamp: string | null = null;
 
   for (const raw of lines) {
     const line = raw.trim();
@@ -38,6 +45,22 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
       continue;
     }
 
+    // Track timestamps for duration calculation
+    const ts = entry.timestamp as string | undefined;
+    if (ts) {
+      if (!firstTimestamp) firstTimestamp = ts;
+      lastTimestamp = ts;
+    }
+
+    // Accumulate token usage from usage objects
+    const usage = (entry.usage ?? (entry.message as Record<string, unknown>)?.usage) as
+      | Record<string, unknown>
+      | undefined;
+    if (usage && typeof usage === "object") {
+      if (typeof usage.input_tokens === "number") inputTokens += usage.input_tokens;
+      if (typeof usage.output_tokens === "number") outputTokens += usage.output_tokens;
+    }
+
     // Normalise: unwrap nested message if present
     const msg = (entry.message as Record<string, unknown>) ?? entry;
     const role = (msg.role as string) ?? (entry.role as string) ?? "";
@@ -45,19 +68,8 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
 
     // Track last user query
     if (role === "user") {
-      if (typeof content === "string" && content.trim()) {
-        lastUserQuery = content.trim();
-      } else if (Array.isArray(content)) {
-        const texts = content
-          .filter(
-            (p): p is Record<string, unknown> =>
-              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
-          )
-          .map((p) => (p.text as string) ?? "")
-          .filter(Boolean);
-        const text = texts.join(" ").trim();
-        if (text) lastUserQuery = text;
-      }
+      const text = extractActionableUserText(content);
+      if (text) lastUserQuery = text;
     }
 
     // Count assistant turns and parse tool use
@@ -72,7 +84,7 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
           toolCalls[toolName] = (toolCalls[toolName] ?? 0) + 1;
           const inp = (b.input as Record<string, unknown>) ?? {};
 
-          // Track SKILL.md reads
+          // Track SKILL.md reads (may be browsing — kept for backwards compat)
           const filePath = (inp.file_path as string) ?? "";
           if (basename(filePath).toUpperCase() === "SKILL.MD") {
             const skillName = basename(dirname(filePath));
@@ -81,6 +93,14 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
             }
           }
 
+          // Track actual Skill tool invocations (high-confidence signal)
+          if (toolName === "Skill") {
+            const skillArg = (inp.skill as string) ?? (inp.name as string) ?? "";
+            if (skillArg && !skillsInvoked.includes(skillArg)) {
+              skillsInvoked.push(skillArg);
+            }
+          }
+
           // Track bash commands
           if (toolName === "Bash") {
             const cmd = ((inp.command as string) ?? "").trim();
@@ -110,15 +130,167 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     }
   }
 
+  // Compute duration from first to last timestamp
+  let durationMs: number | undefined;
+  if (firstTimestamp && lastTimestamp && firstTimestamp !== lastTimestamp) {
+    const start = new Date(firstTimestamp).getTime();
+    const end = new Date(lastTimestamp).getTime();
+    if (!Number.isNaN(start) && !Number.isNaN(end) && end > start) {
+      durationMs = end - start;
+    }
+  }
+
   return {
     tool_calls: toolCalls,
     total_tool_calls: Object.values(toolCalls).reduce((a, b) => a + b, 0),
     bash_commands: bashCommands,
     skills_triggered: skillsTriggered,
+    skills_invoked: skillsInvoked,
     assistant_turns: assistantTurns,
     errors_encountered: errors,
     transcript_chars: totalChars,
     last_user_query: lastUserQuery,
+    ...(inputTokens > 0 ? { input_tokens: inputTokens } : {}),
+    ...(outputTokens > 0 ? { output_tokens: outputTokens } : {}),
+    ...(durationMs !== undefined ? { duration_ms: durationMs } : {}),
+  };
+}
+
+/**
+ * Extract actionable user queries from a Claude transcript.
+ */
+export function extractActionableUserQueries(
+  transcriptPath: string,
+): Array<{ query: string; timestamp: string }> {
+  if (!existsSync(transcriptPath)) return [];
+
+  let content: string;
+  try {
+    content = readFileSync(transcriptPath, "utf-8");
+  } catch {
+    return [];
+  }
+
+  const results: Array<{ query: string; timestamp: string }> = [];
+
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    const msg = (entry.message as Record<string, unknown>) ?? entry;
+    const role = (msg.role as string) ?? (entry.role as string) ?? "";
+    if (role !== "user") continue;
+
+    const text = extractActionableUserText(msg.content ?? entry.content ?? "");
+    if (!text || text.length < 4) continue;
+
+    const timestamp = (entry.timestamp as string) ?? (msg.timestamp as string) ?? "";
+    results.push({ query: text, timestamp });
+  }
+
+  return results;
+}
+
+/**
+ * Recursively find Claude transcript JSONL files under a projects directory.
+ */
+export function findTranscriptFiles(projectsDir: string, since?: Date): string[] {
+  if (!existsSync(projectsDir)) return [];
+
+  const files: string[] = [];
+
+  const walk = (dir: string): void => {
+    let entries: string[];
+    try {
+      entries = readdirSync(dir).sort();
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const entryPath = `${dir}/${entry}`;
+      try {
+        const stats = statSync(entryPath);
+
+        if (stats.isDirectory()) {
+          walk(entryPath);
+          continue;
+        }
+
+        if (!stats.isFile() || !entry.endsWith(".jsonl")) continue;
+        if (since && stats.mtime < since) continue;
+
+        files.push(entryPath);
+      } catch {
+        // Ignore unreadable files and keep scanning.
+      }
+    }
+  };
+
+  walk(projectsDir);
+  return files.sort();
+}
+
+/**
+ * Find a Claude transcript path by session ID.
+ */
+export function findTranscriptPathForSession(
+  sessionId: string,
+  projectsDir: string = CLAUDE_CODE_PROJECTS_DIR,
+): string | null {
+  const filename = `${sessionId}.jsonl`;
+  for (const transcriptPath of findTranscriptFiles(projectsDir)) {
+    if (basename(transcriptPath) === filename) return transcriptPath;
+  }
+  return null;
+}
+
+/**
+ * Build a SessionTelemetryRecord directly from a transcript file.
+ */
+export function buildTelemetryFromTranscript(
+  sessionId: string,
+  transcriptPath: string,
+  source = "claude_code_transcript_fallback",
+): SessionTelemetryRecord | null {
+  if (!existsSync(transcriptPath)) return null;
+
+  const metrics = parseTranscript(transcriptPath);
+  const userQueries = extractActionableUserQueries(transcriptPath);
+
+  let timestamp = userQueries[0]?.timestamp ?? "";
+  if (!timestamp) {
+    try {
+      timestamp = statSync(transcriptPath).mtime.toISOString();
+    } catch {
+      timestamp = new Date().toISOString();
+    }
+  }
+
+  return {
+    timestamp,
+    session_id: sessionId,
+    cwd: "",
+    transcript_path: transcriptPath,
+    tool_calls: metrics.tool_calls,
+    total_tool_calls: metrics.total_tool_calls,
+    bash_commands: metrics.bash_commands,
+    skills_triggered: metrics.skills_triggered,
+    skills_invoked: metrics.skills_invoked,
+    assistant_turns: metrics.assistant_turns,
+    errors_encountered: metrics.errors_encountered,
+    transcript_chars: metrics.transcript_chars,
+    last_user_query: metrics.last_user_query,
+    source,
+    input_tokens: metrics.input_tokens,
+    output_tokens: metrics.output_tokens,
   };
 }
 
@@ -142,14 +314,14 @@ export function getLastUserMessage(transcriptPath: string): string | null {
 
       // Format 1: top-level role field
       if (entry.role === "user") {
-        const text = extractUserText(entry.content);
+        const text = extractActionableUserText(entry.content);
         if (text) return text;
       }
 
       // Format 2: nested message object
       const msg = entry.message as Record<string, unknown> | undefined;
       if (msg && typeof msg === "object" && msg.role === "user") {
-        const text = extractUserText(msg.content);
+        const text = extractActionableUserText(msg.content);
         if (text) return text;
       }
     }
@@ -160,6 +332,40 @@ export function getLastUserMessage(transcriptPath: string): string | null {
   return null;
 }
 
+function extractTextParts(content: unknown): string {
+  if (!Array.isArray(content)) return "";
+
+  return content
+    .filter(
+      (part): part is Record<string, unknown> =>
+        typeof part === "object" &&
+        part !== null &&
+        (part as Record<string, unknown>).type === "text",
+    )
+    .map((part) => (part.text as string) ?? "")
+    .filter(Boolean)
+    .join(" ")
+    .trim();
+}
+
+function summarizeCodexFunctionArguments(argumentsText: unknown): string {
+  if (typeof argumentsText !== "string" || !argumentsText.trim()) return "";
+
+  try {
+    const parsed = JSON.parse(argumentsText) as Record<string, unknown>;
+    return (
+      (typeof parsed.cmd === "string" && parsed.cmd.trim()) ||
+      (typeof parsed.command === "string" && parsed.command.trim()) ||
+      (typeof parsed.file_path === "string" && parsed.file_path.trim()) ||
+      (typeof parsed.path === "string" && parsed.path.trim()) ||
+      (typeof parsed.query === "string" && parsed.query.trim()) ||
+      argumentsText.trim()
+    ).slice(0, 200);
+  } catch {
+    return argumentsText.trim().slice(0, 200);
+  }
+}
+
 /**
  * Parse a transcript into a human-readable excerpt for the grader.
  */
@@ -184,19 +390,13 @@ export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
     const msg = (entry.message as Record<string, unknown>) ?? entry;
     const role = (msg.role as string) ?? (entry.role as string) ?? "";
     const entryContent = msg.content ?? entry.content ?? "";
+    const eventType = (entry.type as string) ?? "";
 
     if (role === "user") {
       if (typeof entryContent === "string") {
         readable.push(`[USER] ${entryContent.slice(0, 200)}`);
       } else if (Array.isArray(entryContent)) {
-        const texts = entryContent
-          .filter(
-            (p): p is Record<string, unknown> =>
-              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
-          )
-          .map((p) => (p.text as string) ?? "")
-          .filter(Boolean);
-        const text = texts.join(" ").trim().slice(0, 200);
+        const text = extractTextParts(entryContent).slice(0, 200);
         if (text) readable.push(`[USER] ${text}`);
       }
     } else if (role === "assistant") {
@@ -218,6 +418,45 @@ export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
           }
         }
       }
+    } else if (eventType === "event_msg") {
+      const payload = (entry.payload as Record<string, unknown>) ?? {};
+      if (payload.type === "user_message") {
+        const text = extractActionableUserText(payload.message)?.slice(0, 200) ?? "";
+        if (text) readable.push(`[USER] ${text}`);
+      }
+    } else if (eventType === "turn.completed") {
+      const text = extractActionableUserText(entry.user_message)?.slice(0, 200) ?? "";
+      if (text) readable.push(`[USER] ${text}`);
+    } else if (eventType === "response_item") {
+      const payload = (entry.payload as Record<string, unknown>) ?? {};
+      const itemType = (payload.type as string) ?? "";
+
+      if (itemType === "function_call") {
+        const name = (payload.name as string) ?? "function_call";
+        const detail = summarizeCodexFunctionArguments(payload.arguments);
+        if (detail) readable.push(`[TOOL:${name}] ${detail}`);
+      } else if (itemType === "agent_reasoning") {
+        const text = ((payload.text as string) ?? "").trim().slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      } else if (itemType === "message" && (payload.role as string) === "assistant") {
+        const text = extractTextParts(payload.content).slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      }
+    } else if (
+      eventType === "item.completed" ||
+      eventType === "item.started" ||
+      eventType === "item.updated"
+    ) {
+      const item = (entry.item as Record<string, unknown>) ?? {};
+      const itemType = (item.item_type as string) ?? (item.type as string) ?? "";
+
+      if (itemType === "command_execution") {
+        const command = ((item.command as string) ?? "").trim().slice(0, 200);
+        if (command) readable.push(`[TOOL:command_execution] ${command}`);
+      } else {
+        const text = ((item.text as string) ?? "").trim().slice(0, 200);
+        if (text) readable.push(`[ASSISTANT] ${text}`);
+      }
     }
   }
 
@@ -269,6 +508,7 @@ function emptyMetrics(): TranscriptMetrics {
     total_tool_calls: 0,
     bash_commands: [],
     skills_triggered: [],
+    skills_invoked: [],
     assistant_turns: 0,
     errors_encountered: 0,
     transcript_chars: 0,
@@ -293,3 +533,9 @@ function extractUserText(content: unknown): string | null {
   }
   return null;
 }
+
+function extractActionableUserText(content: unknown): string | null {
+  const text = extractUserText(content);
+  if (!text) return null;
+  return isActionableQueryText(text) ? text : null;
+}

package/cli/selftune/workflows/discover.ts

@@ -0,0 +1,254 @@
+/**
+ * discover.ts
+ *
+ * Pure analysis functions for discovering multi-skill workflows from
+ * telemetry and usage data. No I/O -- CLI wrapper handles reading JSONL.
+ *
+ * Adapts patterns from composability-v2.ts but removes single-skill scoping
+ * to discover ALL multi-skill workflows across the codebase.
+ */
+
+import type {
+  DiscoveredWorkflow,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+  WorkflowDiscoveryReport,
+} from "../types.js";
+import { clamp } from "../utils/math.js";
+
+/**
+ * Discover multi-skill workflows from telemetry and usage data.
+ *
+ * Algorithm:
+ *  1. Apply window filter to telemetry (sort by timestamp desc, take N)
+ *  2. Build session ID set from filtered telemetry
+ *  3. Filter usage records to in-scope sessions
+ *  4. Group usage by session_id, sort by timestamp, deduplicate consecutive same-skill
+ *  5. Keep sequences with 2+ skills
+ *  6. Count frequency of each unique sequence, filter by minOccurrences (default 3)
+ *  7. For each qualifying sequence compute metrics
+ *  8. If --skill provided, filter to workflows containing that skill
+ *  9. Sort by occurrence_count descending
+ * 10. Return WorkflowDiscoveryReport
+ */
+export function discoverWorkflows(
+  telemetry: SessionTelemetryRecord[],
+  usage: SkillUsageRecord[],
+  options?: { minOccurrences?: number; window?: number; skill?: string },
+): WorkflowDiscoveryReport {
+  const minOccurrences = options?.minOccurrences ?? 3;
+
+  // 1. Apply window: sort by timestamp descending, take last N
+  let sessions = telemetry.filter((r) => r && Array.isArray(r.skills_triggered));
+
+  if (options?.window && options.window > 0) {
+    sessions = sessions
+      .sort((a, b) => (b.timestamp ?? "").localeCompare(a.timestamp ?? ""))
+      .slice(0, options.window);
+  }
+
+  // 2. Build a set of session IDs in scope (after windowing)
+  const sessionIdSet = new Set(sessions.map((s) => s.session_id));
+
+  // 3. Filter usage records to in-scope sessions
+  const usageInScope = usage.filter((u) => sessionIdSet.has(u.session_id));
+
+  // 4. Group usage by session_id
+  const usageBySession = new Map<string, SkillUsageRecord[]>();
+  for (const u of usageInScope) {
+    const group = usageBySession.get(u.session_id);
+    if (group) {
+      group.push(u);
+    } else {
+      usageBySession.set(u.session_id, [u]);
+    }
+  }
+
+  // Build ordered sequences per session (ALL sessions, no target skill filter)
+  const sessionSequences: Array<{
+    skills: string[];
+    sessionId: string;
+    firstQuery: string;
+  }> = [];
+
+  for (const [sessionId, records] of usageBySession) {
+    // Sort by timestamp ascending
+    const sorted = [...records].sort((a, b) =>
+      (a.timestamp ?? "").localeCompare(b.timestamp ?? ""),
+    );
+
+    // Extract skill names, deduplicate consecutive same-skill entries
+    const skills: string[] = [];
+    for (const r of sorted) {
+      if (skills.length === 0 || skills[skills.length - 1] !== r.skill_name) {
+        skills.push(r.skill_name);
+      }
+    }
+
+    // 5. Only record sequences with 2+ skills
+    if (skills.length >= 2) {
+      sessionSequences.push({
+        skills,
+        sessionId,
+        firstQuery: sorted[0]?.query ?? "",
+      });
+    }
+  }
+
+  // 6. Count frequency of each unique sequence (by JSON key)
+  const sequenceCounts = new Map<
+    string,
+    { count: number; query: string; skills: string[]; sessionIds: string[] }
+  >();
+  for (const seq of sessionSequences) {
+    const key = JSON.stringify(seq.skills);
+    const existing = sequenceCounts.get(key);
+    if (existing) {
+      existing.count++;
+      existing.sessionIds.push(seq.sessionId);
+    } else {
+      sequenceCounts.set(key, {
+        count: 1,
+        query: seq.firstQuery,
+        skills: seq.skills,
+        sessionIds: [seq.sessionId],
+      });
+    }
+  }
+
+  // Count all orderings of each skill set (for consistency computation)
+  const skillSetCounts = new Map<string, number>();
+  for (const seq of sessionSequences) {
+    const setKey = JSON.stringify([...seq.skills].sort());
+    skillSetCounts.set(setKey, (skillSetCounts.get(setKey) ?? 0) + 1);
+  }
+
+  // Build telemetry lookup by session_id
+  const telemetryBySession = new Map<string, SessionTelemetryRecord>();
+  for (const s of sessions) {
+    telemetryBySession.set(s.session_id, s);
+  }
+
+  // Compute per-skill solo error rates (for avg_errors_individual)
+  const skillSoloErrors = new Map<string, { totalErrors: number; count: number }>();
+  for (const s of sessions) {
+    if (s.skills_triggered.length === 1) {
+      const skillName = s.skills_triggered[0];
+      const entry = skillSoloErrors.get(skillName);
+      if (entry) {
+        entry.totalErrors += s.errors_encountered ?? 0;
+        entry.count++;
+      } else {
+        skillSoloErrors.set(skillName, {
+          totalErrors: s.errors_encountered ?? 0,
+          count: 1,
+        });
+      }
+    }
+  }
+
+  function getSkillSoloErrorRate(skillName: string): number | undefined {
+    const entry = skillSoloErrors.get(skillName);
+    if (!entry || entry.count === 0) return undefined;
+    return entry.totalErrors / entry.count;
+  }
+
+  // 7. Build workflows, filtered by minOccurrences
+  const workflows: DiscoveredWorkflow[] = [];
+  for (const data of sequenceCounts.values()) {
+    if (data.count < minOccurrences) continue;
+
+    // workflow_id = skills.join("->")
+    const workflowId = data.skills.join("\u2192");
+
+    // Get matching telemetry sessions
+    const matchingSessions = data.sessionIds
+      .map((id) => telemetryBySession.get(id))
+      .filter((s): s is SessionTelemetryRecord => s !== undefined);
+
+    // avg_errors from matching telemetry sessions
+    const avgErrors =
+      matchingSessions.length > 0
+        ? matchingSessions.reduce((sum, r) => sum + (r.errors_encountered ?? 0), 0) /
+          matchingSessions.length
+        : 0;
+
+    const soloRates = data.skills
+      .map((s) => getSkillSoloErrorRate(s))
+      .filter((rate): rate is number => rate !== undefined);
+
+    // avg_errors_individual = max of each skill's solo error rate
+    // Note: This differs from composability-v2.ts which uses a single-skill anchor.
+    // For multi-skill discovery, we conservatively anchor to the worst solo performer.
+    const avgErrorsIndividual = soloRates.length > 0 ? Math.max(...soloRates) : 0;
+
+    // synergy_score = clamp((individual - together) / (individual + 1), -1, 1)
+    // If no solo baseline exists yet, keep the workflow neutral instead of treating missing data as zero.
+    const synergyScore =
+      soloRates.length > 0
+        ? clamp((avgErrorsIndividual - avgErrors) / (avgErrorsIndividual + 1), -1, 1)
+        : 0;
+
+    // sequence_consistency = this_order_count / all_orderings_of_same_set
+    const setKey = JSON.stringify([...data.skills].sort());
+    const totalOrderings = skillSetCounts.get(setKey) ?? data.count;
+    const sequenceConsistency = totalOrderings > 0 ? data.count / totalOrderings : 1;
+
+    // completion_rate = sessions with ALL skills fired / sessions with ANY skill from set
+    const skillSet = new Set(data.skills);
+    let sessionsWithAny = 0;
+    let sessionsWithAll = 0;
+    for (const s of sessions) {
+      const hasAny = s.skills_triggered.some((sk) => skillSet.has(sk));
+      if (hasAny) {
+        sessionsWithAny++;
+        const hasAll = data.skills.every((sk) => s.skills_triggered.includes(sk));
+        if (hasAll) sessionsWithAll++;
+      }
+    }
+    const completionRate = sessionsWithAny > 0 ? sessionsWithAll / sessionsWithAny : 0;
+
+    // representative_query = first query from first matching session
+    const representativeQuery = data.query;
+
+    // first_seen / last_seen from matching sessions
+    const timestamps = matchingSessions
+      .map((s) => s.timestamp)
+      .filter((t) => t)
+      .sort();
+    const firstSeen = timestamps[0] ?? "";
+    const lastSeen = timestamps[timestamps.length - 1] ?? "";
+
+    workflows.push({
+      workflow_id: workflowId,
+      skills: data.skills,
+      occurrence_count: data.count,
+      avg_errors: avgErrors,
+      avg_errors_individual: avgErrorsIndividual,
+      synergy_score: synergyScore,
+      representative_query: representativeQuery,
+      sequence_consistency: sequenceConsistency,
+      completion_rate: completionRate,
+      first_seen: firstSeen,
+      last_seen: lastSeen,
+      session_ids: data.sessionIds,
+    });
+  }
+
+  // 8. If --skill provided, filter to workflows containing that skill
+  let filtered = workflows;
+  if (options?.skill) {
+    const skillFilter = options.skill;
+    filtered = workflows.filter((w) => w.skills.includes(skillFilter));
+  }
+
+  // 9. Sort by occurrence_count descending
+  filtered.sort((a, b) => b.occurrence_count - a.occurrence_count);
+
+  // 10. Return WorkflowDiscoveryReport
+  return {
+    workflows: filtered,
+    total_sessions_analyzed: sessions.length,
+    generated_at: new Date().toISOString(),
+  };
+}