selftune 0.2.16 → 0.2.19

package/cli/selftune/utils/transcript.ts

@@ -6,9 +6,15 @@ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname } from "node:path";
 
 import { CLAUDE_CODE_PROJECTS_DIR } from "../constants.js";
-import type { SessionTelemetryRecord, TranscriptMetrics } from "../types.js";
+import type { SessionTelemetryRecord, SessionType, TranscriptMetrics } from "../types.js";
 import { isActionableQueryText } from "./query-filter.js";
 
+/** Tools that produce durable output artifacts (not reads or exploration). */
+const ARTIFACT_TOOLS = new Set(["Write", "Edit", "WebFetch", "WebSearch", "Skill", "Agent"]);
+
+/** Matches any bash command containing a git invocation. */
+const GIT_CMD_RE = /\bgit\b/;
+
 /**
  * Parse a Claude Code transcript JSONL and extract process metrics.
  *
@@ -32,10 +38,18 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
   let lastUserQuery = "";
   let inputTokens = 0;
   let outputTokens = 0;
+  let cachedInputTokens = 0;
+  let reasoningOutputTokens = 0;
   let firstTimestamp: string | null = null;
   let lastTimestamp: string | null = null;
   let model: string | undefined;
 
+  // File change tracking (Win 2)
+  const changedFiles = new Set<string>();
+  let linesAdded = 0;
+  let linesRemoved = 0;
+  let linesModified = 0;
+
   for (const raw of lines) {
     const line = raw.trim();
     if (!line) continue;
@@ -61,6 +75,14 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     if (usage && typeof usage === "object") {
       if (typeof usage.input_tokens === "number") inputTokens += usage.input_tokens;
       if (typeof usage.output_tokens === "number") outputTokens += usage.output_tokens;
+      // Win 3: Token granularity — cached input tokens
+      if (typeof usage.cache_read_input_tokens === "number")
+        cachedInputTokens += usage.cache_read_input_tokens;
+      if (typeof usage.cache_creation_input_tokens === "number")
+        cachedInputTokens += usage.cache_creation_input_tokens;
+      // Win 3: Reasoning output tokens
+      if (typeof usage.reasoning_output_tokens === "number")
+        reasoningOutputTokens += usage.reasoning_output_tokens;
     }
 
     // Normalise: unwrap nested message if present
@@ -119,6 +141,26 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
             const cmd = ((inp.command as string) ?? "").trim();
             if (cmd) bashCommands.push(cmd);
           }
+
+          // Win 2: Track file changes from Write and Edit tools
+          if (toolName === "Write" || toolName === "Edit") {
+            const fp = (inp.file_path as string) ?? "";
+            if (fp) changedFiles.add(fp);
+          }
+          if (toolName === "Write" && typeof inp.content === "string") {
+            linesAdded += inp.content.split("\n").length;
+          }
+          if (toolName === "Edit") {
+            const oldStr = inp.old_string;
+            const newStr = inp.new_string;
+            if (typeof oldStr === "string" && typeof newStr === "string") {
+              const oldLines = oldStr.split("\n").length;
+              const newLines = newStr.split("\n").length;
+              linesModified += Math.min(oldLines, newLines);
+              linesAdded += Math.max(0, newLines - oldLines);
+              linesRemoved += Math.max(0, oldLines - newLines);
+            }
+          }
         }
       }
     }
@@ -143,6 +185,12 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     }
   }
 
+  // Compute artifact count: output-producing tool calls
+  let artifactCount = 0;
+  for (const [tool, count] of Object.entries(toolCalls)) {
+    if (ARTIFACT_TOOLS.has(tool)) artifactCount += count;
+  }
+
   // Compute duration from first to last timestamp
   let durationMs: number | undefined;
   if (firstTimestamp && lastTimestamp && firstTimestamp !== lastTimestamp) {
@@ -153,6 +201,12 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     }
   }
 
+  // Win 3: Calculate cost from model and token counts
+  const costUsd = calculateCost(model, inputTokens, outputTokens);
+
+  // Infer session type from tool distribution
+  const sessionType = inferSessionType(toolCalls, bashCommands);
+
   return {
     tool_calls: toolCalls,
     total_tool_calls: Object.values(toolCalls).reduce((a, b) => a + b, 0),
@@ -163,8 +217,18 @@ export function parseTranscript(transcriptPath: string): TranscriptMetrics {
     errors_encountered: errors,
     transcript_chars: totalChars,
     last_user_query: lastUserQuery,
+    // Win 2: File change metrics
+    files_changed: changedFiles.size,
+    lines_added: linesAdded,
+    lines_removed: linesRemoved,
+    lines_modified: linesModified,
+    artifact_count: artifactCount,
+    session_type: sessionType,
     ...(inputTokens > 0 ? { input_tokens: inputTokens } : {}),
     ...(outputTokens > 0 ? { output_tokens: outputTokens } : {}),
+    ...(cachedInputTokens > 0 ? { cached_input_tokens: cachedInputTokens } : {}),
+    ...(reasoningOutputTokens > 0 ? { reasoning_output_tokens: reasoningOutputTokens } : {}),
+    ...(costUsd !== undefined ? { cost_usd: costUsd } : {}),
     ...(durationMs !== undefined ? { duration_ms: durationMs } : {}),
     ...(model ? { model } : {}),
     ...(firstTimestamp ? { started_at: firstTimestamp } : {}),
@@ -307,6 +371,16 @@ export function buildTelemetryFromTranscript(
     source,
     input_tokens: metrics.input_tokens,
     output_tokens: metrics.output_tokens,
+    cached_input_tokens: metrics.cached_input_tokens,
+    reasoning_output_tokens: metrics.reasoning_output_tokens,
+    cost_usd: metrics.cost_usd,
+    files_changed: metrics.files_changed,
+    lines_added: metrics.lines_added,
+    lines_removed: metrics.lines_removed,
+    lines_modified: metrics.lines_modified,
+    artifact_count: metrics.artifact_count,
+    session_type: metrics.session_type,
+    agent_summary: generateSessionSummary(metrics),
   };
 }
 
@@ -518,6 +592,141 @@ export function extractTokenUsage(transcriptPath: string): { input: number; outp
   return { input, output };
 }
 
+// ---------------------------------------------------------------------------
+// Win 3: Model cost lookup (USD per million tokens)
+// ---------------------------------------------------------------------------
+
+const MODEL_COSTS: Record<string, { input: number; output: number }> = {
+  "claude-sonnet-4-20250514": { input: 3.0, output: 15.0 },
+  "claude-opus-4-20250514": { input: 15.0, output: 75.0 },
+  "claude-haiku-3-5-20241022": { input: 0.8, output: 4.0 },
+  "claude-3-5-sonnet-20241022": { input: 3.0, output: 15.0 },
+  "claude-3-5-haiku-20241022": { input: 0.8, output: 4.0 },
+  "claude-3-opus-20240229": { input: 15.0, output: 75.0 },
+  "claude-3-sonnet-20240229": { input: 3.0, output: 15.0 },
+  "claude-3-haiku-20240307": { input: 0.25, output: 1.25 },
+};
+
+/**
+ * Calculate estimated cost in USD from model name and token counts.
+ * Returns undefined if the model is unknown or not provided.
+ */
+export function calculateCost(
+  model: string | undefined,
+  inputTokens: number,
+  outputTokens: number,
+): number | undefined {
+  if (!model) return undefined;
+  const costs =
+    MODEL_COSTS[model] ??
+    Object.entries(MODEL_COSTS).find(([k]) =>
+      model.startsWith(k.split("-").slice(0, -1).join("-")),
+    )?.[1];
+  if (!costs) return undefined;
+  return (inputTokens * costs.input + outputTokens * costs.output) / 1_000_000;
+}
+
+/**
+ * Infer session type from tool call distribution.
+ *
+ * - "dev": majority of output tools are Write/Edit/Bash with git commands
+ * - "research": majority are WebFetch/WebSearch/Read
+ * - "content": majority are Write/Edit but no git commands
+ * - "mixed": no clear majority
+ */
+export function inferSessionType(
+  toolCalls: Record<string, number>,
+  bashCommands: string[],
+): "dev" | "research" | "content" | "mixed" {
+  const total = Object.values(toolCalls).reduce((a, b) => a + b, 0);
+  if (total === 0) return "mixed";
+
+  const writeEdit = (toolCalls.Write ?? 0) + (toolCalls.Edit ?? 0);
+  const research = (toolCalls.WebFetch ?? 0) + (toolCalls.WebSearch ?? 0);
+  const bash = toolCalls.Bash ?? 0;
+  const read = toolCalls.Read ?? 0;
+  const hasGit = bashCommands.some((cmd) => GIT_CMD_RE.test(cmd));
+
+  // Dev: file mutations + git commands OR bash-heavy with git
+  if (hasGit && (writeEdit + bash) / total > 0.3) return "dev";
+
+  // Research: web tools + read-heavy, low file mutations
+  if (research > 0 && research / total > 0.2 && writeEdit / total < 0.15) return "research";
+  if (read / total > 0.5 && writeEdit / total < 0.1) return "research";
+
+  // Content: file mutations but no git
+  if (writeEdit / total > 0.2 && !hasGit) return "content";
+
+  return "mixed";
+}
+
+/**
+ * Generate a short heuristic session summary from transcript metrics.
+ * No LLM call — pure template-based approach. Kept under 120 chars.
+ */
+export function generateSessionSummary(metrics: TranscriptMetrics): string {
+  const MAX_LEN = 120;
+  const sessionType: SessionType = metrics.session_type ?? "mixed";
+  const lastQuery = truncateQuery(metrics.last_user_query, 60);
+
+  if (metrics.total_tool_calls === 0 && !lastQuery) {
+    return "Empty session — no tool calls or queries";
+  }
+
+  const topTools = getTopTools(metrics.tool_calls, 2);
+
+  let summary: string;
+  switch (sessionType) {
+    case "dev": {
+      const filesChanged = metrics.files_changed ?? 0;
+      const toolStr = topTools.length > 0 ? ` via ${topTools.join(", ")}` : "";
+      const queryStr = lastQuery ? ` — ${lastQuery}` : "";
+      summary = `${filesChanged} files changed${toolStr}${queryStr}`;
+      break;
+    }
+    case "research": {
+      const searches = (metrics.tool_calls.WebSearch ?? 0) + (metrics.tool_calls.WebFetch ?? 0);
+      const reads = metrics.tool_calls.Read ?? 0;
+      const queryStr = lastQuery ? ` — ${lastQuery}` : "";
+      summary = `${searches} searches + ${reads} reads${queryStr}`;
+      break;
+    }
+    case "content": {
+      const filesChanged = metrics.files_changed ?? 0;
+      const queryStr = lastQuery ? ` — ${lastQuery}` : "";
+      summary = `${filesChanged} files created/edited${queryStr}`;
+      break;
+    }
+    default: {
+      const toolCount = Object.keys(metrics.tool_calls).length;
+      const queryStr = lastQuery ? ` — ${lastQuery}` : "";
+      summary = `${metrics.total_tool_calls} tool calls across ${toolCount} tools${queryStr}`;
+      break;
+    }
+  }
+
+  if (summary.length > MAX_LEN) {
+    return `${summary.slice(0, MAX_LEN - 3)}...`;
+  }
+  return summary;
+}
+
+/** Get the top N tools by call count. */
+function getTopTools(toolCalls: Record<string, number>, n: number): string[] {
+  return Object.entries(toolCalls)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, n)
+    .map(([name]) => name);
+}
+
+/** Truncate a query string to maxLen, adding ellipsis if needed. */
+function truncateQuery(query: string, maxLen: number): string {
+  const trimmed = query.trim();
+  if (!trimmed) return "";
+  if (trimmed.length <= maxLen) return trimmed;
+  return `${trimmed.slice(0, maxLen - 3)}...`;
+}
+
 function emptyMetrics(): TranscriptMetrics {
   return {
     tool_calls: {},

package/cli/selftune/watchlist.ts

@@ -0,0 +1,65 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+
+import { SELFTUNE_CONFIG_DIR, WATCHED_SKILLS_PATH } from "./constants.js";
+
+const CURRENT_WATCHLIST_VERSION = 1;
+
+interface WatchlistPayload {
+  version: typeof CURRENT_WATCHLIST_VERSION;
+  skills: string[];
+}
+
+function normalizeSkills(skills: string[]): string[] {
+  const seen = new Set<string>();
+  const normalized: string[] = [];
+  for (const skill of skills) {
+    const trimmed = skill.trim();
+    if (!trimmed || seen.has(trimmed)) continue;
+    seen.add(trimmed);
+    normalized.push(trimmed);
+  }
+  return normalized;
+}
+
+export function loadWatchedSkills(): string[] {
+  try {
+    if (!existsSync(WATCHED_SKILLS_PATH)) return [];
+    const parsed = JSON.parse(
+      readFileSync(WATCHED_SKILLS_PATH, "utf-8"),
+    ) as Partial<WatchlistPayload>;
+    return parsed.version === CURRENT_WATCHLIST_VERSION && Array.isArray(parsed.skills)
+      ? normalizeSkills(parsed.skills.filter((skill): skill is string => typeof skill === "string"))
+      : [];
+  } catch {
+    return [];
+  }
+}
+
+export function saveWatchedSkills(skills: string[]): string[] {
+  const normalized = normalizeSkills(skills);
+  mkdirSync(SELFTUNE_CONFIG_DIR, { recursive: true });
+  const tempPath = `${WATCHED_SKILLS_PATH}.${process.pid}.${Date.now()}.tmp`;
+  try {
+    writeFileSync(
+      tempPath,
+      JSON.stringify({ version: CURRENT_WATCHLIST_VERSION, skills: normalized }, null, 2),
+      "utf-8",
+    );
+    renameSync(tempPath, WATCHED_SKILLS_PATH);
+  } catch (error) {
+    try {
+      if (existsSync(tempPath)) unlinkSync(tempPath);
+    } catch {
+      // Best-effort cleanup for interrupted temp writes.
+    }
+    throw error;
+  }
+  return normalized;
+}

package/node_modules/@selftune/telemetry-contract/src/types.ts

@@ -143,7 +143,18 @@ export interface CanonicalExecutionFactRecord extends CanonicalSessionRecordBase
   errors_encountered: number;
   input_tokens?: number;
   output_tokens?: number;
+  cached_input_tokens?: number;
+  reasoning_output_tokens?: number;
+  cost_usd?: number;
   duration_ms?: number;
+  files_changed?: number;
+  lines_added?: number;
+  lines_removed?: number;
+  lines_modified?: number;
+  /** Count of output-producing tool calls (Write, Edit, WebFetch, WebSearch, Skill, Agent). */
+  artifact_count?: number;
+  /** Inferred session type based on tool distribution. */
+  session_type?: "dev" | "research" | "content" | "mixed";
   completion_status?: CanonicalCompletionStatus;
   end_reason?: string;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "selftune",
-  "version": "0.2.16",
+  "version": "0.2.19",
   "description": "Self-improving skills CLI for AI agents",
   "keywords": [
     "agent",

package/packages/telemetry-contract/src/types.ts

@@ -143,7 +143,18 @@ export interface CanonicalExecutionFactRecord extends CanonicalSessionRecordBase
   errors_encountered: number;
   input_tokens?: number;
   output_tokens?: number;
+  cached_input_tokens?: number;
+  reasoning_output_tokens?: number;
+  cost_usd?: number;
   duration_ms?: number;
+  files_changed?: number;
+  lines_added?: number;
+  lines_removed?: number;
+  lines_modified?: number;
+  /** Count of output-producing tool calls (Write, Edit, WebFetch, WebSearch, Skill, Agent). */
+  artifact_count?: number;
+  /** Inferred session type based on tool distribution. */
+  session_type?: "dev" | "research" | "content" | "mixed";
   completion_status?: CanonicalCompletionStatus;
   end_reason?: string;
 }