shakers-cli-beta 0.8.0-beta.4 → 0.9.0

package/src/session-normalizers.js

@@ -0,0 +1,174 @@
+'use strict';
+
+// Input normalizers: map a RAW session record into the Claude-shaped internal object model
+// that session-scan harvest + transcript-turns already consume
+// ({ cwd, timestamp, type, message:{ role, content:[{type:'text'|'tool_use', ...}] } }).
+// They ONLY translate input shape — downstream redaction/upload (share.js#derivePayload
+// -> POST usage/reports) is unchanged; no field is read that the Claude path didn't expose.
+// FAIL-CLOSED: every normalizer returns null / [] when a record does not match the expected
+// shape — it never fabricates or guesses fields. A documented format that differs from the
+// real one yields "no data for that tool", never wrong data.
+
+function str(v) {
+  return typeof v === 'string' && v ? v : null;
+}
+
+function textFromParts(parts) {
+  if (typeof parts === 'string') return parts.trim() || null;
+  if (!Array.isArray(parts)) return null;
+  const out = [];
+  for (const p of parts) {
+    if (typeof p === 'string') out.push(p);
+    else if (p && typeof p === 'object' && typeof p.text === 'string') out.push(p.text);
+  }
+  const joined = out.join('').trim();
+  return joined || null;
+}
+
+// --- Codex (~/.codex/sessions/**/rollout-*.jsonl) ---
+// Reads timestamp, cwd (session_meta), user text, and shell commands; a record matching none
+// of those passes through only its own real timestamp/cwd, else is dropped.
+function codexCommand(args) {
+  let a = args;
+  if (typeof a === 'string') {
+    try { a = JSON.parse(a); } catch { return a.trim() || null; }
+  }
+  if (!a || typeof a !== 'object') return null;
+  const cmd = a.command ?? a.cmd ?? a.commandLine;
+  if (Array.isArray(cmd)) return cmd.map(String).join(' ').trim() || null;
+  return str(cmd);
+}
+
+function normalizeCodexRecord(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  const payload = raw.payload && typeof raw.payload === 'object' ? raw.payload : null;
+  const item = payload || raw;
+  const out = {};
+  const ts = str(raw.timestamp) || str(item.timestamp);
+  if (ts) out.timestamp = ts;
+  const cwd = str(raw.cwd) || str(item.cwd);
+  if (cwd) out.cwd = cwd;
+
+  const role = str(item.role);
+  if (role === 'user') {
+    const text = textFromParts(item.content);
+    if (text) {
+      out.type = 'user';
+      out.message = { role: 'user', content: [{ type: 'text', text }] };
+      return out;
+    }
+  }
+
+  const type = str(raw.type) || str(item.type);
+  const name = str(item.name) || (type === 'function_call' ? 'shell' : null);
+  if (name && /shell|exec|bash|terminal|command/i.test(name)) {
+    const command = codexCommand(item.arguments ?? item.input ?? item.args);
+    if (command) {
+      out.message = { content: [{ type: 'tool_use', name: 'Shell', input: { command } }] };
+      return out;
+    }
+  }
+
+  return out.timestamp || out.cwd ? out : null;
+}
+
+// --- Gemini CLI (~/.gemini/tmp/<project-hash>/chats/logs.json | session-*.jsonl) ---
+// logs.json = JSON array of { type:'user'|'model', message, timestamp }; session-*.jsonl =
+// per-line { role, parts }. No cwd is recorded (the dir is a one-way SHA-256 hash), so cwd
+// stays null — global session/time metrics still count; per-repo scoping can't.
+function normalizeGeminiRecord(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  const out = {};
+  const ts = str(raw.timestamp) || str(raw.time);
+  if (ts) out.timestamp = ts;
+
+  const isUser = str(raw.type) === 'user' || str(raw.role) === 'user';
+  if (isUser) {
+    const text = str(raw.message) || textFromParts(raw.parts) || textFromParts(raw.content);
+    if (text) {
+      out.type = 'user';
+      out.message = { role: 'user', content: [{ type: 'text', text }] };
+      return out;
+    }
+  }
+  return out.timestamp ? out : null;
+}
+
+// --- Continue (continue.dev) ~/.continue/sessions/*.json ---
+// Each file is one session object; expand its history into per-turn objects, carrying
+// workspaceDirectory as cwd (good repo attribution).
+function normalizeContinueRecord(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  const cwd = str(raw.workspaceDirectory) || str(raw.workspace) || str(raw.cwd);
+  const history = Array.isArray(raw.history) ? raw.history
+    : (Array.isArray(raw.messages) ? raw.messages : (Array.isArray(raw.conversation) ? raw.conversation : null));
+  // A single message object (not a session) — handle defensively too.
+  if (!history) {
+    const msg = raw.message && typeof raw.message === 'object' ? raw.message : raw;
+    if (str(msg.role) === 'user') {
+      const text = typeof msg.content === 'string' ? msg.content.trim() : textFromParts(msg.content);
+      if (text) return oneUser(text, cwd, str(raw.timestamp) || str(raw.createdAt));
+    }
+    return null;
+  }
+  const out = [];
+  for (const entry of history) {
+    const msg = entry && entry.message && typeof entry.message === 'object' ? entry.message : entry;
+    if (!msg || typeof msg !== 'object') continue;
+    const ts = str(entry.timestamp) || str(entry.createdAt) || str(msg.timestamp);
+    // Only emit on a real user turn (with text) or a record carrying a real timestamp.
+    if (str(msg.role) === 'user') {
+      const text = typeof msg.content === 'string' ? msg.content.trim() : textFromParts(msg.content);
+      if (text) { out.push(oneUser(text, cwd, ts)); continue; }
+    }
+    if (ts) out.push(cwd ? { cwd, timestamp: ts } : { timestamp: ts });
+  }
+  return out.length ? out : null;
+}
+
+// --- Cline / Roo Code: VS Code globalStorage .../tasks/<id>/api_conversation_history.json ---
+// One JSON array of Anthropic messages (role/content); no cwd and no timestamp in this file
+// (ui_messages.json has ts) → cwd/time attribution absent. Emits only real user turns.
+function normalizeClineRecord(raw) {
+  if (!raw || typeof raw !== 'object') return null;
+  if (str(raw.role) !== 'user') return null;
+  const text = typeof raw.content === 'string' ? raw.content.trim() : textFromParts(raw.content);
+  return text ? oneUser(text, null, null) : null;
+}
+
+// --- Aider: per-repo .aider.chat.history.md (markdown) ---
+// Repo-level source → cwd is the repo toplevel (good attribution). Markdown parse:
+// '# aider chat started at <ts>' sets the session time; '#### <text>' lines are user
+// instructions. A file matching neither pattern yields 0 records.
+function parseAiderHistory(text, cwd) {
+  if (typeof text !== 'string' || !text) return [];
+  const out = [];
+  let ts = null;
+  for (const line of text.split('\n')) {
+    const started = /^#\s*aider chat started at\s+(.+?)\s*$/i.exec(line);
+    if (started) {
+      const t = Date.parse(started[1].replace(' ', 'T'));
+      ts = Number.isFinite(t) ? new Date(t).toISOString() : null;
+      if (ts) out.push(cwd ? { cwd, timestamp: ts } : { timestamp: ts });
+      continue;
+    }
+    const user = /^####\s+(.*)$/.exec(line);
+    if (user && user[1].trim()) out.push(oneUser(user[1].trim(), cwd, ts));
+  }
+  return out;
+}
+
+function oneUser(text, cwd, ts) {
+  const o = { type: 'user', message: { role: 'user', content: [{ type: 'text', text }] } };
+  if (cwd) o.cwd = cwd;
+  if (ts) o.timestamp = ts;
+  return o;
+}
+
+module.exports = {
+  normalizeCodexRecord,
+  normalizeGeminiRecord,
+  normalizeContinueRecord,
+  normalizeClineRecord,
+  parseAiderHistory,
+};

package/src/session-scan.js

@@ -4,6 +4,13 @@ const fs = require('fs');
 const path = require('path');
 const { getHomeDir } = require('./env-paths');
 const { scrubSecrets } = require('./agent-synthesis');
+const {
+  normalizeCodexRecord,
+  normalizeGeminiRecord,
+  normalizeContinueRecord,
+  normalizeClineRecord,
+  parseAiderHistory,
+} = require('./session-normalizers');
 
 // GLOBAL, machine-wide inventory of AI coding-tool SESSIONS (hybrid evaluation, Slice 1).
 
@@ -73,16 +80,39 @@ const FILE_TOOL_NAMES = new Set([
 ]);
 const PLANNING_TOOL_NAMES = new Set(['ExitPlanMode', 'TodoWrite']);
 
+// Only logs.json + session-*.jsonl under ~/.gemini/tmp are transcripts (skip checkpoints/config).
+function isGeminiTranscriptFile(name) {
+  return name === 'logs.json' || (name.startsWith('session-') && name.endsWith('.jsonl'));
+}
+
+// Cline / Roo Code conversation file (both forks use this filename under tasks/<id>/).
+function isClineHistoryFile(name) {
+  return name === 'api_conversation_history.json';
+}
+
+// VS Code globalStorage root per OS (Cline/Roo live under <ext-id>/tasks/<id>/ here).
+function vscodeGlobalStorageDir(home) {
+  if (process.platform === 'darwin') return path.join(home, 'Library', 'Application Support', 'Code', 'User', 'globalStorage');
+  if (process.platform === 'win32') return path.join(home, 'AppData', 'Roaming', 'Code', 'User', 'globalStorage');
+  return path.join(home, '.config', 'Code', 'User', 'globalStorage');
+}
+
 function toolSources(env = process.env) {
   const home = getHomeDir(env);
   return [
     { tool: 'claude', dir: path.join(home, '.claude', 'projects') },
-    { tool: 'codex', dir: path.join(home, '.codex', 'sessions') },
+    { tool: 'codex', dir: path.join(home, '.codex', 'sessions'), normalize: normalizeCodexRecord },
+    { tool: 'gemini', dir: path.join(home, '.gemini', 'tmp'), fileFilter: isGeminiTranscriptFile, normalize: normalizeGeminiRecord },
+    // Kept by decision: ~/.cursor/chats is OBSOLETE (real Cursor chat is SQLite in App Support); awaiting a future SQLite extractor, no tool removed.
     { tool: 'cursor', dir: path.join(home, '.cursor', 'chats') },
+    { tool: 'continue', dir: path.join(home, '.continue', 'sessions'), normalize: normalizeContinueRecord },
+    { tool: 'cline', dir: vscodeGlobalStorageDir(home), fileFilter: isClineHistoryFile, normalize: normalizeClineRecord },
+    // Repo-level: .aider.chat.history.md lives INSIDE each git repo, so cwd = the repo toplevel.
+    { tool: 'aider', repoLevel: true, file: '.aider.chat.history.md', parseText: parseAiderHistory },
   ];
 }
 
-function listSessionFiles(root) {
+function listSessionFiles(root, fileFilter = isTranscriptFile) {
   const out = [];
   const stack = [root];
   while (stack.length > 0) {
@@ -97,7 +127,7 @@ function listSessionFiles(root) {
       const full = path.join(dir, entry.name);
       if (entry.isDirectory()) {
         stack.push(full);
-      } else if (entry.isFile() && isTranscriptFile(entry.name)) {
+      } else if (entry.isFile() && fileFilter(entry.name)) {
         out.push(full);
         if (out.length >= MAX_FILES_PER_TOOL) return out;
       }
@@ -106,6 +136,77 @@ function listSessionFiles(root) {
   return out;
 }
 
+// Whole-file JSON reader (bounded) for `.json` transcripts (e.g. Gemini logs.json = array).
+function readJsonArrayFile(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, 'r');
+  } catch {
+    return [];
+  }
+  try {
+    const { size } = fs.fstatSync(fd);
+    const readLen = Math.min(size, MAX_BYTES_PER_FILE);
+    const buf = Buffer.allocUnsafe(readLen);
+    fs.readSync(fd, buf, 0, readLen, 0);
+    const parsed = JSON.parse(buf.toString('utf8'));
+    if (Array.isArray(parsed)) return parsed;
+    return parsed && typeof parsed === 'object' ? [parsed] : [];
+  } catch {
+    return [];
+  } finally {
+    try { fs.closeSync(fd); } catch { /* already closed */ }
+  }
+}
+
+// Parse a file to raw records by extension (.jsonl = per line, .json = whole array).
+function parseRawRecords(file) {
+  return file.endsWith('.jsonl') ? readSessionObjects(file) : readJsonArrayFile(file);
+}
+
+// Read a source's file and normalize its records into the Claude-shaped internal model.
+// A normalizer may return an object, an array (expanded), or null/[] (fail-closed skip).
+function readSourceObjects(source, file) {
+  const raw = parseRawRecords(file);
+  if (typeof source.normalize !== 'function') return raw;
+  const out = [];
+  for (const r of raw) {
+    const o = source.normalize(r);
+    if (Array.isArray(o)) out.push(...o);
+    else if (o) out.push(o);
+  }
+  return out;
+}
+
+// Bounded whole-file text read (for repo-level markdown sources, e.g. Aider).
+function readTextFile(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, 'r');
+  } catch {
+    return null;
+  }
+  try {
+    const { size } = fs.fstatSync(fd);
+    const readLen = Math.min(size, MAX_BYTES_PER_FILE);
+    const buf = Buffer.allocUnsafe(readLen);
+    fs.readSync(fd, buf, 0, readLen, 0);
+    return buf.toString('utf8');
+  } catch {
+    return null;
+  } finally {
+    try { fs.closeSync(fd); } catch { /* already closed */ }
+  }
+}
+
+// Repo-level source: read <toplevel>/<source.file> and parse it with cwd = the toplevel.
+function readRepoLevelObjects(source, toplevel) {
+  const text = readTextFile(path.join(toplevel, source.file));
+  if (text === null) return [];
+  const objects = source.parseText(text, toplevel);
+  return Array.isArray(objects) ? objects : [];
+}
+
 function pushCommand(acc, command) {
   if (typeof command !== 'string') return;
   acc.bashCommandsCount += 1;
@@ -198,16 +299,31 @@ function fileInScope(cwd, selectedCwds) {
 
 // Yields every parsed session object across every known AI-tool session directory on this machine, tagged with the tool it came from.
 function* iterateSessionObjects(env = process.env) {
-  for (const source of toolSources(env)) {
-    const files = listSessionFiles(source.dir);
+  const sources = toolSources(env);
+  // Phase 1: home-dir sources; collect the repo toplevels (cwds) they reveal.
+  const seenCwds = new Set();
+  for (const source of sources) {
+    if (source.repoLevel) continue;
+    const files = listSessionFiles(source.dir, source.fileFilter || isTranscriptFile);
     for (const file of files) {
-      const objects = readSessionObjects(file);
+      const objects = readSourceObjects(source, file);
       const cwd = resolveFileCwd(objects);
+      if (cwd) seenCwds.add(cwd);
       for (const obj of objects) {
         yield { tool: source.tool, obj, cwd, filePath: file };
       }
     }
   }
+  // Phase 2: repo-level sources (e.g. Aider) over the repos just discovered — cwd is known.
+  for (const source of sources) {
+    if (!source.repoLevel) continue;
+    for (const top of seenCwds) {
+      const objects = readRepoLevelObjects(source, top);
+      for (const obj of objects) {
+        yield { tool: source.tool, obj, cwd: top, filePath: path.join(top, source.file) };
+      }
+    }
+  }
 }
 
 // Scan every known AI-tool session directory on this machine.
@@ -303,13 +419,19 @@ function makeSessionsAccumulator() {
 
 function walkSessionSignals(env = process.env, selectedCwds = null, onHumanObject = null) {
   const sessions = makeSessionsAccumulator();
-  for (const source of toolSources(env)) {
-    const files = listSessionFiles(source.dir);
+  const sources = toolSources(env);
+  const seenCwds = new Set();
+  // Phase 1: home-dir sources; remember the repo toplevels (cwds) they reveal.
+  for (const source of sources) {
+    if (source.repoLevel) continue;
+    const files = listSessionFiles(source.dir, source.fileFilter || isTranscriptFile);
     if (files.length === 0) continue;
     sessions.startTool(source.tool);
     for (const f of files) {
-      const objects = readSessionObjects(f);
-      if (!fileInScope(resolveFileCwd(objects), selectedCwds)) continue;
+      const objects = readSourceObjects(source, f);
+      const cwd = resolveFileCwd(objects);
+      if (cwd) seenCwds.add(cwd);
+      if (!fileInScope(cwd, selectedCwds)) continue;
       sessions.addFile(f, objects);
       if (onHumanObject) {
         for (const o of objects) onHumanObject(o);
@@ -317,6 +439,23 @@ function walkSessionSignals(env = process.env, selectedCwds = null, onHumanObjec
     }
     sessions.endTool();
   }
+  // Phase 2: repo-level sources (e.g. Aider) — candidate repos = the selected scope, else
+  // the repos seen in phase 1. cwd is the repo toplevel, so attribution is exact.
+  const toplevels = selectedCwds instanceof Set ? selectedCwds : seenCwds;
+  for (const source of sources) {
+    if (!source.repoLevel) continue;
+    sessions.startTool(source.tool);
+    for (const top of toplevels) {
+      if (!fileInScope(top, selectedCwds)) continue;
+      const objects = readRepoLevelObjects(source, top);
+      if (objects.length === 0) continue;
+      sessions.addFile(path.join(top, source.file), objects);
+      if (onHumanObject) {
+        for (const o of objects) onHumanObject(o);
+      }
+    }
+    sessions.endTool();
+  }
   return sessions.result();
 }
 

package/src/work-streams.js

@@ -2,6 +2,7 @@
 
 const { execFileSync } = require('child_process');
 const { isGitRepo } = require('./git-activity');
+const { takePrefetchedGitLog } = require('./git-log-prefetch');
 
 // Deterministic WORK-STREAM structure for the project where `shakers ai-usage` runs (hybrid evaluation, Slice 2).
 
@@ -9,11 +10,15 @@ const GIT_TIMEOUT_MS = 15000;
 const SESSION_GAP_SECONDS = 4 * 3600;
 const SECONDS_PER_DAY = 86400;
 
+const WORK_STREAMS_LOG_ARGS = ['log', '--no-merges', '--pretty=format:%at'];
+
 function runGitTimestamps(root) {
+  const prefetched = takePrefetchedGitLog(root, WORK_STREAMS_LOG_ARGS);
+  if (prefetched !== undefined) return prefetched;
   try {
     return execFileSync(
       'git',
-      ['-C', root, 'log', '--no-merges', '--pretty=format:%at'],
+      ['-C', root, ...WORK_STREAMS_LOG_ARGS],
       {
         encoding: 'utf8',
         timeout: GIT_TIMEOUT_MS,
@@ -71,4 +76,4 @@ function collectWorkStreams(root) {
   };
 }
 
-module.exports = { collectWorkStreams, SESSION_GAP_SECONDS };
+module.exports = { collectWorkStreams, SESSION_GAP_SECONDS, WORK_STREAMS_LOG_ARGS };