shakers-cli-beta 0.8.0-beta.5 → 0.9.0

package/src/session-scan.js

@@ -4,6 +4,13 @@ const fs = require('fs');
 const path = require('path');
 const { getHomeDir } = require('./env-paths');
 const { scrubSecrets } = require('./agent-synthesis');
+const {
+  normalizeCodexRecord,
+  normalizeGeminiRecord,
+  normalizeContinueRecord,
+  normalizeClineRecord,
+  parseAiderHistory,
+} = require('./session-normalizers');
 
 // GLOBAL, machine-wide inventory of AI coding-tool SESSIONS (hybrid evaluation, Slice 1).
 
@@ -73,16 +80,39 @@ const FILE_TOOL_NAMES = new Set([
 ]);
 const PLANNING_TOOL_NAMES = new Set(['ExitPlanMode', 'TodoWrite']);
 
+// Only logs.json + session-*.jsonl under ~/.gemini/tmp are transcripts (skip checkpoints/config).
+function isGeminiTranscriptFile(name) {
+  return name === 'logs.json' || (name.startsWith('session-') && name.endsWith('.jsonl'));
+}
+
+// Cline / Roo Code conversation file (both forks use this filename under tasks/<id>/).
+function isClineHistoryFile(name) {
+  return name === 'api_conversation_history.json';
+}
+
+// VS Code globalStorage root per OS (Cline/Roo live under <ext-id>/tasks/<id>/ here).
+function vscodeGlobalStorageDir(home) {
+  if (process.platform === 'darwin') return path.join(home, 'Library', 'Application Support', 'Code', 'User', 'globalStorage');
+  if (process.platform === 'win32') return path.join(home, 'AppData', 'Roaming', 'Code', 'User', 'globalStorage');
+  return path.join(home, '.config', 'Code', 'User', 'globalStorage');
+}
+
 function toolSources(env = process.env) {
   const home = getHomeDir(env);
   return [
     { tool: 'claude', dir: path.join(home, '.claude', 'projects') },
-    { tool: 'codex', dir: path.join(home, '.codex', 'sessions') },
+    { tool: 'codex', dir: path.join(home, '.codex', 'sessions'), normalize: normalizeCodexRecord },
+    { tool: 'gemini', dir: path.join(home, '.gemini', 'tmp'), fileFilter: isGeminiTranscriptFile, normalize: normalizeGeminiRecord },
+    // Kept by decision: ~/.cursor/chats is OBSOLETE (real Cursor chat is SQLite in App Support); awaiting a future SQLite extractor, no tool removed.
     { tool: 'cursor', dir: path.join(home, '.cursor', 'chats') },
+    { tool: 'continue', dir: path.join(home, '.continue', 'sessions'), normalize: normalizeContinueRecord },
+    { tool: 'cline', dir: vscodeGlobalStorageDir(home), fileFilter: isClineHistoryFile, normalize: normalizeClineRecord },
+    // Repo-level: .aider.chat.history.md lives INSIDE each git repo, so cwd = the repo toplevel.
+    { tool: 'aider', repoLevel: true, file: '.aider.chat.history.md', parseText: parseAiderHistory },
   ];
 }
 
-function listSessionFiles(root) {
+function listSessionFiles(root, fileFilter = isTranscriptFile) {
   const out = [];
   const stack = [root];
   while (stack.length > 0) {
@@ -97,7 +127,7 @@ function listSessionFiles(root) {
       const full = path.join(dir, entry.name);
       if (entry.isDirectory()) {
         stack.push(full);
-      } else if (entry.isFile() && isTranscriptFile(entry.name)) {
+      } else if (entry.isFile() && fileFilter(entry.name)) {
         out.push(full);
         if (out.length >= MAX_FILES_PER_TOOL) return out;
       }
@@ -106,6 +136,77 @@ function listSessionFiles(root) {
   return out;
 }
 
+// Whole-file JSON reader (bounded) for `.json` transcripts (e.g. Gemini logs.json = array).
+function readJsonArrayFile(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, 'r');
+  } catch {
+    return [];
+  }
+  try {
+    const { size } = fs.fstatSync(fd);
+    const readLen = Math.min(size, MAX_BYTES_PER_FILE);
+    const buf = Buffer.allocUnsafe(readLen);
+    fs.readSync(fd, buf, 0, readLen, 0);
+    const parsed = JSON.parse(buf.toString('utf8'));
+    if (Array.isArray(parsed)) return parsed;
+    return parsed && typeof parsed === 'object' ? [parsed] : [];
+  } catch {
+    return [];
+  } finally {
+    try { fs.closeSync(fd); } catch { /* already closed */ }
+  }
+}
+
+// Parse a file to raw records by extension (.jsonl = per line, .json = whole array).
+function parseRawRecords(file) {
+  return file.endsWith('.jsonl') ? readSessionObjects(file) : readJsonArrayFile(file);
+}
+
+// Read a source's file and normalize its records into the Claude-shaped internal model.
+// A normalizer may return an object, an array (expanded), or null/[] (fail-closed skip).
+function readSourceObjects(source, file) {
+  const raw = parseRawRecords(file);
+  if (typeof source.normalize !== 'function') return raw;
+  const out = [];
+  for (const r of raw) {
+    const o = source.normalize(r);
+    if (Array.isArray(o)) out.push(...o);
+    else if (o) out.push(o);
+  }
+  return out;
+}
+
+// Bounded whole-file text read (for repo-level markdown sources, e.g. Aider).
+function readTextFile(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, 'r');
+  } catch {
+    return null;
+  }
+  try {
+    const { size } = fs.fstatSync(fd);
+    const readLen = Math.min(size, MAX_BYTES_PER_FILE);
+    const buf = Buffer.allocUnsafe(readLen);
+    fs.readSync(fd, buf, 0, readLen, 0);
+    return buf.toString('utf8');
+  } catch {
+    return null;
+  } finally {
+    try { fs.closeSync(fd); } catch { /* already closed */ }
+  }
+}
+
+// Repo-level source: read <toplevel>/<source.file> and parse it with cwd = the toplevel.
+function readRepoLevelObjects(source, toplevel) {
+  const text = readTextFile(path.join(toplevel, source.file));
+  if (text === null) return [];
+  const objects = source.parseText(text, toplevel);
+  return Array.isArray(objects) ? objects : [];
+}
+
 function pushCommand(acc, command) {
   if (typeof command !== 'string') return;
   acc.bashCommandsCount += 1;
@@ -198,16 +299,31 @@ function fileInScope(cwd, selectedCwds) {
 
 // Yields every parsed session object across every known AI-tool session directory on this machine, tagged with the tool it came from.
 function* iterateSessionObjects(env = process.env) {
-  for (const source of toolSources(env)) {
-    const files = listSessionFiles(source.dir);
+  const sources = toolSources(env);
+  // Phase 1: home-dir sources; collect the repo toplevels (cwds) they reveal.
+  const seenCwds = new Set();
+  for (const source of sources) {
+    if (source.repoLevel) continue;
+    const files = listSessionFiles(source.dir, source.fileFilter || isTranscriptFile);
     for (const file of files) {
-      const objects = readSessionObjects(file);
+      const objects = readSourceObjects(source, file);
       const cwd = resolveFileCwd(objects);
+      if (cwd) seenCwds.add(cwd);
       for (const obj of objects) {
         yield { tool: source.tool, obj, cwd, filePath: file };
       }
     }
   }
+  // Phase 2: repo-level sources (e.g. Aider) over the repos just discovered — cwd is known.
+  for (const source of sources) {
+    if (!source.repoLevel) continue;
+    for (const top of seenCwds) {
+      const objects = readRepoLevelObjects(source, top);
+      for (const obj of objects) {
+        yield { tool: source.tool, obj, cwd: top, filePath: path.join(top, source.file) };
+      }
+    }
+  }
 }
 
 // Scan every known AI-tool session directory on this machine.
@@ -303,13 +419,19 @@ function makeSessionsAccumulator() {
 
 function walkSessionSignals(env = process.env, selectedCwds = null, onHumanObject = null) {
   const sessions = makeSessionsAccumulator();
-  for (const source of toolSources(env)) {
-    const files = listSessionFiles(source.dir);
+  const sources = toolSources(env);
+  const seenCwds = new Set();
+  // Phase 1: home-dir sources; remember the repo toplevels (cwds) they reveal.
+  for (const source of sources) {
+    if (source.repoLevel) continue;
+    const files = listSessionFiles(source.dir, source.fileFilter || isTranscriptFile);
     if (files.length === 0) continue;
     sessions.startTool(source.tool);
     for (const f of files) {
-      const objects = readSessionObjects(f);
-      if (!fileInScope(resolveFileCwd(objects), selectedCwds)) continue;
+      const objects = readSourceObjects(source, f);
+      const cwd = resolveFileCwd(objects);
+      if (cwd) seenCwds.add(cwd);
+      if (!fileInScope(cwd, selectedCwds)) continue;
       sessions.addFile(f, objects);
       if (onHumanObject) {
         for (const o of objects) onHumanObject(o);
@@ -317,6 +439,23 @@ function walkSessionSignals(env = process.env, selectedCwds = null, onHumanObjec
     }
     sessions.endTool();
   }
+  // Phase 2: repo-level sources (e.g. Aider) — candidate repos = the selected scope, else
+  // the repos seen in phase 1. cwd is the repo toplevel, so attribution is exact.
+  const toplevels = selectedCwds instanceof Set ? selectedCwds : seenCwds;
+  for (const source of sources) {
+    if (!source.repoLevel) continue;
+    sessions.startTool(source.tool);
+    for (const top of toplevels) {
+      if (!fileInScope(top, selectedCwds)) continue;
+      const objects = readRepoLevelObjects(source, top);
+      if (objects.length === 0) continue;
+      sessions.addFile(path.join(top, source.file), objects);
+      if (onHumanObject) {
+        for (const o of objects) onHumanObject(o);
+      }
+    }
+    sessions.endTool();
+  }
   return sessions.result();
 }
 

package/src/work-streams.js

@@ -2,6 +2,7 @@
 
 const { execFileSync } = require('child_process');
 const { isGitRepo } = require('./git-activity');
+const { takePrefetchedGitLog } = require('./git-log-prefetch');
 
 // Deterministic WORK-STREAM structure for the project where `shakers ai-usage` runs (hybrid evaluation, Slice 2).
 
@@ -9,11 +10,15 @@ const GIT_TIMEOUT_MS = 15000;
 const SESSION_GAP_SECONDS = 4 * 3600;
 const SECONDS_PER_DAY = 86400;
 
+const WORK_STREAMS_LOG_ARGS = ['log', '--no-merges', '--pretty=format:%at'];
+
 function runGitTimestamps(root) {
+  const prefetched = takePrefetchedGitLog(root, WORK_STREAMS_LOG_ARGS);
+  if (prefetched !== undefined) return prefetched;
   try {
     return execFileSync(
       'git',
-      ['-C', root, 'log', '--no-merges', '--pretty=format:%at'],
+      ['-C', root, ...WORK_STREAMS_LOG_ARGS],
       {
         encoding: 'utf8',
         timeout: GIT_TIMEOUT_MS,
@@ -71,4 +76,4 @@ function collectWorkStreams(root) {
   };
 }
 
-module.exports = { collectWorkStreams, SESSION_GAP_SECONDS };
+module.exports = { collectWorkStreams, SESSION_GAP_SECONDS, WORK_STREAMS_LOG_ARGS };