tokenlean 0.1.0 → 0.3.0

package/bin/tl-structure.mjs

@@ -9,9 +9,26 @@
  * Usage: tl-structure [path] [--depth N]
  */
 
+// Prompt info for tl-prompt
+if (process.argv.includes('--prompt')) {
+  console.log(JSON.stringify({
+    name: 'tl-structure',
+    desc: 'Project overview with token estimates',
+    when: 'before-read',
+    example: 'tl-structure'
+  }));
+  process.exit(0);
+}
+
 import { readdirSync, readFileSync, existsSync } from 'fs';
 import { join, basename } from 'path';
-import { estimateTokens, formatTokens } from '../src/output.mjs';
+import {
+  createOutput,
+  parseCommonArgs,
+  estimateTokens,
+  formatTokens,
+  COMMON_OPTIONS_HELP
+} from '../src/output.mjs';
 import {
   getSkipDirs,
   getImportantFiles,
@@ -19,6 +36,25 @@ import {
 } from '../src/project.mjs';
 import { getConfig } from '../src/config.mjs';
 
+const HELP = `
+tl-structure - Smart project overview with context estimates
+
+Usage: tl-structure [path] [options]
+
+Options:
+  --depth N, -d N    Maximum depth to show (default: 3)
+${COMMON_OPTIONS_HELP}
+
+Configure defaults in .tokenleanrc.json:
+  "structure": { "depth": 3, "important": ["src", "lib"] }
+
+Examples:
+  tl-structure                   # Current directory
+  tl-structure src/ -d 2         # Just src, 2 levels deep
+  tl-structure -j                # JSON output
+  tl-structure -q                # Quiet (no headers)
+`;
+
 function getDirStats(dirPath, skipDirs, importantDirs) {
   let totalTokens = 0;
   let fileCount = 0;
@@ -38,80 +74,104 @@ function getDirStats(dirPath, skipDirs, importantDirs) {
             const content = readFileSync(fullPath, 'utf-8');
             totalTokens += estimateTokens(content);
             fileCount++;
-          } catch (e) { /* skip binary */ }
+          } catch { /* skip binary */ }
         }
       }
-    } catch (e) { /* permission error */ }
+    } catch { /* permission error */ }
   }
 
   walk(dirPath);
   return { totalTokens, fileCount };
 }
 
-function printTree(dirPath, prefix, depth, maxDepth, skipDirs, importantDirs, importantFiles) {
-  if (depth > maxDepth) return;
-
-  const entries = readdirSync(dirPath, { withFileTypes: true });
-
-  // Sort: directories first, then by importance, then alphabetically
-  entries.sort((a, b) => {
-    if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1;
-    const aImportant = importantDirs.has(a.name) || importantFiles.has(a.name);
-    const bImportant = importantDirs.has(b.name) || importantFiles.has(b.name);
-    if (aImportant !== bImportant) return aImportant ? -1 : 1;
-    return a.name.localeCompare(b.name);
-  });
+function buildTree(dirPath, depth, maxDepth, skipDirs, importantDirs, importantFiles) {
+  const tree = [];
+  if (depth > maxDepth) return tree;
+
+  try {
+    const entries = readdirSync(dirPath, { withFileTypes: true });
+
+    // Sort: directories first, then by importance, then alphabetically
+    entries.sort((a, b) => {
+      if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1;
+      const aImportant = importantDirs.has(a.name) || importantFiles.has(a.name);
+      const bImportant = importantDirs.has(b.name) || importantFiles.has(b.name);
+      if (aImportant !== bImportant) return aImportant ? -1 : 1;
+      return a.name.localeCompare(b.name);
+    });
+
+    const filtered = entries.filter(e => {
+      if (e.name.startsWith('.') && e.name !== '.claude') return false;
+      if (skipDirs.has(e.name)) return false;
+      return true;
+    });
+
+    for (const entry of filtered) {
+      const fullPath = join(dirPath, entry.name);
+      const isImportant = importantDirs.has(entry.name) || importantFiles.has(entry.name);
+
+      if (entry.isDirectory()) {
+        const stats = getDirStats(fullPath, skipDirs, importantDirs);
+        tree.push({
+          name: entry.name,
+          type: 'dir',
+          important: isImportant,
+          fileCount: stats.fileCount,
+          tokens: stats.totalTokens,
+          children: buildTree(fullPath, depth + 1, maxDepth, skipDirs, importantDirs, importantFiles)
+        });
+      } else {
+        try {
+          const content = readFileSync(fullPath, 'utf-8');
+          const tokens = estimateTokens(content);
+          const lines = content.split('\n').length;
+          tree.push({
+            name: entry.name,
+            type: 'file',
+            important: isImportant,
+            tokens,
+            lines
+          });
+        } catch {
+          tree.push({
+            name: entry.name,
+            type: 'file',
+            important: isImportant,
+            binary: true
+          });
+        }
+      }
+    }
+  } catch { /* permission error */ }
 
-  const filtered = entries.filter(e => {
-    if (e.name.startsWith('.') && e.name !== '.claude') return false;
-    if (skipDirs.has(e.name)) return false;
-    return true;
-  });
+  return tree;
+}
 
-  filtered.forEach((entry, index) => {
-    const isLast = index === filtered.length - 1;
+function printTree(tree, out, prefix = '') {
+  tree.forEach((entry, index) => {
+    const isLast = index === tree.length - 1;
     const connector = isLast ? '└── ' : '├── ';
-    const fullPath = join(dirPath, entry.name);
+    const marker = entry.important ? '*' : ' ';
 
-    const isEntryImportant = importantDirs.has(entry.name) || importantFiles.has(entry.name);
-    const marker = isEntryImportant ? '*' : ' ';
-
-    if (entry.isDirectory()) {
-      const stats = getDirStats(fullPath, skipDirs, importantDirs);
-      const sizeInfo = stats.fileCount > 0
-        ? ` (${stats.fileCount} files, ~${formatTokens(stats.totalTokens)})`
+    if (entry.type === 'dir') {
+      const sizeInfo = entry.fileCount > 0
+        ? ` (${entry.fileCount} files, ~${formatTokens(entry.tokens)})`
         : ' (empty)';
-
-      console.log(`${prefix}${connector}${marker}${entry.name}/${sizeInfo}`);
+      out.add(`${prefix}${connector}${marker}${entry.name}/${sizeInfo}`);
 
       const newPrefix = prefix + (isLast ? '    ' : '│   ');
-      printTree(fullPath, newPrefix, depth + 1, maxDepth, skipDirs, importantDirs, importantFiles);
+      printTree(entry.children, out, newPrefix);
+    } else if (entry.binary) {
+      out.add(`${prefix}${connector}${marker}${entry.name} (binary)`);
     } else {
-      try {
-        const content = readFileSync(fullPath, 'utf-8');
-        const tokens = estimateTokens(content);
-        const lines = content.split('\n').length;
-        console.log(`${prefix}${connector}${marker}${entry.name} (~${formatTokens(tokens)}, ${lines}L)`);
-      } catch (e) {
-        console.log(`${prefix}${connector}${marker}${entry.name} (binary)`);
-      }
+      out.add(`${prefix}${connector}${marker}${entry.name} (~${formatTokens(entry.tokens)}, ${entry.lines}L)`);
     }
   });
 }
 
-// Prompt info for tl-prompt
-if (process.argv.includes('--prompt')) {
-  console.log(JSON.stringify({
-    name: 'tl-structure',
-    desc: 'Project overview with token estimates',
-    when: 'before-read',
-    example: 'tl-structure'
-  }));
-  process.exit(0);
-}
-
 // Main
 const args = process.argv.slice(2);
+const options = parseCommonArgs(args);
 
 // Get config defaults
 const structureConfig = getConfig('structure') || {};
@@ -119,43 +179,52 @@ const structureConfig = getConfig('structure') || {};
 let targetPath = '.';
 let maxDepth = structureConfig.depth || 3;
 
-// Get combined sets (defaults + user config extensions)
-const skipDirs = getSkipDirs();
-const importantDirs = getImportantDirs();
-const importantFiles = getImportantFiles();
-
-for (let i = 0; i < args.length; i++) {
-  if ((args[i] === '--depth' || args[i] === '-d') && args[i + 1]) {
-    maxDepth = parseInt(args[i + 1], 10);
+// Parse tool-specific options
+for (let i = 0; i < options.remaining.length; i++) {
+  const arg = options.remaining[i];
+  if ((arg === '--depth' || arg === '-d') && options.remaining[i + 1]) {
+    maxDepth = parseInt(options.remaining[i + 1], 10);
     i++;
-  } else if (args[i] === '--help' || args[i] === '-h') {
-    console.log(`
-tl-structure - Smart project overview with context estimates
-
-Usage: tl-structure [path] [options]
-
-Options:
-  --depth N, -d N    Maximum depth to show (default: ${structureConfig.depth || 3})
-  --help, -h         Show this help
-
-Configure defaults in .tokenleanrc.json:
-  "structure": { "depth": 3, "important": ["src", "lib"] }
-`);
-    process.exit(0);
-  } else if (!args[i].startsWith('-')) {
-    targetPath = args[i];
+  } else if (!arg.startsWith('-')) {
+    targetPath = arg;
   }
 }
 
+if (options.help) {
+  console.log(HELP);
+  process.exit(0);
+}
+
 if (!existsSync(targetPath)) {
   console.error(`Path not found: ${targetPath}`);
   process.exit(1);
 }
 
-const rootStats = getDirStats(targetPath, skipDirs, importantDirs);
-console.log(`\n${targetPath === '.' ? basename(process.cwd()) : targetPath}`);
-console.log(`Total: ${rootStats.fileCount} files, ~${formatTokens(rootStats.totalTokens)} tokens`);
-console.log(`(* = important for understanding project)\n`);
+// Get combined sets (defaults + user config extensions)
+const skipDirs = getSkipDirs();
+const importantDirs = getImportantDirs();
+const importantFiles = getImportantFiles();
 
-printTree(targetPath, '', 0, maxDepth, skipDirs, importantDirs, importantFiles);
-console.log();
+const out = createOutput(options);
+
+const rootStats = getDirStats(targetPath, skipDirs, importantDirs);
+const tree = buildTree(targetPath, 0, maxDepth, skipDirs, importantDirs, importantFiles);
+
+// Set JSON data
+out.setData('path', targetPath);
+out.setData('totalFiles', rootStats.fileCount);
+out.setData('totalTokens', rootStats.totalTokens);
+out.setData('depth', maxDepth);
+out.setData('tree', tree);
+
+// Headers
+const rootName = targetPath === '.' ? basename(process.cwd()) : targetPath;
+out.header(rootName);
+out.header(`Total: ${rootStats.fileCount} files, ~${formatTokens(rootStats.totalTokens)} tokens`);
+out.header(`(* = important for understanding project)`);
+out.blank();
+
+// Tree output
+printTree(tree, out);
+
+out.print();

package/bin/tl-todo.mjs

@@ -32,6 +32,7 @@ import {
   COMMON_OPTIONS_HELP
 } from '../src/output.mjs';
 import { findProjectRoot, shouldSkip, SKIP_DIRS } from '../src/project.mjs';
+import { withCache } from '../src/cache.mjs';
 
 const HELP = `
 tl-todo - Extract TODOs, FIXMEs, and other markers from codebase
@@ -102,7 +103,13 @@ function findTodos(searchPath, projectRoot) {
     // Require : or ( after marker to avoid false positives like "Todo Extraction"
     const commentPattern = `(${commentPrefixes.join('|')})\\s*(${markerPattern})[:(]`;
     const cmd = `rg -n --no-heading -i "${commentPattern}" "${shellEscape(searchPath)}" ${excludes} 2>/dev/null || true`;
-    const output = execSync(cmd, { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 });
+
+    const cacheKey = { op: 'rg-todo-markers', pattern: commentPattern, path: searchPath };
+    const output = withCache(
+      cacheKey,
+      () => execSync(cmd, { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 }),
+      { projectRoot }
+    );
 
     if (!output.trim()) {
       return todos;

package/bin/tl-unused.mjs

@@ -29,6 +29,7 @@ import {
   COMMON_OPTIONS_HELP
 } from '../src/output.mjs';
 import { findProjectRoot, shouldSkip, isCodeFile } from '../src/project.mjs';
+import { withCache } from '../src/cache.mjs';
 
 const HELP = `
 tl-unused - Find potentially unused exports and unreferenced files
@@ -221,26 +222,35 @@ function extractImports(content) {
 }
 
 function findReferencesWithGrep(name, projectRoot, excludeFile) {
-  // Use ripgrep for fast reference counting
-  const args = [
-    '-l',  // Files only
-    '--type', 'js',
-    '--type', 'ts',
-    '-w',  // Word boundary
-    name,
-    '.'
-  ];
-
-  const result = spawnSync('rg', args, {
-    cwd: projectRoot,
-    encoding: 'utf-8'
-  });
-
-  if (result.error || result.status !== 0) {
-    return 0;
-  }
+  // Use ripgrep for fast reference counting (with caching)
+  const cacheKey = { op: 'rg-ref-count', name, types: 'js,ts' };
+
+  const files = withCache(
+    cacheKey,
+    () => {
+      const args = [
+        '-l',  // Files only
+        '--type', 'js',
+        '--type', 'ts',
+        '-w',  // Word boundary
+        name,
+        '.'
+      ];
+
+      const result = spawnSync('rg', args, {
+        cwd: projectRoot,
+        encoding: 'utf-8'
+      });
+
+      if (result.error || result.status !== 0) {
+        return [];
+      }
+
+      return result.stdout.trim().split('\n').filter(Boolean);
+    },
+    { projectRoot }
+  );
 
-  const files = result.stdout.trim().split('\n').filter(Boolean);
   // Exclude the file that exports it
   const relExclude = relative(projectRoot, excludeFile);
   const otherFiles = files.filter(f => f !== relExclude && !f.includes(relExclude));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokenlean",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Lean CLI tools for AI agents and developers - reduce context, save tokens",
   "type": "module",
   "engines": {
@@ -22,6 +22,7 @@
   "bin": {
     "tl-api": "./bin/tl-api.mjs",
     "tl-blame": "./bin/tl-blame.mjs",
+    "tl-cache": "./bin/tl-cache.mjs",
     "tl-config": "./bin/tl-config.mjs",
     "tl-context": "./bin/tl-context.mjs",
     "tl-coverage": "./bin/tl-coverage.mjs",