tokenlean 0.1.0 → 0.3.0

package/bin/tl-context.mjs

@@ -20,137 +20,186 @@ if (process.argv.includes('--prompt')) {
 
 import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
 import { join, relative } from 'path';
+import {
+  createOutput,
+  parseCommonArgs,
+  estimateTokens,
+  formatTokens,
+  formatTable,
+  COMMON_OPTIONS_HELP
+} from '../src/output.mjs';
+import {
+  findProjectRoot,
+  shouldSkip,
+  getSkipDirs,
+  getImportantDirs
+} from '../src/project.mjs';
+
+const HELP = `
+tl-context - Estimate context token usage for files/directories
+
+Usage: tl-context [path] [options]
+
+Options:
+  --top N, -n N       Show top N files (default: 20, use --all for all)
+  --all               Show all files
+${COMMON_OPTIONS_HELP}
+
+Examples:
+  tl-context src/              # Estimate tokens for src directory
+  tl-context src/ --top 10     # Show top 10 largest files
+  tl-context src/ --all        # Show all files
+  tl-context package.json      # Single file estimate
+  tl-context -j                # JSON output for scripting
+`;
+
+function analyzeDir(dirPath, results = [], skipDirs, importantDirs) {
+  try {
+    const entries = readdirSync(dirPath, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') && !importantDirs.has(entry.name)) continue;
+      if (shouldSkip(entry.name, entry.isDirectory())) continue;
+
+      const fullPath = join(dirPath, entry.name);
+
+      if (entry.isDirectory()) {
+        analyzeDir(fullPath, results, skipDirs, importantDirs);
+      } else {
+        try {
+          const content = readFileSync(fullPath, 'utf-8');
+          const tokens = estimateTokens(content);
+          results.push({ path: fullPath, tokens, lines: content.split('\n').length });
+        } catch {
+          // Skip binary or unreadable files
+        }
+      }
+    }
+  } catch {
+    // Permission error
+  }
 
-const SKIP_DIRS = new Set([
-  'node_modules', '.git', 'android', 'ios', 'dist', 'build',
-  '.expo', '.next', 'coverage', '__pycache__', '.cache'
-]);
-
-const SKIP_EXTENSIONS = new Set([
-  '.png', '.jpg', '.jpeg', '.gif', '.ico', '.svg', '.webp',
-  '.woff', '.woff2', '.ttf', '.eot',
-  '.mp3', '.mp4', '.wav', '.ogg',
-  '.zip', '.tar', '.gz',
-  '.lock', '.log'
-]);
-
-// Rough token estimate: ~4 chars per token for code
-function estimateTokens(content) {
-  return Math.ceil(content.length / 4);
+  return results;
 }
 
-function formatTokens(tokens) {
-  if (tokens >= 1000000) return `${(tokens / 1000000).toFixed(1)}M`;
-  if (tokens >= 1000) return `${(tokens / 1000).toFixed(1)}k`;
-  return String(tokens);
-}
+// Main
+const args = process.argv.slice(2);
+const options = parseCommonArgs(args);
 
-function shouldSkip(name, isDir) {
-  if (isDir && SKIP_DIRS.has(name)) return true;
-  if (!isDir) {
-    const ext = name.substring(name.lastIndexOf('.'));
-    if (SKIP_EXTENSIONS.has(ext)) return true;
+// Parse tool-specific options
+let topN = 20;
+let targetPath = '.';
+
+for (let i = 0; i < options.remaining.length; i++) {
+  const arg = options.remaining[i];
+  if ((arg === '--top' || arg === '-n') && options.remaining[i + 1]) {
+    topN = parseInt(options.remaining[i + 1], 10);
+    i++;
+  } else if (arg === '--all') {
+    topN = null;
+  } else if (!arg.startsWith('-')) {
+    targetPath = arg;
   }
-  return false;
 }
 
-function analyzeDir(dirPath, results = [], depth = 0) {
-  const entries = readdirSync(dirPath, { withFileTypes: true });
+if (options.help) {
+  console.log(HELP);
+  process.exit(0);
+}
 
-  for (const entry of entries) {
-    if (entry.name.startsWith('.') && entry.name !== '.claude') continue;
-    if (shouldSkip(entry.name, entry.isDirectory())) continue;
+if (!existsSync(targetPath)) {
+  console.error(`Path not found: ${targetPath}`);
+  process.exit(1);
+}
 
-    const fullPath = join(dirPath, entry.name);
+const projectRoot = findProjectRoot();
+const skipDirs = getSkipDirs();
+const importantDirs = getImportantDirs();
+const out = createOutput(options);
 
-    if (entry.isDirectory()) {
-      analyzeDir(fullPath, results, depth + 1);
-    } else {
-      try {
-        const content = readFileSync(fullPath, 'utf-8');
-        const tokens = estimateTokens(content);
-        results.push({ path: fullPath, tokens, lines: content.split('\n').length });
-      } catch (e) {
-        // Skip binary or unreadable files
-      }
-    }
-  }
+const stat = statSync(targetPath);
+if (stat.isFile()) {
+  // Single file
+  const content = readFileSync(targetPath, 'utf-8');
+  const tokens = estimateTokens(content);
+  const lines = content.split('\n').length;
 
-  return results;
-}
+  out.setData('file', targetPath);
+  out.setData('tokens', tokens);
+  out.setData('lines', lines);
+
+  out.header(`${targetPath}: ~${formatTokens(tokens)} tokens (${lines} lines)`);
+  out.print();
+} else {
+  // Directory
+  const results = analyzeDir(targetPath, [], skipDirs, importantDirs);
 
-function printResults(results, rootPath, topN) {
   // Sort by tokens descending
   results.sort((a, b) => b.tokens - a.tokens);
 
   const total = results.reduce((sum, r) => sum + r.tokens, 0);
+  const totalLines = results.reduce((sum, r) => sum + r.lines, 0);
 
-  console.log(`\n📊 Context Estimate for: ${rootPath}\n`);
-  console.log(`Total: ~${formatTokens(total)} tokens across ${results.length} files\n`);
-
-  if (topN) {
-    console.log(`Top ${topN} largest files:\n`);
-    results = results.slice(0, topN);
-  }
+  // Set JSON data
+  out.setData('path', targetPath);
+  out.setData('totalTokens', total);
+  out.setData('totalLines', totalLines);
+  out.setData('fileCount', results.length);
 
-  const maxPathLen = Math.min(60, Math.max(...results.map(r => relative(rootPath, r.path).length)));
+  // Header
+  out.header(`Context Estimate: ${targetPath}`);
+  out.header(`Total: ~${formatTokens(total)} tokens across ${results.length} files`);
+  out.blank();
 
-  console.log('  Tokens   Lines  Path');
-  console.log('  ' + '-'.repeat(maxPathLen + 20));
+  // File list
+  const displayResults = topN ? results.slice(0, topN) : results;
 
-  for (const r of results) {
-    const relPath = relative(rootPath, r.path);
-    const truncPath = relPath.length > 60 ? '...' + relPath.slice(-57) : relPath;
-    console.log(`  ${formatTokens(r.tokens).padStart(6)}   ${String(r.lines).padStart(5)}  ${truncPath}`);
+  if (displayResults.length > 0) {
+    if (topN) {
+      out.header(`Top ${Math.min(topN, results.length)} largest files:`);
+    }
+    out.blank();
+
+    // Format as table
+    const rows = displayResults.map(r => {
+      const relPath = relative(targetPath, r.path);
+      const truncPath = relPath.length > 60 ? '...' + relPath.slice(-57) : relPath;
+      return [formatTokens(r.tokens), r.lines, truncPath];
+    });
+
+    const tableLines = formatTable(rows, { indent: '  ', separator: '   ' });
+    out.add('  Tokens   Lines  Path');
+    out.add('  ' + '-'.repeat(70));
+    out.addLines(tableLines);
   }
 
-  console.log();
-
   // Group by directory
   const byDir = {};
   for (const r of results) {
-    const rel = relative(rootPath, r.path);
+    const rel = relative(targetPath, r.path);
     const dir = rel.includes('/') ? rel.split('/')[0] : '.';
     byDir[dir] = (byDir[dir] || 0) + r.tokens;
   }
 
   const sortedDirs = Object.entries(byDir).sort((a, b) => b[1] - a[1]);
 
-  console.log('By top-level directory:\n');
-  for (const [dir, tokens] of sortedDirs.slice(0, 10)) {
-    const pct = ((tokens / total) * 100).toFixed(1);
-    console.log(`  ${formatTokens(tokens).padStart(6)}  ${pct.padStart(5)}%  ${dir}/`);
-  }
-  console.log();
-}
+  out.blank();
+  out.header('By top-level directory:');
+  out.blank();
 
-// Main
-const args = process.argv.slice(2);
-let targetPath = '.';
-let topN = 20;
+  const dirRows = sortedDirs.slice(0, 10).map(([dir, tokens]) => {
+    const pct = ((tokens / total) * 100).toFixed(1) + '%';
+    return [formatTokens(tokens), pct, dir + '/'];
+  });
 
-for (let i = 0; i < args.length; i++) {
-  if (args[i] === '--top' && args[i + 1]) {
-    topN = parseInt(args[i + 1], 10);
-    i++;
-  } else if (args[i] === '--all') {
-    topN = null;
-  } else if (!args[i].startsWith('-')) {
-    targetPath = args[i];
-  }
-}
+  out.addLines(formatTable(dirRows, { indent: '  ', separator: '  ' }));
 
-if (!existsSync(targetPath)) {
-  console.error(`Path not found: ${targetPath}`);
-  process.exit(1);
-}
+  out.setData('byDirectory', Object.fromEntries(sortedDirs));
+  out.setData('files', results.slice(0, 100).map(r => ({
+    path: relative(targetPath, r.path),
+    tokens: r.tokens,
+    lines: r.lines
+  })));
 
-const stat = statSync(targetPath);
-if (stat.isFile()) {
-  const content = readFileSync(targetPath, 'utf-8');
-  const tokens = estimateTokens(content);
-  console.log(`\n${targetPath}: ~${formatTokens(tokens)} tokens (${content.split('\n').length} lines)\n`);
-} else {
-  const results = analyzeDir(targetPath);
-  printResults(results, targetPath, topN);
+  out.print();
 }

package/bin/tl-diff.mjs

@@ -21,6 +21,30 @@ if (process.argv.includes('--prompt')) {
 }
 
 import { execSync } from 'child_process';
+import {
+  createOutput,
+  parseCommonArgs,
+  formatTokens,
+  COMMON_OPTIONS_HELP
+} from '../src/output.mjs';
+
+const HELP = `
+tl-diff - Token-efficient git diff summary
+
+Usage: tl-diff [ref] [options]
+
+Options:
+  --staged             Show staged changes only
+  --stat-only          Show just the summary (no file list)
+${COMMON_OPTIONS_HELP}
+
+Examples:
+  tl-diff                     # Working directory changes
+  tl-diff --staged            # Staged changes
+  tl-diff HEAD~3              # Last 3 commits
+  tl-diff main                # Changes vs main branch
+  tl-diff -j                  # JSON output
+`;
 
 function run(cmd) {
   try {
@@ -30,15 +54,6 @@ function run(cmd) {
   }
 }
 
-function estimateTokens(content) {
-  return Math.ceil(content.length / 4);
-}
-
-function formatTokens(tokens) {
-  if (tokens >= 1000) return `${(tokens / 1000).toFixed(1)}k`;
-  return String(tokens);
-}
-
 function parseDiffStat(stat) {
   const lines = stat.trim().split('\n');
   const files = [];
@@ -96,66 +111,30 @@ function categorizeChanges(files) {
   return categories;
 }
 
-function printSummary(files, categories, options) {
-  const totalChanges = files.reduce((sum, f) => sum + f.changes, 0);
-  const totalAdditions = files.reduce((sum, f) => sum + f.additions, 0);
-  const totalDeletions = files.reduce((sum, f) => sum + f.deletions, 0);
-
-  console.log(`\n📊 Diff Summary`);
-  console.log(`   ${files.length} files changed, ~${formatTokens(totalChanges * 4)} tokens of changes`);
-  console.log(`   +${totalAdditions} additions, -${totalDeletions} deletions\n`);
-
-  const order = ['components', 'hooks', 'store', 'types', 'manuscripts', 'tests', 'config', 'other'];
-  const labels = {
-    components: '🧩 Components',
-    hooks: '🪝 Hooks',
-    store: '📦 Store',
-    types: '📝 Types',
-    manuscripts: '📖 Manuscripts',
-    tests: '🧪 Tests',
-    config: '⚙️  Config',
-    other: '📄 Other'
-  };
-
-  for (const cat of order) {
-    const catFiles = categories[cat];
-    if (catFiles.length === 0) continue;
-
-    console.log(`${labels[cat]} (${catFiles.length})`);
-
-    // Sort by changes descending
-    catFiles.sort((a, b) => b.changes - a.changes);
-
-    for (const f of catFiles.slice(0, 10)) {
-      const bar = '+'.repeat(Math.min(f.additions, 20)) + '-'.repeat(Math.min(f.deletions, 20));
-      console.log(`  ${f.path}`);
-      console.log(`    ${f.changes} changes ${bar}`);
-    }
-
-    if (catFiles.length > 10) {
-      console.log(`  ... and ${catFiles.length - 10} more`);
-    }
-
-    console.log();
-  }
-}
-
 // Main
 const args = process.argv.slice(2);
+const options = parseCommonArgs(args);
+
+// Parse tool-specific options
 let ref = '';
 let staged = false;
 let statOnly = false;
 
-for (let i = 0; i < args.length; i++) {
-  if (args[i] === '--staged') {
+for (const arg of options.remaining) {
+  if (arg === '--staged') {
     staged = true;
-  } else if (args[i] === '--stat-only') {
+  } else if (arg === '--stat-only') {
     statOnly = true;
-  } else if (!args[i].startsWith('-')) {
-    ref = args[i];
+  } else if (!arg.startsWith('-')) {
+    ref = arg;
   }
 }
 
+if (options.help) {
+  console.log(HELP);
+  process.exit(0);
+}
+
 // Build git diff command
 let diffCmd = 'git diff';
 if (staged) {
@@ -167,17 +146,71 @@ diffCmd += ' --stat=200';
 
 const stat = run(diffCmd);
 
+const out = createOutput(options);
+
 if (!stat.trim()) {
-  console.log('\n✨ No changes detected\n');
+  out.header('No changes detected');
+  out.print();
   process.exit(0);
 }
 
 const files = parseDiffStat(stat);
 const categories = categorizeChanges(files);
 
-printSummary(files, categories, { statOnly });
+const totalChanges = files.reduce((sum, f) => sum + f.changes, 0);
+const totalAdditions = files.reduce((sum, f) => sum + f.additions, 0);
+const totalDeletions = files.reduce((sum, f) => sum + f.deletions, 0);
+
+// Set JSON data
+out.setData('files', files);
+out.setData('categories', categories);
+out.setData('totalFiles', files.length);
+out.setData('totalChanges', totalChanges);
+out.setData('estimatedTokens', totalChanges * 4);
+
+// Summary header
+out.header('Diff Summary');
+out.header(`${files.length} files changed, ~${formatTokens(totalChanges * 4)} tokens of changes`);
+out.header(`+${totalAdditions} additions, -${totalDeletions} deletions`);
+out.blank();
 
 if (!statOnly) {
-  console.log('💡 Tip: Use --stat-only for just the summary, or check specific files with:');
-  console.log('   git diff [ref] -- path/to/file.ts\n');
+  const order = ['components', 'hooks', 'store', 'types', 'manuscripts', 'tests', 'config', 'other'];
+  const labels = {
+    components: 'Components',
+    hooks: 'Hooks',
+    store: 'Store',
+    types: 'Types',
+    manuscripts: 'Manuscripts',
+    tests: 'Tests',
+    config: 'Config',
+    other: 'Other'
+  };
+
+  for (const cat of order) {
+    const catFiles = categories[cat];
+    if (catFiles.length === 0) continue;
+
+    out.add(`${labels[cat]} (${catFiles.length})`);
+
+    // Sort by changes descending
+    catFiles.sort((a, b) => b.changes - a.changes);
+
+    for (const f of catFiles.slice(0, 10)) {
+      const bar = '+'.repeat(Math.min(f.additions, 20)) + '-'.repeat(Math.min(f.deletions, 20));
+      out.add(`  ${f.path}`);
+      out.add(`    ${f.changes} changes ${bar}`);
+    }
+
+    if (catFiles.length > 10) {
+      out.add(`  ... and ${catFiles.length - 10} more`);
+    }
+
+    out.blank();
+  }
+
+  out.header('Tip: Use --stat-only for just the summary, or check specific files with:');
+  out.header('   git diff [ref] -- path/to/file.ts');
 }
+
+out.print();

package/bin/tl-entry.mjs

@@ -30,6 +30,7 @@ import {
   COMMON_OPTIONS_HELP
 } from '../src/output.mjs';
 import { findProjectRoot } from '../src/project.mjs';
+import { withCache } from '../src/cache.mjs';
 
 const HELP = `
 tl-entry - Find entry points in a codebase
@@ -130,7 +131,12 @@ function findEntryPoints(searchPath, projectRoot, filterType) {
     for (const { pattern, desc } of config.patterns) {
       try {
         const cmd = `rg -n -g "*.{ts,tsx,js,jsx,mjs}" --no-heading -e "${shellEscape(pattern)}" "${shellEscape(searchPath)}" 2>/dev/null || true`;
-        const output = execSync(cmd, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
+        const cacheKey = { op: 'rg-entry-pattern', pattern, path: searchPath };
+        const output = withCache(
+          cacheKey,
+          () => execSync(cmd, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }),
+          { projectRoot }
+        );
 
         for (const line of output.trim().split('\n')) {
           if (!line) continue;

package/bin/tl-impact.mjs

@@ -33,6 +33,7 @@ import {
   COMMON_OPTIONS_HELP
 } from '../src/output.mjs';
 import { findProjectRoot, categorizeFile } from '../src/project.mjs';
+import { withCache } from '../src/cache.mjs';
 
 const HELP = `
 tl-impact - Analyze the blast radius of changing a file
@@ -79,8 +80,15 @@ function findDirectImporters(filePath, projectRoot) {
   const patterns = searchTerms.map(t => `-e "${rgEscape(t)}"`).join(' ');
 
   try {
-    const rgCommand = `rg -l --type-add 'code:*.{js,jsx,ts,tsx,mjs,mts,cjs}' -t code ${patterns} "${projectRoot}" 2>/dev/null || true`;
-    const result = execSync(rgCommand, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
+    const cacheKey = { op: 'rg-find-candidates', terms: searchTerms.sort() };
+    const result = withCache(
+      cacheKey,
+      () => {
+        const rgCommand = `rg -l --type-add 'code:*.{js,jsx,ts,tsx,mjs,mts,cjs}' -t code ${patterns} "${projectRoot}" 2>/dev/null || true`;
+        return execSync(rgCommand, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
+      },
+      { projectRoot }
+    );
     const candidates = result.trim().split('\n').filter(Boolean);
 
     for (const candidate of candidates) {