tokenlean 0.1.0

package/bin/tl-search.mjs

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+
+/**
+ * tl-search - Run pre-defined search patterns
+ *
+ * Looks for searchPatterns in:
+ *   1. .tokenleanrc.json in project root
+ *   2. ~/.tokenleanrc.json (global)
+ *
+ * Usage: tl-search <pattern-name>
+ */
+
+// Prompt info for tl-prompt
+if (process.argv.includes('--prompt')) {
+  console.log(JSON.stringify({
+    name: 'tl-search',
+    desc: 'Run pre-defined search patterns',
+    when: 'search',
+    example: 'tl-search'
+  }));
+  process.exit(0);
+}
+
+import { spawn, execSync } from 'child_process';
+import { loadConfig, getSearchPatterns, CONFIG_FILENAME } from '../src/config.mjs';
+
+// Check for ripgrep
+try {
+  execSync('which rg', { stdio: 'ignore' });
+} catch {
+  console.error('ripgrep (rg) not found. Install: brew install ripgrep');
+  process.exit(1);
+}
+
+function showHelp(patterns) {
+  const names = Object.keys(patterns);
+
+  if (names.length === 0) {
+    console.log('\nNo search patterns defined.');
+    console.log(`\nAdd patterns to ${CONFIG_FILENAME}:`);
+    console.log(`
+{
+  "searchPatterns": {
+    "hooks": {
+      "description": "Find lifecycle hooks",
+      "pattern": "use(Effect|State|Callback)",
+      "glob": "**/*.{ts,tsx}"
+    }
+  }
+}`);
+    process.exit(0);
+  }
+
+  console.log('\nAvailable search patterns:\n');
+
+  const maxLen = Math.max(...names.map(k => k.length));
+
+  for (const [name, config] of Object.entries(patterns)) {
+    const paddedName = name.padEnd(maxLen);
+    console.log(`  ${paddedName}  ${config.description || '(no description)'}`);
+  }
+
+  console.log('\nUsage: tl-search <pattern-name>');
+  console.log('Example: tl-search hooks\n');
+}
+
+function runSearch(name, config, rootDir) {
+  console.log(`\nSearching: ${config.description || name}`);
+  console.log(`Pattern: ${config.pattern}\n`);
+
+  if (config.type === 'glob-only') {
+    const args = ['--files', '-g', config.pattern];
+    if (config.exclude) {
+      for (const ex of config.exclude) {
+        args.push('-g', `!${ex}`);
+      }
+    }
+    args.push(rootDir);
+    const proc = spawn('rg', args, { stdio: 'inherit' });
+    proc.on('close', code => process.exit(code === 1 ? 0 : code));
+    return;
+  }
+
+  const args = ['--color=always', '-n', '-e', config.pattern];
+
+  if (config.glob) {
+    args.push('-g', config.glob);
+  }
+
+  if (config.exclude) {
+    for (const ex of config.exclude) {
+      args.push('-g', `!${ex}`);
+    }
+  }
+
+  args.push(rootDir);
+
+  const proc = spawn('rg', args, { stdio: 'inherit' });
+  proc.on('close', code => {
+    if (code === 1) {
+      console.log('No matches found.');
+    }
+    process.exit(code === 1 ? 0 : code);
+  });
+}
+
+// Main
+const { config, projectRoot } = loadConfig();
+const patterns = config.searchPatterns || {};
+const patternName = process.argv[2];
+
+if (!patternName || patternName === '--help' || patternName === '-h') {
+  showHelp(patterns);
+  process.exit(0);
+}
+
+if (!patterns[patternName]) {
+  console.error(`\nUnknown pattern: "${patternName}"`);
+  showHelp(patterns);
+  process.exit(1);
+}
+
+runSearch(patternName, patterns[patternName], projectRoot);

package/bin/tl-structure.mjs

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+
+/**
+ * tl-structure - Smart project overview with context estimates
+ *
+ * Shows directory structure with token estimates, highlighting
+ * important directories and files for quick orientation.
+ *
+ * Usage: tl-structure [path] [--depth N]
+ */
+
+import { readdirSync, readFileSync, existsSync } from 'fs';
+import { join, basename } from 'path';
+import { estimateTokens, formatTokens } from '../src/output.mjs';
+import {
+  getSkipDirs,
+  getImportantFiles,
+  getImportantDirs
+} from '../src/project.mjs';
+import { getConfig } from '../src/config.mjs';
+
+function getDirStats(dirPath, skipDirs, importantDirs) {
+  let totalTokens = 0;
+  let fileCount = 0;
+
+  function walk(dir) {
+    try {
+      const entries = readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.name.startsWith('.') && !importantDirs.has(entry.name)) continue;
+        if (skipDirs.has(entry.name)) continue;
+
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+          walk(fullPath);
+        } else {
+          try {
+            const content = readFileSync(fullPath, 'utf-8');
+            totalTokens += estimateTokens(content);
+            fileCount++;
+          } catch (e) { /* skip binary */ }
+        }
+      }
+    } catch (e) { /* permission error */ }
+  }
+
+  walk(dirPath);
+  return { totalTokens, fileCount };
+}
+
+function printTree(dirPath, prefix, depth, maxDepth, skipDirs, importantDirs, importantFiles) {
+  if (depth > maxDepth) return;
+
+  const entries = readdirSync(dirPath, { withFileTypes: true });
+
+  // Sort: directories first, then by importance, then alphabetically
+  entries.sort((a, b) => {
+    if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1;
+    const aImportant = importantDirs.has(a.name) || importantFiles.has(a.name);
+    const bImportant = importantDirs.has(b.name) || importantFiles.has(b.name);
+    if (aImportant !== bImportant) return aImportant ? -1 : 1;
+    return a.name.localeCompare(b.name);
+  });
+
+  const filtered = entries.filter(e => {
+    if (e.name.startsWith('.') && e.name !== '.claude') return false;
+    if (skipDirs.has(e.name)) return false;
+    return true;
+  });
+
+  filtered.forEach((entry, index) => {
+    const isLast = index === filtered.length - 1;
+    const connector = isLast ? '└── ' : '├── ';
+    const fullPath = join(dirPath, entry.name);
+
+    const isEntryImportant = importantDirs.has(entry.name) || importantFiles.has(entry.name);
+    const marker = isEntryImportant ? '*' : ' ';
+
+    if (entry.isDirectory()) {
+      const stats = getDirStats(fullPath, skipDirs, importantDirs);
+      const sizeInfo = stats.fileCount > 0
+        ? ` (${stats.fileCount} files, ~${formatTokens(stats.totalTokens)})`
+        : ' (empty)';
+
+      console.log(`${prefix}${connector}${marker}${entry.name}/${sizeInfo}`);
+
+      const newPrefix = prefix + (isLast ? '    ' : '│   ');
+      printTree(fullPath, newPrefix, depth + 1, maxDepth, skipDirs, importantDirs, importantFiles);
+    } else {
+      try {
+        const content = readFileSync(fullPath, 'utf-8');
+        const tokens = estimateTokens(content);
+        const lines = content.split('\n').length;
+        console.log(`${prefix}${connector}${marker}${entry.name} (~${formatTokens(tokens)}, ${lines}L)`);
+      } catch (e) {
+        console.log(`${prefix}${connector}${marker}${entry.name} (binary)`);
+      }
+    }
+  });
+}
+
+// Prompt info for tl-prompt
+if (process.argv.includes('--prompt')) {
+  console.log(JSON.stringify({
+    name: 'tl-structure',
+    desc: 'Project overview with token estimates',
+    when: 'before-read',
+    example: 'tl-structure'
+  }));
+  process.exit(0);
+}
+
+// Main
+const args = process.argv.slice(2);
+
+// Get config defaults
+const structureConfig = getConfig('structure') || {};
+
+let targetPath = '.';
+let maxDepth = structureConfig.depth || 3;
+
+// Get combined sets (defaults + user config extensions)
+const skipDirs = getSkipDirs();
+const importantDirs = getImportantDirs();
+const importantFiles = getImportantFiles();
+
+for (let i = 0; i < args.length; i++) {
+  if ((args[i] === '--depth' || args[i] === '-d') && args[i + 1]) {
+    maxDepth = parseInt(args[i + 1], 10);
+    i++;
+  } else if (args[i] === '--help' || args[i] === '-h') {
+    console.log(`
+tl-structure - Smart project overview with context estimates
+
+Usage: tl-structure [path] [options]
+
+Options:
+  --depth N, -d N    Maximum depth to show (default: ${structureConfig.depth || 3})
+  --help, -h         Show this help
+
+Configure defaults in .tokenleanrc.json:
+  "structure": { "depth": 3, "important": ["src", "lib"] }
+`);
+    process.exit(0);
+  } else if (!args[i].startsWith('-')) {
+    targetPath = args[i];
+  }
+}
+
+if (!existsSync(targetPath)) {
+  console.error(`Path not found: ${targetPath}`);
+  process.exit(1);
+}
+
+const rootStats = getDirStats(targetPath, skipDirs, importantDirs);
+console.log(`\n${targetPath === '.' ? basename(process.cwd()) : targetPath}`);
+console.log(`Total: ${rootStats.fileCount} files, ~${formatTokens(rootStats.totalTokens)} tokens`);
+console.log(`(* = important for understanding project)\n`);
+
+printTree(targetPath, '', 0, maxDepth, skipDirs, importantDirs, importantFiles);
+console.log();

package/bin/tl-symbols.mjs

@@ -0,0 +1,430 @@
+#!/usr/bin/env node
+
+/**
+ * tl-symbols - Extract function/class/type signatures without bodies
+ *
+ * Shows the API surface of a file in minimal tokens - signatures only,
+ * no implementation details. Perfect for understanding what a file
+ * provides without reading the whole thing.
+ *
+ * Usage: tl-symbols <file> [--exports-only]
+ */
+
+// Prompt info for tl-prompt
+if (process.argv.includes('--prompt')) {
+  console.log(JSON.stringify({
+    name: 'tl-symbols',
+    desc: 'Function/class signatures without bodies',
+    when: 'before-read',
+    example: 'tl-symbols src/utils.ts'
+  }));
+  process.exit(0);
+}
+
+import { readFileSync, existsSync } from 'fs';
+import { basename, extname } from 'path';
+import {
+  createOutput,
+  parseCommonArgs,
+  estimateTokens,
+  formatTokens,
+  COMMON_OPTIONS_HELP
+} from '../src/output.mjs';
+
+const HELP = `
+tl-symbols - Extract function/class/type signatures without bodies
+
+Usage: tl-symbols <file> [options]
+
+Options:
+  --exports-only, -e    Show only exported symbols
+${COMMON_OPTIONS_HELP}
+
+Examples:
+  tl-symbols src/api.ts              # All symbols
+  tl-symbols src/api.ts -e           # Exports only
+  tl-symbols src/api.ts -l 20        # Limit to 20 lines
+  tl-symbols src/api.ts -j           # JSON output
+
+Supported languages:
+  JavaScript/TypeScript (.js, .ts, .jsx, .tsx, .mjs)
+  Python (.py)
+  Go (.go)
+`;
+
+// ─────────────────────────────────────────────────────────────
+// Language Detection
+// ─────────────────────────────────────────────────────────────
+
+const LANG_EXTENSIONS = {
+  js: ['.js', '.mjs', '.cjs', '.jsx', '.ts', '.tsx', '.mts'],
+  python: ['.py'],
+  go: ['.go']
+};
+
+function detectLanguage(filePath) {
+  const ext = extname(filePath).toLowerCase();
+  for (const [lang, exts] of Object.entries(LANG_EXTENSIONS)) {
+    if (exts.includes(ext)) return lang;
+  }
+  return null;
+}
+
+// ─────────────────────────────────────────────────────────────
+// JavaScript/TypeScript Extraction
+// ─────────────────────────────────────────────────────────────
+
+function extractJsSymbols(content, exportsOnly = false) {
+  const symbols = {
+    exports: [],
+    classes: [],
+    functions: [],
+    types: [],
+    constants: []
+  };
+
+  const lines = content.split('\n');
+  let inClass = null;
+  let braceDepth = 0;
+  let currentClassMethods = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmed = line.trim();
+
+    // Skip comments and empty lines
+    if (trimmed.startsWith('//') || trimmed.startsWith('*') || trimmed.startsWith('/*') || !trimmed) {
+      continue;
+    }
+
+    // Track brace depth for class scope
+    const openBraces = (line.match(/\{/g) || []).length;
+    const closeBraces = (line.match(/\}/g) || []).length;
+
+    // Check if we're exiting a class
+    if (inClass && braceDepth === 1 && closeBraces > openBraces) {
+      symbols.classes.push({
+        signature: inClass,
+        methods: currentClassMethods
+      });
+      inClass = null;
+      currentClassMethods = [];
+    }
+
+    braceDepth += openBraces - closeBraces;
+
+    // Export statements
+    if (trimmed.startsWith('export ')) {
+      if (trimmed.includes('export default')) {
+        const match = trimmed.match(/export\s+default\s+(?:class|function|async\s+function)?\s*(\w+)?/);
+        if (match) {
+          symbols.exports.push(trimmed.replace(/\s*\{.*$/, '').trim());
+        }
+      }
+      else if (trimmed.match(/export\s+\{[^}]+\}\s+from/)) {
+        symbols.exports.push(trimmed);
+      }
+      else if (trimmed.match(/export\s+\*\s+from/)) {
+        symbols.exports.push(trimmed);
+      }
+      else if (trimmed.match(/export\s+(interface|type)\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        symbols.types.push(sig);
+        symbols.exports.push(sig);
+      }
+      else if (trimmed.match(/export\s+(?:abstract\s+)?class\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        inClass = sig;
+        currentClassMethods = [];
+        braceDepth = openBraces - closeBraces;
+      }
+      else if (trimmed.match(/export\s+(?:async\s+)?function\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        symbols.functions.push(sig);
+        symbols.exports.push(sig);
+      }
+      else if (trimmed.match(/export\s+const\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        if (trimmed.includes('=>') || trimmed.match(/:\s*\([^)]*\)\s*=>/)) {
+          symbols.functions.push(sig);
+        } else {
+          symbols.constants.push(sig);
+        }
+        symbols.exports.push(sig);
+      }
+      else if (trimmed.match(/export\s+(?:const\s+)?enum\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        symbols.types.push(sig);
+        symbols.exports.push(sig);
+      }
+    }
+    // Non-exported symbols
+    else if (!exportsOnly) {
+      if (trimmed.match(/^interface\s+/)) {
+        symbols.types.push(extractSignatureLine(trimmed));
+      }
+      else if (trimmed.match(/^type\s+\w+/)) {
+        symbols.types.push(extractSignatureLine(trimmed));
+      }
+      else if (trimmed.match(/^(?:abstract\s+)?class\s+/)) {
+        const sig = extractSignatureLine(trimmed);
+        inClass = sig;
+        currentClassMethods = [];
+        braceDepth = openBraces - closeBraces;
+      }
+      else if (trimmed.match(/^(?:async\s+)?function\s+/)) {
+        symbols.functions.push(extractSignatureLine(trimmed));
+      }
+      else if (braceDepth === 0 && trimmed.match(/^const\s+\w+.*=.*=>/)) {
+        symbols.functions.push(extractSignatureLine(trimmed));
+      }
+      // Class methods
+      else if (inClass && braceDepth >= 1) {
+        if (trimmed.match(/^constructor\s*\(/)) {
+          currentClassMethods.push(extractSignatureLine(trimmed));
+        }
+        else if (trimmed.match(/^(?:public\s+|private\s+|protected\s+)?(?:static\s+)?(?:async\s+)?(?:get\s+|set\s+)?(\w+)\s*[(<]/)) {
+          if (!trimmed.includes('=') || trimmed.includes('=>')) {
+            currentClassMethods.push(extractSignatureLine(trimmed));
+          }
+        }
+      }
+    }
+  }
+
+  // Handle last class if file ends inside one
+  if (inClass) {
+    symbols.classes.push({
+      signature: inClass,
+      methods: currentClassMethods
+    });
+  }
+
+  return symbols;
+}
+
+function extractSignatureLine(line) {
+  let sig = line
+    .replace(/\s*\{[\s\S]*$/, '')
+    .replace(/\s*=>\s*[^{].*$/, ' =>')
+    .replace(/\s*=\s*[^=].*$/, '')
+    .trim();
+
+  sig = sig.replace(/[,;]$/, '').trim();
+  return sig;
+}
+
+// ─────────────────────────────────────────────────────────────
+// Python Extraction
+// ─────────────────────────────────────────────────────────────
+
+function extractPythonSymbols(content) {
+  const symbols = { classes: [], functions: [] };
+  const lines = content.split('\n');
+  let inClass = null;
+  let currentClassMethods = [];
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    const classMatch = trimmed.match(/^class\s+(\w+)(?:\([^)]*\))?:/);
+    if (classMatch) {
+      if (inClass) {
+        symbols.classes.push({ signature: inClass, methods: currentClassMethods });
+      }
+      inClass = trimmed.replace(/:$/, '');
+      currentClassMethods = [];
+      continue;
+    }
+
+    const funcMatch = trimmed.match(/^(?:async\s+)?def\s+(\w+)\s*\([^)]*\)(?:\s*->\s*[^:]+)?:/);
+    if (funcMatch) {
+      const sig = trimmed.replace(/:$/, '');
+      if (inClass && line.startsWith('    ')) {
+        currentClassMethods.push(sig);
+      } else {
+        if (inClass) {
+          symbols.classes.push({ signature: inClass, methods: currentClassMethods });
+          inClass = null;
+          currentClassMethods = [];
+        }
+        symbols.functions.push(sig);
+      }
+    }
+  }
+
+  if (inClass) {
+    symbols.classes.push({ signature: inClass, methods: currentClassMethods });
+  }
+
+  return symbols;
+}
+
+// ─────────────────────────────────────────────────────────────
+// Go Extraction
+// ─────────────────────────────────────────────────────────────
+
+function extractGoSymbols(content) {
+  const symbols = { types: [], functions: [] };
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (trimmed.match(/^type\s+\w+\s+(?:struct|interface)/)) {
+      symbols.types.push(trimmed.replace(/\s*\{.*$/, ''));
+    }
+
+    if (trimmed.match(/^func\s+/)) {
+      symbols.functions.push(trimmed.replace(/\s*\{.*$/, ''));
+    }
+  }
+
+  return symbols;
+}
+
+// ─────────────────────────────────────────────────────────────
+// Formatting
+// ─────────────────────────────────────────────────────────────
+
+function formatSymbols(symbols, lang, out) {
+  if (lang === 'js') {
+    if (symbols.exports.length > 0) {
+      out.add('Exports:');
+      const unique = [...new Set(symbols.exports)];
+      unique.forEach(e => out.add('  ' + e));
+      out.blank();
+    }
+
+    if (symbols.classes.length > 0) {
+      out.add('Classes:');
+      for (const cls of symbols.classes) {
+        out.add('  ' + cls.signature);
+        cls.methods.forEach(m => out.add('    ' + m));
+      }
+      out.blank();
+    }
+
+    const nonExportedFuncs = symbols.functions.filter(f => !f.startsWith('export'));
+    if (nonExportedFuncs.length > 0) {
+      out.add('Functions:');
+      nonExportedFuncs.forEach(f => out.add('  ' + f));
+      out.blank();
+    }
+
+    const nonExportedTypes = symbols.types.filter(t => !t.startsWith('export'));
+    if (nonExportedTypes.length > 0) {
+      out.add('Types:');
+      nonExportedTypes.forEach(t => out.add('  ' + t));
+      out.blank();
+    }
+
+    const nonExportedConsts = symbols.constants.filter(c => !c.startsWith('export'));
+    if (nonExportedConsts.length > 0) {
+      out.add('Constants:');
+      nonExportedConsts.forEach(c => out.add('  ' + c));
+      out.blank();
+    }
+  } else if (lang === 'python') {
+    if (symbols.classes.length > 0) {
+      out.add('Classes:');
+      for (const cls of symbols.classes) {
+        out.add('  ' + cls.signature);
+        cls.methods.forEach(m => out.add('    ' + m));
+      }
+      out.blank();
+    }
+
+    if (symbols.functions.length > 0) {
+      out.add('Functions:');
+      symbols.functions.forEach(f => out.add('  ' + f));
+      out.blank();
+    }
+  } else if (lang === 'go') {
+    if (symbols.types.length > 0) {
+      out.add('Types:');
+      symbols.types.forEach(t => out.add('  ' + t));
+      out.blank();
+    }
+
+    if (symbols.functions.length > 0) {
+      out.add('Functions:');
+      symbols.functions.forEach(f => out.add('  ' + f));
+      out.blank();
+    }
+  }
+}
+
+function countSymbols(symbols) {
+  let count = 0;
+  if (symbols.exports) count += symbols.exports.length;
+  if (symbols.classes) {
+    count += symbols.classes.length;
+    symbols.classes.forEach(c => count += c.methods?.length || 0);
+  }
+  if (symbols.functions) count += symbols.functions.length;
+  if (symbols.types) count += symbols.types.length;
+  if (symbols.constants) count += symbols.constants.length;
+  return count;
+}
+
+// ─────────────────────────────────────────────────────────────
+// Main
+// ─────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+const options = parseCommonArgs(args);
+const exportsOnly = options.remaining.includes('--exports-only') || options.remaining.includes('-e');
+const filePath = options.remaining.find(a => !a.startsWith('-'));
+
+if (options.help || !filePath) {
+  console.log(HELP);
+  process.exit(options.help ? 0 : 1);
+}
+
+if (!existsSync(filePath)) {
+  console.error(`File not found: ${filePath}`);
+  process.exit(1);
+}
+
+const lang = detectLanguage(filePath);
+if (!lang) {
+  console.error(`Unsupported file type: ${extname(filePath)}`);
+  console.error('Supported: .js, .ts, .jsx, .tsx, .mjs, .py, .go');
+  process.exit(1);
+}
+
+const content = readFileSync(filePath, 'utf-8');
+const fullFileTokens = estimateTokens(content);
+
+let symbols;
+switch (lang) {
+  case 'js':
+    symbols = extractJsSymbols(content, exportsOnly);
+    break;
+  case 'python':
+    symbols = extractPythonSymbols(content);
+    break;
+  case 'go':
+    symbols = extractGoSymbols(content);
+    break;
+}
+
+const symbolCount = countSymbols(symbols);
+const out = createOutput(options);
+
+// Set JSON data
+out.setData('file', basename(filePath));
+out.setData('language', lang);
+out.setData('symbolCount', symbolCount);
+out.setData('fullFileTokens', fullFileTokens);
+out.setData('symbols', symbols);
+
+// Build text output
+out.header(`\n📦 ${basename(filePath)} (${symbolCount} symbols)`);
+out.header(`   Full file: ~${formatTokens(fullFileTokens)} tokens → Symbols only: ~${formatTokens(Math.ceil(symbolCount * 15))} tokens`);
+out.blank();
+
+formatSymbols(symbols, lang, out);
+
+out.print();