token-pilot 0.7.5 → 0.8.0

package/CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to Token Pilot will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2026-03-07
+
+### Added
+- **`code_audit` tool** — find code quality issues in one call: TODO/FIXME comments (`check="todo"`), deprecated symbols (`check="deprecated"`), structural code patterns via ast-grep (`check="pattern"`), decorator search (`check="annotations"`), or combined audit (`check="all"`).
+- **Incremental index update on file changes** — file watcher now triggers `ast-index update` (debounced 2s) after edits. Keeps index fresh for find_usages, find_unused, code_audit.
+- **ast-index client methods** — `agrep()`, `todo()`, `deprecated()`, `annotations()`, `incrementalUpdate()`.
+
+### Fixed
+- **smart_read on directories** — now returns helpful message instead of EISDIR crash.
+- **MCP instructions** — added "COMBINE BOTH" section for audit tasks (Token Pilot + Grep).
+
+## [0.7.6] - 2026-03-07
+
+### Added
+- **`npx token-pilot init`** — one command creates `.mcp.json` with both token-pilot and context-mode configured. Idempotent — safely updates existing configs without overwriting.
+- **MCP Server Instructions** — protocol-level `instructions` field tells AI agents WHEN to use Token Pilot tools instead of built-in defaults. Works universally on all MCP clients.
+- **Improved tool descriptions** — each tool explicitly states what built-in tool it replaces (e.g. "Use INSTEAD OF Read/cat").
+
+### Fixed
+- **3 high severity vulnerabilities** — updated hono and express-rate-limit.
+- **npm package size** — excluded source maps from package. 505 kB → 286 kB (−43%).
+- **Accurate thresholds** — README and instructions now correctly state smallFileThreshold=200 (was 80).
+- **read_diff documentation** — clarified that smart_read must be called BEFORE editing to create baseline snapshot.
+
+### Changed
+- **README** — honest metrics (60-80%), Quick Start with `init` command, MCP instructions section, Codex/Antigravity support.
+- **npm keywords** — added `codex`, `cline`, `model-context-protocol`, `token-savings`.
+
 ## [0.7.4] - 2026-03-07
 
 ### Added

package/README.md

@@ -13,13 +13,36 @@ Token Pilot:  smart_read("user-service.ts")  →  15-line outline  →  ~200 tok
               After edit: read_diff("user-service.ts")  →  ~20 tokens
 ```
 
-**~80% reduction** in this example. Files under 80 lines are returned in full automatically (no overhead for small files).
+**~80% reduction** in this example. Files under 200 lines are returned in full automatically (no overhead for small files). Real savings start at ~200+ lines.
 
 ## Installation
 
-### npx — Any MCP-compatible AI Assistant
+### Quick Start (recommended)
 
-Zero install. Add to your `.mcp.json` (project-level or `~/.mcp.json` for global):
+One command creates `.mcp.json` with token-pilot + context-mode:
+
+```bash
+npx -y token-pilot init
+```
+
+Safe to run in any project — if `.mcp.json` already exists, only adds missing servers without overwriting existing config.
+
+This generates:
+
+```json
+{
+  "mcpServers": {
+    "token-pilot": { "command": "npx", "args": ["-y", "token-pilot"] },
+    "context-mode": { "command": "npx", "args": ["-y", "claude-context-mode"] }
+  }
+}
+```
+
+**That's it.** Restart your AI assistant. Both packages are downloaded automatically, ast-index binary is fetched on first run. No Rust, no Cargo, no manual setup.
+
+### Manual Setup
+
+Add to your `.mcp.json` (project-level or `~/.mcp.json` for global):
 
 ```json
 {
@@ -32,9 +55,7 @@ Zero install. Add to your `.mcp.json` (project-level or `~/.mcp.json` for global
 }
 ```
 
-**That's it.** npx downloads the package, ast-index binary is fetched automatically on first run. No Rust, no Cargo, no manual setup.
-
-Works with: **Cursor**, **Cline**, **Continue**, **Codex**, **Antigravity**, and any MCP-compatible client.
+Works with: **Claude Code**, **Cursor**, **Codex**, **Antigravity**, **Cline**, and any MCP-compatible client.
 
 #### Cursor
 
@@ -124,7 +145,7 @@ For more control, you can add rules to your project:
 - **Cursor** → `.cursorrules` in project root
 - **Codex** → `AGENTS.md` in project root
 
-## MCP Tools (12)
+## MCP Tools (13)
 
 ### Core Reading
 
@@ -134,7 +155,7 @@ For more control, you can add rules to your project:
 | `read_symbol` | `Read` + scroll | Load source of a specific symbol. Supports `Class.method`. `show` param: full/head/tail/outline. |
 | `read_for_edit` | `Read` before `Edit` | Minimal RAW code around a symbol — copy directly as `old_string` for Edit tool. |
 | `read_range` | `Read` offset | Read a specific line range from a file. |
-| `read_diff` | re-`Read` | Show only what changed since last smart_read. Saves tokens on re-reads. |
+| `read_diff` | re-`Read` | Show only changed hunks since last smart_read. Requires smart_read before editing (for baseline). Works with any edit tool. |
 | `smart_read_many` | multiple `Read` | Batch smart_read for up to 20 files in one call. |
 
 ### Search & Navigation
@@ -146,6 +167,7 @@ For more control, you can add rules to your project:
 | `related_files` | manual explore | Import graph: what a file imports, what imports it, test files. |
 | `outline` | multiple `smart_read` | Compact overview of all code files in a directory. One call instead of 5-6. |
 | `find_unused` | manual | Detect dead code — unused functions, classes, variables. |
+| `code_audit` | multiple `Grep` | Code quality issues in one call: TODO/FIXME comments, deprecated symbols, structural code patterns (via ast-grep), decorator search. |
 
 ### Analytics
 
@@ -158,6 +180,7 @@ For more control, you can add rules to your project:
 ```bash
 token-pilot                      # Start MCP server (uses cwd as project root)
 token-pilot /path/to/project     # Start with specific project root
+token-pilot init [dir]           # Create/update .mcp.json (token-pilot + context-mode)
 token-pilot install-ast-index    # Download ast-index binary (auto on first run)
 token-pilot install-hook [root]  # Install PreToolUse hook
 token-pilot uninstall-hook       # Remove hook
@@ -174,7 +197,7 @@ Create `.token-pilot.json` in your project root to customize behavior:
 ```json
 {
   "smartRead": {
-    "smallFileThreshold": 80,
+    "smallFileThreshold": 200,
     "advisoryReminders": true
   },
   "cache": {
@@ -210,7 +233,7 @@ All fields are optional — sensible defaults are used for anything not specifie
 
 | Option | Default | Description |
 |--------|---------|-------------|
-| `smartRead.smallFileThreshold` | `80` | Files with fewer lines are returned in full (no AST overhead). |
+| `smartRead.smallFileThreshold` | `200` | Files with fewer lines are returned in full (no AST overhead). |
 | `cache.maxSizeMB` | `100` | Max memory for file cache. LRU eviction when exceeded. |
 | `cache.watchFiles` | `true` | Auto-invalidate cache on file changes (chokidar). |
 | `git.watchHead` | `true` | Watch `.git/HEAD` for branch switches, invalidate changed files. |
@@ -232,7 +255,6 @@ When both are configured, Token Pilot automatically:
 - Detects context-mode via `.mcp.json`
 - Suggests context-mode for large non-code files
 - Shows combined architecture info in `session_analytics`
-- Provides `export_ast_index` to feed AST data into context-mode's BM25 index
 
 **Combined savings: 60-80%** in a typical coding session.
 
@@ -320,6 +342,7 @@ src/
     related-files.ts    — related_files handler (import graph)
     outline.ts          — outline handler (directory overview)
     find-unused.ts      — find_unused handler
+    code-audit.ts       — code_audit handler (TODOs, deprecated, patterns)
     project-overview.ts — project_overview (via ast-index map + conventions)
     non-code.ts         — JSON/YAML/MD/TOML structural summaries
     export-ast-index.ts — AST export for context-mode BM25

package/dist/ast-index/client.d.ts

@@ -1,5 +1,5 @@
 import type { FileStructure } from '../types.js';
-import type { AstIndexSymbolDetail, AstIndexSearchResult, AstIndexUsageResult, AstIndexImplementation, AstIndexHierarchyNode, AstIndexRefsResponse, AstIndexMapResponse, AstIndexConventionsResponse, AstIndexCallerEntry, AstIndexCallTreeNode, AstIndexChangedEntry, AstIndexUnusedSymbol, AstIndexImportEntry } from './types.js';
+import type { AstIndexSymbolDetail, AstIndexSearchResult, AstIndexUsageResult, AstIndexImplementation, AstIndexHierarchyNode, AstIndexRefsResponse, AstIndexMapResponse, AstIndexConventionsResponse, AstIndexCallerEntry, AstIndexCallTreeNode, AstIndexChangedEntry, AstIndexUnusedSymbol, AstIndexImportEntry, AstIndexAgrepMatch, AstIndexTodoEntry, AstIndexDeprecatedEntry, AstIndexAnnotationEntry } from './types.js';
 export declare class AstIndexClient {
     private static readonly MAX_INDEX_FILES;
     private binaryPath;
@@ -11,6 +11,7 @@ export declare class AstIndexClient {
     private timeout;
     private configBinaryPath;
     private autoInstall;
+    private astGrepAvailable;
     constructor(projectRoot: string, timeout?: number, options?: {
         binaryPath?: string | null;
         autoInstall?: boolean;
@@ -92,6 +93,25 @@ export declare class AstIndexClient {
      */
     fileImports(filePath: string): Promise<AstIndexImportEntry[]>;
     private parseImportsText;
+    /** Check if ast-grep (sg) is available for structural pattern search */
+    private checkAstGrep;
+    /** Structural pattern search via ast-grep. Requires ast-grep (sg) installed. */
+    agrep(pattern: string, options?: {
+        lang?: string;
+        limit?: number;
+    }): Promise<AstIndexAgrepMatch[]>;
+    private parseAgrepText;
+    /** Find TODO/FIXME/HACK comments in the project */
+    todo(): Promise<AstIndexTodoEntry[]>;
+    private parseTodoText;
+    /** Find @Deprecated symbols in the project */
+    deprecated(): Promise<AstIndexDeprecatedEntry[]>;
+    private parseDeprecatedText;
+    /** Find symbols with a specific annotation/decorator */
+    annotations(name: string): Promise<AstIndexAnnotationEntry[]>;
+    private parseAnnotationsText;
+    /** Trigger incremental index update (called by file watcher after edits) */
+    incrementalUpdate(): Promise<void>;
     isAvailable(): boolean;
     /** Returns true if the index was built but found >50k files (node_modules leak) */
     isOversized(): boolean;

package/dist/ast-index/client.js

@@ -16,6 +16,7 @@ export class AstIndexClient {
     timeout;
     configBinaryPath;
     autoInstall;
+    astGrepAvailable = null;
     constructor(projectRoot, timeout = 5000, options) {
         this.projectRoot = projectRoot;
         this.timeout = timeout;
@@ -636,6 +637,172 @@ export class AstIndexClient {
         }
         return entries;
     }
+    // --- Code audit commands ---
+    /** Check if ast-grep (sg) is available for structural pattern search */
+    async checkAstGrep() {
+        if (this.astGrepAvailable !== null)
+            return this.astGrepAvailable;
+        try {
+            await execFileAsync('sg', ['--version'], { timeout: 3000 });
+            this.astGrepAvailable = true;
+        }
+        catch {
+            this.astGrepAvailable = false;
+        }
+        return this.astGrepAvailable;
+    }
+    /** Structural pattern search via ast-grep. Requires ast-grep (sg) installed. */
+    async agrep(pattern, options) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        const available = await this.checkAstGrep();
+        if (!available) {
+            throw new Error('ast-grep (sg) not installed — required for structural pattern search.\n' +
+                'Install: brew install ast-grep  OR  npm i -g @ast-grep/cli\n' +
+                'Alternative: use Grep/ripgrep for text-based pattern search.');
+        }
+        const limit = options?.limit ?? 50;
+        const args = ['agrep', pattern];
+        if (options?.lang)
+            args.push('--lang', options.lang);
+        args.push('--limit', String(limit));
+        try {
+            const result = await this.exec(args, 15000);
+            return this.parseAgrepText(result).slice(0, limit);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index agrep failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    parseAgrepText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Format: file:line:matched_text  OR  file:line: matched_text
+            const match = line.match(/^(.+?):(\d+):(.*)$/);
+            if (match) {
+                results.push({
+                    file: match[1],
+                    line: parseInt(match[2], 10),
+                    text: match[3].trim(),
+                });
+            }
+        }
+        return results;
+    }
+    /** Find TODO/FIXME/HACK comments in the project */
+    async todo() {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['todo'], 15000);
+            return this.parseTodoText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index todo failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    parseTodoText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try format: file:line: KIND: message  OR  file:line: KIND message
+            const match = line.match(/^(.+?):(\d+):\s*(TODO|FIXME|HACK|XXX|NOTE|WARN(?:ING)?)[:\s]+(.*)$/i);
+            if (match) {
+                results.push({
+                    file: match[1],
+                    line: parseInt(match[2], 10),
+                    kind: match[3].toUpperCase(),
+                    text: match[4].trim(),
+                });
+            }
+        }
+        return results;
+    }
+    /** Find @Deprecated symbols in the project */
+    async deprecated() {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['deprecated'], 15000);
+            return this.parseDeprecatedText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index deprecated failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    parseDeprecatedText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try format: kind name (file:line) - message  OR  kind name (file:line)
+            const match = line.match(/^(\w+)\s+(\S+)\s+\((.+?):(\d+)\)(?:\s*-\s*(.+))?$/);
+            if (match) {
+                results.push({
+                    kind: match[1],
+                    name: match[2],
+                    file: match[3],
+                    line: parseInt(match[4], 10),
+                    message: match[5]?.trim(),
+                });
+            }
+        }
+        return results;
+    }
+    /** Find symbols with a specific annotation/decorator */
+    async annotations(name) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['annotations', name], 15000);
+            return this.parseAnnotationsText(result, name);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index annotations failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    parseAnnotationsText(text, annotationName) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try format: kind name (file:line)  OR  @Annotation kind name (file:line)
+            const match = line.match(/^(?:@\S+\s+)?(\w+)\s+(\S+)\s+\((.+?):(\d+)\)$/);
+            if (match) {
+                results.push({
+                    kind: match[1],
+                    name: match[2],
+                    file: match[3],
+                    line: parseInt(match[4], 10),
+                    annotation: annotationName,
+                });
+            }
+        }
+        return results;
+    }
+    /** Trigger incremental index update (called by file watcher after edits) */
+    async incrementalUpdate() {
+        if (!this.indexed || this.indexDisabled || this.indexOversized)
+            return;
+        try {
+            await this.exec(['update'], 15000);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index incremental update failed: ${err instanceof Error ? err.message : err}`);
+        }
+    }
+    // --- Utility methods ---
     isAvailable() {
         return this.binaryPath !== null;
     }

package/dist/ast-index/types.d.ts

@@ -153,4 +153,33 @@ export interface AstIndexImportEntry {
     isDefault?: boolean;
     isNamespace?: boolean;
 }
+/** ast-index agrep — structural pattern search via ast-grep */
+export interface AstIndexAgrepMatch {
+    file: string;
+    line: number;
+    text: string;
+}
+/** ast-index todo — TODO/FIXME/HACK comments */
+export interface AstIndexTodoEntry {
+    file: string;
+    line: number;
+    kind: string;
+    text: string;
+}
+/** ast-index deprecated — @Deprecated symbols */
+export interface AstIndexDeprecatedEntry {
+    name: string;
+    kind: string;
+    file: string;
+    line: number;
+    message?: string;
+}
+/** ast-index annotations — symbols with specific decorator/annotation */
+export interface AstIndexAnnotationEntry {
+    name: string;
+    kind: string;
+    file: string;
+    line: number;
+    annotation: string;
+}
 //# sourceMappingURL=types.d.ts.map

package/dist/core/validation.d.ts

@@ -76,6 +76,14 @@ export declare function validateFindUnusedArgs(args: unknown): {
     export_only?: boolean;
     limit?: number;
 };
+export interface CodeAuditArgs {
+    check: 'pattern' | 'todo' | 'deprecated' | 'annotations' | 'all';
+    pattern?: string;
+    name?: string;
+    lang?: string;
+    limit?: number;
+}
+export declare function validateCodeAuditArgs(args: unknown): CodeAuditArgs;
 /** Detect roots that would cause ast-index to scan the entire filesystem */
 export declare function isDangerousRoot(root: string): boolean;
 //# sourceMappingURL=validation.d.ts.map

package/dist/core/validation.js

@@ -207,6 +207,33 @@ export function validateFindUnusedArgs(args) {
         limit: optionalNumber(a.limit, 'limit'),
     };
 }
+export function validateCodeAuditArgs(args) {
+    if (!args || typeof args !== 'object') {
+        throw new Error('Arguments must be an object with a "check" parameter.');
+    }
+    const a = args;
+    const validChecks = ['pattern', 'todo', 'deprecated', 'annotations', 'all'];
+    if (typeof a.check !== 'string' || !validChecks.includes(a.check)) {
+        throw new Error(`Required parameter "check" must be one of: ${validChecks.join(', ')}`);
+    }
+    if (a.check === 'pattern') {
+        if (typeof a.pattern !== 'string' || a.pattern.length === 0) {
+            throw new Error('Parameter "pattern" is required when check="pattern". Example: "except:" or "print($$$ARGS)"');
+        }
+    }
+    if (a.check === 'annotations') {
+        if (typeof a.name !== 'string' || a.name.length === 0) {
+            throw new Error('Parameter "name" is required when check="annotations". Example: "Deprecated" or "Controller"');
+        }
+    }
+    return {
+        check: a.check,
+        pattern: optionalString(a.pattern, 'pattern'),
+        name: optionalString(a.name, 'name'),
+        lang: optionalString(a.lang, 'lang'),
+        limit: optionalNumber(a.limit, 'limit'),
+    };
+}
 /** Detect roots that would cause ast-index to scan the entire filesystem */
 export function isDangerousRoot(root) {
     const normalized = root.replace(/\/+$/, '') || '/';

package/dist/git/file-watcher.d.ts

@@ -1,18 +1,27 @@
 import type { FileCache } from '../core/file-cache.js';
 import type { ContextRegistry } from '../core/context-registry.js';
+import type { AstIndexClient } from '../ast-index/client.js';
 /**
  * Watches individual files for changes and auto-invalidates cache.
  * Only watches files that have been explicitly added (via watchFile()),
  * NOT the entire project root — avoids scanning thousands of files
  * and permission errors on Docker volumes, restricted dirs, etc.
+ *
+ * Also triggers debounced ast-index incremental update on file changes
+ * to keep the index fresh for find_usages, find_unused, code_audit.
  */
 export declare class FileWatcher {
+    private static readonly UPDATE_DEBOUNCE_MS;
     private fileCache;
     private contextRegistry;
+    private astIndex;
     private watcher;
     private watchedFiles;
-    constructor(_projectRoot: string, fileCache: FileCache, contextRegistry: ContextRegistry, _ignore: string[]);
+    private updateTimer;
+    constructor(_projectRoot: string, fileCache: FileCache, contextRegistry: ContextRegistry, _ignore: string[], astIndex?: AstIndexClient);
     start(): void;
+    /** Debounced ast-index incremental update after file changes */
+    private scheduleIndexUpdate;
     /** Add a specific file to watch. Called after smart_read/read_symbol loads a file. */
     watchFile(filePath: string): void;
     stop(): void;

package/dist/git/file-watcher.js

@@ -5,15 +5,22 @@ import { resolve } from 'node:path';
  * Only watches files that have been explicitly added (via watchFile()),
  * NOT the entire project root — avoids scanning thousands of files
  * and permission errors on Docker volumes, restricted dirs, etc.
+ *
+ * Also triggers debounced ast-index incremental update on file changes
+ * to keep the index fresh for find_usages, find_unused, code_audit.
  */
 export class FileWatcher {
+    static UPDATE_DEBOUNCE_MS = 2000;
     fileCache;
     contextRegistry;
+    astIndex;
     watcher = null;
     watchedFiles = new Set();
-    constructor(_projectRoot, fileCache, contextRegistry, _ignore) {
+    updateTimer = null;
+    constructor(_projectRoot, fileCache, contextRegistry, _ignore, astIndex) {
         this.fileCache = fileCache;
         this.contextRegistry = contextRegistry;
+        this.astIndex = astIndex ?? null;
     }
     start() {
         // Start with an empty watcher — files are added on demand via watchFile()
@@ -30,14 +37,26 @@ export class FileWatcher {
             if (this.fileCache.get(absPath)) {
                 this.fileCache.invalidate(absPath);
             }
+            this.scheduleIndexUpdate();
         });
         this.watcher.on('unlink', (filePath) => {
             const absPath = resolve(filePath);
             this.fileCache.invalidate(absPath);
             this.contextRegistry.forget(absPath);
             this.watchedFiles.delete(absPath);
+            this.scheduleIndexUpdate();
         });
     }
+    /** Debounced ast-index incremental update after file changes */
+    scheduleIndexUpdate() {
+        if (!this.astIndex)
+            return;
+        if (this.updateTimer)
+            clearTimeout(this.updateTimer);
+        this.updateTimer = setTimeout(() => {
+            this.astIndex?.incrementalUpdate().catch(() => { });
+        }, FileWatcher.UPDATE_DEBOUNCE_MS);
+    }
     /** Add a specific file to watch. Called after smart_read/read_symbol loads a file. */
     watchFile(filePath) {
         const absPath = resolve(filePath);
@@ -49,6 +68,9 @@ export class FileWatcher {
         this.watchedFiles.add(absPath);
     }
     stop() {
+        if (this.updateTimer)
+            clearTimeout(this.updateTimer);
+        this.updateTimer = null;
         this.watcher?.close();
         this.watcher = null;
         this.watchedFiles.clear();

package/dist/handlers/code-audit.d.ts

@@ -0,0 +1,9 @@
+import type { AstIndexClient } from '../ast-index/client.js';
+import type { CodeAuditArgs } from '../core/validation.js';
+export declare function handleCodeAudit(args: CodeAuditArgs, projectRoot: string, astIndex: AstIndexClient): Promise<{
+    content: Array<{
+        type: 'text';
+        text: string;
+    }>;
+}>;
+//# sourceMappingURL=code-audit.d.ts.map