token-pilot 0.8.3 → 0.10.0

package/.claude-plugin/marketplace.json

@@ -1,8 +1,8 @@
 {
   "name": "token-pilot",
   "displayName": "Token Pilot",
-  "description": "Reduces token consumption by 80-95% via AST-aware lazy file reading. 14 MCP tools for structural code reading, symbol navigation, and cross-file search.",
-  "version": "0.1.1",
+  "description": "Reduces token consumption by 80-95% via AST-aware lazy file reading. 16 MCP tools for structural code reading, symbol navigation, and cross-file search.",
+  "version": "0.10.0",
   "author": "Digital-Threads",
   "repository": "https://github.com/Digital-Threads/token-pilot",
   "license": "MIT",

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "token-pilot",
-  "version": "0.7.3",
+  "version": "0.10.0",
   "description": "Reduces token consumption by 80-95% via AST-aware lazy file reading. Returns structural overviews instead of full files.",
   "author": "token-pilot",
   "license": "MIT",

package/CHANGELOG.md

@@ -5,6 +5,43 @@ All notable changes to Token Pilot will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.0] - 2026-03-14
+
+### Added
+- **`smart_diff` tool** — structural git diff with AST symbol mapping. Shows which functions/classes were modified/added/removed instead of raw patch output. Supports scopes: `unstaged`, `staged`, `commit` (ref required), `branch` (ref required). Small diffs (<=30 lines) include actual hunks, large diffs show summary. Returns `rawTokens` for precise savings analytics.
+- **`explore_area` tool** — one-call directory exploration combining outline + imports + tests + git changes. Replaces 3-5 separate tool calls when starting work on an area. Sections: `outline` (recursive depth 2), `imports` (external deps + who imports this area), `tests` (matching test/spec files), `changes` (recent git log). All sections run in parallel via `Promise.allSettled`.
+- **26 new tests** — smart_diff parser (10), symbol mapping (5), validation (11). Total: 170 tests (was 144).
+
+### Changed
+- **16 tools** (was 14) — added `smart_diff`, `explore_area`
+- **MCP instructions** — updated workflow: `project_overview → explore_area → smart_read → read_symbol → read_for_edit → edit → smart_diff`
+- **`outlineDir` and `CODE_EXTENSIONS` exported** from outline.ts for reuse by explore_area
+
+## [0.9.0] - 2026-03-08
+
+### Added
+- **`module_info` tool** — analyze module dependencies, dependents, public API, and unused deps. Uses ast-index v3.27.0 module commands (`modules`, `module-deps`, `module-dependents`, `module-api`, `unused-deps`). Includes degradation check when ast-index is unavailable.
+- **`project_overview` dual-detection** — shows BOTH ast-index type detection AND config-file detection (package.json, composer.json, Cargo.toml, pyproject.toml, go.mod) with CONFIDENCE scoring (high/medium/low/unknown). Detects frameworks, quality tools (PHPStan, ESLint, Vitest, Jest, Biome, etc.), CI pipelines (GitHub Actions, GitLab CI, Jenkins), and Docker.
+- **`project_overview` `include` parameter** — filter sections: `["stack"]` for quick type check, `["quality","ci"]` for tooling overview. Default: all sections.
+- **`find_usages` post-filters** — `scope` (path prefix), `kind` (definitions/imports/usages), `lang` (14 languages by extension), `limit` (per category, 1-500). All filters optional, backward compatible.
+- **`outline` recursive mode** — `recursive=true` with `max_depth` (default 2, max 5) recurses into subdirectories. At max depth shows file counts only.
+- **`src/core/project-detector.ts`** — extracted config-based detection logic into reusable module. Framework detection maps for PHP (7), JS (10), Python (5). Quality tools scanner (13 tools). CI pipeline detector (6 platforms).
+- **ast-index client: 5 module methods** — `modules()`, `moduleDeps()`, `moduleDependents()`, `unusedDeps()`, `moduleApi()` with JSON-first + text fallback parsing.
+- **ast-index types: 4 module interfaces** — `AstIndexModuleEntry`, `AstIndexModuleDep`, `AstIndexUnusedDep`, `AstIndexModuleApi`.
+
+### Fixed
+- **`module_info` token savings** — `tokensWouldBe` was equal to `tokensReturned` (0% savings). Now estimates manual analysis cost correctly.
+- **`outline` recursive overflow** — added `MAX_OUTLINE_LINES=500` guard to prevent runaway output on large projects with `recursive=true`.
+- **`project_overview` "frontend" label** — removed hardcoded "frontend" suffix for secondary stacks (Node.js is not always frontend).
+- **Ruff detection** — no longer double-reads `pyproject.toml`. Checks `ruff.toml`/`.ruff.toml` first, falls back to `pyproject.toml [tool.ruff]` only if needed.
+- **44 new tests** — validators (23) + project-detector (21). Total: 144 tests (was 100).
+
+### Changed
+- **14 tools** (was 13) — added `module_info`
+- **Tool descriptions** — updated with `(v1.1: ...)` version hints for enhanced tools
+- **MCP instructions** — added module_info to "COMBINE BOTH" workflow section
+- **Version sync** — package.json, plugin.json, marketplace.json all at 0.9.0
+
 ## [0.8.3] - 2026-03-08
 
 ### Fixed

package/README.md

@@ -149,7 +149,7 @@ For more control, you can add rules to your project:
 - **Cursor** → `.cursorrules` in project root
 - **Codex** → `AGENTS.md` in project root
 
-## MCP Tools (13)
+## MCP Tools (14)
 
 ### Core Reading
 
@@ -166,12 +166,15 @@ For more control, you can add rules to your project:
 
 | Tool | Instead of | Description |
 |------|-----------|-------------|
-| `find_usages` | `Grep` (refs) | All usages of a symbol: definitions, imports, references. |
-| `project_overview` | `ls` + explore | Project type, architecture, frameworks, directory map. |
+| `find_usages` | `Grep` (refs) | All usages of a symbol: definitions, imports, references. Filters: `scope` (path prefix), `kind` (definitions/imports/usages), `lang`, `limit`. |
+| `project_overview` | `ls` + explore | Dual-detection (ast-index + config files) with confidence scoring. Project type, frameworks, quality tools, CI, architecture, directory map. Filter sections with `include`. |
 | `related_files` | manual explore | Import graph: what a file imports, what imports it, test files. |
-| `outline` | multiple `smart_read` | Compact overview of all code files in a directory. One call instead of 5-6. |
+| `outline` | multiple `smart_read` | Compact overview of all code files in a directory. One call instead of 5-6. Supports `recursive` mode with `max_depth` for deep exploration. |
 | `find_unused` | manual | Detect dead code — unused functions, classes, variables. |
 | `code_audit` | multiple `Grep` | Code quality issues in one call: TODO/FIXME comments, deprecated symbols, structural code patterns (via ast-grep), decorator search. |
+| `module_info` | manual analysis | Module dependency analysis: dependencies, dependents, public API, unused deps. Use for architecture understanding and dependency cleanup. |
+| `smart_diff` | raw `git diff` | Structural diff with AST symbol mapping — shows which functions/classes changed instead of raw patch. Scopes: unstaged, staged, commit, branch. |
+| `explore_area` | outline + related_files + git log | One-call directory exploration: structure, imports, tests, recent changes. Replaces 3-5 separate calls. |
 
 ### Analytics
 
@@ -315,13 +318,13 @@ npm run dev          # TypeScript watch mode
 ```
 src/
   index.ts              — CLI entry point (6 commands)
-  server.ts             — MCP server setup, tool definitions, instructions
+  server.ts             — MCP server setup, 14 tool definitions, instructions
   types.ts              — Core domain types
   ast-index/
-    client.ts           — ast-index CLI wrapper
+    client.ts           — ast-index CLI wrapper (22+ methods)
     binary-manager.ts   — Auto-download & manage ast-index binary
     tar-extract.ts      — Minimal tar extractor (zero deps)
-    types.ts            — ast-index response types
+    types.ts            — ast-index response types (20+ interfaces)
   core/
     file-cache.ts       — LRU file cache with staleness detection
     context-registry.ts — Advisory context tracking + compact reminders
@@ -330,6 +333,7 @@ src/
     session-analytics.ts — Token savings tracking
     validation.ts       — Input validators for all tools
     format-duration.ts  — Shared duration formatter
+    project-detector.ts — Config-based project detection (frameworks, CI, quality tools)
   config/
     loader.ts           — Config loading + deep merge
     defaults.ts         — Default config values
@@ -341,13 +345,16 @@ src/
     read-range.ts       — read_range handler
     read-diff.ts        — read_diff handler (O(n) diff)
     smart-read-many.ts  — Batch smart_read
-    find-usages.ts      — find_usages handler (via ast-index refs)
+    find-usages.ts      — find_usages handler (scope/kind/lang/limit filters)
     read-for-edit.ts    — read_for_edit handler (minimal edit context)
     related-files.ts    — related_files handler (import graph)
-    outline.ts          — outline handler (directory overview)
+    outline.ts          — outline handler (recursive directory overview)
     find-unused.ts      — find_unused handler
     code-audit.ts       — code_audit handler (TODOs, deprecated, patterns)
-    project-overview.ts — project_overview (via ast-index map + conventions)
+    project-overview.ts — project_overview (dual-detection + confidence)
+    module-info.ts      — module_info handler (deps, dependents, API, unused)
+    smart-diff.ts       — smart_diff handler (structural git diff + symbol mapping)
+    explore-area.ts     — explore_area handler (outline + imports + tests + changes)
     non-code.ts         — JSON/YAML/MD/TOML structural summaries
     export-ast-index.ts — AST export for context-mode BM25
   git/

package/dist/ast-index/client.d.ts

@@ -1,5 +1,5 @@
 import type { FileStructure } from '../types.js';
-import type { AstIndexSymbolDetail, AstIndexSearchResult, AstIndexUsageResult, AstIndexImplementation, AstIndexHierarchyNode, AstIndexRefsResponse, AstIndexMapResponse, AstIndexConventionsResponse, AstIndexCallerEntry, AstIndexCallTreeNode, AstIndexChangedEntry, AstIndexUnusedSymbol, AstIndexImportEntry, AstIndexAgrepMatch, AstIndexTodoEntry, AstIndexDeprecatedEntry, AstIndexAnnotationEntry } from './types.js';
+import type { AstIndexSymbolDetail, AstIndexSearchResult, AstIndexUsageResult, AstIndexImplementation, AstIndexHierarchyNode, AstIndexRefsResponse, AstIndexMapResponse, AstIndexConventionsResponse, AstIndexCallerEntry, AstIndexCallTreeNode, AstIndexChangedEntry, AstIndexUnusedSymbol, AstIndexImportEntry, AstIndexAgrepMatch, AstIndexTodoEntry, AstIndexDeprecatedEntry, AstIndexAnnotationEntry, AstIndexModuleEntry, AstIndexModuleDep, AstIndexUnusedDep, AstIndexModuleApi } from './types.js';
 export declare class AstIndexClient {
     private static readonly MAX_INDEX_FILES;
     private binaryPath;
@@ -113,6 +113,20 @@ export declare class AstIndexClient {
     private parseAnnotationsText;
     /** Trigger incremental index update (called by file watcher after edits) */
     incrementalUpdate(): Promise<void>;
+    /** List project modules matching optional pattern */
+    modules(pattern?: string): Promise<AstIndexModuleEntry[]>;
+    /** Get dependencies of a module */
+    moduleDeps(module: string): Promise<AstIndexModuleDep[]>;
+    /** Get modules that depend on this module */
+    moduleDependents(module: string): Promise<AstIndexModuleDep[]>;
+    /** Find unused dependencies of a module */
+    unusedDeps(module: string): Promise<AstIndexUnusedDep[]>;
+    /** Get public API of a module */
+    moduleApi(module: string): Promise<AstIndexModuleApi[]>;
+    private parseModuleListText;
+    private parseModuleDepText;
+    private parseUnusedDepsText;
+    private parseModuleApiText;
     isAvailable(): boolean;
     /** Returns true if the index was built but found >50k files (node_modules leak) */
     isOversized(): boolean;

package/dist/ast-index/client.js

@@ -817,6 +817,185 @@ export class AstIndexClient {
             console.error(`[token-pilot] ast-index incremental update failed: ${err instanceof Error ? err.message : err}`);
         }
     }
+    // --- Module analysis methods (ast-index v3.27.0) ---
+    /** List project modules matching optional pattern */
+    async modules(pattern) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const cmdArgs = pattern ? ['module', pattern] : ['module'];
+            const result = await this.exec(cmdArgs, 15000);
+            return this.parseModuleListText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index module failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    /** Get dependencies of a module */
+    async moduleDeps(module) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['deps', module], 15000);
+            return this.parseModuleDepText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index deps failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    /** Get modules that depend on this module */
+    async moduleDependents(module) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['dependents', module], 15000);
+            return this.parseModuleDepText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index dependents failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    /** Find unused dependencies of a module */
+    async unusedDeps(module) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['unused-deps', module], 15000);
+            return this.parseUnusedDepsText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index unused-deps failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    /** Get public API of a module */
+    async moduleApi(module) {
+        if (this.indexDisabled || this.indexOversized)
+            return [];
+        await this.ensureIndex();
+        try {
+            const result = await this.exec(['api', module], 15000);
+            return this.parseModuleApiText(result);
+        }
+        catch (err) {
+            console.error(`[token-pilot] ast-index api failed: ${err instanceof Error ? err.message : err}`);
+            return [];
+        }
+    }
+    // Parsers for module commands (text format — JSON may not be supported for all)
+    parseModuleListText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try JSON first
+            try {
+                const parsed = JSON.parse(line);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+            catch { /* not JSON, parse as text */ }
+            // Format: name (path) — N files  OR  name (path)  OR  path
+            const match = line.match(/^(\S+)\s+\((.+?)\)(?:\s*—\s*(\d+)\s+files?)?$/);
+            if (match) {
+                results.push({
+                    name: match[1],
+                    path: match[2],
+                    file_count: match[3] ? parseInt(match[3], 10) : undefined,
+                });
+            }
+            else {
+                // Fallback: treat entire line as a path-based module
+                const trimmed = line.trim();
+                if (trimmed && !trimmed.startsWith('#') && !trimmed.startsWith('─')) {
+                    const name = trimmed.split('/').pop() ?? trimmed;
+                    results.push({ name, path: trimmed });
+                }
+            }
+        }
+        return results;
+    }
+    parseModuleDepText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try JSON first
+            try {
+                const parsed = JSON.parse(line);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+            catch { /* not JSON, parse as text */ }
+            // Format: → name (path)  OR  ← name (path)  OR  name (path)  OR  name
+            const match = line.match(/^[→←\-\s]*(\S+)(?:\s+\((.+?)\))?(?:\s+\[(direct|transitive)\])?$/);
+            if (match) {
+                results.push({
+                    name: match[1],
+                    path: match[2] ?? match[1],
+                    type: match[3],
+                });
+            }
+        }
+        return results;
+    }
+    parseUnusedDepsText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try JSON first
+            try {
+                const parsed = JSON.parse(line);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+            catch { /* not JSON, parse as text */ }
+            // Format: ⚠ name (path) — reason  OR  name (path)  OR  name — reason
+            const match = line.match(/^[⚠!\s]*(\S+)(?:\s+\((.+?)\))?(?:\s*[—\-]+\s*(.+))?$/);
+            if (match) {
+                results.push({
+                    name: match[1],
+                    path: match[2] ?? match[1],
+                    reason: match[3]?.trim(),
+                });
+            }
+        }
+        return results;
+    }
+    parseModuleApiText(text) {
+        const results = [];
+        for (const line of text.split('\n')) {
+            if (!line.trim())
+                continue;
+            // Try JSON first
+            try {
+                const parsed = JSON.parse(line);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+            catch { /* not JSON, parse as text */ }
+            // Format: kind name (file:line)  OR  kind name signature (file:line)
+            const match = line.match(/^(\w+)\s+(\S+)(?:\s+(.*?))?\s+\((.+?):(\d+)\)$/);
+            if (match) {
+                results.push({
+                    kind: match[1],
+                    name: match[2],
+                    signature: match[3]?.trim() || undefined,
+                    file: match[4],
+                    line: parseInt(match[5], 10),
+                });
+            }
+        }
+        return results;
+    }
     // --- Utility methods ---
     isAvailable() {
         return this.binaryPath !== null;

package/dist/ast-index/types.d.ts

@@ -1,6 +1,6 @@
 /**
  * Types for ast-index CLI JSON output.
- * These map to the ACTUAL JSON responses from ast-index v3.24.0 commands.
+ * These map to the ACTUAL JSON responses from ast-index v3.27.0 commands.
  */
 /** ast-index outline — parsed from text output (JSON not supported) */
 export interface AstIndexOutlineEntry {
@@ -182,4 +182,30 @@ export interface AstIndexAnnotationEntry {
     line: number;
     annotation: string;
 }
+/** ast-index module — list project modules */
+export interface AstIndexModuleEntry {
+    name: string;
+    path: string;
+    file_count?: number;
+}
+/** ast-index deps / dependents — module dependency */
+export interface AstIndexModuleDep {
+    name: string;
+    path: string;
+    type?: string;
+}
+/** ast-index unused-deps — unused module dependency */
+export interface AstIndexUnusedDep {
+    name: string;
+    path: string;
+    reason?: string;
+}
+/** ast-index api — public API of a module */
+export interface AstIndexModuleApi {
+    name: string;
+    kind: string;
+    signature?: string;
+    file: string;
+    line: number;
+}
 //# sourceMappingURL=types.d.ts.map

package/dist/ast-index/types.js

@@ -1,6 +1,6 @@
 /**
  * Types for ast-index CLI JSON output.
- * These map to the ACTUAL JSON responses from ast-index v3.24.0 commands.
+ * These map to the ACTUAL JSON responses from ast-index v3.27.0 commands.
  */
 export {};
 //# sourceMappingURL=types.js.map

package/dist/core/project-detector.d.ts

@@ -0,0 +1,42 @@
+export interface DetectedStack {
+    type: string;
+    name: string;
+    version: string;
+    langVersion?: string;
+    framework?: string;
+    description?: string;
+}
+export interface ProjectDetection {
+    /** Project name (from primary config or dirname) */
+    projectName: string;
+    /** Project version (from primary config) */
+    projectVersion: string;
+    /** Project description */
+    projectDescription?: string;
+    /** What ast-index map() says about project type */
+    astIndexType?: string;
+    /** All detected stacks from config files */
+    configStacks: DetectedStack[];
+    /** Primary stack (most source files) */
+    primaryStack?: DetectedStack;
+    /** Confidence of detection */
+    confidence: 'high' | 'medium' | 'low' | 'unknown';
+    /** Quality tools found */
+    qualityTools: string[];
+    /** CI pipelines found */
+    ciPipelines: string[];
+    /** Docker presence */
+    hasDocker: boolean;
+}
+/**
+ * Detect project stacks by reading all config files in parallel.
+ * Returns dual-detection data: ast-index type + config-based stacks.
+ */
+export declare function detectProject(projectRoot: string, astIndexType?: string): Promise<ProjectDetection>;
+/** Detect quality/linting tools present in project root */
+export declare function detectQualityTools(projectRoot: string): Promise<string[]>;
+/** Detect CI/CD pipelines present in project */
+export declare function detectCI(projectRoot: string): Promise<string[]>;
+/** Detect Docker presence */
+export declare function detectDocker(projectRoot: string): Promise<boolean>;
+//# sourceMappingURL=project-detector.d.ts.map