token-pilot 0.14.2 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +25 -8
  3. package/dist/ast-index/client.d.ts +2 -89
  4. package/dist/ast-index/client.js +49 -742
  5. package/dist/ast-index/enricher.d.ts +10 -0
  6. package/dist/ast-index/enricher.js +202 -0
  7. package/dist/ast-index/parser.d.ts +31 -0
  8. package/dist/ast-index/parser.js +340 -0
  9. package/dist/ast-index/regex-parser-python.d.ts +8 -0
  10. package/dist/ast-index/regex-parser-python.js +132 -0
  11. package/dist/ast-index/regex-parser.d.ts +8 -0
  12. package/dist/ast-index/regex-parser.js +118 -0
  13. package/dist/config/defaults.js +1 -0
  14. package/dist/core/session-analytics.d.ts +2 -2
  15. package/dist/core/session-analytics.js +78 -61
  16. package/dist/core/symbol-resolver.d.ts +0 -1
  17. package/dist/core/symbol-resolver.js +3 -12
  18. package/dist/core/validation.d.ts +12 -0
  19. package/dist/core/validation.js +62 -2
  20. package/dist/handlers/code-audit.js +2 -2
  21. package/dist/handlers/find-unused.js +1 -1
  22. package/dist/handlers/find-usages.d.ts +1 -1
  23. package/dist/handlers/find-usages.js +93 -25
  24. package/dist/handlers/read-for-edit.d.ts +1 -0
  25. package/dist/handlers/read-for-edit.js +65 -0
  26. package/dist/handlers/read-symbols.d.ts +18 -0
  27. package/dist/handlers/read-symbols.js +142 -0
  28. package/dist/handlers/smart-diff.js +23 -0
  29. package/dist/handlers/smart-read.js +14 -1
  30. package/dist/index.d.ts +1 -1
  31. package/dist/index.js +6 -5
  32. package/dist/server/token-estimates.d.ts +31 -0
  33. package/dist/server/token-estimates.js +204 -0
  34. package/dist/server/tool-definitions.d.ts +1070 -0
  35. package/dist/server/tool-definitions.js +316 -0
  36. package/dist/server.js +23 -480
  37. package/dist/types.d.ts +1 -0
  38. package/package.json +1 -1
  39. package/skills/guide/SKILL.md +64 -0
@@ -1,9 +1,12 @@
1
1
  import { execFile } from 'node:child_process';
2
2
  import { promisify } from 'node:util';
3
- import { stat } from 'node:fs/promises';
4
- import { createHash } from 'node:crypto';
5
- import { readFile } from 'node:fs/promises';
6
3
  import { findBinary, installBinary } from './binary-manager.js';
4
+ import { parseFileCount, parseOutlineText, parseImportsText, parseImplementationsText, parseHierarchyText, parseAgrepText, parseTodoText, parseDeprecatedText, parseAnnotationsText, parseModuleListText, parseModuleDepText, parseUnusedDepsText, parseModuleApiText, } from './parser.js';
5
+ import { buildFileStructure } from './enricher.js';
6
+ import { parseTypeScriptRegex } from './regex-parser.js';
7
+ import { parsePythonRegex } from './regex-parser-python.js';
8
+ const TS_JS_EXTENSIONS = new Set(['ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs']);
9
+ const PYTHON_EXTENSIONS = new Set(['py', 'pyw']);
7
10
  const execFileAsync = promisify(execFile);
8
11
  export class AstIndexClient {
9
12
  static MAX_INDEX_FILES = 50_000;
@@ -51,12 +54,10 @@ export class AstIndexClient {
51
54
  async ensureIndex() {
52
55
  if (this.indexed)
53
56
  return;
54
- // Project root is too broad (/, home dir) — refuse to build
55
57
  if (this.indexDisabled) {
56
58
  throw new Error('ast-index: index build disabled — project root is too broad (e.g. /). ' +
57
59
  'Configure mcpServers with "args": ["/path/to/project"] to set the correct project root.');
58
60
  }
59
- // If a previous build found >50k files, don't retry
60
61
  if (this.indexOversized) {
61
62
  throw new Error('ast-index disabled: previous build indexed >50k files (likely node_modules). ' +
62
63
  'Ensure node_modules is in .gitignore, then restart the MCP server.');
@@ -73,14 +74,12 @@ export class AstIndexClient {
73
74
  }
74
75
  }
75
76
  async buildIndex() {
76
- // Check if index already exists and has files
77
77
  let existingFileCount = 0;
78
78
  try {
79
79
  const stats = await this.exec(['--format', 'json', 'stats']);
80
- existingFileCount = this.parseFileCount(stats);
80
+ existingFileCount = parseFileCount(stats);
81
81
  }
82
82
  catch { /* no index yet */ }
83
- // Guard: existing index is oversized (node_modules leak from previous build)
84
83
  if (existingFileCount > AstIndexClient.MAX_INDEX_FILES) {
85
84
  console.error(`[token-pilot] ast-index: existing index has ${existingFileCount} files (>${AstIndexClient.MAX_INDEX_FILES}) — likely includes node_modules. Clearing.`);
86
85
  try {
@@ -88,19 +87,15 @@ export class AstIndexClient {
88
87
  }
89
88
  catch { /* best effort */ }
90
89
  existingFileCount = 0;
91
- // Fall through to rebuild — maybe .gitignore was fixed
92
90
  }
93
91
  if (existingFileCount > 0) {
94
- // Index exists — use incremental update (fast)
95
92
  console.error(`[token-pilot] ast-index: updating index (${existingFileCount} files)...`);
96
93
  try {
97
94
  await this.exec(['update'], 30000);
98
- // Re-check count after update
99
95
  try {
100
- existingFileCount = this.parseFileCount(await this.exec(['--format', 'json', 'stats']));
96
+ existingFileCount = parseFileCount(await this.exec(['--format', 'json', 'stats']));
101
97
  }
102
98
  catch { /* keep previous count */ }
103
- // Guard: update may have grown index beyond limit
104
99
  if (existingFileCount > AstIndexClient.MAX_INDEX_FILES) {
105
100
  return this.handleOversizedIndex(existingFileCount);
106
101
  }
@@ -112,12 +107,10 @@ export class AstIndexClient {
112
107
  console.error(`[token-pilot] ast-index: update failed, falling back to rebuild — ${updateErr instanceof Error ? updateErr.message : updateErr}`);
113
108
  }
114
109
  }
115
- // No index or update failed — full rebuild
116
110
  console.error('[token-pilot] ast-index: building index (this may take a moment)...');
117
111
  try {
118
112
  await this.exec(['rebuild'], 120000);
119
- const fileCount = this.parseFileCount(await this.exec(['--format', 'json', 'stats']).catch(() => ''));
120
- // Guard: rebuild produced oversized index
113
+ const fileCount = parseFileCount(await this.exec(['--format', 'json', 'stats']).catch(() => ''));
121
114
  if (fileCount > AstIndexClient.MAX_INDEX_FILES) {
122
115
  return this.handleOversizedIndex(fileCount);
123
116
  }
@@ -125,10 +118,9 @@ export class AstIndexClient {
125
118
  console.error(`[token-pilot] ast-index: index built (${fileCount} files)`);
126
119
  }
127
120
  catch (buildErr) {
128
- // If rebuild failed due to lock, check if index is usable anyway
129
121
  const errMsg = buildErr instanceof Error ? buildErr.message : String(buildErr);
130
122
  if (errMsg.includes('lock') || errMsg.includes('already running')) {
131
- const count = this.parseFileCount(await this.exec(['--format', 'json', 'stats']).catch(() => ''));
123
+ const count = parseFileCount(await this.exec(['--format', 'json', 'stats']).catch(() => ''));
132
124
  if (count > 0 && count <= AstIndexClient.MAX_INDEX_FILES) {
133
125
  this.indexed = true;
134
126
  console.error(`[token-pilot] ast-index: using existing index (${count} files, rebuild skipped due to lock)`);
@@ -142,7 +134,6 @@ export class AstIndexClient {
142
134
  throw buildErr;
143
135
  }
144
136
  }
145
- /** Mark index as oversized — disables index-dependent tools, outline still works */
146
137
  async handleOversizedIndex(fileCount) {
147
138
  this.indexOversized = true;
148
139
  this.indexed = false;
@@ -156,117 +147,52 @@ export class AstIndexClient {
156
147
  ` → Tools disabled: find_unused, find_usages, related_files, project_overview\n` +
157
148
  ` → Tools still working: outline, smart_read, smart_read_many, read_symbol`);
158
149
  }
159
- /** Extract file count from stats output (JSON or text) */
160
- parseFileCount(statsText) {
161
- // Try JSON first (--format json)
162
- try {
163
- const json = JSON.parse(statsText);
164
- if (json?.stats?.file_count !== undefined)
165
- return json.stats.file_count;
166
- }
167
- catch { /* not JSON, fall through */ }
168
- // Fallback: text format
169
- const match = statsText.match(/Files:\s*(\d+)/);
170
- return match ? parseInt(match[1], 10) : 0;
171
- }
172
150
  async outline(filePath) {
173
- // outline parses a single file — try directly without requiring full index
174
151
  try {
175
152
  const result = await this.exec(['outline', filePath]);
176
- const entries = this.parseOutlineText(result);
153
+ const entries = parseOutlineText(result);
177
154
  if (entries.length === 0)
178
155
  return null;
179
- return await this.buildFileStructure(filePath, entries);
156
+ return await buildFileStructure(filePath, entries);
180
157
  }
181
158
  catch {
182
- // Direct call failed — try building index first (unless disabled/oversized)
183
159
  if (this.indexDisabled || this.indexOversized)
184
- return null;
160
+ return this.regexFallback(filePath);
185
161
  try {
186
162
  await this.ensureIndex();
187
163
  const result = await this.exec(['outline', filePath]);
188
- const entries = this.parseOutlineText(result);
164
+ const entries = parseOutlineText(result);
189
165
  if (entries.length === 0)
190
166
  return null;
191
- return await this.buildFileStructure(filePath, entries);
167
+ return await buildFileStructure(filePath, entries);
192
168
  }
193
169
  catch (err) {
194
170
  console.error(`[token-pilot] ast-index outline failed for ${filePath}: ${err instanceof Error ? err.message : err}`);
195
- return null;
171
+ return this.regexFallback(filePath);
196
172
  }
197
173
  }
198
174
  }
199
- /**
200
- * Parse text output from `ast-index outline`:
201
- * Outline of src/file.ts:
202
- * :10 ClassName [class]
203
- * :11 propName [property]
204
- * :14 methodName [function]
205
- */
206
- parseOutlineText(text) {
207
- const lines = text.split('\n');
208
- const entries = [];
209
- const classStack = [];
210
- for (const line of lines) {
211
- // Match: optional whitespace, :LINE_NUM, SYMBOL_NAME, [KIND]
212
- const match = line.match(/^(\s*):(\d+)\s+(\S+)\s+\[(\w+)\]/);
213
- if (!match)
214
- continue;
215
- const indent = match[1].length;
216
- const entry = {
217
- name: match[3],
218
- kind: match[4],
219
- start_line: parseInt(match[2], 10),
220
- end_line: 0, // computed later
221
- };
222
- // Pop stack until we find a parent with less indent
223
- while (classStack.length > 0 && classStack[classStack.length - 1].indent >= indent) {
224
- classStack.pop();
225
- }
226
- if (classStack.length > 0) {
227
- // This is a child of the top of stack
228
- const parent = classStack[classStack.length - 1].entry;
229
- if (!parent.children)
230
- parent.children = [];
231
- parent.children.push(entry);
232
- }
233
- else {
234
- entries.push(entry);
235
- }
236
- // Push classes/interfaces onto stack as potential parents
237
- if (['class', 'interface', 'struct', 'enum', 'impl', 'trait', 'namespace', 'module'].includes(entry.kind.toLowerCase())) {
238
- classStack.push({ entry, indent });
239
- }
175
+ /** Regex-based fallback for TS/JS/Python when ast-index binary is unavailable. */
176
+ async regexFallback(filePath) {
177
+ const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
178
+ const parser = TS_JS_EXTENSIONS.has(ext) ? parseTypeScriptRegex
179
+ : PYTHON_EXTENSIONS.has(ext) ? parsePythonRegex
180
+ : null;
181
+ if (!parser)
182
+ return null;
183
+ try {
184
+ const { readFile } = await import('node:fs/promises');
185
+ const content = await readFile(filePath, 'utf-8');
186
+ const entries = parser(content);
187
+ if (entries.length === 0)
188
+ return null;
189
+ return await buildFileStructure(filePath, entries);
240
190
  }
241
- // Compute end_line for all entries
242
- this.computeEndLines(entries);
243
- return entries;
244
- }
245
- /** Compute end_line from sequential start positions */
246
- computeEndLines(entries) {
247
- for (let i = 0; i < entries.length; i++) {
248
- // Children first (recursive)
249
- if (entries[i].children?.length) {
250
- this.computeEndLines(entries[i].children);
251
- }
252
- if (i < entries.length - 1) {
253
- // end = next sibling's start - 1
254
- entries[i].end_line = entries[i + 1].start_line - 1;
255
- }
256
- else {
257
- // last entry: estimate based on children or use start + reasonable default
258
- const children = entries[i].children;
259
- if (children?.length) {
260
- entries[i].end_line = children[children.length - 1].end_line + 1;
261
- }
262
- else {
263
- entries[i].end_line = entries[i].start_line + 10; // estimated
264
- }
265
- }
191
+ catch {
192
+ return null;
266
193
  }
267
194
  }
268
195
  async symbol(name) {
269
- // Try directly first (works if index exists from a previous session)
270
196
  try {
271
197
  const result = await this.exec(['symbol', name, '--format', 'json']);
272
198
  const raw = JSON.parse(result);
@@ -276,7 +202,6 @@ export class AstIndexClient {
276
202
  }
277
203
  }
278
204
  catch { /* fall through to ensureIndex path */ }
279
- // Direct call failed — try building index (unless disabled/oversized)
280
205
  if (this.indexDisabled || this.indexOversized)
281
206
  return null;
282
207
  try {
@@ -317,7 +242,6 @@ export class AstIndexClient {
317
242
  })) : []),
318
243
  ...(Array.isArray(parsed.references) ? parsed.references : []),
319
244
  ];
320
- // Fallback: if parsed is an array directly
321
245
  const matches = all.length > 0 ? all : (Array.isArray(parsed) ? parsed : []);
322
246
  const mapped = matches
323
247
  .map((m) => ({
@@ -326,7 +250,7 @@ export class AstIndexClient {
326
250
  text: m.content ?? m.text ?? m.signature ?? '',
327
251
  }))
328
252
  .filter(r => r.file !== '' && r.text !== '');
329
- // Deduplicate by file:line (merge of 4 categories creates dupes)
253
+ // Deduplicate by file:line
330
254
  const seen = new Set();
331
255
  return mapped.filter(r => {
332
256
  const key = `${r.file}:${r.line}`;
@@ -348,12 +272,7 @@ export class AstIndexClient {
348
272
  const raw = JSON.parse(result);
349
273
  if (!Array.isArray(raw))
350
274
  return [];
351
- return raw.map(u => ({
352
- file: u.path,
353
- line: u.line,
354
- text: u.context,
355
- kind: 'reference',
356
- }));
275
+ return raw.map(u => ({ file: u.path, line: u.line, text: u.context, kind: 'reference' }));
357
276
  }
358
277
  catch (err) {
359
278
  console.error(`[token-pilot] ast-index usages failed: ${err instanceof Error ? err.message : err}`);
@@ -368,8 +287,7 @@ export class AstIndexClient {
368
287
  return JSON.parse(result);
369
288
  }
370
289
  catch {
371
- // JSON parse failed — parse text format as fallback
372
- return this.parseImplementationsText(result);
290
+ return parseImplementationsText(result);
373
291
  }
374
292
  }
375
293
  catch (err) {
@@ -385,8 +303,7 @@ export class AstIndexClient {
385
303
  return JSON.parse(result);
386
304
  }
387
305
  catch {
388
- // JSON parse failed — parse text format as fallback
389
- return this.parseHierarchyText(result, name);
306
+ return parseHierarchyText(result, name);
390
307
  }
391
308
  }
392
309
  catch (err) {
@@ -394,62 +311,6 @@ export class AstIndexClient {
394
311
  return null;
395
312
  }
396
313
  }
397
- parseImplementationsText(text) {
398
- const results = [];
399
- // Parse lines like: "class ClassName (file.php:42)"
400
- for (const line of text.split('\n')) {
401
- const m = line.match(/^\s*(class|interface|trait|struct|impl)\s+(\S+)\s+\((.+):(\d+)\)/);
402
- if (m) {
403
- results.push({ kind: m[1], name: m[2], file: m[3], line: parseInt(m[4], 10) });
404
- }
405
- }
406
- return results;
407
- }
408
- parseHierarchyText(text, rootName) {
409
- if (!text.trim())
410
- return null;
411
- // Parse ast-index hierarchy text output:
412
- // Hierarchy for 'ClassName':
413
- // Parents:
414
- // ParentClass (extends)
415
- // Children:
416
- // ChildClass (implements) (file.ts:42)
417
- const lines = text.split('\n');
418
- const parents = [];
419
- const childNodes = [];
420
- let section = 'none';
421
- for (const line of lines) {
422
- const trimmed = line.trim();
423
- if (trimmed === 'Parents:') {
424
- section = 'parents';
425
- continue;
426
- }
427
- if (trimmed === 'Children:') {
428
- section = 'children';
429
- continue;
430
- }
431
- if (trimmed.startsWith('Hierarchy for') || !trimmed)
432
- continue;
433
- // Match: SymbolName (relationship) (file:line) — file:line is optional
434
- const m = trimmed.match(/^(\S+)\s+\((\w+)\)(?:\s+\((.+):(\d+)\))?/);
435
- if (m && section !== 'none') {
436
- const node = {
437
- name: m[1],
438
- kind: m[2], // extends, implements, etc.
439
- children: [],
440
- file: m[3],
441
- line: m[4] ? parseInt(m[4], 10) : undefined,
442
- };
443
- if (section === 'parents')
444
- parents.push(node);
445
- else
446
- childNodes.push(node);
447
- }
448
- }
449
- if (parents.length === 0 && childNodes.length === 0)
450
- return null;
451
- return { name: rootName, kind: 'class', children: childNodes, parents };
452
- }
453
314
  async stats() {
454
315
  try {
455
316
  return await this.exec(['stats']);
@@ -458,10 +319,6 @@ export class AstIndexClient {
458
319
  return null;
459
320
  }
460
321
  }
461
- /**
462
- * List all files known to the ast-index.
463
- * Parses the `files` command output which lists one file per line.
464
- */
465
322
  async listFiles() {
466
323
  try {
467
324
  await this.ensureIndex();
@@ -473,10 +330,6 @@ export class AstIndexClient {
473
330
  return [];
474
331
  }
475
332
  }
476
- /**
477
- * Cross-references: definitions + imports + usages in one call.
478
- * Replaces separate symbol() + usages() calls.
479
- */
480
333
  async refs(symbolName, limit = 20) {
481
334
  await this.ensureIndex();
482
335
  try {
@@ -488,9 +341,6 @@ export class AstIndexClient {
488
341
  return { definitions: [], imports: [], usages: [] };
489
342
  }
490
343
  }
491
- /**
492
- * Project map: directory structure with file counts and symbol kinds.
493
- */
494
344
  async map(options) {
495
345
  await this.ensureIndex();
496
346
  try {
@@ -507,9 +357,6 @@ export class AstIndexClient {
507
357
  return null;
508
358
  }
509
359
  }
510
- /**
511
- * Detect project conventions: architecture, frameworks, naming patterns.
512
- */
513
360
  async conventions() {
514
361
  await this.ensureIndex();
515
362
  try {
@@ -521,9 +368,6 @@ export class AstIndexClient {
521
368
  return null;
522
369
  }
523
370
  }
524
- /**
525
- * Find callers of a function.
526
- */
527
371
  async callers(functionName, limit = 50) {
528
372
  await this.ensureIndex();
529
373
  try {
@@ -536,9 +380,6 @@ export class AstIndexClient {
536
380
  return [];
537
381
  }
538
382
  }
539
- /**
540
- * Show call hierarchy tree (callers tree up).
541
- */
542
383
  async callTree(functionName, depth = 3) {
543
384
  await this.ensureIndex();
544
385
  try {
@@ -550,9 +391,6 @@ export class AstIndexClient {
550
391
  return null;
551
392
  }
552
393
  }
553
- /**
554
- * Show changed symbols since base branch (git diff).
555
- */
556
394
  async changed(base) {
557
395
  await this.ensureIndex();
558
396
  try {
@@ -568,9 +406,6 @@ export class AstIndexClient {
568
406
  return [];
569
407
  }
570
408
  }
571
- /**
572
- * Find potentially unused symbols.
573
- */
574
409
  async unusedSymbols(options) {
575
410
  await this.ensureIndex();
576
411
  try {
@@ -590,72 +425,27 @@ export class AstIndexClient {
590
425
  return [];
591
426
  }
592
427
  }
593
- /**
594
- * Get imports for a specific file.
595
- * Parses text output: " { X, Y } from 'source';"
596
- */
597
428
  async fileImports(filePath) {
598
429
  await this.ensureIndex();
599
430
  try {
600
431
  const result = await this.exec(['imports', filePath]);
601
- return this.parseImportsText(result);
432
+ return parseImportsText(result);
602
433
  }
603
434
  catch (err) {
604
435
  console.error(`[token-pilot] ast-index imports failed: ${err instanceof Error ? err.message : err}`);
605
436
  return [];
606
437
  }
607
438
  }
608
- parseImportsText(text) {
609
- const entries = [];
610
- for (const line of text.split('\n')) {
611
- const trimmed = line.trim();
612
- if (!trimmed || trimmed.startsWith('Imports in') || trimmed.startsWith('Total:'))
613
- continue;
614
- // Match: { X, Y } from 'source'
615
- const braceMatch = trimmed.match(/^\{\s*(.+?)\s*\}\s+from\s+['"](.+?)['"]/);
616
- if (braceMatch) {
617
- entries.push({
618
- specifiers: braceMatch[1].split(',').map(s => s.trim()),
619
- source: braceMatch[2],
620
- });
621
- continue;
622
- }
623
- // Match: * as X from 'source'
624
- const nsMatch = trimmed.match(/^\*\s+as\s+(\S+)\s+from\s+['"](.+?)['"]/);
625
- if (nsMatch) {
626
- entries.push({
627
- specifiers: [nsMatch[1]],
628
- source: nsMatch[2],
629
- isNamespace: true,
630
- });
631
- continue;
632
- }
633
- // Match: X from 'source' (default import)
634
- const defaultMatch = trimmed.match(/^(\w+)\s+from\s+['"](.+?)['"]/);
635
- if (defaultMatch) {
636
- entries.push({
637
- specifiers: [defaultMatch[1]],
638
- source: defaultMatch[2],
639
- isDefault: true,
640
- });
641
- continue;
642
- }
643
- }
644
- return entries;
645
- }
646
439
  // --- Code audit commands ---
647
- /** Check if ast-grep (sg) is available for structural pattern search */
648
440
  async checkAstGrep() {
649
441
  if (this.astGrepAvailable !== null)
650
442
  return this.astGrepAvailable;
651
- // Try system PATH first
652
443
  try {
653
444
  await execFileAsync('sg', ['--version'], { timeout: 3000 });
654
445
  this.astGrepAvailable = true;
655
446
  return true;
656
447
  }
657
448
  catch { /* not in PATH */ }
658
- // Try node_modules/.bin/sg (from optionalDependencies or source installs)
659
449
  try {
660
450
  const localBinDir = new URL('../../node_modules/.bin', import.meta.url).pathname;
661
451
  await execFileAsync(localBinDir + '/sg', ['--version'], { timeout: 3000 });
@@ -667,7 +457,6 @@ export class AstIndexClient {
667
457
  this.astGrepAvailable = false;
668
458
  return false;
669
459
  }
670
- /** Structural pattern search via ast-grep. Requires ast-grep (sg) installed. */
671
460
  async agrep(pattern, options) {
672
461
  if (this.indexDisabled || this.indexOversized)
673
462
  return [];
@@ -684,129 +473,52 @@ export class AstIndexClient {
684
473
  args.push('--lang', options.lang);
685
474
  try {
686
475
  const result = await this.exec(args, 15000);
687
- return this.parseAgrepText(result).slice(0, limit);
476
+ return parseAgrepText(result).slice(0, limit);
688
477
  }
689
478
  catch (err) {
690
479
  console.error(`[token-pilot] ast-index agrep failed: ${err instanceof Error ? err.message : err}`);
691
480
  return [];
692
481
  }
693
482
  }
694
- parseAgrepText(text) {
695
- const results = [];
696
- for (const line of text.split('\n')) {
697
- if (!line.trim())
698
- continue;
699
- // Format: file:line:matched_text OR file:line: matched_text
700
- const match = line.match(/^(.+?):(\d+):(.*)$/);
701
- if (match) {
702
- results.push({
703
- file: match[1],
704
- line: parseInt(match[2], 10),
705
- text: match[3].trim(),
706
- });
707
- }
708
- }
709
- return results;
710
- }
711
- /** Find TODO/FIXME/HACK comments in the project */
712
483
  async todo() {
713
484
  if (this.indexDisabled || this.indexOversized)
714
485
  return [];
715
486
  await this.ensureIndex();
716
487
  try {
717
488
  const result = await this.exec(['todo'], 15000);
718
- return this.parseTodoText(result);
489
+ return parseTodoText(result);
719
490
  }
720
491
  catch (err) {
721
492
  console.error(`[token-pilot] ast-index todo failed: ${err instanceof Error ? err.message : err}`);
722
493
  return [];
723
494
  }
724
495
  }
725
- parseTodoText(text) {
726
- const results = [];
727
- for (const line of text.split('\n')) {
728
- if (!line.trim())
729
- continue;
730
- // Try format: file:line: KIND: message OR file:line: KIND message
731
- const match = line.match(/^(.+?):(\d+):\s*(TODO|FIXME|HACK|XXX|NOTE|WARN(?:ING)?)[:\s]+(.*)$/i);
732
- if (match) {
733
- results.push({
734
- file: match[1],
735
- line: parseInt(match[2], 10),
736
- kind: match[3].toUpperCase(),
737
- text: match[4].trim(),
738
- });
739
- }
740
- }
741
- return results;
742
- }
743
- /** Find @Deprecated symbols in the project */
744
496
  async deprecated() {
745
497
  if (this.indexDisabled || this.indexOversized)
746
498
  return [];
747
499
  await this.ensureIndex();
748
500
  try {
749
501
  const result = await this.exec(['deprecated'], 15000);
750
- return this.parseDeprecatedText(result);
502
+ return parseDeprecatedText(result);
751
503
  }
752
504
  catch (err) {
753
505
  console.error(`[token-pilot] ast-index deprecated failed: ${err instanceof Error ? err.message : err}`);
754
506
  return [];
755
507
  }
756
508
  }
757
- parseDeprecatedText(text) {
758
- const results = [];
759
- for (const line of text.split('\n')) {
760
- if (!line.trim())
761
- continue;
762
- // Try format: kind name (file:line) - message OR kind name (file:line)
763
- const match = line.match(/^(\w+)\s+(\S+)\s+\((.+?):(\d+)\)(?:\s*-\s*(.+))?$/);
764
- if (match) {
765
- results.push({
766
- kind: match[1],
767
- name: match[2],
768
- file: match[3],
769
- line: parseInt(match[4], 10),
770
- message: match[5]?.trim(),
771
- });
772
- }
773
- }
774
- return results;
775
- }
776
- /** Find symbols with a specific annotation/decorator */
777
509
  async annotations(name) {
778
510
  if (this.indexDisabled || this.indexOversized)
779
511
  return [];
780
512
  await this.ensureIndex();
781
513
  try {
782
514
  const result = await this.exec(['annotations', name], 15000);
783
- return this.parseAnnotationsText(result, name);
515
+ return parseAnnotationsText(result, name);
784
516
  }
785
517
  catch (err) {
786
518
  console.error(`[token-pilot] ast-index annotations failed: ${err instanceof Error ? err.message : err}`);
787
519
  return [];
788
520
  }
789
521
  }
790
- parseAnnotationsText(text, annotationName) {
791
- const results = [];
792
- for (const line of text.split('\n')) {
793
- if (!line.trim())
794
- continue;
795
- // Try format: kind name (file:line) OR @Annotation kind name (file:line)
796
- const match = line.match(/^(?:@\S+\s+)?(\w+)\s+(\S+)\s+\((.+?):(\d+)\)$/);
797
- if (match) {
798
- results.push({
799
- kind: match[1],
800
- name: match[2],
801
- file: match[3],
802
- line: parseInt(match[4], 10),
803
- annotation: annotationName,
804
- });
805
- }
806
- }
807
- return results;
808
- }
809
- /** Trigger incremental index update (called by file watcher after edits) */
810
522
  async incrementalUpdate() {
811
523
  if (!this.indexed || this.indexDisabled || this.indexOversized)
812
524
  return;
@@ -817,8 +529,7 @@ export class AstIndexClient {
817
529
  console.error(`[token-pilot] ast-index incremental update failed: ${err instanceof Error ? err.message : err}`);
818
530
  }
819
531
  }
820
- // --- Module analysis methods (ast-index v3.27.0) ---
821
- /** List project modules matching optional pattern */
532
+ // --- Module analysis methods ---
822
533
  async modules(pattern) {
823
534
  if (this.indexDisabled || this.indexOversized)
824
535
  return [];
@@ -826,207 +537,89 @@ export class AstIndexClient {
826
537
  try {
827
538
  const cmdArgs = pattern ? ['module', pattern] : ['module'];
828
539
  const result = await this.exec(cmdArgs, 15000);
829
- return this.parseModuleListText(result);
540
+ return parseModuleListText(result);
830
541
  }
831
542
  catch (err) {
832
543
  console.error(`[token-pilot] ast-index module failed: ${err instanceof Error ? err.message : err}`);
833
544
  return [];
834
545
  }
835
546
  }
836
- /** Get dependencies of a module */
837
547
  async moduleDeps(module) {
838
548
  if (this.indexDisabled || this.indexOversized)
839
549
  return [];
840
550
  await this.ensureIndex();
841
551
  try {
842
552
  const result = await this.exec(['deps', module], 15000);
843
- return this.parseModuleDepText(result);
553
+ return parseModuleDepText(result);
844
554
  }
845
555
  catch (err) {
846
556
  console.error(`[token-pilot] ast-index deps failed: ${err instanceof Error ? err.message : err}`);
847
557
  return [];
848
558
  }
849
559
  }
850
- /** Get modules that depend on this module */
851
560
  async moduleDependents(module) {
852
561
  if (this.indexDisabled || this.indexOversized)
853
562
  return [];
854
563
  await this.ensureIndex();
855
564
  try {
856
565
  const result = await this.exec(['dependents', module], 15000);
857
- return this.parseModuleDepText(result);
566
+ return parseModuleDepText(result);
858
567
  }
859
568
  catch (err) {
860
569
  console.error(`[token-pilot] ast-index dependents failed: ${err instanceof Error ? err.message : err}`);
861
570
  return [];
862
571
  }
863
572
  }
864
- /** Find unused dependencies of a module */
865
573
  async unusedDeps(module) {
866
574
  if (this.indexDisabled || this.indexOversized)
867
575
  return [];
868
576
  await this.ensureIndex();
869
577
  try {
870
578
  const result = await this.exec(['unused-deps', module], 15000);
871
- return this.parseUnusedDepsText(result);
579
+ return parseUnusedDepsText(result);
872
580
  }
873
581
  catch (err) {
874
582
  console.error(`[token-pilot] ast-index unused-deps failed: ${err instanceof Error ? err.message : err}`);
875
583
  return [];
876
584
  }
877
585
  }
878
- /** Get public API of a module */
879
586
  async moduleApi(module) {
880
587
  if (this.indexDisabled || this.indexOversized)
881
588
  return [];
882
589
  await this.ensureIndex();
883
590
  try {
884
591
  const result = await this.exec(['api', module], 15000);
885
- return this.parseModuleApiText(result);
592
+ return parseModuleApiText(result);
886
593
  }
887
594
  catch (err) {
888
595
  console.error(`[token-pilot] ast-index api failed: ${err instanceof Error ? err.message : err}`);
889
596
  return [];
890
597
  }
891
598
  }
892
- // Parsers for module commands (text format — JSON may not be supported for all)
893
- parseModuleListText(text) {
894
- const results = [];
895
- for (const line of text.split('\n')) {
896
- if (!line.trim())
897
- continue;
898
- // Try JSON first
899
- try {
900
- const parsed = JSON.parse(line);
901
- if (Array.isArray(parsed))
902
- return parsed;
903
- }
904
- catch { /* not JSON, parse as text */ }
905
- // Format: name (path) — N files OR name (path) OR path
906
- const match = line.match(/^(\S+)\s+\((.+?)\)(?:\s*—\s*(\d+)\s+files?)?$/);
907
- if (match) {
908
- results.push({
909
- name: match[1],
910
- path: match[2],
911
- file_count: match[3] ? parseInt(match[3], 10) : undefined,
912
- });
913
- }
914
- else {
915
- // Fallback: treat entire line as a path-based module
916
- const trimmed = line.trim();
917
- if (trimmed && !trimmed.startsWith('#') && !trimmed.startsWith('─')) {
918
- const name = trimmed.split('/').pop() ?? trimmed;
919
- results.push({ name, path: trimmed });
920
- }
921
- }
922
- }
923
- return results;
924
- }
925
- parseModuleDepText(text) {
926
- const results = [];
927
- for (const line of text.split('\n')) {
928
- if (!line.trim())
929
- continue;
930
- // Try JSON first
931
- try {
932
- const parsed = JSON.parse(line);
933
- if (Array.isArray(parsed))
934
- return parsed;
935
- }
936
- catch { /* not JSON, parse as text */ }
937
- // Format: → name (path) OR ← name (path) OR name (path) OR name
938
- const match = line.match(/^[→←\-\s]*(\S+)(?:\s+\((.+?)\))?(?:\s+\[(direct|transitive)\])?$/);
939
- if (match) {
940
- results.push({
941
- name: match[1],
942
- path: match[2] ?? match[1],
943
- type: match[3],
944
- });
945
- }
946
- }
947
- return results;
948
- }
949
- parseUnusedDepsText(text) {
950
- const results = [];
951
- for (const line of text.split('\n')) {
952
- if (!line.trim())
953
- continue;
954
- // Try JSON first
955
- try {
956
- const parsed = JSON.parse(line);
957
- if (Array.isArray(parsed))
958
- return parsed;
959
- }
960
- catch { /* not JSON, parse as text */ }
961
- // Format: ⚠ name (path) — reason OR name (path) OR name — reason
962
- const match = line.match(/^[⚠!\s]*(\S+)(?:\s+\((.+?)\))?(?:\s*[—\-]+\s*(.+))?$/);
963
- if (match) {
964
- results.push({
965
- name: match[1],
966
- path: match[2] ?? match[1],
967
- reason: match[3]?.trim(),
968
- });
969
- }
970
- }
971
- return results;
972
- }
973
- parseModuleApiText(text) {
974
- const results = [];
975
- for (const line of text.split('\n')) {
976
- if (!line.trim())
977
- continue;
978
- // Try JSON first
979
- try {
980
- const parsed = JSON.parse(line);
981
- if (Array.isArray(parsed))
982
- return parsed;
983
- }
984
- catch { /* not JSON, parse as text */ }
985
- // Format: kind name (file:line) OR kind name signature (file:line)
986
- const match = line.match(/^(\w+)\s+(\S+)(?:\s+(.*?))?\s+\((.+?):(\d+)\)$/);
987
- if (match) {
988
- results.push({
989
- kind: match[1],
990
- name: match[2],
991
- signature: match[3]?.trim() || undefined,
992
- file: match[4],
993
- line: parseInt(match[5], 10),
994
- });
995
- }
996
- }
997
- return results;
998
- }
999
599
  // --- Utility methods ---
1000
600
  isAvailable() {
1001
601
  return this.binaryPath !== null;
1002
602
  }
1003
- /** Returns true if the index was built but found >50k files (node_modules leak) */
1004
603
  isOversized() {
1005
604
  return this.indexOversized;
1006
605
  }
1007
- /** Returns true if index building is disabled (dangerous root like /) */
1008
606
  isDisabled() {
1009
607
  return this.indexDisabled;
1010
608
  }
1011
- /** Disable index building (e.g. project root is / or home dir) */
1012
609
  disableIndex() {
1013
610
  this.indexDisabled = true;
1014
611
  }
1015
- /** Re-enable index building after auto-detecting a valid project root */
1016
612
  enableIndex() {
1017
613
  this.indexDisabled = false;
1018
614
  }
1019
- /** Update project root (e.g. after auto-detecting from file path) */
1020
615
  updateProjectRoot(newRoot) {
1021
616
  this.projectRoot = newRoot;
1022
- this.indexed = false; // Force re-check
617
+ this.indexed = false;
1023
618
  }
1024
619
  async exec(args, timeoutMs) {
1025
620
  if (!this.binaryPath) {
1026
621
  throw new Error('ast-index not initialized. Call init() first.');
1027
622
  }
1028
- // If ast-grep was found in node_modules/.bin, inject it into PATH
1029
- // so ast-index can find sg when running agrep
1030
623
  const env = this.astGrepBinDir
1031
624
  ? { ...process.env, PATH: `${this.astGrepBinDir}:${process.env.PATH ?? ''}` }
1032
625
  : undefined;
@@ -1041,291 +634,5 @@ export class AstIndexClient {
1041
634
  }
1042
635
  return stdout;
1043
636
  }
1044
- async buildFileStructure(filePath, entries) {
1045
- const content = await readFile(filePath, 'utf-8');
1046
- const lines = content.split('\n');
1047
- const fileStat = await stat(filePath);
1048
- // Fix last entry end_line to use actual file line count
1049
- this.fixLastEndLine(entries, lines.length);
1050
- // Enrich classes that ast-index returned without children (language-specific)
1051
- const lang = this.detectLanguage(filePath);
1052
- if (lang === 'Python') {
1053
- this.enrichPythonClassMethods(entries, lines);
1054
- }
1055
- else if (lang === 'PHP') {
1056
- this.enrichPHPClassMethods(entries, lines);
1057
- }
1058
- // Enrich entries with signatures from file content
1059
- this.enrichSignatures(entries, lines);
1060
- return {
1061
- path: filePath,
1062
- language: lang,
1063
- meta: {
1064
- lines: lines.length,
1065
- bytes: fileStat.size,
1066
- lastModified: fileStat.mtimeMs,
1067
- contentHash: createHash('sha256').update(content).digest('hex'),
1068
- },
1069
- imports: [],
1070
- exports: [],
1071
- symbols: entries.map(e => this.mapOutlineEntry(e)),
1072
- };
1073
- }
1074
- /**
1075
- * Python: ast-index doesn't return methods inside classes.
1076
- * Parse file content to extract `def` methods for classes without children.
1077
- */
1078
- enrichPythonClassMethods(entries, lines) {
1079
- for (const entry of entries) {
1080
- if (entry.kind.toLowerCase() !== 'class')
1081
- continue;
1082
- if (entry.children && entry.children.length > 0)
1083
- continue;
1084
- const classStartIdx = entry.start_line - 1; // 0-based
1085
- const classEndIdx = entry.end_line - 1;
1086
- // Detect class body indent: look for first `def ` inside class range
1087
- let bodyIndent = -1;
1088
- for (let i = classStartIdx + 1; i <= classEndIdx && i < lines.length; i++) {
1089
- const defMatch = lines[i].match(/^(\s+)def\s/);
1090
- if (defMatch) {
1091
- bodyIndent = defMatch[1].length;
1092
- break;
1093
- }
1094
- }
1095
- if (bodyIndent < 0)
1096
- continue; // no methods found
1097
- const methods = [];
1098
- for (let i = classStartIdx + 1; i <= classEndIdx && i < lines.length; i++) {
1099
- const line = lines[i];
1100
- // Match `def method_name(` at the detected indent level
1101
- const match = line.match(new RegExp(`^\\s{${bodyIndent}}def\\s+(\\w+)\\s*\\(`));
1102
- if (!match)
1103
- continue;
1104
- const methodName = match[1];
1105
- const methodLine = i + 1; // 1-based
1106
- // Check for async/static/decorators
1107
- const isAsync = line.includes('async def');
1108
- const isStatic = i > 0 && /^\s*@staticmethod/.test(lines[i - 1]);
1109
- const isClassMethod = i > 0 && /^\s*@classmethod/.test(lines[i - 1]);
1110
- // Collect decorators above
1111
- const decorators = [];
1112
- for (let d = i - 1; d >= classStartIdx; d--) {
1113
- const decMatch = lines[d].match(new RegExp(`^\\s{${bodyIndent}}@(\\w+)`));
1114
- if (decMatch) {
1115
- decorators.unshift(`@${decMatch[1]}`);
1116
- }
1117
- else {
1118
- break;
1119
- }
1120
- }
1121
- // Determine visibility from name
1122
- const visibility = methodName.startsWith('__') && !methodName.endsWith('__')
1123
- ? 'private'
1124
- : methodName.startsWith('_')
1125
- ? 'protected'
1126
- : 'public';
1127
- methods.push({
1128
- name: methodName,
1129
- kind: isStatic || isClassMethod ? 'function' : 'method',
1130
- start_line: methodLine,
1131
- end_line: 0, // computed below
1132
- signature: line.trim(),
1133
- visibility,
1134
- is_async: isAsync,
1135
- is_static: isStatic,
1136
- decorators: decorators.length > 0 ? decorators : undefined,
1137
- });
1138
- }
1139
- // Compute end_lines for methods
1140
- for (let m = 0; m < methods.length; m++) {
1141
- if (m < methods.length - 1) {
1142
- // End before next method (or its first decorator)
1143
- const nextStart = methods[m + 1].start_line;
1144
- // Walk back from next method to skip decorators/blank lines
1145
- let endLine = nextStart - 1;
1146
- for (let k = nextStart - 2; k >= methods[m].start_line; k--) {
1147
- const l = lines[k];
1148
- if (l.trim() === '' || new RegExp(`^\\s{${bodyIndent}}@`).test(l)) {
1149
- endLine = k; // 0-based → will be used as 1-based below
1150
- }
1151
- else {
1152
- break;
1153
- }
1154
- }
1155
- methods[m].end_line = endLine;
1156
- }
1157
- else {
1158
- // Last method ends at class end
1159
- methods[m].end_line = entry.end_line;
1160
- }
1161
- }
1162
- entry.children = methods;
1163
- }
1164
- }
1165
- /**
1166
- * PHP: ast-index doesn't return methods inside classes.
1167
- * Parse file content to extract `function` methods for classes without children.
1168
- */
1169
- enrichPHPClassMethods(entries, lines) {
1170
- for (const entry of entries) {
1171
- if (entry.kind.toLowerCase() !== 'class')
1172
- continue;
1173
- if (entry.children && entry.children.length > 0)
1174
- continue;
1175
- const classStartIdx = entry.start_line - 1;
1176
- const classEndIdx = entry.end_line - 1;
1177
- // Detect class body indent: look for first `function ` inside class range
1178
- let bodyIndent = -1;
1179
- for (let i = classStartIdx + 1; i <= classEndIdx && i < lines.length; i++) {
1180
- const fnMatch = lines[i].match(/^(\s+)(?:public|private|protected|static|\s)*function\s/);
1181
- if (fnMatch) {
1182
- bodyIndent = fnMatch[1].length;
1183
- break;
1184
- }
1185
- }
1186
- if (bodyIndent < 0)
1187
- continue;
1188
- const methods = [];
1189
- for (let i = classStartIdx + 1; i <= classEndIdx && i < lines.length; i++) {
1190
- const line = lines[i];
1191
- // Match PHP method: [visibility] [static] function name(
1192
- const match = line.match(new RegExp(`^\\s{${bodyIndent}}(?:(public|private|protected)\\s+)?(?:(static)\\s+)?function\\s+(\\w+)\\s*\\(`));
1193
- if (!match)
1194
- continue;
1195
- const visibility = match[1] ?? 'public';
1196
- const isStatic = !!match[2];
1197
- const methodName = match[3];
1198
- const methodLine = i + 1;
1199
- methods.push({
1200
- name: methodName,
1201
- kind: isStatic ? 'function' : 'method',
1202
- start_line: methodLine,
1203
- end_line: 0,
1204
- signature: line.trim(),
1205
- visibility,
1206
- is_static: isStatic,
1207
- });
1208
- }
1209
- // Compute end_lines
1210
- for (let m = 0; m < methods.length; m++) {
1211
- if (m < methods.length - 1) {
1212
- methods[m].end_line = methods[m + 1].start_line - 1;
1213
- }
1214
- else {
1215
- methods[m].end_line = entry.end_line;
1216
- }
1217
- }
1218
- entry.children = methods;
1219
- }
1220
- }
1221
- /** Fix the last entry's end_line to use actual file line count */
1222
- fixLastEndLine(entries, totalLines) {
1223
- if (entries.length === 0)
1224
- return;
1225
- const last = entries[entries.length - 1];
1226
- last.end_line = totalLines;
1227
- // Recursively fix children
1228
- if (last.children?.length) {
1229
- this.fixLastEndLine(last.children, last.end_line - 1);
1230
- }
1231
- }
1232
- /** Read actual signature lines from file content */
1233
- enrichSignatures(entries, lines) {
1234
- for (const entry of entries) {
1235
- if (!entry.signature) {
1236
- const lineIdx = entry.start_line - 1;
1237
- if (lineIdx >= 0 && lineIdx < lines.length) {
1238
- entry.signature = lines[lineIdx].trim();
1239
- }
1240
- }
1241
- if (entry.children?.length) {
1242
- this.enrichSignatures(entry.children, lines);
1243
- }
1244
- }
1245
- }
1246
- mapOutlineEntry(entry) {
1247
- return {
1248
- name: entry.name,
1249
- qualifiedName: entry.name, // Will be enriched with parent context
1250
- kind: this.mapKind(entry.kind),
1251
- signature: entry.signature ?? entry.name,
1252
- location: {
1253
- startLine: entry.start_line,
1254
- endLine: entry.end_line,
1255
- lineCount: entry.end_line - entry.start_line + 1,
1256
- },
1257
- visibility: this.mapVisibility(entry.visibility),
1258
- async: entry.is_async ?? false,
1259
- static: entry.is_static ?? false,
1260
- decorators: entry.decorators ?? [],
1261
- children: (entry.children ?? []).map(c => this.mapOutlineEntry(c)),
1262
- doc: entry.doc ?? null,
1263
- references: [],
1264
- };
1265
- }
1266
- mapKind(kind) {
1267
- const map = {
1268
- function: 'function',
1269
- class: 'class',
1270
- method: 'method',
1271
- property: 'property',
1272
- variable: 'variable',
1273
- type: 'type',
1274
- interface: 'interface',
1275
- enum: 'enum',
1276
- constant: 'constant',
1277
- namespace: 'namespace',
1278
- struct: 'class',
1279
- trait: 'interface',
1280
- impl: 'class',
1281
- module: 'namespace',
1282
- };
1283
- return map[kind.toLowerCase()] ?? 'function';
1284
- }
1285
- mapVisibility(vis) {
1286
- if (!vis)
1287
- return 'default';
1288
- const map = {
1289
- public: 'public',
1290
- private: 'private',
1291
- protected: 'protected',
1292
- pub: 'public',
1293
- export: 'public',
1294
- };
1295
- return map[vis.toLowerCase()] ?? 'default';
1296
- }
1297
- detectLanguage(filePath) {
1298
- const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
1299
- const map = {
1300
- ts: 'TypeScript', tsx: 'TypeScript',
1301
- js: 'JavaScript', jsx: 'JavaScript', mjs: 'JavaScript',
1302
- py: 'Python',
1303
- go: 'Go',
1304
- rs: 'Rust',
1305
- java: 'Java',
1306
- kt: 'Kotlin', kts: 'Kotlin',
1307
- swift: 'Swift',
1308
- cs: 'C#',
1309
- cpp: 'C++', cc: 'C++', cxx: 'C++', hpp: 'C++',
1310
- c: 'C', h: 'C',
1311
- php: 'PHP',
1312
- rb: 'Ruby',
1313
- scala: 'Scala',
1314
- dart: 'Dart',
1315
- lua: 'Lua',
1316
- sh: 'Bash', bash: 'Bash',
1317
- sql: 'SQL',
1318
- r: 'R',
1319
- vue: 'Vue',
1320
- svelte: 'Svelte',
1321
- pl: 'Perl', pm: 'Perl',
1322
- ex: 'Elixir', exs: 'Elixir',
1323
- groovy: 'Groovy',
1324
- m: 'Objective-C',
1325
- proto: 'Protocol Buffers',
1326
- bsl: 'BSL',
1327
- };
1328
- return map[ext] ?? 'Unknown';
1329
- }
1330
637
  }
1331
638
  //# sourceMappingURL=client.js.map