token-pilot 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +28 -10
- package/dist/ast-index/client.d.ts +3 -0
- package/dist/ast-index/client.d.ts.map +1 -1
- package/dist/ast-index/client.js +80 -14
- package/dist/ast-index/client.js.map +1 -1
- package/dist/core/context-window-tracker.d.ts +89 -0
- package/dist/core/context-window-tracker.d.ts.map +1 -0
- package/dist/core/context-window-tracker.js +161 -0
- package/dist/core/context-window-tracker.js.map +1 -0
- package/dist/core/context-window-tracker.test.d.ts +2 -0
- package/dist/core/context-window-tracker.test.d.ts.map +1 -0
- package/dist/core/context-window-tracker.test.js +238 -0
- package/dist/core/context-window-tracker.test.js.map +1 -0
- package/dist/core/diff-engine.d.ts +64 -0
- package/dist/core/diff-engine.d.ts.map +1 -0
- package/dist/core/diff-engine.js +185 -0
- package/dist/core/diff-engine.js.map +1 -0
- package/dist/core/diff-engine.test.d.ts +2 -0
- package/dist/core/diff-engine.test.d.ts.map +1 -0
- package/dist/core/diff-engine.test.js +351 -0
- package/dist/core/diff-engine.test.js.map +1 -0
- package/dist/core/persistent-cache.d.ts +153 -0
- package/dist/core/persistent-cache.d.ts.map +1 -0
- package/dist/core/persistent-cache.js +555 -0
- package/dist/core/persistent-cache.js.map +1 -0
- package/dist/core/persistent-cache.test.d.ts +2 -0
- package/dist/core/persistent-cache.test.d.ts.map +1 -0
- package/dist/core/persistent-cache.test.js +251 -0
- package/dist/core/persistent-cache.test.js.map +1 -0
- package/dist/core/real-token-estimator.d.ts +49 -0
- package/dist/core/real-token-estimator.d.ts.map +1 -0
- package/dist/core/real-token-estimator.js +93 -0
- package/dist/core/real-token-estimator.js.map +1 -0
- package/dist/core/symbol-resolver.d.ts +1 -0
- package/dist/core/symbol-resolver.d.ts.map +1 -1
- package/dist/core/symbol-resolver.js +11 -9
- package/dist/core/symbol-resolver.js.map +1 -1
- package/dist/formatters/context-markup.d.ts +40 -0
- package/dist/formatters/context-markup.d.ts.map +1 -0
- package/dist/formatters/context-markup.js +55 -0
- package/dist/formatters/context-markup.js.map +1 -0
- package/dist/formatters/smart-read-xml.d.ts +20 -0
- package/dist/formatters/smart-read-xml.d.ts.map +1 -0
- package/dist/formatters/smart-read-xml.js +163 -0
- package/dist/formatters/smart-read-xml.js.map +1 -0
- package/dist/handlers/project-overview.d.ts.map +1 -1
- package/dist/handlers/project-overview.js +57 -24
- package/dist/handlers/project-overview.js.map +1 -1
- package/dist/handlers/read-symbol.d.ts +2 -1
- package/dist/handlers/read-symbol.d.ts.map +1 -1
- package/dist/handlers/read-symbol.js +7 -3
- package/dist/handlers/read-symbol.js.map +1 -1
- package/dist/index.js +21 -1
- package/dist/index.js.map +1 -1
- package/dist/server.js +8 -4
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { PersistentFileCache } from './persistent-cache';
|
|
6
|
+
describe('PersistentFileCache', () => {
|
|
7
|
+
let cache;
|
|
8
|
+
let tempDir;
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
tempDir = mkdtempSync(join(tmpdir(), 'token-pilot-test-'));
|
|
11
|
+
cache = new PersistentFileCache(tempDir, 100, 80); // 100MB test size, 80 line threshold
|
|
12
|
+
});
|
|
13
|
+
afterEach(() => {
|
|
14
|
+
cache.close();
|
|
15
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
16
|
+
});
|
|
17
|
+
const createMockEntry = (content = 'test content', hash = 'hash1') => ({
|
|
18
|
+
structure: {
|
|
19
|
+
path: 'test.ts',
|
|
20
|
+
language: 'typescript',
|
|
21
|
+
meta: { lines: 10, bytes: content.length, lastModified: Date.now(), contentHash: hash },
|
|
22
|
+
imports: [],
|
|
23
|
+
exports: [],
|
|
24
|
+
symbols: []
|
|
25
|
+
},
|
|
26
|
+
content,
|
|
27
|
+
lines: content.split('\n'),
|
|
28
|
+
mtime: Date.now(),
|
|
29
|
+
hash,
|
|
30
|
+
lastAccess: Date.now()
|
|
31
|
+
});
|
|
32
|
+
describe('L1 Cache (Memory)', () => {
|
|
33
|
+
it('should store entry in L1 memory cache', () => {
|
|
34
|
+
const entry = createMockEntry();
|
|
35
|
+
cache.set('file1.ts', entry);
|
|
36
|
+
const retrieved = cache.get('file1.ts');
|
|
37
|
+
expect(retrieved).not.toBeNull();
|
|
38
|
+
expect(retrieved?.hash).toBe(entry.hash);
|
|
39
|
+
});
|
|
40
|
+
it('should retrieve from L1 cache in less than 5ms', () => {
|
|
41
|
+
const entry = createMockEntry();
|
|
42
|
+
cache.set('file1.ts', entry);
|
|
43
|
+
const start = performance.now();
|
|
44
|
+
const result = cache.get('file1.ts');
|
|
45
|
+
const elapsed = performance.now() - start;
|
|
46
|
+
expect(result).not.toBeNull();
|
|
47
|
+
expect(elapsed).toBeLessThan(5);
|
|
48
|
+
});
|
|
49
|
+
it('should return null for non-existent file', () => {
|
|
50
|
+
const result = cache.get('nonexistent.ts');
|
|
51
|
+
expect(result).toBeNull();
|
|
52
|
+
});
|
|
53
|
+
it('should update lastAccess when reading', () => {
|
|
54
|
+
const entry = createMockEntry();
|
|
55
|
+
cache.set('file1.ts', entry);
|
|
56
|
+
const retrieved1 = cache.get('file1.ts');
|
|
57
|
+
const accessTime1 = retrieved1?.lastAccess || 0;
|
|
58
|
+
// Wait a bit then read again
|
|
59
|
+
const retrieved2 = cache.get('file1.ts');
|
|
60
|
+
const accessTime2 = retrieved2?.lastAccess || 0;
|
|
61
|
+
expect(accessTime2).toBeGreaterThanOrEqual(accessTime1);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
describe('L2 Cache (SQLite)', () => {
|
|
65
|
+
it('should persist entry to SQLite', () => {
|
|
66
|
+
const entry = createMockEntry();
|
|
67
|
+
cache.set('file1.ts', entry);
|
|
68
|
+
// Create new cache instance pointing to same DB
|
|
69
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
70
|
+
const retrieved = cache2.get('file1.ts');
|
|
71
|
+
cache2.close();
|
|
72
|
+
expect(retrieved).not.toBeNull();
|
|
73
|
+
expect(retrieved?.hash).toBe(entry.hash);
|
|
74
|
+
});
|
|
75
|
+
it('should retrieve from L2 cache in ~100ms', () => {
|
|
76
|
+
const entry = createMockEntry();
|
|
77
|
+
cache.set('file1.ts', entry);
|
|
78
|
+
// Create new instance to force L2 read
|
|
79
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
80
|
+
const start = performance.now();
|
|
81
|
+
const result = cache2.get('file1.ts');
|
|
82
|
+
const elapsed = performance.now() - start;
|
|
83
|
+
cache2.close();
|
|
84
|
+
expect(result).not.toBeNull();
|
|
85
|
+
expect(elapsed).toBeLessThan(200); // L2 should be < 200ms (more generous)
|
|
86
|
+
});
|
|
87
|
+
it('should promote L2 entry to L1 on read', () => {
|
|
88
|
+
const entry = createMockEntry();
|
|
89
|
+
cache.set('file1.ts', entry);
|
|
90
|
+
// Create new instance to force L2 read
|
|
91
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
92
|
+
cache2.get('file1.ts');
|
|
93
|
+
// Second read should be from L1 (fast)
|
|
94
|
+
const start = performance.now();
|
|
95
|
+
cache2.get('file1.ts');
|
|
96
|
+
const elapsed = performance.now() - start;
|
|
97
|
+
cache2.close();
|
|
98
|
+
expect(elapsed).toBeLessThan(5);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
describe('Persistence across restarts', () => {
|
|
102
|
+
it('should survive cache instance restart', () => {
|
|
103
|
+
const entry = createMockEntry();
|
|
104
|
+
cache.set('file1.ts', entry);
|
|
105
|
+
cache.close();
|
|
106
|
+
// Create new instance (simulates restart)
|
|
107
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
108
|
+
const retrieved = cache2.get('file1.ts');
|
|
109
|
+
cache2.close();
|
|
110
|
+
expect(retrieved).not.toBeNull();
|
|
111
|
+
expect(retrieved?.hash).toBe(entry.hash);
|
|
112
|
+
});
|
|
113
|
+
it('should handle multiple files across restarts', () => {
|
|
114
|
+
cache.set('file1.ts', createMockEntry('content1', 'hash1'));
|
|
115
|
+
cache.set('file2.ts', createMockEntry('content2', 'hash2'));
|
|
116
|
+
cache.set('file3.ts', createMockEntry('content3', 'hash3'));
|
|
117
|
+
cache.close();
|
|
118
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
119
|
+
expect(cache2.get('file1.ts')?.hash).toBe('hash1');
|
|
120
|
+
expect(cache2.get('file2.ts')?.hash).toBe('hash2');
|
|
121
|
+
expect(cache2.get('file3.ts')?.hash).toBe('hash3');
|
|
122
|
+
cache2.close();
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
describe('Semantic caching (by hash)', () => {
|
|
126
|
+
it('should retrieve entry by content hash', () => {
|
|
127
|
+
const entry = createMockEntry('shared content', 'shared-hash');
|
|
128
|
+
cache.set('file1.ts', entry);
|
|
129
|
+
const retrieved = cache.getByHash('shared-hash');
|
|
130
|
+
expect(retrieved).not.toBeNull();
|
|
131
|
+
expect(retrieved?.hash).toBe('shared-hash');
|
|
132
|
+
});
|
|
133
|
+
it('should return null for non-existent hash', () => {
|
|
134
|
+
const retrieved = cache.getByHash('nonexistent-hash');
|
|
135
|
+
expect(retrieved).toBeNull();
|
|
136
|
+
});
|
|
137
|
+
it('should find same content from different paths', () => {
|
|
138
|
+
const content = 'identical content';
|
|
139
|
+
const hash = 'same-hash';
|
|
140
|
+
cache.set('file1.ts', createMockEntry(content, hash));
|
|
141
|
+
cache.set('file2.ts', createMockEntry(content, hash));
|
|
142
|
+
const byHash = cache.getByHash(hash);
|
|
143
|
+
expect(byHash).not.toBeNull();
|
|
144
|
+
expect(byHash?.hash).toBe(hash);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
describe('LRU Eviction', () => {
|
|
148
|
+
it('should evict oldest entries when size limit exceeded', () => {
|
|
149
|
+
const cache3 = new PersistentFileCache(tempDir, 0.001); // Very small 1KB limit
|
|
150
|
+
// Add multiple entries
|
|
151
|
+
for (let i = 0; i < 5; i++) {
|
|
152
|
+
cache3.set(`file${i}.ts`, createMockEntry(`content ${i}`.repeat(100), `hash${i}`));
|
|
153
|
+
}
|
|
154
|
+
// Newest entry should still exist
|
|
155
|
+
expect(cache3.get('file4.ts')).not.toBeNull();
|
|
156
|
+
// Oldest entries should be evicted
|
|
157
|
+
const stats = cache3.getStats();
|
|
158
|
+
expect(stats.entryCount).toBeLessThan(5); // Some should be evicted
|
|
159
|
+
cache3.close();
|
|
160
|
+
});
|
|
161
|
+
it('should respect L1 max size', () => {
|
|
162
|
+
const cache2 = new PersistentFileCache(tempDir, 100);
|
|
163
|
+
const originalMaxSize = cache2.l1MaxSize;
|
|
164
|
+
// Add more than L1MaxSize entries
|
|
165
|
+
for (let i = 0; i < originalMaxSize + 10; i++) {
|
|
166
|
+
cache2.set(`file${i}.ts`, createMockEntry(`content${i}`, `hash${i}`));
|
|
167
|
+
}
|
|
168
|
+
const stats = cache2.getStats();
|
|
169
|
+
expect(stats.l1Size).toBeLessThanOrEqual(originalMaxSize);
|
|
170
|
+
cache2.close();
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
describe('Cache management', () => {
|
|
174
|
+
it('should delete entry from L1 and L2', () => {
|
|
175
|
+
const entry = createMockEntry();
|
|
176
|
+
cache.set('file1.ts', entry);
|
|
177
|
+
expect(cache.get('file1.ts')).not.toBeNull();
|
|
178
|
+
cache.delete('file1.ts');
|
|
179
|
+
expect(cache.get('file1.ts')).toBeNull();
|
|
180
|
+
});
|
|
181
|
+
it('should clear all cache', () => {
|
|
182
|
+
cache.set('file1.ts', createMockEntry('c1', 'h1'));
|
|
183
|
+
cache.set('file2.ts', createMockEntry('c2', 'h2'));
|
|
184
|
+
const stats1 = cache.getStats();
|
|
185
|
+
expect(stats1.entryCount).toBeGreaterThan(0);
|
|
186
|
+
cache.clear();
|
|
187
|
+
const stats2 = cache.getStats();
|
|
188
|
+
expect(stats2.entryCount).toBe(0);
|
|
189
|
+
expect(cache.get('file1.ts')).toBeNull();
|
|
190
|
+
});
|
|
191
|
+
it('should report accurate stats', () => {
|
|
192
|
+
cache.set('file1.ts', createMockEntry('content1', 'hash1'));
|
|
193
|
+
cache.set('file2.ts', createMockEntry('content2', 'hash2'));
|
|
194
|
+
const stats = cache.getStats();
|
|
195
|
+
expect(stats.entryCount).toBeGreaterThanOrEqual(2);
|
|
196
|
+
expect(stats.totalBytes).toBeGreaterThan(0);
|
|
197
|
+
expect(stats.l1Size).toBeGreaterThan(0);
|
|
198
|
+
expect(stats.maxBytes).toBe(100 * 1024 * 1024);
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
describe('Cache Performance Metrics', () => {
|
|
202
|
+
it('should track L1 hits', () => {
|
|
203
|
+
const entry = createMockEntry();
|
|
204
|
+
cache.set('file1.ts', entry);
|
|
205
|
+
// First hit: L1
|
|
206
|
+
cache.get('file1.ts');
|
|
207
|
+
const metrics = cache.getPerformanceMetrics?.();
|
|
208
|
+
if (metrics) {
|
|
209
|
+
expect(metrics.l1Hits).toBeGreaterThan(0);
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
it('should track cache misses', () => {
|
|
213
|
+
cache.get('nonexistent.ts');
|
|
214
|
+
const metrics = cache.getPerformanceMetrics?.();
|
|
215
|
+
if (metrics) {
|
|
216
|
+
expect(metrics.misses).toBeGreaterThan(0);
|
|
217
|
+
}
|
|
218
|
+
});
|
|
219
|
+
it('should calculate hit rate correctly', () => {
|
|
220
|
+
const entry = createMockEntry();
|
|
221
|
+
cache.set('file1.ts', entry);
|
|
222
|
+
cache.get('file1.ts'); // hit
|
|
223
|
+
cache.get('nonexistent.ts'); // miss
|
|
224
|
+
const metrics = cache.getPerformanceMetrics?.();
|
|
225
|
+
if (metrics) {
|
|
226
|
+
expect(metrics.totalRequests).toBeGreaterThanOrEqual(2);
|
|
227
|
+
expect(metrics.hitRate).toBeLessThanOrEqual(1);
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
describe('Concurrency & Performance', () => {
|
|
232
|
+
it('should handle concurrent operations', () => {
|
|
233
|
+
const entries = Array.from({ length: 20 }, (_, i) => cache.set(`file${i}.ts`, createMockEntry(`content${i}`, `hash${i}`)));
|
|
234
|
+
// Read all entries
|
|
235
|
+
for (let i = 0; i < 20; i++) {
|
|
236
|
+
expect(cache.get(`file${i}.ts`)).not.toBeNull();
|
|
237
|
+
}
|
|
238
|
+
expect(cache.getStats().entryCount).toBeGreaterThanOrEqual(20);
|
|
239
|
+
});
|
|
240
|
+
it('should handle insert/update of same file', () => {
|
|
241
|
+
const entry1 = createMockEntry('version1', 'hash1');
|
|
242
|
+
cache.set('file.ts', entry1);
|
|
243
|
+
expect(cache.get('file.ts')?.hash).toBe('hash1');
|
|
244
|
+
const entry2 = createMockEntry('version2', 'hash2');
|
|
245
|
+
cache.set('file.ts', entry2);
|
|
246
|
+
expect(cache.get('file.ts')?.hash).toBe('hash2');
|
|
247
|
+
expect(cache.getStats().entryCount).toBeGreaterThanOrEqual(1);
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
});
|
|
251
|
+
//# sourceMappingURL=persistent-cache.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"persistent-cache.test.js","sourceRoot":"","sources":["../../src/core/persistent-cache.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AAGxD,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,IAAI,KAA0B,CAAA;IAC9B,IAAI,OAAe,CAAA;IAEnB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAA;QAC1D,KAAK,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA,CAAC,qCAAqC;IACzF,CAAC,CAAC,CAAA;IAEF,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,MAAM,eAAe,GAAG,CAAC,UAAkB,cAAc,EAAE,OAAe,OAAO,EAAc,EAAE,CAAC,CAAC;QACjG,SAAS,EAAE;YACT,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,YAAY;YACtB,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;YACvF,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;SACZ;QACD,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1B,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;QACjB,IAAI;QACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAA;IAEF,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACvC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACpC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;YAEzC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAC7B,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACxC,MAAM,WAAW,GAAG,UAAU,EAAE,UAAU,IAAI,CAAC,CAAA;YAE/C,6BAA6B;YAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACxC,MAAM,WAAW,GAAG,UAAU,EAAE,UAAU,IAAI,CAAC,CAAA;YAE/C,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAA;QACzD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,gDAAgD;YAChD,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACxC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,uCAAuC;YACvC,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACrC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;YACzC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAC7B,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA,CAAC,uCAAuC;QAC3E,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,uCAAuC;YACvC,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YAEtB,uCAAuC;YACvC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YAC/B,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACtB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;YACzC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAC5B,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,0CAA0C;YAC1C,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACxC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;YAC3D,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;YAC3D,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;YAC3D,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAClD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAClD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,EAAE,CAAA;QAChB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,eAAe,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAA;YAC9D,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;YAChD,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAA;YACrD,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC9B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,OAAO,GAAG,mBAAmB,CAAA;YACnC,MAAM,IAAI,GAAG,WAAW,CAAA;YAExB,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;YACrD,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;YAErD,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAC7B,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA,CAAC,uBAAuB;YAE9E,uBAAuB;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;YACpF,CAAC;YAED,kCAAkC;YAClC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAE7C,mCAAmC;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAC/B,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA,CAAC,yBAAyB;YAElE,MAAM,CAAC,KAAK,EAAE,CAAA;QAChB,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YACpD,MAAM,eAAe,GAAI,MAAc,CAAC,SAAS,CAAA;YAEjD,kCAAkC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,UAAU,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;YACvE,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAA;YAEzD,MAAM,CAAC,KAAK,EAAE,CAAA;QAChB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAC5B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YAE5C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;YACxB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAClD,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAElD,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAE5C,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;YAC3D,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;YAE3D,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAA;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAE5B,gBAAgB;YAChB,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACrB,MAAM,OAAO,GAAI,KAAa,CAAC,qBAAqB,EAAE,EAAE,CAAA;YAExD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAC3B,MAAM,OAAO,GAAI,KAAa,CAAC,qBAAqB,EAAE,EAAE,CAAA;YAExD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;YAC/B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;YAC5B,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA,CAAC,MAAM;YAC5B,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA,CAAC,OAAO;YAEnC,MAAM,OAAO,GAAI,KAAa,CAAC,qBAAqB,EAAE,EAAE,CAAA;YAExD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;gBACvD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAA;YAChD,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAClD,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,UAAU,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CACrE,CAAA;YAED,mBAAmB;YACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAA;YACjD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAA;QAChE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;YACnD,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC5B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEhD,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;YACnD,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC5B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEhD,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export type SupportedModel = 'gpt-4' | 'gpt-4-turbo' | 'gpt-3.5-turbo' | 'claude-3-opus' | 'claude-3-sonnet' | 'claude-3-haiku';
|
|
2
|
+
/**
|
|
3
|
+
* Real token estimator using js-tiktoken
|
|
4
|
+
* Provides accurate token counts per model instead of rough character-based estimation
|
|
5
|
+
*
|
|
6
|
+
* Replaces: old estimateTokens() which used character count / 3.5 (very inaccurate ±50%)
|
|
7
|
+
* Improvement: ±5% accuracy vs actual OpenAI/Anthropic tokenization
|
|
8
|
+
*/
|
|
9
|
+
export declare class RealTokenEstimator {
|
|
10
|
+
private encoders;
|
|
11
|
+
private defaultModel;
|
|
12
|
+
private supportedModels;
|
|
13
|
+
constructor(defaultModel?: SupportedModel);
|
|
14
|
+
/**
|
|
15
|
+
* Estimate tokens in text for specific model
|
|
16
|
+
* @param text - Text to estimate
|
|
17
|
+
* @param model - Model (default: gpt-4)
|
|
18
|
+
* @returns Number of tokens
|
|
19
|
+
*/
|
|
20
|
+
estimateTokens(text: string, model?: SupportedModel): number;
|
|
21
|
+
/**
|
|
22
|
+
* Estimate tokens for multiple pieces of text
|
|
23
|
+
*/
|
|
24
|
+
estimateTokensForParts(parts: Array<{
|
|
25
|
+
text: string;
|
|
26
|
+
model?: SupportedModel;
|
|
27
|
+
}>, baseModel?: SupportedModel): number;
|
|
28
|
+
/**
|
|
29
|
+
* Get all supported models
|
|
30
|
+
*/
|
|
31
|
+
getSupportedModels(): SupportedModel[];
|
|
32
|
+
/**
|
|
33
|
+
* Check if model is supported
|
|
34
|
+
*/
|
|
35
|
+
supportsModel(model: string): model is SupportedModel;
|
|
36
|
+
/**
|
|
37
|
+
* Clear encoder cache if needed
|
|
38
|
+
*/
|
|
39
|
+
clearCache(): void;
|
|
40
|
+
/**
|
|
41
|
+
* Get model-specific token limit (context window)
|
|
42
|
+
*/
|
|
43
|
+
getTokenLimit(model: SupportedModel): number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Singleton instance for global use
|
|
47
|
+
*/
|
|
48
|
+
export declare const tokenEstimator: RealTokenEstimator;
|
|
49
|
+
//# sourceMappingURL=real-token-estimator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"real-token-estimator.d.ts","sourceRoot":"","sources":["../../src/core/real-token-estimator.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,cAAc,GACtB,OAAO,GACP,aAAa,GACb,eAAe,GACf,eAAe,GACf,iBAAiB,GACjB,gBAAgB,CAAA;AAEpB;;;;;;GAMG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAsC;IACtD,OAAO,CAAC,YAAY,CAA0B;IAC9C,OAAO,CAAC,eAAe,CAOtB;gBAEW,YAAY,GAAE,cAAwB;IAIlD;;;;;OAKG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,GAAE,cAAkC,GAAG,MAAM;IAoB/E;;OAEG;IACH,sBAAsB,CACpB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC,EACtD,SAAS,CAAC,EAAE,cAAc,GACzB,MAAM;IAOT;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,cAAc;IAIrD;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM;CAW7C;AAED;;GAEG;AACH,eAAO,MAAM,cAAc,oBAA2B,CAAA"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { encoding_for_model } from 'js-tiktoken';
|
|
2
|
+
/**
|
|
3
|
+
* Real token estimator using js-tiktoken
|
|
4
|
+
* Provides accurate token counts per model instead of rough character-based estimation
|
|
5
|
+
*
|
|
6
|
+
* Replaces: old estimateTokens() which used character count / 3.5 (very inaccurate ±50%)
|
|
7
|
+
* Improvement: ±5% accuracy vs actual OpenAI/Anthropic tokenization
|
|
8
|
+
*/
|
|
9
|
+
export class RealTokenEstimator {
|
|
10
|
+
encoders = new Map();
|
|
11
|
+
defaultModel = 'gpt-4';
|
|
12
|
+
supportedModels = [
|
|
13
|
+
'gpt-4',
|
|
14
|
+
'gpt-4-turbo',
|
|
15
|
+
'gpt-3.5-turbo',
|
|
16
|
+
'claude-3-opus',
|
|
17
|
+
'claude-3-sonnet',
|
|
18
|
+
'claude-3-haiku'
|
|
19
|
+
];
|
|
20
|
+
constructor(defaultModel = 'gpt-4') {
|
|
21
|
+
this.defaultModel = defaultModel;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Estimate tokens in text for specific model
|
|
25
|
+
* @param text - Text to estimate
|
|
26
|
+
* @param model - Model (default: gpt-4)
|
|
27
|
+
* @returns Number of tokens
|
|
28
|
+
*/
|
|
29
|
+
estimateTokens(text, model = this.defaultModel) {
|
|
30
|
+
if (!text)
|
|
31
|
+
return 0;
|
|
32
|
+
try {
|
|
33
|
+
// Get or create encoder for model
|
|
34
|
+
if (!this.encoders.has(model)) {
|
|
35
|
+
this.encoders.set(model, encoding_for_model(model));
|
|
36
|
+
}
|
|
37
|
+
const encoder = this.encoders.get(model);
|
|
38
|
+
const tokens = encoder.encode(text);
|
|
39
|
+
return tokens.length;
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
// Fallback to character-based if js-tiktoken fails
|
|
43
|
+
console.warn(`[token-pilot] Token estimation failed for ${model}, using fallback`);
|
|
44
|
+
return Math.ceil(text.length / 3.5);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Estimate tokens for multiple pieces of text
|
|
49
|
+
*/
|
|
50
|
+
estimateTokensForParts(parts, baseModel) {
|
|
51
|
+
return parts.reduce((total, part) => {
|
|
52
|
+
const model = part.model || baseModel || this.defaultModel;
|
|
53
|
+
return total + this.estimateTokens(part.text, model);
|
|
54
|
+
}, 0);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Get all supported models
|
|
58
|
+
*/
|
|
59
|
+
getSupportedModels() {
|
|
60
|
+
return [...this.supportedModels];
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Check if model is supported
|
|
64
|
+
*/
|
|
65
|
+
supportsModel(model) {
|
|
66
|
+
return this.supportedModels.includes(model);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Clear encoder cache if needed
|
|
70
|
+
*/
|
|
71
|
+
clearCache() {
|
|
72
|
+
this.encoders.clear();
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Get model-specific token limit (context window)
|
|
76
|
+
*/
|
|
77
|
+
getTokenLimit(model) {
|
|
78
|
+
const limits = {
|
|
79
|
+
'gpt-4': 8192,
|
|
80
|
+
'gpt-4-turbo': 128000,
|
|
81
|
+
'gpt-3.5-turbo': 4096,
|
|
82
|
+
'claude-3-opus': 200000,
|
|
83
|
+
'claude-3-sonnet': 200000,
|
|
84
|
+
'claude-3-haiku': 200000
|
|
85
|
+
};
|
|
86
|
+
return limits[model] || 8192;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Singleton instance for global use
|
|
91
|
+
*/
|
|
92
|
+
export const tokenEstimator = new RealTokenEstimator();
|
|
93
|
+
//# sourceMappingURL=real-token-estimator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"real-token-estimator.js","sourceRoot":"","sources":["../../src/core/real-token-estimator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAUhD;;;;;;GAMG;AACH,MAAM,OAAO,kBAAkB;IACrB,QAAQ,GAA6B,IAAI,GAAG,EAAE,CAAA;IAC9C,YAAY,GAAmB,OAAO,CAAA;IACtC,eAAe,GAAqB;QAC1C,OAAO;QACP,aAAa;QACb,eAAe;QACf,eAAe;QACf,iBAAiB;QACjB,gBAAgB;KACjB,CAAA;IAED,YAAY,eAA+B,OAAO;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IAClC,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,IAAY,EAAE,QAAwB,IAAI,CAAC,YAAY;QACpE,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAA;QAEnB,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;YACrD,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAEnC,OAAO,MAAM,CAAC,MAAM,CAAA;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mDAAmD;YACnD,OAAO,CAAC,IAAI,CAAC,6CAA6C,KAAK,kBAAkB,CAAC,CAAA;YAClF,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,sBAAsB,CACpB,KAAsD,EACtD,SAA0B;QAE1B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,IAAI,IAAI,CAAC,YAAY,CAAA;YAC1D,OAAO,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtD,CAAC,EAAE,CAAC,CAAC,CAAA;IACP,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAA;IAClC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAa;QACzB,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAuB,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IACvB,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAqB;QACjC,MAAM,MAAM,GAAmC;YAC7C,OAAO,EAAE,IAAI;YACb,aAAa,EAAE,MAAM;YACrB,eAAe,EAAE,IAAI;YACrB,eAAe,EAAE,MAAM;YACvB,iBAAiB,EAAE,MAAM;YACzB,gBAAgB,EAAE,MAAM;SACzB,CAAA;QACD,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAA;IAC9B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"symbol-resolver.d.ts","sourceRoot":"","sources":["../../src/core/symbol-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAc,aAAa,EAAc,MAAM,aAAa,CAAC;AAEzF,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAiB;gBAErB,QAAQ,EAAE,cAAc;IAIpC;;;OAGG;IACG,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAoD/F;;OAEG;IACH,aAAa,CACX,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9D,MAAM;IAeT,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,eAAe;
|
|
1
|
+
{"version":3,"file":"symbol-resolver.d.ts","sourceRoot":"","sources":["../../src/core/symbol-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAc,aAAa,EAAc,MAAM,aAAa,CAAC;AAEzF,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAiB;gBAErB,QAAQ,EAAE,cAAc;IAIpC;;;OAGG;IACG,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAoD/F;;OAEG;IACH,aAAa,CACX,QAAQ,EAAE,cAAc,EACxB,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9D,MAAM;IAeT,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,WAAW;CAcpB"}
|
|
@@ -81,20 +81,22 @@ export class SymbolResolver {
|
|
|
81
81
|
return map[kind.toLowerCase()] ?? 'function';
|
|
82
82
|
}
|
|
83
83
|
findInStructure(qualifiedName, symbols) {
|
|
84
|
-
|
|
84
|
+
// Support both . and :: separators (PHP uses ::)
|
|
85
|
+
const parts = qualifiedName.includes('::')
|
|
86
|
+
? qualifiedName.split('::')
|
|
87
|
+
: qualifiedName.split('.');
|
|
88
|
+
return this.findByParts(parts, symbols);
|
|
89
|
+
}
|
|
90
|
+
findByParts(parts, symbols) {
|
|
85
91
|
for (const sym of symbols) {
|
|
86
92
|
if (parts.length === 1 && sym.name === parts[0]) {
|
|
87
93
|
return sym;
|
|
88
94
|
}
|
|
89
|
-
if (parts.length
|
|
90
|
-
const
|
|
91
|
-
if (
|
|
92
|
-
return
|
|
95
|
+
if (parts.length >= 2 && sym.name === parts[0]) {
|
|
96
|
+
const found = this.findByParts(parts.slice(1), sym.children);
|
|
97
|
+
if (found)
|
|
98
|
+
return found;
|
|
93
99
|
}
|
|
94
|
-
// Recurse
|
|
95
|
-
const found = this.findInStructure(qualifiedName, sym.children);
|
|
96
|
-
if (found)
|
|
97
|
-
return found;
|
|
98
100
|
}
|
|
99
101
|
return null;
|
|
100
102
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"symbol-resolver.js","sourceRoot":"","sources":["../../src/core/symbol-resolver.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,cAAc;IACjB,QAAQ,CAAiB;IAEjC,YAAY,QAAwB;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,aAAqB,EAAE,SAAyB;QAC5D,sBAAsB;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,CAAC;YACX,uEAAuE;YACvE,IAAI,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;YACrC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;gBACrE,IAAI,KAAK;oBAAE,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC9C,CAAC;YAED,OAAO;gBACL,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,aAAa,EAAE,aAAa;oBAC5B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;oBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI;oBAC1C,QAAQ,EAAE;wBACR,SAAS,EAAE,MAAM,CAAC,UAAU;wBAC5B,OAAO;wBACP,SAAS,EAAE,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC;qBAC3C;oBACD,UAAU,EAAE,SAAS;oBACrB,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,EAAE;oBACZ,GAAG,EAAE,IAAI;oBACT,UAAU,EAAE,EAAE;iBACf;gBACD,QAAQ,EAAE,MAAM,CAAC,IAAI;gBACrB,SAAS,EAAE,MAAM,CAAC,UAAU;gBAC5B,OAAO;aACR,CAAC;QACJ,CAAC;QAED,yCAAyC;QACzC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;YACrE,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE,SAAS,CAAC,IAAI;oBACxB,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;oBACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;iBAChC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,aAAa,CACX,QAAwB,EACxB,KAAe,EACf,UAA6D,EAAE;QAE/D,MAAM,EAAE,aAAa,GAAG,CAAC,EAAE,YAAY,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAExD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,GAAG,YAAY,CAAC,CAAC;QAEpE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAEO,OAAO,CAAC,IAAY;QAC1B,MAAM,GAAG,GAA+B;YACtC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;YACtD,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;YACxD,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU;YAC1D,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW;YAC3D,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW;SACnC,CAAC;QACF,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,UAAU,CAAC;IAC/C,CAAC;IAEO,eAAe,CAAC,aAAqB,EAAE,OAAqB;QAClE,MAAM,KAAK,GAAG,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"symbol-resolver.js","sourceRoot":"","sources":["../../src/core/symbol-resolver.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,cAAc;IACjB,QAAQ,CAAiB;IAEjC,YAAY,QAAwB;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,aAAqB,EAAE,SAAyB;QAC5D,sBAAsB;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,CAAC;YACX,uEAAuE;YACvE,IAAI,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC;YACrC,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;gBACrE,IAAI,KAAK;oBAAE,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC9C,CAAC;YAED,OAAO;gBACL,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,aAAa,EAAE,aAAa;oBAC5B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;oBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI;oBAC1C,QAAQ,EAAE;wBACR,SAAS,EAAE,MAAM,CAAC,UAAU;wBAC5B,OAAO;wBACP,SAAS,EAAE,OAAO,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC;qBAC3C;oBACD,UAAU,EAAE,SAAS;oBACrB,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,EAAE;oBACZ,GAAG,EAAE,IAAI;oBACT,UAAU,EAAE,EAAE;iBACf;gBACD,QAAQ,EAAE,MAAM,CAAC,IAAI;gBACrB,SAAS,EAAE,MAAM,CAAC,UAAU;gBAC5B,OAAO;aACR,CAAC;QACJ,CAAC;QAED,yCAAyC;QACzC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;YACrE,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE,SAAS,CAAC,IAAI;oBACxB,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;oBACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;iBAChC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,aAAa,CACX,QAAwB,EACxB,KAAe,EACf,UAA6D,EAAE;QAE/D,MAAM,EAAE,aAAa,GAAG,CAAC,EAAE,YAAY,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAExD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,GAAG,YAAY,CAAC,CAAC;QAEpE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAEO,OAAO,CAAC,IAAY;QAC1B,MAAM,GAAG,GAA+B;YACtC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;YACtD,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;YACxD,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU;YAC1D,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW;YAC3D,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW;SACnC,CAAC;QACF,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,UAAU,CAAC;IAC/C,CAAC;IAEO,eAAe,CAAC,aAAqB,EAAE,OAAqB;QAClE,iDAAiD;QACjD,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC;YACxC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC;YAC3B,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE7B,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;IAEO,WAAW,CAAC,KAAe,EAAE,OAAqB;QACxD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChD,OAAO,GAAG,CAAC;YACb,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC7D,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context markup utilities for AI assistants
|
|
3
|
+
* Ensures AI cannot ignore or forget that structured overviews are STRUCTURAL_OVERVIEW, not full code
|
|
4
|
+
*
|
|
5
|
+
* Problem: Advisory context "here's an overview" can be ignored
|
|
6
|
+
* Solution: Explicit XML structure that marks region type unmistakably
|
|
7
|
+
*/
|
|
8
|
+
export interface ContextRegion {
|
|
9
|
+
type: 'structure' | 'symbol' | 'range' | 'full' | 'diff';
|
|
10
|
+
source: string;
|
|
11
|
+
startLine?: number;
|
|
12
|
+
endLine?: number;
|
|
13
|
+
symbolName?: string;
|
|
14
|
+
warning: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Create context warning for structural overview
|
|
18
|
+
*/
|
|
19
|
+
export declare function createStructureOverviewWarning(): string;
|
|
20
|
+
/**
|
|
21
|
+
* Create context warning for symbol read
|
|
22
|
+
*/
|
|
23
|
+
export declare function createSymbolReadWarning(symbolName: string, filePath: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* Create context warning for range read
|
|
26
|
+
*/
|
|
27
|
+
export declare function createRangeReadWarning(startLine: number, endLine: number, filePath: string): string;
|
|
28
|
+
/**
|
|
29
|
+
* Create context warning for diff
|
|
30
|
+
*/
|
|
31
|
+
export declare function createDiffWarning(filePath: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Add context warning to response
|
|
34
|
+
*/
|
|
35
|
+
export declare function addContextWarning(response: string, region: ContextRegion): string;
|
|
36
|
+
/**
|
|
37
|
+
* Create full context notice
|
|
38
|
+
*/
|
|
39
|
+
export declare function createContextNotice(type: 'structure' | 'symbol' | 'range' | 'diff', metadata: any): string;
|
|
40
|
+
//# sourceMappingURL=context-markup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-markup.d.ts","sourceRoot":"","sources":["../../src/formatters/context-markup.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAA;IACxD,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;GAEG;AACH,wBAAgB,8BAA8B,IAAI,MAAM,CAGvD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGpF;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGnG;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAG1D;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,MAAM,CAGjF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,MAAM,CAS1G"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context markup utilities for AI assistants
|
|
3
|
+
* Ensures AI cannot ignore or forget that structured overviews are STRUCTURAL_OVERVIEW, not full code
|
|
4
|
+
*
|
|
5
|
+
* Problem: Advisory context "here's an overview" can be ignored
|
|
6
|
+
* Solution: Explicit XML structure that marks region type unmistakably
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Create context warning for structural overview
|
|
10
|
+
*/
|
|
11
|
+
export function createStructureOverviewWarning() {
|
|
12
|
+
return 'This is a STRUCTURAL_OVERVIEW of file contents. It shows classes, functions, and imports only. ' +
|
|
13
|
+
'To see full implementation, use read_symbol() or read_range().';
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Create context warning for symbol read
|
|
17
|
+
*/
|
|
18
|
+
export function createSymbolReadWarning(symbolName, filePath) {
|
|
19
|
+
return `This is the FULL source code of symbol "${symbolName}" in ${filePath}. ` +
|
|
20
|
+
'This is complete implementation, not an overview.';
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Create context warning for range read
|
|
24
|
+
*/
|
|
25
|
+
export function createRangeReadWarning(startLine, endLine, filePath) {
|
|
26
|
+
return `This is source code from lines ${startLine}-${endLine} of ${filePath}. ` +
|
|
27
|
+
'This is a partial file excerpt, not the complete file.';
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Create context warning for diff
|
|
31
|
+
*/
|
|
32
|
+
export function createDiffWarning(filePath) {
|
|
33
|
+
return `This shows what CHANGED in ${filePath} since last read. ` +
|
|
34
|
+
'Only modified symbols are shown, not the entire file.';
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Add context warning to response
|
|
38
|
+
*/
|
|
39
|
+
export function addContextWarning(response, region) {
|
|
40
|
+
const xmlWarning = `<!-- CONTEXT: ${region.warning} -->`;
|
|
41
|
+
return xmlWarning + '\n' + response;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Create full context notice
|
|
45
|
+
*/
|
|
46
|
+
export function createContextNotice(type, metadata) {
|
|
47
|
+
const notices = {
|
|
48
|
+
structure: createStructureOverviewWarning(),
|
|
49
|
+
symbol: createSymbolReadWarning(metadata.symbolName, metadata.filePath),
|
|
50
|
+
range: createRangeReadWarning(metadata.startLine, metadata.endLine, metadata.filePath),
|
|
51
|
+
diff: createDiffWarning(metadata.filePath)
|
|
52
|
+
};
|
|
53
|
+
return notices[type] || 'Unknown context type';
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=context-markup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-markup.js","sourceRoot":"","sources":["../../src/formatters/context-markup.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH;;GAEG;AACH,MAAM,UAAU,8BAA8B;IAC5C,OAAO,iGAAiG;QACjG,gEAAgE,CAAA;AACzE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,UAAkB,EAAE,QAAgB;IAC1E,OAAO,2CAA2C,UAAU,QAAQ,QAAQ,IAAI;QACzE,mDAAmD,CAAA;AAC5D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,SAAiB,EAAE,OAAe,EAAE,QAAgB;IACzF,OAAO,kCAAkC,SAAS,IAAI,OAAO,OAAO,QAAQ,IAAI;QACzE,wDAAwD,CAAA;AACjE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,OAAO,8BAA8B,QAAQ,oBAAoB;QAC1D,uDAAuD,CAAA;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,MAAqB;IACvE,MAAM,UAAU,GAAG,iBAAiB,MAAM,CAAC,OAAO,MAAM,CAAA;IACxD,OAAO,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAA;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAA+C,EAAE,QAAa;IAChG,MAAM,OAAO,GAA2B;QACtC,SAAS,EAAE,8BAA8B,EAAE;QAC3C,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACvE,KAAK,EAAE,sBAAsB,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACtF,IAAI,EAAE,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC;KAC3C,CAAA;IAED,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,sBAAsB,CAAA;AAChD,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { FileStructure } from '../types';
|
|
2
|
+
/**
|
|
3
|
+
* Format FileStructure as XML for explicit structural marking
|
|
4
|
+
* Replaces JSON which can be misunderstood as code content
|
|
5
|
+
*
|
|
6
|
+
* Ensures AI understands:
|
|
7
|
+
* - This is STRUCTURAL_OVERVIEW, not full code
|
|
8
|
+
* - Symbols shown are just names + signatures, not implementations
|
|
9
|
+
* - Usage instructions are explicit
|
|
10
|
+
*/
|
|
11
|
+
export declare function formatSmartReadAsXml(structure: FileStructure): string;
|
|
12
|
+
/**
|
|
13
|
+
* Format symbol read as XML
|
|
14
|
+
*/
|
|
15
|
+
export declare function formatSymbolReadAsXml(symbolName: string, filePath: string, sourceCode: string, startLine: number, endLine: number): string;
|
|
16
|
+
/**
|
|
17
|
+
* Format range read as XML
|
|
18
|
+
*/
|
|
19
|
+
export declare function formatRangeReadAsXml(filePath: string, sourceCode: string, startLine: number, endLine: number): string;
|
|
20
|
+
//# sourceMappingURL=smart-read-xml.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"smart-read-xml.d.ts","sourceRoot":"","sources":["../../src/formatters/smart-read-xml.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAE7C;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAyErE;AAcD;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,MAAM,CAiCR;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,MAAM,CAgCR"}
|