token-pilot 0.1.6 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +32 -1
- package/dist/ast-index/client.d.ts +3 -0
- package/dist/ast-index/client.d.ts.map +1 -1
- package/dist/ast-index/client.js +80 -14
- package/dist/ast-index/client.js.map +1 -1
- package/dist/core/context-window-tracker.d.ts +89 -0
- package/dist/core/context-window-tracker.d.ts.map +1 -0
- package/dist/core/context-window-tracker.js +161 -0
- package/dist/core/context-window-tracker.js.map +1 -0
- package/dist/core/context-window-tracker.test.d.ts +2 -0
- package/dist/core/context-window-tracker.test.d.ts.map +1 -0
- package/dist/core/context-window-tracker.test.js +238 -0
- package/dist/core/context-window-tracker.test.js.map +1 -0
- package/dist/core/diff-engine.d.ts +64 -0
- package/dist/core/diff-engine.d.ts.map +1 -0
- package/dist/core/diff-engine.js +185 -0
- package/dist/core/diff-engine.js.map +1 -0
- package/dist/core/diff-engine.test.d.ts +2 -0
- package/dist/core/diff-engine.test.d.ts.map +1 -0
- package/dist/core/diff-engine.test.js +351 -0
- package/dist/core/diff-engine.test.js.map +1 -0
- package/dist/core/persistent-cache.d.ts +153 -0
- package/dist/core/persistent-cache.d.ts.map +1 -0
- package/dist/core/persistent-cache.js +555 -0
- package/dist/core/persistent-cache.js.map +1 -0
- package/dist/core/persistent-cache.test.d.ts +2 -0
- package/dist/core/persistent-cache.test.d.ts.map +1 -0
- package/dist/core/persistent-cache.test.js +251 -0
- package/dist/core/persistent-cache.test.js.map +1 -0
- package/dist/core/real-token-estimator.d.ts +49 -0
- package/dist/core/real-token-estimator.d.ts.map +1 -0
- package/dist/core/real-token-estimator.js +93 -0
- package/dist/core/real-token-estimator.js.map +1 -0
- package/dist/core/symbol-resolver.d.ts +1 -0
- package/dist/core/symbol-resolver.d.ts.map +1 -1
- package/dist/core/symbol-resolver.js +11 -9
- package/dist/core/symbol-resolver.js.map +1 -1
- package/dist/formatters/context-markup.d.ts +40 -0
- package/dist/formatters/context-markup.d.ts.map +1 -0
- package/dist/formatters/context-markup.js +55 -0
- package/dist/formatters/context-markup.js.map +1 -0
- package/dist/formatters/smart-read-xml.d.ts +20 -0
- package/dist/formatters/smart-read-xml.d.ts.map +1 -0
- package/dist/formatters/smart-read-xml.js +163 -0
- package/dist/formatters/smart-read-xml.js.map +1 -0
- package/dist/handlers/project-overview.d.ts.map +1 -1
- package/dist/handlers/project-overview.js +57 -24
- package/dist/handlers/project-overview.js.map +1 -1
- package/dist/handlers/read-symbol.d.ts +2 -1
- package/dist/handlers/read-symbol.d.ts.map +1 -1
- package/dist/handlers/read-symbol.js +7 -3
- package/dist/handlers/read-symbol.js.map +1 -1
- package/dist/index.js +21 -1
- package/dist/index.js.map +1 -1
- package/dist/server.js +8 -4
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
- package/start.sh +27 -27
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
2
|
+
import { ContextWindowTracker } from './context-window-tracker';
|
|
3
|
+
describe('ContextWindowTracker', () => {
|
|
4
|
+
let tracker;
|
|
5
|
+
beforeEach(() => {
|
|
6
|
+
tracker = new ContextWindowTracker('gpt-4');
|
|
7
|
+
});
|
|
8
|
+
describe('basic tracking', () => {
|
|
9
|
+
it('should start with zero tokens', () => {
|
|
10
|
+
expect(tracker.getUsedTokens()).toBe(0);
|
|
11
|
+
expect(tracker.getStatus().usedTokens).toBe(0);
|
|
12
|
+
});
|
|
13
|
+
it('should track tokens for a source', () => {
|
|
14
|
+
tracker.track('file1.ts', 100);
|
|
15
|
+
expect(tracker.getUsedTokens()).toBe(100);
|
|
16
|
+
});
|
|
17
|
+
it('should sum tokens from multiple sources', () => {
|
|
18
|
+
tracker.track('file1.ts', 100);
|
|
19
|
+
tracker.track('file2.ts', 200);
|
|
20
|
+
tracker.track('symbol.ts', 50);
|
|
21
|
+
expect(tracker.getUsedTokens()).toBe(350);
|
|
22
|
+
});
|
|
23
|
+
it('should update total when overwriting a source', () => {
|
|
24
|
+
tracker.track('file1.ts', 100);
|
|
25
|
+
expect(tracker.getUsedTokens()).toBe(100);
|
|
26
|
+
tracker.track('file1.ts', 150);
|
|
27
|
+
expect(tracker.getUsedTokens()).toBe(150);
|
|
28
|
+
});
|
|
29
|
+
it('should untrack sources', () => {
|
|
30
|
+
tracker.track('file1.ts', 100);
|
|
31
|
+
tracker.track('file2.ts', 200);
|
|
32
|
+
expect(tracker.getUsedTokens()).toBe(300);
|
|
33
|
+
tracker.untrack('file1.ts');
|
|
34
|
+
expect(tracker.getUsedTokens()).toBe(200);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
describe('canLoad() safety checks', () => {
|
|
38
|
+
it('should allow loading within safe zone', () => {
|
|
39
|
+
// GPT-4 has 8192 tokens, safe zone is 80% = 6553 tokens
|
|
40
|
+
tracker.track('file1.ts', 5000);
|
|
41
|
+
expect(tracker.canLoad(1000)).toBe(true);
|
|
42
|
+
});
|
|
43
|
+
it('should prevent loading beyond safe zone', () => {
|
|
44
|
+
// GPT-4 safe zone: 8192 * 0.8 = 6553
|
|
45
|
+
tracker.track('file1.ts', 6000);
|
|
46
|
+
expect(tracker.canLoad(1000)).toBe(false);
|
|
47
|
+
});
|
|
48
|
+
it('should prevent loading when already at safe limit', () => {
|
|
49
|
+
// GPT-4 safe zone: 8192 * 0.8 = 6553
|
|
50
|
+
tracker.track('file1.ts', 6553);
|
|
51
|
+
expect(tracker.canLoad(1)).toBe(false);
|
|
52
|
+
});
|
|
53
|
+
it('should allow loading exactly at safe limit', () => {
|
|
54
|
+
tracker.track('file1.ts', 6000);
|
|
55
|
+
const safeRemaining = tracker.getSafeRemaining();
|
|
56
|
+
expect(tracker.canLoad(safeRemaining)).toBe(true);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
describe('model switching', () => {
|
|
60
|
+
it('should support different models with different limits', () => {
|
|
61
|
+
tracker = new ContextWindowTracker('gpt-4');
|
|
62
|
+
expect(tracker.getTotalTokens()).toBe(8192);
|
|
63
|
+
tracker.setModel('gpt-4-turbo');
|
|
64
|
+
expect(tracker.getTotalTokens()).toBe(128000);
|
|
65
|
+
tracker.setModel('claude-3-opus');
|
|
66
|
+
expect(tracker.getTotalTokens()).toBe(200000);
|
|
67
|
+
});
|
|
68
|
+
it('should reset usage when switching models', () => {
|
|
69
|
+
tracker.track('file1.ts', 1000);
|
|
70
|
+
expect(tracker.getUsedTokens()).toBe(1000);
|
|
71
|
+
tracker.setModel('claude-3-opus');
|
|
72
|
+
expect(tracker.getUsedTokens()).toBe(0);
|
|
73
|
+
expect(tracker.getEntries()).toHaveLength(0);
|
|
74
|
+
});
|
|
75
|
+
it('should update safety calculations for new model', () => {
|
|
76
|
+
tracker = new ContextWindowTracker('gpt-4');
|
|
77
|
+
tracker.track('file1.ts', 5000);
|
|
78
|
+
expect(tracker.canLoad(2000)).toBe(false);
|
|
79
|
+
tracker.setModel('claude-3-opus');
|
|
80
|
+
expect(tracker.getUsedTokens()).toBe(0);
|
|
81
|
+
expect(tracker.canLoad(2000)).toBe(true);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
describe('safety margin', () => {
|
|
85
|
+
it('should respect safety margin', () => {
|
|
86
|
+
tracker.setSafetyMargin(0.9);
|
|
87
|
+
// GPT-4 safe zone: 8192 * 0.9 = 7372
|
|
88
|
+
tracker.track('file1.ts', 7000);
|
|
89
|
+
expect(tracker.canLoad(500)).toBe(false);
|
|
90
|
+
});
|
|
91
|
+
it('should allow customizing safety margin', () => {
|
|
92
|
+
tracker.setSafetyMargin(0.5);
|
|
93
|
+
// GPT-4 safe zone: 8192 * 0.5 = 4096
|
|
94
|
+
tracker.track('file1.ts', 4000);
|
|
95
|
+
expect(tracker.canLoad(100)).toBe(false);
|
|
96
|
+
});
|
|
97
|
+
it('should reject invalid safety margins', () => {
|
|
98
|
+
expect(() => tracker.setSafetyMargin(-0.1)).toThrow();
|
|
99
|
+
expect(() => tracker.setSafetyMargin(1.1)).toThrow();
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
describe('warning zone detection', () => {
|
|
103
|
+
it('should detect when in warning zone', () => {
|
|
104
|
+
// Safe zone is 80%, warning is 80-99%
|
|
105
|
+
tracker.track('file1.ts', 6600); // 80.5% of 8192
|
|
106
|
+
expect(tracker.isInWarningZone()).toBe(true);
|
|
107
|
+
});
|
|
108
|
+
it('should not warn when below warning zone', () => {
|
|
109
|
+
tracker.track('file1.ts', 5000); // ~61% of 8192
|
|
110
|
+
expect(tracker.isInWarningZone()).toBe(false);
|
|
111
|
+
});
|
|
112
|
+
it('should not warn at exact safe margin', () => {
|
|
113
|
+
const safeLimit = 8192 * 0.8; // 6553
|
|
114
|
+
tracker.track('file1.ts', safeLimit);
|
|
115
|
+
expect(tracker.isInWarningZone()).toBe(false);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
describe('overflow detection', () => {
|
|
119
|
+
it('should detect overflow (>99% of limit)', () => {
|
|
120
|
+
// 99% of GPT-4 is 8110
|
|
121
|
+
tracker.track('file1.ts', 8100);
|
|
122
|
+
expect(tracker.isOverflowing()).toBe(true);
|
|
123
|
+
});
|
|
124
|
+
it('should not detect overflow below 99%', () => {
|
|
125
|
+
// 98% of GPT-4 is 8027
|
|
126
|
+
tracker.track('file1.ts', 8000);
|
|
127
|
+
expect(tracker.isOverflowing()).toBe(false);
|
|
128
|
+
});
|
|
129
|
+
it('should detect overflow at exactly 99%', () => {
|
|
130
|
+
const hardLimit = 8192 * 0.99; // 8110
|
|
131
|
+
tracker.track('file1.ts', hardLimit);
|
|
132
|
+
expect(tracker.isOverflowing()).toBe(true);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
describe('status reporting', () => {
|
|
136
|
+
it('should report accurate status', () => {
|
|
137
|
+
tracker.track('file1.ts', 1000);
|
|
138
|
+
tracker.track('file2.ts', 500);
|
|
139
|
+
const status = tracker.getStatus();
|
|
140
|
+
expect(status.model).toBe('gpt-4');
|
|
141
|
+
expect(status.usedTokens).toBe(1500);
|
|
142
|
+
expect(status.totalTokens).toBe(8192);
|
|
143
|
+
expect(status.availableTokens).toBe(6692);
|
|
144
|
+
expect(status.isSafe).toBe(true);
|
|
145
|
+
expect(status.timestamp).toBeGreaterThan(0);
|
|
146
|
+
});
|
|
147
|
+
it('should mark as unsafe when approaching limit', () => {
|
|
148
|
+
tracker.track('file1.ts', 6700); // Beyond safe margin
|
|
149
|
+
const status = tracker.getStatus();
|
|
150
|
+
expect(status.isSafe).toBe(false);
|
|
151
|
+
});
|
|
152
|
+
it('should include all tracked entries in status', () => {
|
|
153
|
+
tracker.track('file1.ts', 100);
|
|
154
|
+
tracker.track('file2.ts', 200);
|
|
155
|
+
tracker.track('symbol.ts', 300);
|
|
156
|
+
const status = tracker.getStatus();
|
|
157
|
+
expect(status.entries).toHaveLength(3);
|
|
158
|
+
expect(status.entries).toContainEqual({ source: 'file1.ts', tokens: 100 });
|
|
159
|
+
expect(status.entries).toContainEqual({ source: 'file2.ts', tokens: 200 });
|
|
160
|
+
expect(status.entries).toContainEqual({ source: 'symbol.ts', tokens: 300 });
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
describe('percentage tracking', () => {
|
|
164
|
+
it('should calculate percentage used correctly', () => {
|
|
165
|
+
tracker.track('file1.ts', 8192 / 2); // 50%
|
|
166
|
+
expect(tracker.getPercentageUsed()).toBe(50);
|
|
167
|
+
});
|
|
168
|
+
it('should report 0% when empty', () => {
|
|
169
|
+
expect(tracker.getPercentageUsed()).toBe(0);
|
|
170
|
+
});
|
|
171
|
+
it('should report correct percentage in status', () => {
|
|
172
|
+
tracker.track('file1.ts', 4096); // 50% of 8192
|
|
173
|
+
const status = tracker.getStatus();
|
|
174
|
+
expect(status.percentageUsed).toBe(50);
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
describe('entries management', () => {
|
|
178
|
+
it('should return all tracked entries', () => {
|
|
179
|
+
tracker.track('file1.ts', 100);
|
|
180
|
+
tracker.track('file2.ts', 200);
|
|
181
|
+
const entries = tracker.getEntries();
|
|
182
|
+
expect(entries).toHaveLength(2);
|
|
183
|
+
expect(entries[0].source).toBe('file1.ts');
|
|
184
|
+
expect(entries[0].tokens).toBe(100);
|
|
185
|
+
});
|
|
186
|
+
it('should return empty entries after reset', () => {
|
|
187
|
+
tracker.track('file1.ts', 100);
|
|
188
|
+
tracker.reset();
|
|
189
|
+
expect(tracker.getEntries()).toHaveLength(0);
|
|
190
|
+
expect(tracker.getUsedTokens()).toBe(0);
|
|
191
|
+
});
|
|
192
|
+
});
|
|
193
|
+
describe('safe remaining calculation', () => {
|
|
194
|
+
it('should calculate safe remaining tokens', () => {
|
|
195
|
+
// GPT-4: 8192, safe zone: 6553.6
|
|
196
|
+
tracker.track('file1.ts', 1000);
|
|
197
|
+
const remaining = tracker.getSafeRemaining();
|
|
198
|
+
const safeZone = 8192 * 0.8;
|
|
199
|
+
expect(remaining).toBe(safeZone - 1000);
|
|
200
|
+
});
|
|
201
|
+
it('should return 0 when already beyond safe zone', () => {
|
|
202
|
+
tracker.track('file1.ts', 7000);
|
|
203
|
+
const remaining = tracker.getSafeRemaining();
|
|
204
|
+
expect(remaining).toBe(0);
|
|
205
|
+
});
|
|
206
|
+
it('should return full safe zone when empty', () => {
|
|
207
|
+
const remaining = tracker.getSafeRemaining();
|
|
208
|
+
const expectedSafeZone = 8192 * 0.8;
|
|
209
|
+
expect(remaining).toBe(expectedSafeZone);
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
describe('all supported models', () => {
|
|
213
|
+
it('should support all declared models', () => {
|
|
214
|
+
const models = [
|
|
215
|
+
'gpt-4',
|
|
216
|
+
'gpt-4-turbo',
|
|
217
|
+
'gpt-3.5-turbo',
|
|
218
|
+
'claude-3-opus',
|
|
219
|
+
'claude-3-sonnet',
|
|
220
|
+
'claude-3-haiku'
|
|
221
|
+
];
|
|
222
|
+
const expectedLimits = {
|
|
223
|
+
'gpt-4': 8192,
|
|
224
|
+
'gpt-4-turbo': 128000,
|
|
225
|
+
'gpt-3.5-turbo': 4096,
|
|
226
|
+
'claude-3-opus': 200000,
|
|
227
|
+
'claude-3-sonnet': 200000,
|
|
228
|
+
'claude-3-haiku': 200000
|
|
229
|
+
};
|
|
230
|
+
models.forEach(model => {
|
|
231
|
+
tracker.setModel(model);
|
|
232
|
+
expect(tracker.getTotalTokens()).toBe(expectedLimits[model]);
|
|
233
|
+
expect(tracker.getModel()).toBe(model);
|
|
234
|
+
});
|
|
235
|
+
});
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
//# sourceMappingURL=context-window-tracker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-window-tracker.test.js","sourceRoot":"","sources":["../../src/core/context-window-tracker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAE/D,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,OAA6B,CAAA;IAEjC,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEzC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEzC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;YAC3B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,wDAAwD;YACxD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,qCAAqC;YACrC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,qCAAqC;YACrC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;YAChD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;YAC3C,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAE3C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAE7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAE1C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAA;YAC3C,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEzC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;YAC5B,qCAAqC;YACrC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;YAC5B,qCAAqC;YACrC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;YACrD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;QACtD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,sCAAsC;YACtC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA,CAAC,gBAAgB;YAChD,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA,CAAC,eAAe;YAC/C,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,GAAG,CAAA,CAAC,OAAO;YACpC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;YACpC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,uBAAuB;YACvB,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,uBAAuB;YACvB,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,IAAI,CAAA,CAAC,OAAO;YACrC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;YACpC,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAA;YAClC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAClC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA,CAAC,qBAAqB;YACrD,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAA;YAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;YAE/B,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAA;YAClC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;YAC1E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;YAC1E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;QAC7E,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,GAAG,CAAC,CAAC,CAAA,CAAC,MAAM;YAC1C,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA,CAAC,cAAc;YAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAA;YAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,EAAE,CAAA;YACpC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;YAC9B,OAAO,CAAC,KAAK,EAAE,CAAA;YAEf,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;YAC5C,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,iCAAiC;YACjC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,GAAG,CAAA;YAC3B,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;YAC/B,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;YAC5C,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;YAC5C,MAAM,gBAAgB,GAAG,IAAI,GAAG,GAAG,CAAA;YACnC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,MAAM,GAA8G;gBACxH,OAAO;gBACP,aAAa;gBACb,eAAe;gBACf,eAAe;gBACf,iBAAiB;gBACjB,gBAAgB;aACjB,CAAA;YAED,MAAM,cAAc,GAA0C;gBAC5D,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,MAAM;gBACrB,eAAe,EAAE,IAAI;gBACrB,eAAe,EAAE,MAAM;gBACvB,iBAAiB,EAAE,MAAM;gBACzB,gBAAgB,EAAE,MAAM;aACzB,CAAA;YAED,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;gBACvB,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAA;gBAC5D,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACxC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { FileStructure, SymbolInfo } from '../types';
|
|
2
|
+
export interface DiffOperation {
|
|
3
|
+
type: 'add' | 'remove' | 'modify';
|
|
4
|
+
oldStartLine: number;
|
|
5
|
+
oldEndLine: number;
|
|
6
|
+
newStartLine: number;
|
|
7
|
+
newEndLine: number;
|
|
8
|
+
count: number;
|
|
9
|
+
}
|
|
10
|
+
export interface DiffResult {
|
|
11
|
+
operations: DiffOperation[];
|
|
12
|
+
totalAddedLines: number;
|
|
13
|
+
totalRemovedLines: number;
|
|
14
|
+
totalModifiedLines: number;
|
|
15
|
+
changePercentage: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Diff Engine using Myers Algorithm (via diff library)
|
|
19
|
+
*
|
|
20
|
+
* Optimizes re-reading of changed files:
|
|
21
|
+
* - Identifies exactly which lines changed
|
|
22
|
+
* - Extracts which symbols were affected
|
|
23
|
+
* - Returns only changed symbols (~20 tokens vs 3000 tokens for full file)
|
|
24
|
+
*
|
|
25
|
+
* Problem: read_diff returns full file causing unnecessary token usage
|
|
26
|
+
* Solution: Compute line-level diff and extract only changed symbols
|
|
27
|
+
*/
|
|
28
|
+
export declare class DiffEngine {
|
|
29
|
+
/**
|
|
30
|
+
* Compute diff between two file versions
|
|
31
|
+
* Uses Myers algorithm (via diff library) for optimal edit distance
|
|
32
|
+
*
|
|
33
|
+
* Performance: O(n+m) where n,m are line counts (vs O(n*m) naive approach)
|
|
34
|
+
*/
|
|
35
|
+
computeDiff(oldContent: string, newContent: string): DiffResult;
|
|
36
|
+
/**
|
|
37
|
+
* Extract symbols that have changed based on diff and symbol locations
|
|
38
|
+
* Filters to only symbols that intersect with changed line ranges
|
|
39
|
+
*/
|
|
40
|
+
extractChangedSymbols(diff: DiffResult, oldStructure: FileStructure, newStructure: FileStructure): SymbolInfo[];
|
|
41
|
+
/**
|
|
42
|
+
* Get which lines were marked as changed in diff
|
|
43
|
+
*/
|
|
44
|
+
private getChangedLineRanges;
|
|
45
|
+
/**
|
|
46
|
+
* Check if two line ranges overlap
|
|
47
|
+
*/
|
|
48
|
+
private rangesOverlap;
|
|
49
|
+
/**
|
|
50
|
+
* Merge adjacent remove + add operations into modify operations
|
|
51
|
+
* This is more semantically accurate than separate add/remove
|
|
52
|
+
*/
|
|
53
|
+
private mergeAdjacentChanges;
|
|
54
|
+
/**
|
|
55
|
+
* Calculate token savings when using delta read vs full read
|
|
56
|
+
* Assumes full file tokens vs changed symbols only
|
|
57
|
+
*/
|
|
58
|
+
estimateTokenSavings(fullFileTokens: number, changedSymbols: SymbolInfo[]): number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Singleton instance for global use
|
|
62
|
+
*/
|
|
63
|
+
export declare const diffEngine: DiffEngine;
|
|
64
|
+
//# sourceMappingURL=diff-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff-engine.d.ts","sourceRoot":"","sources":["../../src/core/diff-engine.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAgB,MAAM,UAAU,CAAA;AAEvE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACjC,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;IACzB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,UAAU;IACrB;;;;;OAKG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU;IAyE/D;;;OAGG;IACH,qBAAqB,CACnB,IAAI,EAAE,UAAU,EAChB,YAAY,EAAE,aAAa,EAC3B,YAAY,EAAE,aAAa,GAC1B,UAAU,EAAE;IA2Bf;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAoB5B;;OAEG;IACH,OAAO,CAAC,aAAa;IAOrB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAkC5B;;;OAGG;IACH,oBAAoB,CAClB,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,UAAU,EAAE,GAC3B,MAAM;CAOV;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,YAAmB,CAAA"}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { diffLines } from 'diff';
|
|
2
|
+
/**
|
|
3
|
+
* Diff Engine using Myers Algorithm (via diff library)
|
|
4
|
+
*
|
|
5
|
+
* Optimizes re-reading of changed files:
|
|
6
|
+
* - Identifies exactly which lines changed
|
|
7
|
+
* - Extracts which symbols were affected
|
|
8
|
+
* - Returns only changed symbols (~20 tokens vs 3000 tokens for full file)
|
|
9
|
+
*
|
|
10
|
+
* Problem: read_diff returns full file causing unnecessary token usage
|
|
11
|
+
* Solution: Compute line-level diff and extract only changed symbols
|
|
12
|
+
*/
|
|
13
|
+
export class DiffEngine {
|
|
14
|
+
/**
|
|
15
|
+
* Compute diff between two file versions
|
|
16
|
+
* Uses Myers algorithm (via diff library) for optimal edit distance
|
|
17
|
+
*
|
|
18
|
+
* Performance: O(n+m) where n,m are line counts (vs O(n*m) naive approach)
|
|
19
|
+
*/
|
|
20
|
+
computeDiff(oldContent, newContent) {
|
|
21
|
+
if (oldContent === newContent) {
|
|
22
|
+
return {
|
|
23
|
+
operations: [],
|
|
24
|
+
totalAddedLines: 0,
|
|
25
|
+
totalRemovedLines: 0,
|
|
26
|
+
totalModifiedLines: 0,
|
|
27
|
+
changePercentage: 0
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const oldLines = oldContent.split('\n');
|
|
31
|
+
const newLines = newContent.split('\n');
|
|
32
|
+
const changes = diffLines(oldContent, newContent);
|
|
33
|
+
const operations = [];
|
|
34
|
+
let oldLineNum = 1;
|
|
35
|
+
let newLineNum = 1;
|
|
36
|
+
let addedCount = 0;
|
|
37
|
+
let removedCount = 0;
|
|
38
|
+
let modifiedCount = 0;
|
|
39
|
+
for (const change of changes) {
|
|
40
|
+
const lineCount = change.count || 0;
|
|
41
|
+
if (change.added) {
|
|
42
|
+
addedCount += lineCount;
|
|
43
|
+
const op = {
|
|
44
|
+
type: 'add',
|
|
45
|
+
oldStartLine: oldLineNum,
|
|
46
|
+
oldEndLine: oldLineNum - 1,
|
|
47
|
+
newStartLine: newLineNum,
|
|
48
|
+
newEndLine: newLineNum + lineCount - 1,
|
|
49
|
+
count: lineCount
|
|
50
|
+
};
|
|
51
|
+
operations.push(op);
|
|
52
|
+
newLineNum += lineCount;
|
|
53
|
+
}
|
|
54
|
+
else if (change.removed) {
|
|
55
|
+
removedCount += lineCount;
|
|
56
|
+
const op = {
|
|
57
|
+
type: 'remove',
|
|
58
|
+
oldStartLine: oldLineNum,
|
|
59
|
+
oldEndLine: oldLineNum + lineCount - 1,
|
|
60
|
+
newStartLine: newLineNum,
|
|
61
|
+
newEndLine: newLineNum - 1,
|
|
62
|
+
count: lineCount
|
|
63
|
+
};
|
|
64
|
+
operations.push(op);
|
|
65
|
+
oldLineNum += lineCount;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// unchanged lines
|
|
69
|
+
oldLineNum += lineCount;
|
|
70
|
+
newLineNum += lineCount;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Detect modifications (adjacent remove + add = modification)
|
|
74
|
+
const mergedOps = this.mergeAdjacentChanges(operations);
|
|
75
|
+
const modifiedOps = mergedOps.filter(op => op.type === 'modify');
|
|
76
|
+
modifiedCount = modifiedOps.reduce((sum, op) => sum + op.count, 0);
|
|
77
|
+
const totalLines = Math.max(oldLines.length, newLines.length);
|
|
78
|
+
const changePercentage = totalLines > 0 ? ((addedCount + removedCount) / totalLines) * 100 : 0;
|
|
79
|
+
return {
|
|
80
|
+
operations: mergedOps,
|
|
81
|
+
totalAddedLines: addedCount,
|
|
82
|
+
totalRemovedLines: removedCount,
|
|
83
|
+
totalModifiedLines: modifiedCount,
|
|
84
|
+
changePercentage
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Extract symbols that have changed based on diff and symbol locations
|
|
89
|
+
* Filters to only symbols that intersect with changed line ranges
|
|
90
|
+
*/
|
|
91
|
+
extractChangedSymbols(diff, oldStructure, newStructure) {
|
|
92
|
+
const changedLines = this.getChangedLineRanges(diff);
|
|
93
|
+
// Find symbols in new structure that overlap with changed lines
|
|
94
|
+
const changedSymbols = [];
|
|
95
|
+
const seenNames = new Set();
|
|
96
|
+
for (const symbol of newStructure.symbols) {
|
|
97
|
+
const symbolRange = {
|
|
98
|
+
start: symbol.location.startLine,
|
|
99
|
+
end: symbol.location.endLine
|
|
100
|
+
};
|
|
101
|
+
// Check if symbol overlaps with any changed lines
|
|
102
|
+
const isChanged = changedLines.some(range => this.rangesOverlap(symbolRange, range));
|
|
103
|
+
if (isChanged && !seenNames.has(symbol.name)) {
|
|
104
|
+
changedSymbols.push(symbol);
|
|
105
|
+
seenNames.add(symbol.name);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return changedSymbols;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Get which lines were marked as changed in diff
|
|
112
|
+
*/
|
|
113
|
+
getChangedLineRanges(diff) {
|
|
114
|
+
const ranges = [];
|
|
115
|
+
for (const op of diff.operations) {
|
|
116
|
+
if (op.type === 'add' || op.type === 'modify') {
|
|
117
|
+
ranges.push({
|
|
118
|
+
start: op.newStartLine,
|
|
119
|
+
end: op.newEndLine
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
else if (op.type === 'remove') {
|
|
123
|
+
ranges.push({
|
|
124
|
+
start: op.oldStartLine,
|
|
125
|
+
end: op.oldEndLine
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return ranges;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Check if two line ranges overlap
|
|
133
|
+
*/
|
|
134
|
+
rangesOverlap(range1, range2) {
|
|
135
|
+
return !(range1.end < range2.start || range2.end < range1.start);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Merge adjacent remove + add operations into modify operations
|
|
139
|
+
* This is more semantically accurate than separate add/remove
|
|
140
|
+
*/
|
|
141
|
+
mergeAdjacentChanges(operations) {
|
|
142
|
+
if (operations.length <= 1)
|
|
143
|
+
return operations;
|
|
144
|
+
const merged = [];
|
|
145
|
+
for (let i = 0; i < operations.length; i++) {
|
|
146
|
+
const current = operations[i];
|
|
147
|
+
// Check if next operation is adjacent (remove followed by add)
|
|
148
|
+
if (i + 1 < operations.length &&
|
|
149
|
+
current.type === 'remove' &&
|
|
150
|
+
operations[i + 1].type === 'add' &&
|
|
151
|
+
current.oldEndLine >= operations[i + 1].newStartLine - 2) {
|
|
152
|
+
const next = operations[i + 1];
|
|
153
|
+
const modify = {
|
|
154
|
+
type: 'modify',
|
|
155
|
+
oldStartLine: current.oldStartLine,
|
|
156
|
+
oldEndLine: current.oldEndLine,
|
|
157
|
+
newStartLine: next.newStartLine,
|
|
158
|
+
newEndLine: next.newEndLine,
|
|
159
|
+
count: Math.max(current.count, next.count)
|
|
160
|
+
};
|
|
161
|
+
merged.push(modify);
|
|
162
|
+
i++; // Skip next operation since we merged it
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
merged.push(current);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return merged;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Calculate token savings when using delta read vs full read
|
|
172
|
+
* Assumes full file tokens vs changed symbols only
|
|
173
|
+
*/
|
|
174
|
+
estimateTokenSavings(fullFileTokens, changedSymbols) {
|
|
175
|
+
// Each symbol signature is roughly 20-50 tokens
|
|
176
|
+
const avgTokensPerSymbol = 35;
|
|
177
|
+
const changedTokens = changedSymbols.length * avgTokensPerSymbol;
|
|
178
|
+
return Math.max(0, fullFileTokens - changedTokens);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Singleton instance for global use
|
|
183
|
+
*/
|
|
184
|
+
export const diffEngine = new DiffEngine();
|
|
185
|
+
//# sourceMappingURL=diff-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff-engine.js","sourceRoot":"","sources":["../../src/core/diff-engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAA;AAoBhC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,UAAU;IACrB;;;;;OAKG;IACH,WAAW,CAAC,UAAkB,EAAE,UAAkB;QAChD,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;YAC9B,OAAO;gBACL,UAAU,EAAE,EAAE;gBACd,eAAe,EAAE,CAAC;gBAClB,iBAAiB,EAAE,CAAC;gBACpB,kBAAkB,EAAE,CAAC;gBACrB,gBAAgB,EAAE,CAAC;aACpB,CAAA;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QAEjD,MAAM,UAAU,GAAoB,EAAE,CAAA;QACtC,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,aAAa,GAAG,CAAC,CAAA;QAErB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAA;YAEnC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,UAAU,IAAI,SAAS,CAAA;gBACvB,MAAM,EAAE,GAAkB;oBACxB,IAAI,EAAE,KAAK;oBACX,YAAY,EAAE,UAAU;oBACxB,UAAU,EAAE,UAAU,GAAG,CAAC;oBAC1B,YAAY,EAAE,UAAU;oBACxB,UAAU,EAAE,UAAU,GAAG,SAAS,GAAG,CAAC;oBACtC,KAAK,EAAE,SAAS;iBACjB,CAAA;gBACD,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACnB,UAAU,IAAI,SAAS,CAAA;YACzB,CAAC;iBAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1B,YAAY,IAAI,SAAS,CAAA;gBACzB,MAAM,EAAE,GAAkB;oBACxB,IAAI,EAAE,QAAQ;oBACd,YAAY,EAAE,UAAU;oBACxB,UAAU,EAAE,UAAU,GAAG,SAAS,GAAG,CAAC;oBACtC,YAAY,EAAE,UAAU;oBACxB,UAAU,EAAE,UAAU,GAAG,CAAC;oBAC1B,KAAK,EAAE,SAAS;iBACjB,CAAA;gBACD,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACnB,UAAU,IAAI,SAAS,CAAA;YACzB,CAAC;iBAAM,CAAC;gBACN,kBAAkB;gBAClB,UAAU,IAAI,SAAS,CAAA;gBACvB,UAAU,IAAI,SAAS,CAAA;YACzB,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAA;QACvD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAA;QAChE,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAElE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAE9F,OAAO;YACL,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,UAAU;YAC3B,iBAAiB,EAAE,YAAY;YAC/B,kBAAkB,EAAE,aAAa;YACjC,gBAAgB;SACjB,CAAA;IACH,CAAC;IAED;;;OAGG;IACH,qBAAqB,CACnB,IAAgB,EAChB,YAA2B,EAC3B,YAA2B;QAE3B,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAA;QAEpD,gEAAgE;QAChE,MAAM,cAAc,GAAiB,EAAE,CAAA;QACvC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QAEnC,KAAK,MAAM,MAAM,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,WAAW,GAAG;gBAClB,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;gBAChC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;aAC7B,CAAA;YAED,kDAAkD;YAClD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CACjC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,KAAK,CAAC,CAChD,CAAA;YAED,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC3B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,cAAc,CAAA;IACvB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,IAAgB;QAC3C,MAAM,MAAM,GAA0C,EAAE,CAAA;QAExD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,EAAE,CAAC,YAAY;oBACtB,GAAG,EAAE,EAAE,CAAC,UAAU;iBACnB,CAAC,CAAA;YACJ,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,EAAE,CAAC,YAAY;oBACtB,GAAG,EAAE,EAAE,CAAC,UAAU;iBACnB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,MAAsC,EACtC,MAAsC;QAEtC,OAAO,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;IAClE,CAAC;IAED;;;OAGG;IACK,oBAAoB,CAAC,UAA2B;QACtD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,UAAU,CAAA;QAE7C,MAAM,MAAM,GAAoB,EAAE,CAAA;QAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE7B,+DAA+D;YAC/D,IACE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM;gBACzB,OAAO,CAAC,IAAI,KAAK,QAAQ;gBACzB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK;gBAChC,OAAO,CAAC,UAAU,IAAI,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,EACxD,CAAC;gBACD,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9B,MAAM,MAAM,GAAkB;oBAC5B,IAAI,EAAE,QAAQ;oBACd,YAAY,EAAE,OAAO,CAAC,YAAY;oBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;iBAC3C,CAAA;gBACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACnB,CAAC,EAAE,CAAA,CAAC,yCAAyC;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACtB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAClB,cAAsB,EACtB,cAA4B;QAE5B,gDAAgD;QAChD,MAAM,kBAAkB,GAAG,EAAE,CAAA;QAC7B,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,GAAG,kBAAkB,CAAA;QAEhE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,GAAG,aAAa,CAAC,CAAA;IACpD,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff-engine.test.d.ts","sourceRoot":"","sources":["../../src/core/diff-engine.test.ts"],"names":[],"mappings":""}
|