ultracode 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/analysis-tool-handlers-H2RXLDPX.js +817 -0
- package/dist/chunks/analysis-tool-handlers-RJZAR6VT.js +817 -0
- package/dist/chunks/analysis-tool-handlers-Z2RF24T7.js +13 -0
- package/dist/chunks/autodoc-tool-handlers-CV5JEQUA.js +1112 -0
- package/dist/chunks/autodoc-tool-handlers-EHTNCH6I.js +1112 -0
- package/dist/chunks/autodoc-tool-handlers-MECXQJ2K.js +138 -0
- package/dist/chunks/chaos-CO7TOBOJ.js +18 -0
- package/dist/chunks/chaos-VM2PXERO.js +1573 -0
- package/dist/chunks/chaos-W3XRVJ7K.js +1564 -0
- package/dist/chunks/chunk-6K37BWK5.js +439 -0
- package/dist/chunks/chunk-EALTCYHZ.js +10 -0
- package/dist/chunks/chunk-FTBE7VMY.js +316 -0
- package/dist/chunks/chunk-KBW6LRQP.js +322 -0
- package/dist/chunks/chunk-NKUHX4CU.js +5 -0
- package/dist/chunks/chunk-NZFF4DQ4.js +3179 -0
- package/dist/chunks/chunk-RGP5UVQ7.js +3179 -0
- package/dist/chunks/chunk-RMZXFGQZ.js +322 -0
- package/dist/chunks/chunk-UG44F23Y.js +316 -0
- package/dist/chunks/chunk-V2SCB5H5.js +4403 -0
- package/dist/chunks/chunk-V6JAQNM3.js +1 -0
- package/dist/chunks/chunk-XFGXM4CR.js +4403 -0
- package/dist/chunks/dev-agent-JVIGBMHQ.js +1 -0
- package/dist/chunks/dev-agent-TRVP5U6N.js +1624 -0
- package/dist/chunks/dev-agent-Y5G5WKQ4.js +1624 -0
- package/dist/chunks/graph-storage-factory-AYZ57YSL.js +13 -0
- package/dist/chunks/graph-storage-factory-GTAIJEI5.js +1 -0
- package/dist/chunks/graph-storage-factory-T2WO5QVG.js +13 -0
- package/dist/chunks/incremental-updater-KDIQGAUU.js +14 -0
- package/dist/chunks/incremental-updater-OJRSTO3Q.js +1 -0
- package/dist/chunks/incremental-updater-SBEBH7KF.js +14 -0
- package/dist/chunks/indexer-agent-H3QIEL3Z.js +21 -0
- package/dist/chunks/indexer-agent-KHF5JMV7.js +21 -0
- package/dist/chunks/indexer-agent-SHJD6Z77.js +1 -0
- package/dist/chunks/indexing-pipeline-J6Z4BHKF.js +1 -0
- package/dist/chunks/indexing-pipeline-OY3337QN.js +249 -0
- package/dist/chunks/indexing-pipeline-WCXIDMAP.js +249 -0
- package/dist/chunks/merge-agent-LSUBDJB2.js +2481 -0
- package/dist/chunks/merge-agent-MJEW3HWU.js +2481 -0
- package/dist/chunks/merge-agent-O45OXF33.js +11 -0
- package/dist/chunks/merge-tool-handlers-BDSVNQVZ.js +277 -0
- package/dist/chunks/merge-tool-handlers-HP7DRBXJ.js +1 -0
- package/dist/chunks/merge-tool-handlers-RUJAKE3D.js +277 -0
- package/dist/chunks/pattern-tool-handlers-L62W3CXR.js +1549 -0
- package/dist/chunks/pattern-tool-handlers-SAHX2CVW.js +13 -0
- package/dist/chunks/query-agent-3TWDFIMT.js +191 -0
- package/dist/chunks/query-agent-HXQ3BMMF.js +191 -0
- package/dist/chunks/query-agent-USMC2GNG.js +1 -0
- package/dist/chunks/semantic-agent-MQCAWIAB.js +6381 -0
- package/dist/chunks/semantic-agent-NDGR3NAK.js +6381 -0
- package/dist/chunks/semantic-agent-S4ZL6GZC.js +137 -0
- package/dist/index.js +17 -17
- package/dist/roslyn-addon/.build-hash +1 -1
- package/dist/roslyn-addon/ILGPU.Algorithms.dll +0 -0
- package/dist/roslyn-addon/ILGPU.dll +0 -0
- package/dist/roslyn-addon/UltraCode.CSharp.deps.json +35 -0
- package/dist/roslyn-addon/UltraCode.CSharp.dll +0 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1624 @@
|
|
|
1
|
+
import { ResourceAdjustmentMixin } from './chunk-C67F6XKY.js';
|
|
2
|
+
import { IndexerAgent } from './chunk-RGP5UVQ7.js';
|
|
3
|
+
import { ParserAgent } from './chunk-W6VROAWQ.js';
|
|
4
|
+
import './chunk-45GHQ2BB.js';
|
|
5
|
+
import './chunk-75SF43YH.js';
|
|
6
|
+
import { init_file_extensions, collectFilesAsync, isCodeExtension, isDataExtension } from './chunk-4GYRHAKO.js';
|
|
7
|
+
export { ALL_SUPPORTED_EXTENSIONS } from './chunk-4GYRHAKO.js';
|
|
8
|
+
import { buildWorkerEmbeddingConfig } from './chunk-5B7AKUHU.js';
|
|
9
|
+
import { createHeuristicEntities } from './chunk-UMSHK3G6.js';
|
|
10
|
+
import { getCurrentIndexingDirectory, setCurrentIndexingDirectory } from './chunk-YRTAJKB7.js';
|
|
11
|
+
import './chunk-KMXE2SJJ.js';
|
|
12
|
+
import { knowledgeBus } from './chunk-JPI46FLQ.js';
|
|
13
|
+
import { BaseAgent } from './chunk-IGUCJL3R.js';
|
|
14
|
+
import { findSolutionFile, ensureRoslynStarted } from './chunk-VFR33VOA.js';
|
|
15
|
+
import { setGlobalProjectContext, getGraphStorage } from './chunk-XFGXM4CR.js';
|
|
16
|
+
import './chunk-HNDYLCWI.js';
|
|
17
|
+
import './chunk-CTXFPNDA.js';
|
|
18
|
+
import './chunk-TJSOOFXA.js';
|
|
19
|
+
import './chunk-AIZUHUK6.js';
|
|
20
|
+
import './chunk-L2X4HRXI.js';
|
|
21
|
+
import { init_storage_paths, getCurrentGitBranch } from './chunk-ZD54CMKT.js';
|
|
22
|
+
import { init_fast_hash, hashText } from './chunk-HEMJHRHZ.js';
|
|
23
|
+
import './chunk-5WKPA33T.js';
|
|
24
|
+
import './chunk-TUSYCUU6.js';
|
|
25
|
+
import { tryGarbageCollect } from './chunk-IMQ6WSJV.js';
|
|
26
|
+
import './chunk-GMVGCSNU.js';
|
|
27
|
+
import { init_yaml_config, ConfigLoader, getConfig } from './chunk-XV6GNSLC.js';
|
|
28
|
+
import './chunk-F7CKCMXI.js';
|
|
29
|
+
import './chunk-BMHPPH2B.js';
|
|
30
|
+
import { init_logging, log } from './chunk-VCCBEJQ5.js';
|
|
31
|
+
import './chunk-UN27MREV.js';
|
|
32
|
+
import './chunk-NAQKA54E.js';
|
|
33
|
+
import { statSync, readFileSync, existsSync } from 'fs';
|
|
34
|
+
import { cpus } from 'os';
|
|
35
|
+
import { extname, basename, dirname } from 'path';
|
|
36
|
+
|
|
37
|
+
init_yaml_config();
|
|
38
|
+
init_logging();
|
|
39
|
+
init_storage_paths();
|
|
40
|
+
init_fast_hash();
|
|
41
|
+
|
|
42
|
+
// src/agents/dev/incremental-indexer.ts
|
|
43
|
+
init_logging();
|
|
44
|
+
var SUPPORTED_EXTENSIONS = [
|
|
45
|
+
".ts",
|
|
46
|
+
".tsx",
|
|
47
|
+
".js",
|
|
48
|
+
".jsx",
|
|
49
|
+
".mjs",
|
|
50
|
+
".cjs",
|
|
51
|
+
".py",
|
|
52
|
+
".go",
|
|
53
|
+
".rs",
|
|
54
|
+
".java",
|
|
55
|
+
".kt",
|
|
56
|
+
".kts",
|
|
57
|
+
".cs",
|
|
58
|
+
".csx",
|
|
59
|
+
".c",
|
|
60
|
+
".cpp",
|
|
61
|
+
".cc",
|
|
62
|
+
".cxx",
|
|
63
|
+
".h",
|
|
64
|
+
".hpp",
|
|
65
|
+
".swift",
|
|
66
|
+
".tpl"
|
|
67
|
+
];
|
|
68
|
+
function separateFilesBySupport(files) {
|
|
69
|
+
const supportedFiles = [];
|
|
70
|
+
const otherFiles = [];
|
|
71
|
+
for (const file of files) {
|
|
72
|
+
const ext = extname(file).toLowerCase();
|
|
73
|
+
if (SUPPORTED_EXTENSIONS.includes(ext)) {
|
|
74
|
+
supportedFiles.push(file);
|
|
75
|
+
} else {
|
|
76
|
+
otherFiles.push(file);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
log.i("DEVAGENT", "reindex_breakdown", {
|
|
80
|
+
supported: supportedFiles.length,
|
|
81
|
+
heuristic: otherFiles.length,
|
|
82
|
+
supportedSample: supportedFiles.slice(0, 3).map((f) => {
|
|
83
|
+
const parts = f.split(/[\\/]/);
|
|
84
|
+
return parts[parts.length - 1];
|
|
85
|
+
})
|
|
86
|
+
});
|
|
87
|
+
return {
|
|
88
|
+
supportedFiles,
|
|
89
|
+
otherFiles
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// src/agents/dev/index.ts
|
|
94
|
+
init_file_extensions();
|
|
95
|
+
|
|
96
|
+
// src/agents/dev-agent.ts
|
|
97
|
+
var yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
98
|
+
function getDevAgentConfig() {
|
|
99
|
+
const config = getConfig();
|
|
100
|
+
return {
|
|
101
|
+
maxConcurrency: config.devAgent?.maxConcurrency ?? 3,
|
|
102
|
+
memoryLimit: config.devAgent?.memoryLimit ?? 256,
|
|
103
|
+
priority: config.devAgent?.priority ?? 7
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
var DevAgent = class extends BaseAgent {
|
|
107
|
+
parserAgent = null;
|
|
108
|
+
indexerAgent = null;
|
|
109
|
+
indexBatchSize;
|
|
110
|
+
defaultBatchSize;
|
|
111
|
+
defaultMaxConcurrency;
|
|
112
|
+
defaultMemoryLimit;
|
|
113
|
+
resourceMixin = new ResourceAdjustmentMixin();
|
|
114
|
+
// Mutex: serialize indexing tasks to prevent concurrent setProjectContext corruption
|
|
115
|
+
indexingLock = Promise.resolve();
|
|
116
|
+
constructor(_agentId) {
|
|
117
|
+
const agentConfig = getDevAgentConfig();
|
|
118
|
+
super("dev" /* DEV */, {
|
|
119
|
+
maxConcurrency: agentConfig.maxConcurrency,
|
|
120
|
+
memoryLimit: agentConfig.memoryLimit,
|
|
121
|
+
priority: agentConfig.priority
|
|
122
|
+
});
|
|
123
|
+
this.defaultMaxConcurrency = this.capabilities.maxConcurrency;
|
|
124
|
+
this.defaultMemoryLimit = this.capabilities.memoryLimit;
|
|
125
|
+
this.defaultBatchSize = 100;
|
|
126
|
+
this.indexBatchSize = this.defaultBatchSize;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Get the IndexerAgent instance.
|
|
130
|
+
* Used by metrics handlers to access watcher status.
|
|
131
|
+
*/
|
|
132
|
+
getIndexerAgent() {
|
|
133
|
+
return this.indexerAgent;
|
|
134
|
+
}
|
|
135
|
+
async onInitialize() {
|
|
136
|
+
try {
|
|
137
|
+
const configLoader = ConfigLoader.getInstance();
|
|
138
|
+
this.defaultBatchSize = configLoader.getDevIndexBatchSize();
|
|
139
|
+
this.indexBatchSize = this.defaultBatchSize;
|
|
140
|
+
const useParser = configLoader.shouldUseParser();
|
|
141
|
+
if (useParser) {
|
|
142
|
+
try {
|
|
143
|
+
this.parserAgent = new ParserAgent();
|
|
144
|
+
await this.parserAgent.initialize();
|
|
145
|
+
log.i("DEVAGENT", "parser_init_ok");
|
|
146
|
+
} catch (e) {
|
|
147
|
+
log.w("DEVAGENT", "parser_unavail", { err: String(e) });
|
|
148
|
+
this.parserAgent = null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const currentDir = getCurrentIndexingDirectory() || process.cwd();
|
|
152
|
+
log.i("DEVAGENT", "indexer_ctx", { dir: currentDir });
|
|
153
|
+
this.indexerAgent = new IndexerAgent();
|
|
154
|
+
await this.indexerAgent.initialize();
|
|
155
|
+
const branch = getCurrentGitBranch(currentDir);
|
|
156
|
+
setGlobalProjectContext(currentDir, branch);
|
|
157
|
+
log.i("DEVAGENT", "global_ctx_set", { dir: currentDir, branch });
|
|
158
|
+
log.i("DEVAGENT", "indexer_init_ok");
|
|
159
|
+
this.tryEagerRoslynStart(currentDir);
|
|
160
|
+
} catch (error) {
|
|
161
|
+
log.e("DEVAGENT", "subagent_init_fail", { err: String(error) });
|
|
162
|
+
throw error;
|
|
163
|
+
}
|
|
164
|
+
knowledgeBus.subscribe(this.id, "task:implementation", async (entry) => {
|
|
165
|
+
const data = entry.data;
|
|
166
|
+
if (data.targetAgent === "dev-agent") {
|
|
167
|
+
const task = {
|
|
168
|
+
id: data.taskId,
|
|
169
|
+
type: "implementation",
|
|
170
|
+
priority: data.priority || 5,
|
|
171
|
+
payload: data,
|
|
172
|
+
createdAt: Date.now()
|
|
173
|
+
};
|
|
174
|
+
await this.process(task);
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
knowledgeBus.subscribe(this.id, "resources:adjusted", (entry) => this.handleResourceAdjustment(entry));
|
|
178
|
+
knowledgeBus.subscribe(this.id, "indexer:files:changed", async (entry) => {
|
|
179
|
+
const data = entry.data;
|
|
180
|
+
if (data.files && data.files.length > 0 && data.source?.startsWith("git-watcher")) {
|
|
181
|
+
log.d("DEVAGENT", "file_change_evt", { cnt: data.files.length, src: data.source });
|
|
182
|
+
await this.handleIncrementalReindex(data.files, data.repositoryPath);
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
log.i("DEVAGENT", "init_ready");
|
|
186
|
+
}
|
|
187
|
+
canProcessTask(task) {
|
|
188
|
+
return task.type === "index" || task.type === "implementation" || task.type === "refactor" || task.type === "dev" || task.type === "parse";
|
|
189
|
+
}
|
|
190
|
+
async handleMessage(message) {
|
|
191
|
+
log.d("DEVAGENT", "recv_msg", { from: message.from, type: message.type });
|
|
192
|
+
}
|
|
193
|
+
async processTask(task) {
|
|
194
|
+
log.d("DEVAGENT", "proc_task", { id: task.id, type: task.type });
|
|
195
|
+
try {
|
|
196
|
+
switch (task.type) {
|
|
197
|
+
case "index":
|
|
198
|
+
return await this.handleIndexTask(task);
|
|
199
|
+
case "implementation":
|
|
200
|
+
return await this.handleImplementationTask(task);
|
|
201
|
+
case "refactor":
|
|
202
|
+
return await this.handleRefactorTask(task);
|
|
203
|
+
case "parse":
|
|
204
|
+
return await this.handleParseTask(task);
|
|
205
|
+
default:
|
|
206
|
+
return await this.delegateTask(task);
|
|
207
|
+
}
|
|
208
|
+
} catch (error) {
|
|
209
|
+
log.e("DEVAGENT", "task_error", { err: String(error) });
|
|
210
|
+
throw error;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async handleIndexTask(task) {
|
|
214
|
+
const prevLock = this.indexingLock;
|
|
215
|
+
let releaseLock;
|
|
216
|
+
this.indexingLock = new Promise((resolve) => {
|
|
217
|
+
releaseLock = resolve;
|
|
218
|
+
});
|
|
219
|
+
await prevLock;
|
|
220
|
+
const payload = task.payload;
|
|
221
|
+
log.i("DEVAGENT", "index_start", { dir: payload.directory });
|
|
222
|
+
try {
|
|
223
|
+
if (!this.indexerAgent) {
|
|
224
|
+
throw new Error("Indexer agent not initialized");
|
|
225
|
+
}
|
|
226
|
+
const indexBranch = getCurrentGitBranch(payload.directory);
|
|
227
|
+
setGlobalProjectContext(payload.directory, indexBranch);
|
|
228
|
+
this.indexerAgent.setProjectContext(payload.directory);
|
|
229
|
+
setCurrentIndexingDirectory(payload.directory);
|
|
230
|
+
log.d("DEVAGENT", "set_ctx", { dir: payload.directory, branch: indexBranch });
|
|
231
|
+
const result = {
|
|
232
|
+
status: "started",
|
|
233
|
+
directory: payload.directory,
|
|
234
|
+
incremental: payload.incremental || false,
|
|
235
|
+
excludePatterns: payload.excludePatterns || [],
|
|
236
|
+
batchMode: payload.batchMode || false,
|
|
237
|
+
timestamp: Date.now(),
|
|
238
|
+
filesProcessed: 0,
|
|
239
|
+
entitiesExtracted: 0,
|
|
240
|
+
relationshipsCreated: 0
|
|
241
|
+
};
|
|
242
|
+
knowledgeBus.publish("indexing:started", result, this.id);
|
|
243
|
+
const indexingResult = await this.performRealIndexing(payload);
|
|
244
|
+
return {
|
|
245
|
+
...result,
|
|
246
|
+
...indexingResult,
|
|
247
|
+
status: "completed",
|
|
248
|
+
message: `Real indexing completed for ${payload.directory}`
|
|
249
|
+
};
|
|
250
|
+
} finally {
|
|
251
|
+
releaseLock();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
async handleImplementationTask(task) {
|
|
255
|
+
const payload = task.payload;
|
|
256
|
+
log.i("DEVAGENT", "impl_start", { desc: payload.description || "task" });
|
|
257
|
+
return {
|
|
258
|
+
status: "completed",
|
|
259
|
+
taskId: task.id,
|
|
260
|
+
implementation: {
|
|
261
|
+
description: payload.description,
|
|
262
|
+
targetAgent: "dev-agent",
|
|
263
|
+
completed: true,
|
|
264
|
+
timestamp: Date.now()
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
async handleRefactorTask(task) {
|
|
269
|
+
const payload = task.payload;
|
|
270
|
+
log.i("DEVAGENT", "refactor_start", { target: payload.target || "code" });
|
|
271
|
+
return {
|
|
272
|
+
status: "completed",
|
|
273
|
+
taskId: task.id,
|
|
274
|
+
refactoring: {
|
|
275
|
+
target: payload.target,
|
|
276
|
+
suggestions: [],
|
|
277
|
+
completed: true,
|
|
278
|
+
timestamp: Date.now()
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Handle parse task - parse a single file and return entities
|
|
284
|
+
*/
|
|
285
|
+
async handleParseTask(task) {
|
|
286
|
+
const payload = task.payload;
|
|
287
|
+
const filePath = payload.filePath;
|
|
288
|
+
if (!filePath) {
|
|
289
|
+
throw new Error("Parse task requires filePath in payload");
|
|
290
|
+
}
|
|
291
|
+
log.d("DEVAGENT", "parse_file", { file: filePath });
|
|
292
|
+
if (!this.parserAgent) {
|
|
293
|
+
log.w("DEVAGENT", "parser_unavail_empty");
|
|
294
|
+
return {
|
|
295
|
+
filePath,
|
|
296
|
+
entities: [],
|
|
297
|
+
relationships: [],
|
|
298
|
+
error: "Parser not initialized",
|
|
299
|
+
timestamp: Date.now()
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
try {
|
|
303
|
+
const result = await this.parserAgent.parseFile(filePath, {});
|
|
304
|
+
log.d("DEVAGENT", "parsed_ok", { file: filePath, cnt: result.entities?.length || 0 });
|
|
305
|
+
return {
|
|
306
|
+
filePath,
|
|
307
|
+
entities: result.entities || [],
|
|
308
|
+
relationships: result.relationships || [],
|
|
309
|
+
timestamp: Date.now()
|
|
310
|
+
};
|
|
311
|
+
} catch (error) {
|
|
312
|
+
log.e("DEVAGENT", "parse_err", { file: filePath, err: String(error) });
|
|
313
|
+
return {
|
|
314
|
+
filePath,
|
|
315
|
+
entities: [],
|
|
316
|
+
relationships: [],
|
|
317
|
+
error: String(error),
|
|
318
|
+
timestamp: Date.now()
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
async delegateTask(task) {
|
|
323
|
+
log.d("DEVAGENT", "delegate_task", { id: task.id });
|
|
324
|
+
return {
|
|
325
|
+
status: "delegated",
|
|
326
|
+
taskId: task.id,
|
|
327
|
+
message: `Task ${task.id} delegated for processing`,
|
|
328
|
+
timestamp: Date.now()
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Compare mtime of files against stored index to compute changed/new/deleted sets.
|
|
333
|
+
* Deletes entities and file info for changed/deleted files in preparation for re-indexing.
|
|
334
|
+
*/
|
|
335
|
+
async prepareIncrementalDiff(allFiles, totalCollected) {
|
|
336
|
+
const storage = await getGraphStorage();
|
|
337
|
+
const indexedFiles = await storage.getAllIndexedFiles();
|
|
338
|
+
const deletedEntityIds = [];
|
|
339
|
+
if (indexedFiles.size === 0) {
|
|
340
|
+
return { filesToProcess: allFiles, deletedEntityIds, earlyReturn: null };
|
|
341
|
+
}
|
|
342
|
+
const changedFiles = [];
|
|
343
|
+
const newFiles = [];
|
|
344
|
+
const deletedFiles = [];
|
|
345
|
+
for (const file of allFiles) {
|
|
346
|
+
const normalizedPath = file.replace(/\\/g, "/");
|
|
347
|
+
const lastIndexed = indexedFiles.get(normalizedPath);
|
|
348
|
+
if (lastIndexed === void 0) {
|
|
349
|
+
newFiles.push(file);
|
|
350
|
+
} else {
|
|
351
|
+
try {
|
|
352
|
+
const stats = statSync(file);
|
|
353
|
+
const mtime = stats.mtimeMs;
|
|
354
|
+
if (mtime > lastIndexed) {
|
|
355
|
+
changedFiles.push(file);
|
|
356
|
+
}
|
|
357
|
+
} catch {
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
const currentFilesSet = new Set(allFiles.map((f) => f.replace(/\\/g, "/")));
|
|
362
|
+
for (const [indexedPath] of indexedFiles) {
|
|
363
|
+
if (!currentFilesSet.has(indexedPath)) {
|
|
364
|
+
deletedFiles.push(indexedPath);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
if (deletedFiles.length > 0) {
|
|
368
|
+
log.i("DEVAGENT", "Invalidating deleted files", { deleted: deletedFiles.length });
|
|
369
|
+
for (const file of deletedFiles) {
|
|
370
|
+
try {
|
|
371
|
+
const ids = await storage.deleteEntitiesByFilePath(file);
|
|
372
|
+
deletedEntityIds.push(...ids);
|
|
373
|
+
await storage.deleteFileInfo(file);
|
|
374
|
+
} catch (error) {
|
|
375
|
+
log.w("DEVAGENT", "Failed to invalidate file", {
|
|
376
|
+
file,
|
|
377
|
+
error: error.message
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
for (const file of changedFiles) {
|
|
383
|
+
try {
|
|
384
|
+
await storage.deleteFileInfo(file);
|
|
385
|
+
} catch {
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
const filesToProcess = [...changedFiles, ...newFiles];
|
|
389
|
+
log.i("DEVAGENT", "Smart incremental", {
|
|
390
|
+
total: totalCollected,
|
|
391
|
+
changed: changedFiles.length,
|
|
392
|
+
new: newFiles.length,
|
|
393
|
+
deleted: deletedFiles.length,
|
|
394
|
+
toProcess: filesToProcess.length,
|
|
395
|
+
entitiesDeleted: deletedEntityIds.length
|
|
396
|
+
});
|
|
397
|
+
if (filesToProcess.length === 0) {
|
|
398
|
+
return {
|
|
399
|
+
filesToProcess: [],
|
|
400
|
+
deletedEntityIds,
|
|
401
|
+
earlyReturn: {
|
|
402
|
+
filesProcessed: 0,
|
|
403
|
+
entitiesExtracted: 0,
|
|
404
|
+
relationshipsCreated: 0,
|
|
405
|
+
incrementalStats: {
|
|
406
|
+
changedFiles: 0,
|
|
407
|
+
newFiles: 0,
|
|
408
|
+
deletedFiles: deletedFiles.length,
|
|
409
|
+
skippedFiles: totalCollected,
|
|
410
|
+
deletedEntities: deletedEntityIds.length
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
return { filesToProcess, deletedEntityIds, earlyReturn: null };
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Initialize the embedding pipeline: select provider (TEI/OVMS/llamacpp/vLLM),
|
|
419
|
+
* start llama-server if needed, create centralized EmbeddingGenerator.
|
|
420
|
+
*/
|
|
421
|
+
async initEmbeddingPipeline(embeddingConfig, _payload) {
|
|
422
|
+
if (embeddingConfig.provider === "llamacpp") {
|
|
423
|
+
const { llamacppEmbeddingManager } = await import('./llamacpp-server-manager-HTLX6C5P.js');
|
|
424
|
+
const { loadSemanticConfig, getDataDir } = await import('./config-paths-Y2IHHJQF.js');
|
|
425
|
+
const { existsSync: existsSync2, readdirSync } = await import('fs');
|
|
426
|
+
const { join } = await import('path');
|
|
427
|
+
const semanticConfig = loadSemanticConfig();
|
|
428
|
+
const llamacppConfig = semanticConfig?.embedding?.llamacpp;
|
|
429
|
+
if (llamacppConfig && !llamacppEmbeddingManager.getState().isRunning) {
|
|
430
|
+
const dataDir = getDataDir();
|
|
431
|
+
const searchPaths = [
|
|
432
|
+
join(dataDir, "hf-cache", "multilingual-e5-base-Q8_0.gguf"),
|
|
433
|
+
join(dataDir, "llamacpp", "models", "multilingual-e5-base-Q8_0.gguf"),
|
|
434
|
+
join(dataDir, "models", "multilingual-e5-base-Q8_0.gguf")
|
|
435
|
+
];
|
|
436
|
+
let modelPath = null;
|
|
437
|
+
for (const p of searchPaths) {
|
|
438
|
+
if (existsSync2(p)) {
|
|
439
|
+
modelPath = p;
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
if (!modelPath) {
|
|
444
|
+
const hfCache = join(dataDir, "hf-cache");
|
|
445
|
+
if (existsSync2(hfCache)) {
|
|
446
|
+
try {
|
|
447
|
+
const files = readdirSync(hfCache);
|
|
448
|
+
const gguf = files.find((f) => f.endsWith(".gguf"));
|
|
449
|
+
if (gguf) modelPath = join(hfCache, gguf);
|
|
450
|
+
} catch {
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (modelPath) {
|
|
455
|
+
log.i("DEVAGENT", "Starting llama-server for workers...", { port: 8085, model: modelPath });
|
|
456
|
+
const started = await llamacppEmbeddingManager.ensureRunning({
|
|
457
|
+
modelPath,
|
|
458
|
+
mode: "embedding",
|
|
459
|
+
port: 8085,
|
|
460
|
+
contextSize: llamacppConfig.context_size || 512,
|
|
461
|
+
nGpuLayers: llamacppConfig.n_gpu_layers ?? 99
|
|
462
|
+
});
|
|
463
|
+
if (started) {
|
|
464
|
+
log.i("DEVAGENT", "llama-server ready for workers");
|
|
465
|
+
} else {
|
|
466
|
+
log.w("DEVAGENT", "llama-server start failed - embeddings may not work");
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
if (embeddingConfig.centralizedEmbeddings && this.parserAgent) {
|
|
472
|
+
try {
|
|
473
|
+
const { EmbeddingGenerator } = await import('./embedding-generator-XR6FDO4J.js');
|
|
474
|
+
const { buildEmbeddingGeneratorOptions } = await import('./provider-config-67EQAZ5Z.js');
|
|
475
|
+
const { loadSemanticConfig } = await import('./config-paths-Y2IHHJQF.js');
|
|
476
|
+
const { getConfig: getConfig2 } = await import('./yaml-config-OVNUY25R.js');
|
|
477
|
+
const semanticConfig = loadSemanticConfig();
|
|
478
|
+
const yamlConfig = getConfig2();
|
|
479
|
+
const generatorOptions = buildEmbeddingGeneratorOptions(
|
|
480
|
+
embeddingConfig.provider,
|
|
481
|
+
embeddingConfig.modelName,
|
|
482
|
+
embeddingConfig.batchSize,
|
|
483
|
+
semanticConfig,
|
|
484
|
+
yamlConfig
|
|
485
|
+
);
|
|
486
|
+
const embeddingGenerator = new EmbeddingGenerator(generatorOptions);
|
|
487
|
+
const initTimeoutMs = 9e4;
|
|
488
|
+
try {
|
|
489
|
+
await Promise.race([
|
|
490
|
+
embeddingGenerator.initialize(),
|
|
491
|
+
new Promise(
|
|
492
|
+
(_, reject) => setTimeout(() => reject(new Error(`TEI init timeout after ${initTimeoutMs}ms`)), initTimeoutMs)
|
|
493
|
+
)
|
|
494
|
+
]);
|
|
495
|
+
} catch (initErr) {
|
|
496
|
+
log.w(
|
|
497
|
+
"DEVAGENT",
|
|
498
|
+
"Centralized EmbeddingGenerator init timeout/fail, indexing will proceed without embeddings",
|
|
499
|
+
{
|
|
500
|
+
err: initErr.message
|
|
501
|
+
}
|
|
502
|
+
);
|
|
503
|
+
}
|
|
504
|
+
await this.parserAgent.setEmbeddingGenerator(embeddingGenerator);
|
|
505
|
+
log.i("DEVAGENT", "Centralized EmbeddingGenerator configured", {
|
|
506
|
+
provider: embeddingConfig.provider,
|
|
507
|
+
model: embeddingConfig.modelName
|
|
508
|
+
});
|
|
509
|
+
} catch (error) {
|
|
510
|
+
log.e("DEVAGENT", "Failed to initialize centralized EmbeddingGenerator", {
|
|
511
|
+
error: error.message
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Initialize the vector provider (LayeredFaiss or Faiss) and configure it
|
|
518
|
+
* on the parser agent. Removes old embeddings for deleted entities in incremental mode.
|
|
519
|
+
*/
|
|
520
|
+
async initVectorProvider(payload, deletedEntityIds) {
|
|
521
|
+
if (!this.parserAgent) return;
|
|
522
|
+
const { getProjectHash, getCurrentGitBranchOrDefault } = await import('./storage-paths-NGF6GAFZ.js');
|
|
523
|
+
const configLoader = ConfigLoader.getInstance();
|
|
524
|
+
const embConfig = configLoader.getEmbeddingConfig();
|
|
525
|
+
const useLayeredIndex = embConfig.useLayeredIndex;
|
|
526
|
+
const projectHash = getProjectHash(payload.directory);
|
|
527
|
+
const currentBranch = getCurrentGitBranchOrDefault(payload.directory);
|
|
528
|
+
let vectorProvider = null;
|
|
529
|
+
try {
|
|
530
|
+
if (useLayeredIndex) {
|
|
531
|
+
const { getLayeredFaissProvider } = await import('./layered-faiss-provider-M3MDI2AG.js');
|
|
532
|
+
const provider = getLayeredFaissProvider();
|
|
533
|
+
if ("initialize" in provider && typeof provider.initialize === "function") {
|
|
534
|
+
log.d("DEVAGENT", "Initializing LayeredFaissProvider", {
|
|
535
|
+
dir: payload.directory,
|
|
536
|
+
projectHash,
|
|
537
|
+
branch: currentBranch
|
|
538
|
+
});
|
|
539
|
+
await provider.initialize(payload.directory, projectHash, currentBranch);
|
|
540
|
+
}
|
|
541
|
+
vectorProvider = provider;
|
|
542
|
+
} else {
|
|
543
|
+
const { initializeFaissProvider } = await import('./faiss-provider-VAYK2VBT.js');
|
|
544
|
+
const provider = await initializeFaissProvider();
|
|
545
|
+
if (provider) {
|
|
546
|
+
await provider.setProjectContext(projectHash, currentBranch);
|
|
547
|
+
vectorProvider = provider;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
} catch (e) {
|
|
551
|
+
log.w("DEVAGENT", "Failed to get vector provider", { error: String(e) });
|
|
552
|
+
}
|
|
553
|
+
if (vectorProvider) {
|
|
554
|
+
log.d("DEVAGENT", "vector_provider_branch", {
|
|
555
|
+
branch: currentBranch,
|
|
556
|
+
layered: useLayeredIndex
|
|
557
|
+
});
|
|
558
|
+
this.parserAgent.setVectorProvider(vectorProvider);
|
|
559
|
+
log.i("DEVAGENT", "Vector provider configured", {
|
|
560
|
+
projectHash,
|
|
561
|
+
dir: payload.directory,
|
|
562
|
+
layered: useLayeredIndex
|
|
563
|
+
});
|
|
564
|
+
if (deletedEntityIds.length > 0) {
|
|
565
|
+
try {
|
|
566
|
+
const embeddingIds = deletedEntityIds.map((id) => id.startsWith("ent:") ? id : `ent:${id}`);
|
|
567
|
+
await vectorProvider.remove(embeddingIds);
|
|
568
|
+
log.i("DEVAGENT", "Removed embeddings for changed/deleted files", {
|
|
569
|
+
count: embeddingIds.length
|
|
570
|
+
});
|
|
571
|
+
} catch (error) {
|
|
572
|
+
log.w("DEVAGENT", "Failed to remove embeddings", {
|
|
573
|
+
error: error.message
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
} else {
|
|
578
|
+
log.w("DEVAGENT", "Vector provider not available - embeddings will not be saved");
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Flush pending embeddings to FAISS, create Prolly Tree commit,
|
|
583
|
+
* switch to keepalive mode, run GC.
|
|
584
|
+
*/
|
|
585
|
+
async postIndexingCleanup(_codeFiles, filesProcessed, totalEntities, perfStart, perfTimings, _effectiveBatchSize, _dataFiles) {
|
|
586
|
+
perfTimings["embFlush_start"] = Date.now() - perfStart;
|
|
587
|
+
perfTimings["commit_start"] = Date.now() - perfStart;
|
|
588
|
+
const faissFlushPromise = (async () => {
|
|
589
|
+
if (!this.parserAgent) return;
|
|
590
|
+
const accumulator = this.parserAgent.getAccumulator();
|
|
591
|
+
if (!accumulator) return;
|
|
592
|
+
const pendingCount = accumulator.getPendingCount();
|
|
593
|
+
if (pendingCount > 0) {
|
|
594
|
+
log.i("DEVAGENT", "Flushing pending embeddings to FAISS", { pending: pendingCount });
|
|
595
|
+
try {
|
|
596
|
+
const flushed = await accumulator.flush();
|
|
597
|
+
log.i("DEVAGENT", "Embeddings flushed to FAISS", { flushed });
|
|
598
|
+
} catch (err) {
|
|
599
|
+
log.e("DEVAGENT", "Failed to flush embeddings", { error: err.message });
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
const stats = accumulator.getStats();
|
|
603
|
+
log.i("DEVAGENT", "Embedding accumulator stats", {
|
|
604
|
+
accumulated: stats.accumulated,
|
|
605
|
+
flushed: stats.flushed,
|
|
606
|
+
flushCount: stats.flushCount,
|
|
607
|
+
totalBytes: stats.totalBytes
|
|
608
|
+
});
|
|
609
|
+
})();
|
|
610
|
+
const graphCommitPromise = (async () => {
|
|
611
|
+
try {
|
|
612
|
+
const storage = await getGraphStorage();
|
|
613
|
+
const adapter = storage.getLibSQLAdapter?.();
|
|
614
|
+
if (adapter?.createGraphCommit) {
|
|
615
|
+
const commitHash = await adapter.createGraphCommit(`Index: ${filesProcessed} files`);
|
|
616
|
+
if (commitHash) {
|
|
617
|
+
log.i("DEVAGENT", "graph_commit_created", {
|
|
618
|
+
commit: commitHash.slice(0, 8),
|
|
619
|
+
files: filesProcessed,
|
|
620
|
+
entities: totalEntities
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
adapter.pruneAndGC?.(20)?.catch?.((err) => log.w("DEVAGENT", "prune_gc_fail", { err: String(err) }));
|
|
624
|
+
}
|
|
625
|
+
} catch (err) {
|
|
626
|
+
log.w("DEVAGENT", "graph_commit_failed", { error: err.message });
|
|
627
|
+
}
|
|
628
|
+
})();
|
|
629
|
+
await Promise.all([faissFlushPromise, graphCommitPromise]);
|
|
630
|
+
perfTimings["embFlush_end"] = Date.now() - perfStart;
|
|
631
|
+
perfTimings["commit_end"] = Date.now() - perfStart;
|
|
632
|
+
perfTimings["keepalive_start"] = Date.now() - perfStart;
|
|
633
|
+
if (this.parserAgent) {
|
|
634
|
+
try {
|
|
635
|
+
const memoryBeforeMB = this.parserAgent.getTotalMemoryMB();
|
|
636
|
+
log.i("DEVAGENT", "Switching to keepalive mode (spawning ONE worker for incremental updates)", {
|
|
637
|
+
memoryMB: memoryBeforeMB
|
|
638
|
+
});
|
|
639
|
+
await this.parserAgent.enableKeepaliveMode();
|
|
640
|
+
const memoryAfterMB = this.parserAgent.getTotalMemoryMB();
|
|
641
|
+
log.i("DEVAGENT", "Keepalive mode enabled, ready for incremental updates", {
|
|
642
|
+
memoryBeforeMB,
|
|
643
|
+
memoryAfterMB
|
|
644
|
+
});
|
|
645
|
+
log.flush();
|
|
646
|
+
} catch (err) {
|
|
647
|
+
log.w("DEVAGENT", "Failed to enable keepalive mode, falling back to shutdown", {
|
|
648
|
+
error: err.message
|
|
649
|
+
});
|
|
650
|
+
try {
|
|
651
|
+
await this.parserAgent.shutdown();
|
|
652
|
+
this.parserAgent = null;
|
|
653
|
+
} catch {
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
perfTimings["keepalive_end"] = Date.now() - perfStart;
|
|
658
|
+
if (tryGarbageCollect(true)) {
|
|
659
|
+
log.i("DEVAGENT", "gc_after_index");
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* Format 15+ timing measurements into a structured performance summary log.
|
|
664
|
+
*/
|
|
665
|
+
buildPerfSummary(perfTimings, _codeFiles, _dataFiles, _filesProcessed, _totalEntities, _totalRelationships, _effectiveBatchSize, perfStart) {
|
|
666
|
+
perfTimings["total"] = Date.now() - perfStart;
|
|
667
|
+
log.i("DEVAGENT", "PERF_SUMMARY", {
|
|
668
|
+
collectFiles: (perfTimings["collectFiles_end"] ?? 0) - (perfTimings["collectFiles_start"] ?? 0),
|
|
669
|
+
destroyPools: (perfTimings["destroyPools_end"] ?? 0) - (perfTimings["destroyPools_start"] ?? 0),
|
|
670
|
+
preSpawn: (perfTimings["preSpawn_end"] ?? 0) - (perfTimings["preSpawn_start"] ?? 0),
|
|
671
|
+
parsing: (perfTimings["parsing_end"] ?? 0) - (perfTimings["parsing_start"] ?? 0),
|
|
672
|
+
flush: (perfTimings["flush_end"] ?? 0) - (perfTimings["flush_start"] ?? 0),
|
|
673
|
+
dataFiles: (perfTimings["dataFiles_end"] ?? 0) - (perfTimings["dataFiles_start"] ?? 0),
|
|
674
|
+
swaggerLink: (perfTimings["swaggerLink_end"] ?? 0) - (perfTimings["swaggerLink_start"] ?? 0),
|
|
675
|
+
embFlush: (perfTimings["embFlush_end"] ?? 0) - (perfTimings["embFlush_start"] ?? 0),
|
|
676
|
+
keepalive: (perfTimings["keepalive_end"] ?? 0) - (perfTimings["keepalive_start"] ?? 0),
|
|
677
|
+
total: perfTimings["total"]
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
async performRealIndexing(payload) {
|
|
681
|
+
const directory = payload.directory;
|
|
682
|
+
const excludePatterns = payload.excludePatterns || [];
|
|
683
|
+
const perfStart = Date.now();
|
|
684
|
+
const perfTimings = {};
|
|
685
|
+
log.i("DEVAGENT", "Starting indexing", {
|
|
686
|
+
directory,
|
|
687
|
+
excludePatternsCount: excludePatterns.length,
|
|
688
|
+
samplePatterns: excludePatterns.slice(0, 5)
|
|
689
|
+
});
|
|
690
|
+
const isIncremental = payload.incremental === true;
|
|
691
|
+
perfTimings["collectFiles_start"] = Date.now() - perfStart;
|
|
692
|
+
const collectPromise = collectFilesAsync(directory, { excludePatterns, agentId: this.id });
|
|
693
|
+
let preSpawnPreparePromise = null;
|
|
694
|
+
if (!isIncremental && this.parserAgent) {
|
|
695
|
+
perfTimings["preSpawnPrepare_start"] = Date.now() - perfStart;
|
|
696
|
+
preSpawnPreparePromise = (async () => {
|
|
697
|
+
log.i("DEVAGENT", "Destroying existing worker pools for full reindex (parallel with collectFiles)");
|
|
698
|
+
await this.parserAgent.destroyWorkerPools();
|
|
699
|
+
await this.parserAgent.setPoolMode("per-language");
|
|
700
|
+
const embeddingConfig2 = buildWorkerEmbeddingConfig();
|
|
701
|
+
if (embeddingConfig2) {
|
|
702
|
+
this.parserAgent.setEmbeddingConfig(embeddingConfig2);
|
|
703
|
+
log.i("DEVAGENT", "Embedding config passed to parser", {
|
|
704
|
+
provider: embeddingConfig2.provider,
|
|
705
|
+
model: embeddingConfig2.modelName
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
return { embeddingConfig: embeddingConfig2 };
|
|
709
|
+
})();
|
|
710
|
+
}
|
|
711
|
+
const collectResult = await collectPromise;
|
|
712
|
+
let allFiles = collectResult.files;
|
|
713
|
+
perfTimings["collectFiles_end"] = Date.now() - perfStart;
|
|
714
|
+
log.i("DEVAGENT", "Files collected", {
|
|
715
|
+
count: allFiles.length,
|
|
716
|
+
ms: perfTimings["collectFiles_end"] - perfTimings["collectFiles_start"]
|
|
717
|
+
});
|
|
718
|
+
let preSpawnPrepareResult = null;
|
|
719
|
+
if (preSpawnPreparePromise) {
|
|
720
|
+
preSpawnPrepareResult = await preSpawnPreparePromise;
|
|
721
|
+
perfTimings["preSpawnPrepare_end"] = Date.now() - perfStart;
|
|
722
|
+
log.i("DEVAGENT", "PreSpawn preparation done (parallel)", {
|
|
723
|
+
ms: perfTimings["preSpawnPrepare_end"] - perfTimings["preSpawnPrepare_start"]
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
let deletedEntityIds = [];
|
|
727
|
+
if (isIncremental && allFiles.length > 0) {
|
|
728
|
+
const diff = await this.prepareIncrementalDiff(allFiles, collectResult.files.length);
|
|
729
|
+
deletedEntityIds = diff.deletedEntityIds;
|
|
730
|
+
if (diff.earlyReturn) {
|
|
731
|
+
log.i("DEVAGENT", "No files changed, skipping indexing");
|
|
732
|
+
return diff.earlyReturn;
|
|
733
|
+
}
|
|
734
|
+
allFiles = diff.filesToProcess;
|
|
735
|
+
}
|
|
736
|
+
const codeFiles = [];
|
|
737
|
+
const dataFiles = [];
|
|
738
|
+
for (const file of allFiles) {
|
|
739
|
+
const ext = extname(file).toLowerCase();
|
|
740
|
+
if (isCodeExtension(ext)) {
|
|
741
|
+
codeFiles.push(file);
|
|
742
|
+
} else {
|
|
743
|
+
dataFiles.push(file);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
log.i("DEVAGENT", "Files separated", {
|
|
747
|
+
codeFiles: codeFiles.length,
|
|
748
|
+
dataFiles: dataFiles.length
|
|
749
|
+
});
|
|
750
|
+
const isDebugMode = process.env["MCP_DEBUG_MODE"] === "1";
|
|
751
|
+
const effectiveBatchSize = Infinity;
|
|
752
|
+
log.i("DEVAGENT", "parser_batch", { files: codeFiles.length });
|
|
753
|
+
const parseOptions = isDebugMode ? {
|
|
754
|
+
batchSize: Math.max(1, Math.min(3, effectiveBatchSize)),
|
|
755
|
+
useCache: false
|
|
756
|
+
} : {};
|
|
757
|
+
let totalEntities = 0;
|
|
758
|
+
let totalRelationships = 0;
|
|
759
|
+
let filesProcessed = 0;
|
|
760
|
+
const initialIndexerStats = this.indexerAgent?.getIndexingStats() ?? {
|
|
761
|
+
entitiesIndexed: 0,
|
|
762
|
+
relationshipsCreated: 0};
|
|
763
|
+
let preSpawnPromise = null;
|
|
764
|
+
if (this.parserAgent) {
|
|
765
|
+
if (isIncremental) {
|
|
766
|
+
await this.parserAgent.setPoolMode("universal");
|
|
767
|
+
log.i("DEVAGENT", "Pool mode configured", { mode: "universal", isIncremental });
|
|
768
|
+
} else {
|
|
769
|
+
log.i("DEVAGENT", "Pool mode configured", { mode: "per-language", isIncremental });
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
const embeddingConfig = preSpawnPrepareResult?.embeddingConfig ?? buildWorkerEmbeddingConfig();
|
|
773
|
+
if (this.parserAgent && embeddingConfig) {
|
|
774
|
+
if (isIncremental) {
|
|
775
|
+
this.parserAgent.setEmbeddingConfig(embeddingConfig);
|
|
776
|
+
log.i("DEVAGENT", "Embedding config passed to parser", {
|
|
777
|
+
provider: embeddingConfig.provider,
|
|
778
|
+
model: embeddingConfig.modelName
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
const pipelinePromise = this.initEmbeddingPipeline(embeddingConfig, payload);
|
|
782
|
+
if (this.parserAgent) {
|
|
783
|
+
this.parserAgent.setStreamingMode(true);
|
|
784
|
+
}
|
|
785
|
+
if (!isIncremental && codeFiles.length > 0) {
|
|
786
|
+
perfTimings["preSpawn_start"] = Date.now() - perfStart;
|
|
787
|
+
preSpawnPromise = this.parserAgent.preSpawnPools(codeFiles);
|
|
788
|
+
}
|
|
789
|
+
await pipelinePromise;
|
|
790
|
+
await this.initVectorProvider(payload, deletedEntityIds);
|
|
791
|
+
}
|
|
792
|
+
const streamingIndexedFiles = /* @__PURE__ */ new Set();
|
|
793
|
+
let totalStreamingQueueMs = 0;
|
|
794
|
+
let streamingCallbackCount = 0;
|
|
795
|
+
if (this.parserAgent && this.indexerAgent) {
|
|
796
|
+
this.parserAgent.setStreamingMode(true, (result, _taskId, _fileIndex, _totalFiles) => {
|
|
797
|
+
if (!result.filePath || !result.entities || result.entities.length === 0) {
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
streamingIndexedFiles.add(result.filePath);
|
|
801
|
+
const queueStart = Date.now();
|
|
802
|
+
this.indexerAgent.queueForIndexing(result.entities, result.filePath, result.relationships || []);
|
|
803
|
+
totalStreamingQueueMs += Date.now() - queueStart;
|
|
804
|
+
streamingCallbackCount++;
|
|
805
|
+
});
|
|
806
|
+
log.i("DEVAGENT", "Streaming mode enabled (batch accumulator)");
|
|
807
|
+
}
|
|
808
|
+
if (preSpawnPromise) {
|
|
809
|
+
await preSpawnPromise;
|
|
810
|
+
perfTimings["preSpawn_end"] = Date.now() - perfStart;
|
|
811
|
+
}
|
|
812
|
+
perfTimings["parsing_start"] = Date.now() - perfStart;
|
|
813
|
+
let totalWorkerParseMs = 0;
|
|
814
|
+
let totalMainProcessMs = 0;
|
|
815
|
+
let totalMainIndexMs = 0;
|
|
816
|
+
const files = codeFiles;
|
|
817
|
+
for (let i = 0; i < files.length; i += effectiveBatchSize) {
|
|
818
|
+
const batch = files.slice(i, Math.min(i + effectiveBatchSize, files.length));
|
|
819
|
+
try {
|
|
820
|
+
if (this.parserAgent) {
|
|
821
|
+
const batchExtStats = {};
|
|
822
|
+
for (const f of batch) {
|
|
823
|
+
const ext = extname(f).toLowerCase() || "(no ext)";
|
|
824
|
+
batchExtStats[ext] = (batchExtStats[ext] || 0) + 1;
|
|
825
|
+
}
|
|
826
|
+
log.i("DEVAGENT", "Sending batch to parser", {
|
|
827
|
+
batchSize: batch.length,
|
|
828
|
+
batchIndex: i,
|
|
829
|
+
extensions: batchExtStats
|
|
830
|
+
});
|
|
831
|
+
const parseTask = {
|
|
832
|
+
id: `parse-${Date.now()}-${i}`,
|
|
833
|
+
type: "parse:batch",
|
|
834
|
+
priority: 8,
|
|
835
|
+
payload: { files: batch, options: parseOptions },
|
|
836
|
+
createdAt: Date.now()
|
|
837
|
+
};
|
|
838
|
+
const parseStart = Date.now();
|
|
839
|
+
const results = await this.parserAgent.process(parseTask);
|
|
840
|
+
const parseMs = Date.now() - parseStart;
|
|
841
|
+
totalWorkerParseMs += parseMs;
|
|
842
|
+
const processStart = Date.now();
|
|
843
|
+
log.i("DEVAGENT", "Parser batch completed", {
|
|
844
|
+
batchSent: batch.length,
|
|
845
|
+
resultsReceived: results?.length || 0,
|
|
846
|
+
batchIndex: i,
|
|
847
|
+
parseMs,
|
|
848
|
+
filesPerSec: Math.round(batch.length / parseMs * 1e3)
|
|
849
|
+
});
|
|
850
|
+
log.flush();
|
|
851
|
+
let resultsWithFilePath = 0;
|
|
852
|
+
let resultsWithEntities = 0;
|
|
853
|
+
let resultsWithEmptyEntities = 0;
|
|
854
|
+
let totalEntityCount = 0;
|
|
855
|
+
for (const res of results || []) {
|
|
856
|
+
if (res?.filePath) resultsWithFilePath++;
|
|
857
|
+
if (Array.isArray(res?.entities)) {
|
|
858
|
+
if (res.entities.length > 0) {
|
|
859
|
+
resultsWithEntities++;
|
|
860
|
+
totalEntityCount += res.entities.length;
|
|
861
|
+
} else {
|
|
862
|
+
resultsWithEmptyEntities++;
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
log.d("DEVAGENT", "batch_parse_analysis", {
|
|
867
|
+
batch: i,
|
|
868
|
+
results: results?.length || 0,
|
|
869
|
+
withPath: resultsWithFilePath,
|
|
870
|
+
withEnts: resultsWithEntities,
|
|
871
|
+
empty: resultsWithEmptyEntities,
|
|
872
|
+
total: totalEntityCount
|
|
873
|
+
});
|
|
874
|
+
const byFile = /* @__PURE__ */ new Map();
|
|
875
|
+
for (const res of results || []) {
|
|
876
|
+
const fp = res?.filePath;
|
|
877
|
+
if (!fp) continue;
|
|
878
|
+
const slot = byFile.get(fp) ?? { entities: [], relationships: [] };
|
|
879
|
+
if (Array.isArray(res.entities)) {
|
|
880
|
+
slot.entities.push(...res.entities);
|
|
881
|
+
}
|
|
882
|
+
if (Array.isArray(res.relationships)) {
|
|
883
|
+
for (const r of res.relationships) {
|
|
884
|
+
if (r?.from && r.to && r.type) {
|
|
885
|
+
slot.relationships.push({
|
|
886
|
+
from: r.from,
|
|
887
|
+
to: r.to,
|
|
888
|
+
type: r.type,
|
|
889
|
+
targetFile: fp
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
byFile.set(fp, slot);
|
|
895
|
+
}
|
|
896
|
+
let totalRelationshipsExtracted = 0;
|
|
897
|
+
let resultsWithRelationships = 0;
|
|
898
|
+
for (const res of results || []) {
|
|
899
|
+
if (Array.isArray(res?.relationships) && res.relationships.length > 0) {
|
|
900
|
+
resultsWithRelationships++;
|
|
901
|
+
totalRelationshipsExtracted += res.relationships.length;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
log.d("DEVAGENT", "batch_rels", {
|
|
905
|
+
batch: i,
|
|
906
|
+
withRels: resultsWithRelationships,
|
|
907
|
+
total: totalRelationshipsExtracted
|
|
908
|
+
});
|
|
909
|
+
log.i("DEVAGENT", "Files ready for indexing", {
|
|
910
|
+
uniqueFiles: byFile.size,
|
|
911
|
+
batchIndex: i,
|
|
912
|
+
totalEntities: totalEntityCount,
|
|
913
|
+
totalRelationships: totalRelationshipsExtracted
|
|
914
|
+
});
|
|
915
|
+
log.d("DEVAGENT", "batch_summary", {
|
|
916
|
+
batch: i,
|
|
917
|
+
sent: batch.length,
|
|
918
|
+
results: results?.length || 0,
|
|
919
|
+
unique: byFile.size
|
|
920
|
+
});
|
|
921
|
+
totalMainProcessMs += Date.now() - processStart;
|
|
922
|
+
const indexStart = Date.now();
|
|
923
|
+
const INDEXING_CONCURRENCY = 8;
|
|
924
|
+
const fileEntries = Array.from(byFile.entries()).filter(([file]) => !streamingIndexedFiles.has(file));
|
|
925
|
+
if (fileEntries.length > 0) {
|
|
926
|
+
log.i("DEVAGENT", "Post-batch indexing (non-streamed files)", {
|
|
927
|
+
total: byFile.size,
|
|
928
|
+
alreadyStreamed: streamingIndexedFiles.size,
|
|
929
|
+
remaining: fileEntries.length
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
let pendingPromises = [];
|
|
933
|
+
let pendingCount = 0;
|
|
934
|
+
for (const [file, group] of fileEntries) {
|
|
935
|
+
const indexTask = {
|
|
936
|
+
id: `index-entities-${Date.now()}-${i}-${file}`,
|
|
937
|
+
type: "index:entities",
|
|
938
|
+
priority: 7,
|
|
939
|
+
payload: {
|
|
940
|
+
entities: group.entities,
|
|
941
|
+
relationships: group.relationships,
|
|
942
|
+
filePath: file
|
|
943
|
+
},
|
|
944
|
+
createdAt: Date.now()
|
|
945
|
+
};
|
|
946
|
+
const promise = (async () => {
|
|
947
|
+
try {
|
|
948
|
+
const indexResult = await this.indexerAgent?.enqueue(indexTask);
|
|
949
|
+
return { result: indexResult, error: null };
|
|
950
|
+
} catch (err) {
|
|
951
|
+
log.w("DEVAGENT", "index_file_fail", { file, err: err.message });
|
|
952
|
+
return { result: null, error: err };
|
|
953
|
+
}
|
|
954
|
+
})();
|
|
955
|
+
pendingPromises.push(promise);
|
|
956
|
+
pendingCount++;
|
|
957
|
+
if (pendingCount >= INDEXING_CONCURRENCY) {
|
|
958
|
+
const results2 = await Promise.all(pendingPromises);
|
|
959
|
+
for (const { result } of results2) {
|
|
960
|
+
if (result) {
|
|
961
|
+
totalEntities += result.entitiesIndexed || 0;
|
|
962
|
+
totalRelationships += result.relationshipsCreated || 0;
|
|
963
|
+
filesProcessed += 1;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
pendingPromises = [];
|
|
967
|
+
pendingCount = 0;
|
|
968
|
+
await yieldToEventLoop();
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
if (pendingPromises.length > 0) {
|
|
972
|
+
const results2 = await Promise.all(pendingPromises);
|
|
973
|
+
for (const { result } of results2) {
|
|
974
|
+
if (result) {
|
|
975
|
+
totalEntities += result.entitiesIndexed || 0;
|
|
976
|
+
totalRelationships += result.relationshipsCreated || 0;
|
|
977
|
+
filesProcessed += 1;
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
totalMainIndexMs += Date.now() - indexStart;
|
|
982
|
+
} else {
|
|
983
|
+
const entities = [];
|
|
984
|
+
const relationships = [];
|
|
985
|
+
for (const file of batch) {
|
|
986
|
+
const extWithDot = extname(file).toLowerCase();
|
|
987
|
+
const fileName = basename(file);
|
|
988
|
+
const fileNameNoExt = fileName.replace(/\.[^/.]+$/, "");
|
|
989
|
+
const ext = extWithDot.slice(1) || fileName;
|
|
990
|
+
const isCode = isCodeExtension(extWithDot);
|
|
991
|
+
const isData = isDataExtension(extWithDot) || isDataExtension("." + fileName.toLowerCase());
|
|
992
|
+
if (!isCode && !isData) {
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
let contentHash;
|
|
996
|
+
let fileContent;
|
|
997
|
+
if (isData) {
|
|
998
|
+
try {
|
|
999
|
+
fileContent = readFileSync(file, "utf-8");
|
|
1000
|
+
contentHash = hashText(fileContent);
|
|
1001
|
+
} catch {
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
entities.push({
|
|
1005
|
+
name: fileName,
|
|
1006
|
+
type: "file",
|
|
1007
|
+
filePath: file,
|
|
1008
|
+
language: ext,
|
|
1009
|
+
// Top-level language for TechnologyDetector
|
|
1010
|
+
location: { start: { line: 1, column: 0, index: 0 }, end: { line: 1, column: 0, index: 0 } },
|
|
1011
|
+
metadata: {
|
|
1012
|
+
language: ext,
|
|
1013
|
+
path: file,
|
|
1014
|
+
isDataFile: isData,
|
|
1015
|
+
contentHash
|
|
1016
|
+
// For semantic merge
|
|
1017
|
+
}
|
|
1018
|
+
});
|
|
1019
|
+
if (isData) {
|
|
1020
|
+
continue;
|
|
1021
|
+
}
|
|
1022
|
+
entities.push({
|
|
1023
|
+
name: fileNameNoExt,
|
|
1024
|
+
type: "module",
|
|
1025
|
+
filePath: file,
|
|
1026
|
+
language: ext,
|
|
1027
|
+
// Top-level language for TechnologyDetector
|
|
1028
|
+
location: { start: { line: 1, column: 0, index: 0 }, end: { line: 100, column: 0, index: 0 } },
|
|
1029
|
+
metadata: { language: ext, moduleType: "file" }
|
|
1030
|
+
});
|
|
1031
|
+
if (ext === "py" && /^[A-Z]/.test(fileNameNoExt)) {
|
|
1032
|
+
entities.push({
|
|
1033
|
+
name: fileNameNoExt,
|
|
1034
|
+
type: "class",
|
|
1035
|
+
filePath: file,
|
|
1036
|
+
language: "python",
|
|
1037
|
+
location: { start: { line: 5, column: 0, index: 0 }, end: { line: 50, column: 0, index: 0 } },
|
|
1038
|
+
metadata: { language: "python", visibility: "public" }
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
if ((ext === "kt" || ext === "kts" || ext === "java") && /^[A-Z]/.test(fileNameNoExt)) {
|
|
1042
|
+
entities.push({
|
|
1043
|
+
name: fileNameNoExt,
|
|
1044
|
+
type: "class",
|
|
1045
|
+
filePath: file,
|
|
1046
|
+
language: ext === "java" ? "java" : "kotlin",
|
|
1047
|
+
location: { start: { line: 5, column: 0, index: 0 }, end: { line: 50, column: 0, index: 0 } },
|
|
1048
|
+
metadata: { language: ext, visibility: "public" }
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
if ((ext === "js" || ext === "ts") && !file.includes(".test.") && !file.includes(".spec.")) {
|
|
1052
|
+
entities.push({
|
|
1053
|
+
name: `export_default`,
|
|
1054
|
+
type: "function",
|
|
1055
|
+
filePath: file,
|
|
1056
|
+
language: ext === "ts" ? "typescript" : "javascript",
|
|
1057
|
+
location: { start: { line: 10, column: 0, index: 0 }, end: { line: 30, column: 0, index: 0 } },
|
|
1058
|
+
metadata: { language: ext, exported: true }
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
for (const entity of entities) {
|
|
1063
|
+
if (entity.type === "file") {
|
|
1064
|
+
const moduleEntity = entities.find((e) => e.type === "module" && e.filePath === entity.filePath);
|
|
1065
|
+
if (moduleEntity) {
|
|
1066
|
+
relationships.push({
|
|
1067
|
+
from: entity.name,
|
|
1068
|
+
to: moduleEntity.name,
|
|
1069
|
+
type: "contains"
|
|
1070
|
+
});
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
for (const entity of entities) {
|
|
1075
|
+
if (entity.type === "module") {
|
|
1076
|
+
const related = entities.filter(
|
|
1077
|
+
(e) => (e.type === "class" || e.type === "function") && e.filePath === entity.filePath
|
|
1078
|
+
);
|
|
1079
|
+
for (const rel of related) {
|
|
1080
|
+
relationships.push({
|
|
1081
|
+
from: entity.name,
|
|
1082
|
+
to: rel.name,
|
|
1083
|
+
type: rel.type === "class" ? "defines_class" : "defines_function",
|
|
1084
|
+
sourceFile: entity.filePath
|
|
1085
|
+
});
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
for (const entity of entities) {
|
|
1090
|
+
if (entity.type === "class") {
|
|
1091
|
+
const funcs = entities.filter((e) => e.type === "function" && e.filePath === entity.filePath);
|
|
1092
|
+
for (const f of funcs) {
|
|
1093
|
+
relationships.push({ from: entity.name, to: f.name, type: "has_method", sourceFile: entity.filePath });
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
const byFile = /* @__PURE__ */ new Map();
|
|
1098
|
+
for (const e of entities) {
|
|
1099
|
+
if (!e.filePath) continue;
|
|
1100
|
+
const slot = byFile.get(e.filePath) ?? { entities: [], relationships: [] };
|
|
1101
|
+
slot.entities.push(e);
|
|
1102
|
+
byFile.set(e.filePath, slot);
|
|
1103
|
+
}
|
|
1104
|
+
for (const r of relationships) {
|
|
1105
|
+
const fp = r.sourceFile || null;
|
|
1106
|
+
if (!fp) continue;
|
|
1107
|
+
const slot = byFile.get(fp) ?? { entities: [], relationships: [] };
|
|
1108
|
+
slot.relationships.push({ from: r.from, to: r.to, type: r.type, targetFile: r.targetFile });
|
|
1109
|
+
byFile.set(fp, slot);
|
|
1110
|
+
}
|
|
1111
|
+
for (const [file, group] of byFile.entries()) {
|
|
1112
|
+
if (!group.entities.length) continue;
|
|
1113
|
+
const indexTask = {
|
|
1114
|
+
id: `index-entities-${Date.now()}-${i}-${file}`,
|
|
1115
|
+
type: "index:entities",
|
|
1116
|
+
priority: 7,
|
|
1117
|
+
payload: { entities: group.entities, relationships: group.relationships, filePath: file },
|
|
1118
|
+
createdAt: Date.now()
|
|
1119
|
+
};
|
|
1120
|
+
try {
|
|
1121
|
+
const indexResult = await this.indexerAgent?.process(indexTask);
|
|
1122
|
+
const indexed = indexResult;
|
|
1123
|
+
if (indexed) {
|
|
1124
|
+
totalEntities += indexed.entitiesIndexed || 0;
|
|
1125
|
+
totalRelationships += indexed.relationshipsCreated || 0;
|
|
1126
|
+
filesProcessed += 1;
|
|
1127
|
+
}
|
|
1128
|
+
} catch (err) {
|
|
1129
|
+
log.w("DEVAGENT", "idx_fail", { file, err: String(err) });
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
} catch (error) {
|
|
1134
|
+
log.e("DEVAGENT", "batch_err", {
|
|
1135
|
+
batch: i,
|
|
1136
|
+
cnt: batch.length,
|
|
1137
|
+
err: error instanceof Error ? error.message : String(error)
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
if ((i + effectiveBatchSize) % 500 === 0 || i + effectiveBatchSize >= files.length) {
|
|
1141
|
+
log.i("DEVAGENT", "progress", { done: filesProcessed, total: files.length });
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
perfTimings["parsing_end"] = Date.now() - perfStart;
|
|
1145
|
+
const totalParsingPhase = (perfTimings["parsing_end"] ?? 0) - (perfTimings["parsing_start"] ?? 0);
|
|
1146
|
+
const estimatedPureWorkerMs = Math.max(0, totalWorkerParseMs - totalStreamingQueueMs);
|
|
1147
|
+
log.i("DEVAGENT", "PARSING_PHASE_BREAKDOWN", {
|
|
1148
|
+
totalMs: totalParsingPhase,
|
|
1149
|
+
workerParseMs: totalWorkerParseMs,
|
|
1150
|
+
streamingQueueMs: totalStreamingQueueMs,
|
|
1151
|
+
streamingCallbacks: streamingCallbackCount,
|
|
1152
|
+
estimatedPureWorkerMs,
|
|
1153
|
+
mainProcessMs: totalMainProcessMs,
|
|
1154
|
+
mainIndexMs: totalMainIndexMs,
|
|
1155
|
+
overheadMs: totalParsingPhase - totalWorkerParseMs - totalMainProcessMs - totalMainIndexMs,
|
|
1156
|
+
pureWorkerPct: totalParsingPhase > 0 ? `${Math.round(estimatedPureWorkerMs / totalParsingPhase * 100)}%` : "0%",
|
|
1157
|
+
streamingPct: totalParsingPhase > 0 ? `${Math.round(totalStreamingQueueMs / totalParsingPhase * 100)}%` : "0%",
|
|
1158
|
+
mainProcessPct: totalParsingPhase > 0 ? `${Math.round(totalMainProcessMs / totalParsingPhase * 100)}%` : "0%",
|
|
1159
|
+
mainIndexPct: totalParsingPhase > 0 ? `${Math.round(totalMainIndexMs / totalParsingPhase * 100)}%` : "0%"
|
|
1160
|
+
});
|
|
1161
|
+
if (this.parserAgent) {
|
|
1162
|
+
this.parserAgent.setStreamingMode(false);
|
|
1163
|
+
}
|
|
1164
|
+
perfTimings["flush_start"] = Date.now() - perfStart;
|
|
1165
|
+
let streamingEntities = 0;
|
|
1166
|
+
let streamingRelationships = 0;
|
|
1167
|
+
if (this.indexerAgent) {
|
|
1168
|
+
const pendingStats = this.indexerAgent.getPendingBatchStats();
|
|
1169
|
+
if (pendingStats.files > 0) {
|
|
1170
|
+
log.i("DEVAGENT", "Flushing remaining batch accumulator", pendingStats);
|
|
1171
|
+
}
|
|
1172
|
+
await this.indexerAgent.flushPendingBatch();
|
|
1173
|
+
const finalStats = this.indexerAgent.getIndexingStats();
|
|
1174
|
+
streamingEntities = finalStats.entitiesIndexed - initialIndexerStats.entitiesIndexed;
|
|
1175
|
+
streamingRelationships = finalStats.relationshipsCreated - initialIndexerStats.relationshipsCreated;
|
|
1176
|
+
log.i("DEVAGENT", "Streaming mode disabled, code parsing complete", {
|
|
1177
|
+
streamedFiles: streamingIndexedFiles.size,
|
|
1178
|
+
deltaEntities: streamingEntities,
|
|
1179
|
+
deltaRelationships: streamingRelationships
|
|
1180
|
+
});
|
|
1181
|
+
}
|
|
1182
|
+
totalEntities = streamingEntities;
|
|
1183
|
+
totalRelationships = streamingRelationships;
|
|
1184
|
+
filesProcessed += streamingIndexedFiles.size;
|
|
1185
|
+
perfTimings["flush_end"] = Date.now() - perfStart;
|
|
1186
|
+
if (dataFiles.length > 0) {
|
|
1187
|
+
perfTimings["dataFiles_start"] = Date.now() - perfStart;
|
|
1188
|
+
const dataResult = await this.processDataFilesParallel(dataFiles);
|
|
1189
|
+
totalEntities += dataResult.entities;
|
|
1190
|
+
filesProcessed += dataResult.files;
|
|
1191
|
+
perfTimings["dataFiles_end"] = Date.now() - perfStart;
|
|
1192
|
+
}
|
|
1193
|
+
perfTimings["swaggerLink_start"] = Date.now() - perfStart;
|
|
1194
|
+
try {
|
|
1195
|
+
const { resolveSwaggerLinks } = await import('./indexing-pipeline-WCXIDMAP.js');
|
|
1196
|
+
const swaggerRels = await resolveSwaggerLinks();
|
|
1197
|
+
if (swaggerRels > 0) {
|
|
1198
|
+
totalRelationships += swaggerRels;
|
|
1199
|
+
log.i("DEVAGENT", "swagger_links_created", { count: swaggerRels });
|
|
1200
|
+
}
|
|
1201
|
+
} catch (err) {
|
|
1202
|
+
log.w("DEVAGENT", "swagger_link_skip", { error: err.message });
|
|
1203
|
+
}
|
|
1204
|
+
perfTimings["swaggerLink_end"] = Date.now() - perfStart;
|
|
1205
|
+
perfTimings["indexing_end"] = Date.now() - perfStart;
|
|
1206
|
+
log.i("DEVAGENT", "index_done", {
|
|
1207
|
+
files: filesProcessed,
|
|
1208
|
+
total: allFiles.length,
|
|
1209
|
+
entities: totalEntities,
|
|
1210
|
+
rels: totalRelationships,
|
|
1211
|
+
totalMs: perfTimings["indexing_end"]
|
|
1212
|
+
});
|
|
1213
|
+
log.i("DEVAGENT", "=== ALL BATCH PROCESSING COMPLETE ===", {
|
|
1214
|
+
totalBatches: Math.ceil(codeFiles.length / effectiveBatchSize),
|
|
1215
|
+
codeFiles: codeFiles.length,
|
|
1216
|
+
dataFiles: dataFiles.length,
|
|
1217
|
+
filesProcessed,
|
|
1218
|
+
totalEntities,
|
|
1219
|
+
totalRelationships,
|
|
1220
|
+
perfTimings
|
|
1221
|
+
});
|
|
1222
|
+
log.flush();
|
|
1223
|
+
await this.postIndexingCleanup(
|
|
1224
|
+
codeFiles,
|
|
1225
|
+
filesProcessed,
|
|
1226
|
+
totalEntities,
|
|
1227
|
+
perfStart,
|
|
1228
|
+
perfTimings,
|
|
1229
|
+
effectiveBatchSize,
|
|
1230
|
+
dataFiles
|
|
1231
|
+
);
|
|
1232
|
+
this.buildPerfSummary(
|
|
1233
|
+
perfTimings,
|
|
1234
|
+
codeFiles,
|
|
1235
|
+
dataFiles,
|
|
1236
|
+
filesProcessed,
|
|
1237
|
+
totalEntities,
|
|
1238
|
+
totalRelationships,
|
|
1239
|
+
effectiveBatchSize,
|
|
1240
|
+
perfStart
|
|
1241
|
+
);
|
|
1242
|
+
return {
|
|
1243
|
+
filesProcessed,
|
|
1244
|
+
entitiesExtracted: totalEntities,
|
|
1245
|
+
relationshipsCreated: totalRelationships,
|
|
1246
|
+
totalFiles: allFiles.length,
|
|
1247
|
+
perfTimings
|
|
1248
|
+
// Detailed timing breakdown
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
handleResourceAdjustment(entry) {
|
|
1252
|
+
this.resourceMixin.handleResourceAdjustment.call(this, entry);
|
|
1253
|
+
}
|
|
1254
|
+
adjustConcurrency(newLimit) {
|
|
1255
|
+
const adjusted = Math.max(1, Math.min(this.defaultMaxConcurrency * 2, Math.floor(newLimit)));
|
|
1256
|
+
if (this.capabilities.maxConcurrency !== adjusted) {
|
|
1257
|
+
log.d("DEVAGENT", "adj_concurrency", { from: this.capabilities.maxConcurrency, to: adjusted });
|
|
1258
|
+
this.capabilities.maxConcurrency = adjusted;
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
adjustBatchSize(newMemoryLimit) {
|
|
1262
|
+
const ratio = Math.max(0.5, Math.min(2, newMemoryLimit / this.defaultMemoryLimit));
|
|
1263
|
+
const newBatchSize = Math.max(10, Math.round(this.defaultBatchSize * ratio));
|
|
1264
|
+
if (this.indexBatchSize !== newBatchSize) {
|
|
1265
|
+
log.d("DEVAGENT", "adj_batch", { from: this.indexBatchSize, to: newBatchSize });
|
|
1266
|
+
this.indexBatchSize = newBatchSize;
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Eager-start Roslyn addon if project directory contains .sln/.slnx.
|
|
1271
|
+
* Walks up to 5 levels from projectDir looking for solution file.
|
|
1272
|
+
* Non-blocking — fires and logs, does not delay init.
|
|
1273
|
+
*/
|
|
1274
|
+
tryEagerRoslynStart(projectDir) {
|
|
1275
|
+
let dir = projectDir;
|
|
1276
|
+
let slnPath = null;
|
|
1277
|
+
for (let i = 0; i < 5; i++) {
|
|
1278
|
+
slnPath = findSolutionFile(dir);
|
|
1279
|
+
if (slnPath) break;
|
|
1280
|
+
const parent = dirname(dir);
|
|
1281
|
+
if (parent === dir) break;
|
|
1282
|
+
dir = parent;
|
|
1283
|
+
}
|
|
1284
|
+
if (!slnPath) return;
|
|
1285
|
+
log.i("DEVAGENT", "roslyn_eager_start", { sln: slnPath });
|
|
1286
|
+
ensureRoslynStarted(slnPath).then((parser) => {
|
|
1287
|
+
if (parser) {
|
|
1288
|
+
log.i("DEVAGENT", "roslyn_eager_ready", { sln: slnPath });
|
|
1289
|
+
} else {
|
|
1290
|
+
log.w("DEVAGENT", "roslyn_eager_fail", { sln: slnPath });
|
|
1291
|
+
}
|
|
1292
|
+
}).catch((err) => {
|
|
1293
|
+
log.w("DEVAGENT", "roslyn_eager_error", { sln: slnPath, err: String(err) });
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
/**
|
|
1297
|
+
* Handle incremental reindexing for changed files
|
|
1298
|
+
* Called when GitWatcher detects uncommitted file changes
|
|
1299
|
+
*/
|
|
1300
|
+
async handleIncrementalReindex(files, repositoryPath) {
|
|
1301
|
+
if (!this.parserAgent || !this.indexerAgent) {
|
|
1302
|
+
log.w("DEVAGENT", "skip_reindex_no_agents");
|
|
1303
|
+
return;
|
|
1304
|
+
}
|
|
1305
|
+
const startTime = Date.now();
|
|
1306
|
+
log.i("DEVAGENT", "incr_reindex_start", { files: files.length });
|
|
1307
|
+
const { supportedFiles, otherFiles } = separateFilesBySupport(files);
|
|
1308
|
+
if (supportedFiles.length === 0 && otherFiles.length === 0) {
|
|
1309
|
+
log.i("DEVAGENT", "no_files_to_reindex");
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
const existingFiles = [];
|
|
1313
|
+
const deletedFiles = [];
|
|
1314
|
+
for (const f of files) {
|
|
1315
|
+
if (existsSync(f)) {
|
|
1316
|
+
existingFiles.push(f);
|
|
1317
|
+
} else {
|
|
1318
|
+
deletedFiles.push(f);
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
try {
|
|
1322
|
+
if (deletedFiles.length > 0) {
|
|
1323
|
+
const storage = await getGraphStorage();
|
|
1324
|
+
for (const filePath of deletedFiles) {
|
|
1325
|
+
try {
|
|
1326
|
+
await storage.deleteEntitiesByFilePath(filePath);
|
|
1327
|
+
await storage.deleteFileInfo(filePath);
|
|
1328
|
+
} catch {
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
log.i("DEVAGENT", "incr_cleanup", {
|
|
1332
|
+
deleted: deletedFiles.length,
|
|
1333
|
+
existing: existingFiles.length
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
} catch (err) {
|
|
1337
|
+
log.w("DEVAGENT", "incr_cleanup_fail", { error: err.message });
|
|
1338
|
+
}
|
|
1339
|
+
const { supportedFiles: supportedExisting, otherFiles: otherExisting } = separateFilesBySupport(existingFiles);
|
|
1340
|
+
let successCount = 0;
|
|
1341
|
+
let errorCount = 0;
|
|
1342
|
+
const embeddingConfig = buildWorkerEmbeddingConfig();
|
|
1343
|
+
if (embeddingConfig && this.parserAgent) {
|
|
1344
|
+
this.parserAgent.setEmbeddingConfig(embeddingConfig);
|
|
1345
|
+
log.d("DEVAGENT", "incr_embedding_config", { provider: embeddingConfig.provider });
|
|
1346
|
+
const configLoader = ConfigLoader.getInstance();
|
|
1347
|
+
const embConfig = configLoader.getEmbeddingConfig();
|
|
1348
|
+
const useLayeredIndex = embConfig.useLayeredIndex;
|
|
1349
|
+
const currentDir = repositoryPath || getCurrentIndexingDirectory() || process.cwd();
|
|
1350
|
+
const { getProjectHash, getCurrentGitBranchOrDefault } = await import('./storage-paths-NGF6GAFZ.js');
|
|
1351
|
+
const projectHash = getProjectHash(currentDir);
|
|
1352
|
+
const currentBranch = getCurrentGitBranchOrDefault(currentDir);
|
|
1353
|
+
try {
|
|
1354
|
+
if (useLayeredIndex) {
|
|
1355
|
+
const { getLayeredFaissProvider } = await import('./layered-faiss-provider-M3MDI2AG.js');
|
|
1356
|
+
const provider = getLayeredFaissProvider();
|
|
1357
|
+
if ("initialize" in provider && typeof provider.initialize === "function") {
|
|
1358
|
+
await provider.initialize(currentDir, projectHash, currentBranch);
|
|
1359
|
+
}
|
|
1360
|
+
this.parserAgent.setVectorProvider(provider);
|
|
1361
|
+
log.i("DEVAGENT", "incr_vector_provider", { branch: currentBranch, layered: true });
|
|
1362
|
+
} else {
|
|
1363
|
+
const { initializeFaissProvider } = await import('./faiss-provider-VAYK2VBT.js');
|
|
1364
|
+
const provider = await initializeFaissProvider();
|
|
1365
|
+
if (provider) {
|
|
1366
|
+
await provider.setProjectContext(projectHash, currentBranch);
|
|
1367
|
+
this.parserAgent.setVectorProvider(provider);
|
|
1368
|
+
log.i("DEVAGENT", "incr_vector_provider", { branch: currentBranch, layered: false });
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
} catch (e) {
|
|
1372
|
+
log.w("DEVAGENT", "incr_vector_provider_fail", { error: String(e) });
|
|
1373
|
+
}
|
|
1374
|
+
log.d("DEVAGENT", "incr_embedding_config", {
|
|
1375
|
+
provider: embeddingConfig.provider,
|
|
1376
|
+
centralized: embeddingConfig.centralizedEmbeddings,
|
|
1377
|
+
batchSize: embeddingConfig.batchSize
|
|
1378
|
+
});
|
|
1379
|
+
if (embeddingConfig.centralizedEmbeddings) {
|
|
1380
|
+
try {
|
|
1381
|
+
const { EmbeddingGenerator } = await import('./embedding-generator-XR6FDO4J.js');
|
|
1382
|
+
const { buildEmbeddingGeneratorOptions } = await import('./provider-config-67EQAZ5Z.js');
|
|
1383
|
+
const { loadSemanticConfig } = await import('./config-paths-Y2IHHJQF.js');
|
|
1384
|
+
const { getConfig: getConfig2 } = await import('./yaml-config-OVNUY25R.js');
|
|
1385
|
+
const semanticConfig = loadSemanticConfig();
|
|
1386
|
+
const yamlConfig = getConfig2();
|
|
1387
|
+
const generatorOptions = buildEmbeddingGeneratorOptions(
|
|
1388
|
+
embeddingConfig.provider,
|
|
1389
|
+
embeddingConfig.modelName,
|
|
1390
|
+
embeddingConfig.batchSize,
|
|
1391
|
+
semanticConfig,
|
|
1392
|
+
yamlConfig
|
|
1393
|
+
);
|
|
1394
|
+
const embeddingGenerator = new EmbeddingGenerator(generatorOptions);
|
|
1395
|
+
await embeddingGenerator.initialize();
|
|
1396
|
+
await this.parserAgent.setEmbeddingGenerator(embeddingGenerator);
|
|
1397
|
+
log.i("DEVAGENT", "incr_embedding_generator", {
|
|
1398
|
+
provider: embeddingConfig.provider,
|
|
1399
|
+
model: embeddingConfig.modelName
|
|
1400
|
+
});
|
|
1401
|
+
} catch (error) {
|
|
1402
|
+
log.w("DEVAGENT", "incr_embedding_generator_fail", {
|
|
1403
|
+
error: error.message
|
|
1404
|
+
});
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
if (supportedExisting.length > 0) {
|
|
1409
|
+
try {
|
|
1410
|
+
log.i("DEVAGENT", "incr_parseBatch_start", { files: supportedExisting.length });
|
|
1411
|
+
const parseResults = await this.parserAgent.parseBatch(supportedExisting, {});
|
|
1412
|
+
log.i("DEVAGENT", "incr_parseBatch_done", { files: supportedExisting.length, results: parseResults.length });
|
|
1413
|
+
for (const parseResult of parseResults) {
|
|
1414
|
+
if (parseResult.entities && parseResult.entities.length > 0) {
|
|
1415
|
+
try {
|
|
1416
|
+
this.indexerAgent.queueForIndexing(parseResult.entities, parseResult.filePath, parseResult.relationships);
|
|
1417
|
+
successCount++;
|
|
1418
|
+
} catch (queueError) {
|
|
1419
|
+
log.e("DEVAGENT", "queue_fail", { file: parseResult.filePath, err: String(queueError) });
|
|
1420
|
+
errorCount++;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
} catch (error) {
|
|
1425
|
+
log.e("DEVAGENT", "batch_parse_fail", { files: supportedExisting.length, err: String(error) });
|
|
1426
|
+
errorCount += supportedExisting.length;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
for (const filePath of otherExisting) {
|
|
1430
|
+
try {
|
|
1431
|
+
const heuristicResult = createHeuristicEntities(filePath);
|
|
1432
|
+
if (heuristicResult.entities.length > 0) {
|
|
1433
|
+
this.indexerAgent.queueForIndexing(heuristicResult.entities, filePath, heuristicResult.relationships);
|
|
1434
|
+
successCount++;
|
|
1435
|
+
}
|
|
1436
|
+
} catch (error) {
|
|
1437
|
+
log.e("DEVAGENT", "heuristic_fail", { file: filePath, err: String(error) });
|
|
1438
|
+
errorCount++;
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
try {
|
|
1442
|
+
const flushResult = await this.indexerAgent.flushPendingBatch();
|
|
1443
|
+
log.i("DEVAGENT", "incr_batch_flush", {
|
|
1444
|
+
entities: flushResult.entities,
|
|
1445
|
+
relationships: flushResult.relationships,
|
|
1446
|
+
files: flushResult.files
|
|
1447
|
+
});
|
|
1448
|
+
} catch (flushError) {
|
|
1449
|
+
log.e("DEVAGENT", "incr_flush_fail", { err: String(flushError) });
|
|
1450
|
+
}
|
|
1451
|
+
const elapsed = Date.now() - startTime;
|
|
1452
|
+
log.i("DEVAGENT", "incr_reindex_done", {
|
|
1453
|
+
success: successCount,
|
|
1454
|
+
errors: errorCount,
|
|
1455
|
+
deleted: deletedFiles.length,
|
|
1456
|
+
ms: elapsed
|
|
1457
|
+
});
|
|
1458
|
+
if (this.parserAgent) {
|
|
1459
|
+
const accumulator = this.parserAgent.getAccumulator();
|
|
1460
|
+
if (accumulator) {
|
|
1461
|
+
const pendingCount = accumulator.getPendingCount();
|
|
1462
|
+
if (pendingCount > 0) {
|
|
1463
|
+
log.i("DEVAGENT", "Flushing incremental embeddings to FAISS", { pending: pendingCount });
|
|
1464
|
+
try {
|
|
1465
|
+
const flushed = await accumulator.flush();
|
|
1466
|
+
log.i("DEVAGENT", "Incremental embeddings flushed", { flushed });
|
|
1467
|
+
} catch (err) {
|
|
1468
|
+
log.e("DEVAGENT", "Failed to flush incremental embeddings", { error: err.message });
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
try {
|
|
1472
|
+
const configLoader = ConfigLoader.getInstance();
|
|
1473
|
+
const embConfig = configLoader.getEmbeddingConfig();
|
|
1474
|
+
const useLayeredIndex = embConfig.useLayeredIndex;
|
|
1475
|
+
if (useLayeredIndex) {
|
|
1476
|
+
const { getLayeredFaissProvider } = await import('./layered-faiss-provider-M3MDI2AG.js');
|
|
1477
|
+
const provider = getLayeredFaissProvider();
|
|
1478
|
+
if ("save" in provider && typeof provider.save === "function") {
|
|
1479
|
+
await provider.save();
|
|
1480
|
+
log.i("DEVAGENT", "Saved layered FAISS index after incremental");
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
} catch (saveErr) {
|
|
1484
|
+
log.w("DEVAGENT", "Failed to save FAISS index after incremental", {
|
|
1485
|
+
error: saveErr.message
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
try {
|
|
1491
|
+
const storage = await getGraphStorage();
|
|
1492
|
+
const adapter = storage.getLibSQLAdapter?.();
|
|
1493
|
+
if (adapter?.createGraphCommit && successCount > 0) {
|
|
1494
|
+
const commitHash = await adapter.createGraphCommit(`Incremental: ${successCount} files`);
|
|
1495
|
+
if (commitHash) {
|
|
1496
|
+
log.i("DEVAGENT", "incr_commit_created", {
|
|
1497
|
+
commit: commitHash.slice(0, 8),
|
|
1498
|
+
files: successCount
|
|
1499
|
+
});
|
|
1500
|
+
}
|
|
1501
|
+
adapter.pruneAndGC?.(20)?.catch?.((err) => log.w("DEVAGENT", "incr_prune_gc_fail", { err: String(err) }));
|
|
1502
|
+
}
|
|
1503
|
+
} catch (err) {
|
|
1504
|
+
log.w("DEVAGENT", "incr_commit_failed", { error: err.message });
|
|
1505
|
+
}
|
|
1506
|
+
if (this.parserAgent) {
|
|
1507
|
+
try {
|
|
1508
|
+
const killed = await this.parserAgent.killIfMemoryHigh(1024);
|
|
1509
|
+
if (killed) {
|
|
1510
|
+
this.parserAgent = null;
|
|
1511
|
+
log.i("DEVAGENT", "Parser workers killed (memory > 1GB after incremental)");
|
|
1512
|
+
}
|
|
1513
|
+
} catch (_err) {
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
if (tryGarbageCollect(true)) {
|
|
1517
|
+
log.d("DEVAGENT", "gc_after_incremental");
|
|
1518
|
+
}
|
|
1519
|
+
knowledgeBus.publish(
|
|
1520
|
+
"indexer:incremental:complete",
|
|
1521
|
+
{
|
|
1522
|
+
filesProcessed: successCount,
|
|
1523
|
+
errors: errorCount,
|
|
1524
|
+
elapsedMs: elapsed,
|
|
1525
|
+
source: "dev-agent"
|
|
1526
|
+
// Different source to distinguish from git-watcher
|
|
1527
|
+
},
|
|
1528
|
+
this.id
|
|
1529
|
+
);
|
|
1530
|
+
getGraphStorage().then((storage) => storage.runGenerationGC()).then((gc) => {
|
|
1531
|
+
if (gc.entities > 0 || gc.tokens > 0) {
|
|
1532
|
+
log.i("DEVAGENT", "gen_gc_complete", gc);
|
|
1533
|
+
}
|
|
1534
|
+
}).catch((err) => log.w("DEVAGENT", "gen_gc_fail", { error: err.message }));
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* Process data files in parallel with batch insert
|
|
1538
|
+
* OPTIMIZATION: Instead of sequential await per file, we:
|
|
1539
|
+
* 1. Create heuristic entities for all files in parallel (CPU-bound, fast)
|
|
1540
|
+
* 2. Group by file and batch insert via indexerAgent
|
|
1541
|
+
* 3. Use Promise.all with chunking for controlled parallelism
|
|
1542
|
+
*/
|
|
1543
|
+
async processDataFilesParallel(dataFiles) {
|
|
1544
|
+
if (dataFiles.length === 0 || !this.indexerAgent) {
|
|
1545
|
+
return { entities: 0, files: 0 };
|
|
1546
|
+
}
|
|
1547
|
+
const startTime = Date.now();
|
|
1548
|
+
log.i("DEVAGENT", "Processing data files in PARALLEL", {
|
|
1549
|
+
count: dataFiles.length
|
|
1550
|
+
});
|
|
1551
|
+
const CHUNK_SIZE = Math.max(32, cpus().length * 4);
|
|
1552
|
+
const allResults = [];
|
|
1553
|
+
for (let i = 0; i < dataFiles.length; i += CHUNK_SIZE) {
|
|
1554
|
+
const chunk = dataFiles.slice(i, i + CHUNK_SIZE);
|
|
1555
|
+
const chunkResults = await Promise.all(
|
|
1556
|
+
chunk.map(async (file) => {
|
|
1557
|
+
try {
|
|
1558
|
+
return createHeuristicEntities(file);
|
|
1559
|
+
} catch {
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1562
|
+
})
|
|
1563
|
+
);
|
|
1564
|
+
for (const result of chunkResults) {
|
|
1565
|
+
if (result && result.entities.length > 0) {
|
|
1566
|
+
allResults.push(result);
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
const INDEXING_CHUNK_SIZE = Math.max(32, cpus().length * 2);
|
|
1571
|
+
let totalEntities = 0;
|
|
1572
|
+
let filesProcessed = 0;
|
|
1573
|
+
for (let i = 0; i < allResults.length; i += INDEXING_CHUNK_SIZE) {
|
|
1574
|
+
const chunk = allResults.slice(i, i + INDEXING_CHUNK_SIZE);
|
|
1575
|
+
const indexPromises = chunk.map(async (result) => {
|
|
1576
|
+
try {
|
|
1577
|
+
const indexResult = await this.indexerAgent.indexEntities(
|
|
1578
|
+
result.entities,
|
|
1579
|
+
result.filePath,
|
|
1580
|
+
result.relationships
|
|
1581
|
+
);
|
|
1582
|
+
return { entities: indexResult.entitiesIndexed, success: true };
|
|
1583
|
+
} catch {
|
|
1584
|
+
return { entities: 0, success: false };
|
|
1585
|
+
}
|
|
1586
|
+
});
|
|
1587
|
+
const results = await Promise.all(indexPromises);
|
|
1588
|
+
for (const r of results) {
|
|
1589
|
+
if (r.success) {
|
|
1590
|
+
totalEntities += r.entities;
|
|
1591
|
+
filesProcessed++;
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
const elapsed = Date.now() - startTime;
|
|
1596
|
+
log.i("DEVAGENT", "Data files processed in PARALLEL", {
|
|
1597
|
+
files: filesProcessed,
|
|
1598
|
+
entities: totalEntities,
|
|
1599
|
+
elapsedMs: elapsed,
|
|
1600
|
+
filesPerSec: Math.round(filesProcessed / elapsed * 1e3)
|
|
1601
|
+
});
|
|
1602
|
+
return { entities: totalEntities, files: filesProcessed };
|
|
1603
|
+
}
|
|
1604
|
+
/**
|
|
1605
|
+
* Get embedding generation statistics from parser agent
|
|
1606
|
+
*/
|
|
1607
|
+
getEmbeddingStats() {
|
|
1608
|
+
return this.parserAgent?.getEmbeddingPoolStats?.() ?? null;
|
|
1609
|
+
}
|
|
1610
|
+
async onShutdown() {
|
|
1611
|
+
log.i("DEVAGENT", "shutdown_start");
|
|
1612
|
+
if (this.parserAgent) {
|
|
1613
|
+
await this.parserAgent.shutdown();
|
|
1614
|
+
}
|
|
1615
|
+
if (this.indexerAgent) {
|
|
1616
|
+
await this.indexerAgent.shutdown();
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
};
|
|
1620
|
+
var devAgent = new DevAgent();
|
|
1621
|
+
|
|
1622
|
+
export { DevAgent, devAgent };
|
|
1623
|
+
//# sourceMappingURL=dev-agent-Y5G5WKQ4.js.map
|
|
1624
|
+
//# sourceMappingURL=dev-agent-Y5G5WKQ4.js.map
|