ultracode 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/analysis-tool-handlers-H2RXLDPX.js +817 -0
- package/dist/chunks/analysis-tool-handlers-RJZAR6VT.js +817 -0
- package/dist/chunks/analysis-tool-handlers-Z2RF24T7.js +13 -0
- package/dist/chunks/autodoc-tool-handlers-CV5JEQUA.js +1112 -0
- package/dist/chunks/autodoc-tool-handlers-EHTNCH6I.js +1112 -0
- package/dist/chunks/autodoc-tool-handlers-MECXQJ2K.js +138 -0
- package/dist/chunks/chaos-CO7TOBOJ.js +18 -0
- package/dist/chunks/chaos-VM2PXERO.js +1573 -0
- package/dist/chunks/chaos-W3XRVJ7K.js +1564 -0
- package/dist/chunks/chunk-6K37BWK5.js +439 -0
- package/dist/chunks/chunk-EALTCYHZ.js +10 -0
- package/dist/chunks/chunk-FTBE7VMY.js +316 -0
- package/dist/chunks/chunk-KBW6LRQP.js +322 -0
- package/dist/chunks/chunk-NKUHX4CU.js +5 -0
- package/dist/chunks/chunk-NZFF4DQ4.js +3179 -0
- package/dist/chunks/chunk-RGP5UVQ7.js +3179 -0
- package/dist/chunks/chunk-RMZXFGQZ.js +322 -0
- package/dist/chunks/chunk-UG44F23Y.js +316 -0
- package/dist/chunks/chunk-V2SCB5H5.js +4403 -0
- package/dist/chunks/chunk-V6JAQNM3.js +1 -0
- package/dist/chunks/chunk-XFGXM4CR.js +4403 -0
- package/dist/chunks/dev-agent-JVIGBMHQ.js +1 -0
- package/dist/chunks/dev-agent-TRVP5U6N.js +1624 -0
- package/dist/chunks/dev-agent-Y5G5WKQ4.js +1624 -0
- package/dist/chunks/graph-storage-factory-AYZ57YSL.js +13 -0
- package/dist/chunks/graph-storage-factory-GTAIJEI5.js +1 -0
- package/dist/chunks/graph-storage-factory-T2WO5QVG.js +13 -0
- package/dist/chunks/incremental-updater-KDIQGAUU.js +14 -0
- package/dist/chunks/incremental-updater-OJRSTO3Q.js +1 -0
- package/dist/chunks/incremental-updater-SBEBH7KF.js +14 -0
- package/dist/chunks/indexer-agent-H3QIEL3Z.js +21 -0
- package/dist/chunks/indexer-agent-KHF5JMV7.js +21 -0
- package/dist/chunks/indexer-agent-SHJD6Z77.js +1 -0
- package/dist/chunks/indexing-pipeline-J6Z4BHKF.js +1 -0
- package/dist/chunks/indexing-pipeline-OY3337QN.js +249 -0
- package/dist/chunks/indexing-pipeline-WCXIDMAP.js +249 -0
- package/dist/chunks/merge-agent-LSUBDJB2.js +2481 -0
- package/dist/chunks/merge-agent-MJEW3HWU.js +2481 -0
- package/dist/chunks/merge-agent-O45OXF33.js +11 -0
- package/dist/chunks/merge-tool-handlers-BDSVNQVZ.js +277 -0
- package/dist/chunks/merge-tool-handlers-HP7DRBXJ.js +1 -0
- package/dist/chunks/merge-tool-handlers-RUJAKE3D.js +277 -0
- package/dist/chunks/pattern-tool-handlers-L62W3CXR.js +1549 -0
- package/dist/chunks/pattern-tool-handlers-SAHX2CVW.js +13 -0
- package/dist/chunks/query-agent-3TWDFIMT.js +191 -0
- package/dist/chunks/query-agent-HXQ3BMMF.js +191 -0
- package/dist/chunks/query-agent-USMC2GNG.js +1 -0
- package/dist/chunks/semantic-agent-MQCAWIAB.js +6381 -0
- package/dist/chunks/semantic-agent-NDGR3NAK.js +6381 -0
- package/dist/chunks/semantic-agent-S4ZL6GZC.js +137 -0
- package/dist/index.js +17 -17
- package/dist/roslyn-addon/.build-hash +1 -1
- package/dist/roslyn-addon/ILGPU.Algorithms.dll +0 -0
- package/dist/roslyn-addon/ILGPU.dll +0 -0
- package/dist/roslyn-addon/UltraCode.CSharp.deps.json +35 -0
- package/dist/roslyn-addon/UltraCode.CSharp.dll +0 -0
- package/package.json +1 -1
|
@@ -0,0 +1,4403 @@
|
|
|
1
|
+
import { ProllyNodeStore, CommitManager, BranchDiffCache } from './chunk-HNDYLCWI.js';
|
|
2
|
+
import { ProllyTree, serializeEntity } from './chunk-AIZUHUK6.js';
|
|
3
|
+
import { detectBaseBranch } from './chunk-L2X4HRXI.js';
|
|
4
|
+
import { init_storage_paths, getCurrentGitBranchOrDefault, normalizeBranchName, getProjectHash, getGlobalDbPaths } from './chunk-ZD54CMKT.js';
|
|
5
|
+
import { init_logging, log } from './chunk-VCCBEJQ5.js';
|
|
6
|
+
import { DatabaseCorruptionError, DEFAULT_CONFIG, CACHE_CONFIG, normalizeToSupportedDimension, getEmbeddingColumn, isCorruptionError } from './chunk-UN27MREV.js';
|
|
7
|
+
import { existsSync, mkdirSync, unlinkSync, statSync } from 'fs';
|
|
8
|
+
import { join, dirname } from 'path';
|
|
9
|
+
import { nanoid } from 'nanoid';
|
|
10
|
+
import xxhash from 'xxhash-wasm';
|
|
11
|
+
import * as cbor from 'cbor-x';
|
|
12
|
+
import { LRUCache } from 'lru-cache';
|
|
13
|
+
import { AsyncLocalStorage } from 'async_hooks';
|
|
14
|
+
|
|
15
|
+
// src/storage/graph-storage-factory.ts
|
|
16
|
+
init_logging();
|
|
17
|
+
init_storage_paths();
|
|
18
|
+
|
|
19
|
+
// src/storage/graph-storage-libsql.ts
|
|
20
|
+
init_logging();
|
|
21
|
+
init_storage_paths();
|
|
22
|
+
var ID_LENGTH = 12;
|
|
23
|
+
var DEFAULT_QUERY_LIMIT = 100;
|
|
24
|
+
var MAX_QUERY_LIMIT = 5e4;
|
|
25
|
+
var MAX_SUBGRAPH_DEPTH = 5;
|
|
26
|
+
function createProjectContext(projectPath, branchName) {
|
|
27
|
+
const resolvedBranch = branchName ?? getCurrentGitBranchOrDefault(projectPath);
|
|
28
|
+
const normalizedBranch = normalizeBranchName(resolvedBranch);
|
|
29
|
+
const baseBranch = detectBaseBranch(projectPath);
|
|
30
|
+
return {
|
|
31
|
+
projectHash: getProjectHash(projectPath),
|
|
32
|
+
branchName: normalizedBranch,
|
|
33
|
+
// Set baseBranch for layered reads if current branch is different from base
|
|
34
|
+
baseBranch: baseBranch && baseBranch !== normalizedBranch ? baseBranch : void 0
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
var GraphStorageLibSQL = class {
|
|
38
|
+
adapter;
|
|
39
|
+
xxhashInstance = null;
|
|
40
|
+
constructor(adapter) {
|
|
41
|
+
this.adapter = adapter;
|
|
42
|
+
}
|
|
43
|
+
// ===========================================================================
|
|
44
|
+
// INITIALIZATION
|
|
45
|
+
// ===========================================================================
|
|
46
|
+
async initialize() {
|
|
47
|
+
const startTime = Date.now();
|
|
48
|
+
log.t("STORAGE", `[GraphStorageLibSQL] \u25B6 initialize() START`);
|
|
49
|
+
this.xxhashInstance = await xxhash();
|
|
50
|
+
log.t("STORAGE", `[GraphStorageLibSQL] \u25C0 initialize() END (${Date.now() - startTime}ms)`);
|
|
51
|
+
log.i("GRAPHSTORAGE", "init_xxhash");
|
|
52
|
+
}
|
|
53
|
+
// ===========================================================================
|
|
54
|
+
// PROJECT CONTEXT
|
|
55
|
+
// ===========================================================================
|
|
56
|
+
setProjectContext(context) {
|
|
57
|
+
this.adapter.setProjectContext(context);
|
|
58
|
+
}
|
|
59
|
+
setProject(projectPath, branchName) {
|
|
60
|
+
const ctx = createProjectContext(projectPath, branchName);
|
|
61
|
+
log.w("STORAGE", "setProject", {
|
|
62
|
+
path: projectPath,
|
|
63
|
+
hash: ctx.projectHash,
|
|
64
|
+
branch: ctx.branchName,
|
|
65
|
+
base: ctx.baseBranch || "none"
|
|
66
|
+
});
|
|
67
|
+
this.adapter.setProjectContext(ctx);
|
|
68
|
+
this.adapter.loadGenerationCache().catch((err) => {
|
|
69
|
+
log.w("STORAGE", "gen_cache_load_fail", { error: err.message });
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
getProjectContext() {
|
|
73
|
+
return this.adapter.getProjectContext();
|
|
74
|
+
}
|
|
75
|
+
// ===========================================================================
|
|
76
|
+
// ENTITY OPERATIONS
|
|
77
|
+
// ===========================================================================
|
|
78
|
+
async insertEntity(entity) {
|
|
79
|
+
const id = this.stableEntityId(entity);
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
const entityWithId = {
|
|
82
|
+
...entity,
|
|
83
|
+
id,
|
|
84
|
+
complexityScore: entity.complexityScore ?? this.calculateComplexity(entity),
|
|
85
|
+
language: entity.language ?? this.detectLanguage(entity.filePath),
|
|
86
|
+
sizeBytes: entity.sizeBytes ?? 0,
|
|
87
|
+
createdAt: entity.createdAt || now,
|
|
88
|
+
updatedAt: entity.updatedAt || now
|
|
89
|
+
};
|
|
90
|
+
await this.adapter.insertEntity(entityWithId);
|
|
91
|
+
}
|
|
92
|
+
async insertEntities(entities) {
|
|
93
|
+
const startTime = Date.now();
|
|
94
|
+
log.t("STORAGE", `[GraphStorageLibSQL] \u25B6 insertEntities (${entities.length} entities)`);
|
|
95
|
+
const now = Date.now();
|
|
96
|
+
const seen = /* @__PURE__ */ new Set();
|
|
97
|
+
const unique = [];
|
|
98
|
+
for (const e of entities) {
|
|
99
|
+
const key = this.entityKey(e);
|
|
100
|
+
if (!seen.has(key)) {
|
|
101
|
+
seen.add(key);
|
|
102
|
+
unique.push(e);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const entitiesWithIds = unique.map((entity) => ({
|
|
106
|
+
...entity,
|
|
107
|
+
id: this.stableEntityId(entity),
|
|
108
|
+
complexityScore: entity.complexityScore ?? this.calculateComplexity(entity),
|
|
109
|
+
language: entity.language ?? this.detectLanguage(entity.filePath),
|
|
110
|
+
sizeBytes: entity.sizeBytes ?? 0,
|
|
111
|
+
createdAt: entity.createdAt || now,
|
|
112
|
+
updatedAt: entity.updatedAt || now
|
|
113
|
+
}));
|
|
114
|
+
const result = await this.adapter.insertEntities(entitiesWithIds);
|
|
115
|
+
log.t("STORAGE", `insertEntities`, { count: unique.length, ms: Date.now() - startTime });
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
async updateEntity(id, updates) {
|
|
119
|
+
const existing = await this.adapter.getEntity(id);
|
|
120
|
+
if (!existing) {
|
|
121
|
+
throw new Error(`Entity ${id} not found`);
|
|
122
|
+
}
|
|
123
|
+
const updated = {
|
|
124
|
+
...existing,
|
|
125
|
+
...updates,
|
|
126
|
+
updatedAt: Date.now(),
|
|
127
|
+
complexityScore: updates.complexityScore ?? this.calculateComplexity({ ...existing, ...updates }),
|
|
128
|
+
language: updates.language ?? this.detectLanguage(updates.filePath || existing.filePath)
|
|
129
|
+
};
|
|
130
|
+
await this.adapter.insertEntity(updated);
|
|
131
|
+
}
|
|
132
|
+
async deleteEntity(id) {
|
|
133
|
+
await this.adapter.deleteEntity(id);
|
|
134
|
+
}
|
|
135
|
+
async getEntity(id) {
|
|
136
|
+
return await this.adapter.getEntity(id);
|
|
137
|
+
}
|
|
138
|
+
async getEntitiesBatch(ids) {
|
|
139
|
+
return await this.adapter.getEntitiesBatch(ids);
|
|
140
|
+
}
|
|
141
|
+
async getEntityFromBranch(id, targetBranch) {
|
|
142
|
+
const currentContext = this.adapter.getProjectContext();
|
|
143
|
+
this.adapter.setProjectContext({
|
|
144
|
+
projectHash: currentContext.projectHash,
|
|
145
|
+
branchName: normalizeBranchName(targetBranch)
|
|
146
|
+
});
|
|
147
|
+
try {
|
|
148
|
+
return await this.adapter.getEntity(id);
|
|
149
|
+
} finally {
|
|
150
|
+
this.adapter.setProjectContext(currentContext);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async findEntities(query) {
|
|
154
|
+
return await this.adapter.findEntities({
|
|
155
|
+
filters: query.filters,
|
|
156
|
+
limit: Math.min(query.limit || DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT),
|
|
157
|
+
offset: query.offset || 0
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Count entities by language (efficient SQL aggregation for TechnologyDetector).
|
|
162
|
+
* Excludes external placeholder entities.
|
|
163
|
+
*/
|
|
164
|
+
async countByLanguage() {
|
|
165
|
+
return await this.adapter.countByLanguage();
|
|
166
|
+
}
|
|
167
|
+
async findEntitiesInBranch(query, targetBranch) {
|
|
168
|
+
const currentContext = this.adapter.getProjectContext();
|
|
169
|
+
this.adapter.setProjectContext({
|
|
170
|
+
projectHash: currentContext.projectHash,
|
|
171
|
+
branchName: normalizeBranchName(targetBranch)
|
|
172
|
+
});
|
|
173
|
+
try {
|
|
174
|
+
return await this.adapter.findEntities({
|
|
175
|
+
filters: query.filters,
|
|
176
|
+
limit: Math.min(query.limit || DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT),
|
|
177
|
+
offset: query.offset || 0
|
|
178
|
+
});
|
|
179
|
+
} finally {
|
|
180
|
+
this.adapter.setProjectContext(currentContext);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async compareEntitiesBetweenBranches(namePattern, branch1, branch2) {
|
|
184
|
+
const query = {
|
|
185
|
+
type: "entity",
|
|
186
|
+
filters: { name: new RegExp(namePattern) },
|
|
187
|
+
limit: MAX_QUERY_LIMIT
|
|
188
|
+
};
|
|
189
|
+
const [entities1, entities2] = await Promise.all([
|
|
190
|
+
this.findEntitiesInBranch(query, branch1),
|
|
191
|
+
this.findEntitiesInBranch(query, branch2)
|
|
192
|
+
]);
|
|
193
|
+
const map1 = new Map(entities1.map((e) => [this.entityKey(e), e]));
|
|
194
|
+
const matched = [];
|
|
195
|
+
for (const e2 of entities2) {
|
|
196
|
+
const key = this.entityKey(e2);
|
|
197
|
+
const e1 = map1.get(key);
|
|
198
|
+
if (e1) {
|
|
199
|
+
matched.push([e1, e2]);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return { branch1Entities: entities1, branch2Entities: entities2, matched };
|
|
203
|
+
}
|
|
204
|
+
async getAllEntities() {
|
|
205
|
+
return await this.adapter.getAllEntities();
|
|
206
|
+
}
|
|
207
|
+
async searchEntities(options) {
|
|
208
|
+
return await this.adapter.searchEntities(options);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Search entities by directory path (LIKE pattern)
|
|
212
|
+
*/
|
|
213
|
+
async searchEntitiesInDirectory(directoryPath) {
|
|
214
|
+
return await this.adapter.searchEntitiesInDirectory(directoryPath);
|
|
215
|
+
}
|
|
216
|
+
// ===========================================================================
|
|
217
|
+
// RELATIONSHIP OPERATIONS
|
|
218
|
+
// ===========================================================================
|
|
219
|
+
async insertRelationship(relationship) {
|
|
220
|
+
const id = this.stableRelationshipId(relationship);
|
|
221
|
+
const now = Date.now();
|
|
222
|
+
const relWithId = {
|
|
223
|
+
...relationship,
|
|
224
|
+
id,
|
|
225
|
+
createdAt: relationship.createdAt ?? now
|
|
226
|
+
};
|
|
227
|
+
await this.adapter.insertRelationship(relWithId);
|
|
228
|
+
const reverseRels = this.generateReverseRelationships([relationship]);
|
|
229
|
+
for (const rev of reverseRels) {
|
|
230
|
+
const revWithId = {
|
|
231
|
+
...rev,
|
|
232
|
+
id: this.stableRelationshipId(rev),
|
|
233
|
+
createdAt: now
|
|
234
|
+
};
|
|
235
|
+
await this.adapter.insertRelationship(revWithId);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
async insertRelationships(relationships) {
|
|
239
|
+
const startTime = Date.now();
|
|
240
|
+
log.t("STORAGE", `[GraphStorageLibSQL] \u25B6 insertRelationships (${relationships.length} relationships)`);
|
|
241
|
+
const now = Date.now();
|
|
242
|
+
const seen = /* @__PURE__ */ new Set();
|
|
243
|
+
const unique = [];
|
|
244
|
+
for (const r of relationships) {
|
|
245
|
+
const key = this.relationshipKey(r);
|
|
246
|
+
if (!seen.has(key)) {
|
|
247
|
+
seen.add(key);
|
|
248
|
+
unique.push(r);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
const reverseRelationships = this.generateReverseRelationships(unique);
|
|
252
|
+
for (const r of reverseRelationships) {
|
|
253
|
+
const key = this.relationshipKey(r);
|
|
254
|
+
if (!seen.has(key)) {
|
|
255
|
+
seen.add(key);
|
|
256
|
+
unique.push(r);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
const relsWithIds = unique.map((r) => ({
|
|
260
|
+
...r,
|
|
261
|
+
id: this.stableRelationshipId(r),
|
|
262
|
+
createdAt: r.createdAt ?? now
|
|
263
|
+
}));
|
|
264
|
+
const result = await this.adapter.insertRelationships(relsWithIds);
|
|
265
|
+
log.t("STORAGE", `insertRelationships`, { count: unique.length, ms: Date.now() - startTime });
|
|
266
|
+
return result;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Generate reverse relationships for bidirectional graph traversal
|
|
270
|
+
* For each A → calls → B, creates B → called_by → A
|
|
271
|
+
*/
|
|
272
|
+
generateReverseRelationships(relationships) {
|
|
273
|
+
const reverseMap = {
|
|
274
|
+
["calls" /* CALLS */]: "called_by" /* CALLED_BY */,
|
|
275
|
+
["imports" /* IMPORTS */]: "imported_by" /* IMPORTED_BY */,
|
|
276
|
+
["references" /* REFERENCES */]: "referenced_by" /* REFERENCED_BY */,
|
|
277
|
+
["extends" /* EXTENDS */]: "extended_by" /* EXTENDED_BY */,
|
|
278
|
+
["implements" /* IMPLEMENTS */]: "implemented_by" /* IMPLEMENTED_BY */,
|
|
279
|
+
// No reverse for these (already bidirectional or self-referential)
|
|
280
|
+
["called_by" /* CALLED_BY */]: null,
|
|
281
|
+
["imported_by" /* IMPORTED_BY */]: null,
|
|
282
|
+
["referenced_by" /* REFERENCED_BY */]: null,
|
|
283
|
+
["extended_by" /* EXTENDED_BY */]: null,
|
|
284
|
+
["implemented_by" /* IMPLEMENTED_BY */]: null,
|
|
285
|
+
["exports" /* EXPORTS */]: null,
|
|
286
|
+
["contains" /* CONTAINS */]: null,
|
|
287
|
+
["depends_on" /* DEPENDS_ON */]: null,
|
|
288
|
+
["member_of" /* MEMBER_OF */]: null,
|
|
289
|
+
["documents" /* DOCUMENTS */]: null,
|
|
290
|
+
["dispatches_action" /* DISPATCHES_ACTION */]: null,
|
|
291
|
+
["listens_to_action" /* LISTENS_TO_ACTION */]: null,
|
|
292
|
+
["handles_action" /* HANDLES_ACTION */]: null,
|
|
293
|
+
["selects_state" /* SELECTS_STATE */]: null,
|
|
294
|
+
["modifies_state" /* MODIFIES_STATE */]: null,
|
|
295
|
+
["produces_api" /* PRODUCES_API */]: null,
|
|
296
|
+
["consumes_api" /* CONSUMES_API */]: null,
|
|
297
|
+
["generated_from" /* GENERATED_FROM */]: null
|
|
298
|
+
};
|
|
299
|
+
const reverse = [];
|
|
300
|
+
for (const r of relationships) {
|
|
301
|
+
const reverseType = reverseMap[r.type];
|
|
302
|
+
if (reverseType) {
|
|
303
|
+
reverse.push({
|
|
304
|
+
id: "",
|
|
305
|
+
// Will be assigned by stableRelationshipId
|
|
306
|
+
fromId: r.toId,
|
|
307
|
+
toId: r.fromId,
|
|
308
|
+
type: reverseType,
|
|
309
|
+
metadata: {
|
|
310
|
+
...r.metadata,
|
|
311
|
+
isReverse: true,
|
|
312
|
+
originalType: r.type
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return reverse;
|
|
318
|
+
}
|
|
319
|
+
async deleteRelationship(id) {
|
|
320
|
+
await this.adapter.deleteRelationship(id);
|
|
321
|
+
}
|
|
322
|
+
async getRelationshipsForEntity(entityId, type) {
|
|
323
|
+
return await this.adapter.getRelationshipsForEntity(entityId, type);
|
|
324
|
+
}
|
|
325
|
+
async findRelationships(query) {
|
|
326
|
+
return await this.adapter.findRelationships({
|
|
327
|
+
filters: query.filters,
|
|
328
|
+
limit: Math.min(query.limit || DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT),
|
|
329
|
+
offset: query.offset || 0
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
async getAllRelationships() {
|
|
333
|
+
return await this.adapter.getAllRelationships();
|
|
334
|
+
}
|
|
335
|
+
async getRelationships(sourceId, type) {
|
|
336
|
+
return this.getRelationshipsForEntity(sourceId, type);
|
|
337
|
+
}
|
|
338
|
+
async getRelationshipsFromBranch(entityId, targetBranch, type) {
|
|
339
|
+
const currentContext = this.adapter.getProjectContext();
|
|
340
|
+
this.adapter.setProjectContext({
|
|
341
|
+
projectHash: currentContext.projectHash,
|
|
342
|
+
branchName: normalizeBranchName(targetBranch)
|
|
343
|
+
});
|
|
344
|
+
try {
|
|
345
|
+
return await this.adapter.getRelationshipsForEntity(entityId, type);
|
|
346
|
+
} finally {
|
|
347
|
+
this.adapter.setProjectContext(currentContext);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
async findIncomingRelationshipsByName(entityName, types) {
|
|
351
|
+
const entities = await this.adapter.searchEntities({ namePattern: entityName });
|
|
352
|
+
const results = [];
|
|
353
|
+
for (const entity of entities) {
|
|
354
|
+
const rels = await this.adapter.getRelationshipsForEntity(entity.id);
|
|
355
|
+
const incoming = rels.filter((r) => r.toId === entity.id);
|
|
356
|
+
if (types && types.length > 0) {
|
|
357
|
+
results.push(...incoming.filter((r) => types.includes(r.type)));
|
|
358
|
+
} else {
|
|
359
|
+
results.push(...incoming);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return results;
|
|
363
|
+
}
|
|
364
|
+
// ===========================================================================
|
|
365
|
+
// FILE OPERATIONS
|
|
366
|
+
// ===========================================================================
|
|
367
|
+
async updateFileInfo(info) {
|
|
368
|
+
await this.adapter.updateFileInfo(info);
|
|
369
|
+
}
|
|
370
|
+
async batchUpdateFileInfo(infos) {
|
|
371
|
+
await this.adapter.batchUpdateFileInfo(infos);
|
|
372
|
+
}
|
|
373
|
+
async getFileInfo(path) {
|
|
374
|
+
return await this.adapter.getFileInfo(path);
|
|
375
|
+
}
|
|
376
|
+
async getOutdatedFiles(since) {
|
|
377
|
+
return await this.adapter.getOutdatedFiles(since);
|
|
378
|
+
}
|
|
379
|
+
async getAllIndexedFiles() {
|
|
380
|
+
return await this.adapter.getAllIndexedFiles();
|
|
381
|
+
}
|
|
382
|
+
async deleteFileInfo(path) {
|
|
383
|
+
await this.adapter.deleteFileInfo(path);
|
|
384
|
+
}
|
|
385
|
+
// ===========================================================================
|
|
386
|
+
// ENTITY OPERATIONS BY FILE PATH (for incremental indexing)
|
|
387
|
+
// ===========================================================================
|
|
388
|
+
async getEntityIdsByFilePath(filePath) {
|
|
389
|
+
return await this.adapter.getEntityIdsByFilePath(filePath);
|
|
390
|
+
}
|
|
391
|
+
async deleteEntitiesByFilePath(filePath) {
|
|
392
|
+
return await this.adapter.deleteEntitiesByFilePath(filePath);
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Invalidate file generation (for deleted files).
|
|
396
|
+
* Marks all entities for this file as stale without blocking DELETE.
|
|
397
|
+
*/
|
|
398
|
+
async invalidateFileGeneration(filePath) {
|
|
399
|
+
await this.adapter.getGenerationManager().invalidateFileGeneration(filePath);
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Run generation-based GC: clean stale entities and orphan name_tokens.
|
|
403
|
+
* Call after incremental reindex completes.
|
|
404
|
+
*/
|
|
405
|
+
async runGenerationGC() {
|
|
406
|
+
return await this.adapter.getGenerationManager().runFullGC();
|
|
407
|
+
}
|
|
408
|
+
// ===========================================================================
|
|
409
|
+
// QUERY OPERATIONS
|
|
410
|
+
// ===========================================================================
|
|
411
|
+
async executeQuery(query) {
|
|
412
|
+
const start = Date.now();
|
|
413
|
+
const [entities, relationships] = await Promise.all([this.findEntities(query), this.findRelationships(query)]);
|
|
414
|
+
const stats = await this.adapter.getStats();
|
|
415
|
+
return {
|
|
416
|
+
entities,
|
|
417
|
+
relationships,
|
|
418
|
+
stats: {
|
|
419
|
+
totalEntities: stats.totalEntities,
|
|
420
|
+
totalRelationships: stats.totalRelationships,
|
|
421
|
+
queryTimeMs: Date.now() - start
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
async getSubgraph(entityId, depth) {
|
|
426
|
+
const start = Date.now();
|
|
427
|
+
const maxDepth = Math.min(depth, MAX_SUBGRAPH_DEPTH);
|
|
428
|
+
const entities = /* @__PURE__ */ new Map();
|
|
429
|
+
const relationships = /* @__PURE__ */ new Map();
|
|
430
|
+
const visited = /* @__PURE__ */ new Set();
|
|
431
|
+
const queue = [{ id: entityId, level: 0 }];
|
|
432
|
+
while (queue.length > 0) {
|
|
433
|
+
const { id, level } = queue.shift();
|
|
434
|
+
if (visited.has(id) || level > maxDepth) continue;
|
|
435
|
+
visited.add(id);
|
|
436
|
+
const entity = await this.adapter.getEntity(id);
|
|
437
|
+
if (entity) {
|
|
438
|
+
entities.set(id, entity);
|
|
439
|
+
const rels = await this.adapter.getRelationshipsForEntity(id);
|
|
440
|
+
for (const rel of rels) {
|
|
441
|
+
relationships.set(rel.id, rel);
|
|
442
|
+
if (level < maxDepth) {
|
|
443
|
+
const nextId = rel.fromId === id ? rel.toId : rel.fromId;
|
|
444
|
+
if (!visited.has(nextId)) {
|
|
445
|
+
queue.push({ id: nextId, level: level + 1 });
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
entities: Array.from(entities.values()),
|
|
453
|
+
relationships: Array.from(relationships.values()),
|
|
454
|
+
stats: {
|
|
455
|
+
totalEntities: entities.size,
|
|
456
|
+
totalRelationships: relationships.size,
|
|
457
|
+
queryTimeMs: Date.now() - start
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
// ===========================================================================
|
|
462
|
+
// MAINTENANCE OPERATIONS
|
|
463
|
+
// ===========================================================================
|
|
464
|
+
async vacuum() {
|
|
465
|
+
log.i("GRAPHSTORAGE", "vacuum_requested");
|
|
466
|
+
}
|
|
467
|
+
async analyze() {
|
|
468
|
+
log.i("GRAPHSTORAGE", "analyze_requested");
|
|
469
|
+
}
|
|
470
|
+
async getMetrics() {
|
|
471
|
+
const stats = await this.adapter.getStats();
|
|
472
|
+
const memoryUsage = process.memoryUsage();
|
|
473
|
+
return {
|
|
474
|
+
totalEntities: stats.totalEntities,
|
|
475
|
+
totalRelationships: stats.totalRelationships,
|
|
476
|
+
totalFiles: stats.totalFiles,
|
|
477
|
+
databaseSizeMB: 0,
|
|
478
|
+
// Not easily available from libsql
|
|
479
|
+
indexSizeMB: 0,
|
|
480
|
+
cacheHitRate: 0,
|
|
481
|
+
lastVacuum: 0,
|
|
482
|
+
totalEmbeddings: stats.totalEmbeddings,
|
|
483
|
+
vectorSearchEnabled: true,
|
|
484
|
+
performanceMetricsCount: 0,
|
|
485
|
+
memoryUsageMB: Math.round(memoryUsage.heapUsed / 1024 / 1024),
|
|
486
|
+
concurrentConnections: 1,
|
|
487
|
+
averageQueryTimeMs: 0
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
async getStatistics() {
|
|
491
|
+
const stats = await this.adapter.getStats();
|
|
492
|
+
return {
|
|
493
|
+
totalEntities: stats.totalEntities,
|
|
494
|
+
totalRelationships: stats.totalRelationships,
|
|
495
|
+
totalFiles: stats.totalFiles
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
// ===========================================================================
|
|
499
|
+
// PROJECT METADATA
|
|
500
|
+
// ===========================================================================
|
|
501
|
+
async updateProjectMetadata(projectPath, isFullIndex = false) {
|
|
502
|
+
await this.adapter.updateProjectMetadata(projectPath, isFullIndex);
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Get incremental tracking info for deciding if full rebuild is needed
|
|
506
|
+
*/
|
|
507
|
+
async getIncrementalTrackingInfo() {
|
|
508
|
+
return await this.adapter.getIncrementalTrackingInfo();
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* Record incremental file changes after an incremental update
|
|
512
|
+
*/
|
|
513
|
+
async recordIncrementalChanges(changedFileCount) {
|
|
514
|
+
await this.adapter.recordIncrementalChanges(changedFileCount);
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Reset incremental tracking after a full index
|
|
518
|
+
*/
|
|
519
|
+
async resetIncrementalTracking() {
|
|
520
|
+
await this.adapter.resetIncrementalTracking();
|
|
521
|
+
}
|
|
522
|
+
async listProjects() {
|
|
523
|
+
return await this.adapter.listProjects();
|
|
524
|
+
}
|
|
525
|
+
async listBranches() {
|
|
526
|
+
return await this.adapter.listBranches();
|
|
527
|
+
}
|
|
528
|
+
// ===========================================================================
|
|
529
|
+
// CLEAR OPERATIONS
|
|
530
|
+
// ===========================================================================
|
|
531
|
+
async clear() {
|
|
532
|
+
await this.adapter.clear();
|
|
533
|
+
}
|
|
534
|
+
async clearAll() {
|
|
535
|
+
await this.adapter.clearAll();
|
|
536
|
+
}
|
|
537
|
+
async deleteProject(projectPath) {
|
|
538
|
+
const currentContext = this.adapter.getProjectContext();
|
|
539
|
+
const targetHash = getProjectHash(projectPath);
|
|
540
|
+
const branches = await this.adapter.listBranches();
|
|
541
|
+
for (const branch of branches) {
|
|
542
|
+
this.adapter.setProjectContext({ projectHash: targetHash, branchName: branch });
|
|
543
|
+
await this.adapter.clear();
|
|
544
|
+
}
|
|
545
|
+
this.adapter.setProjectContext(currentContext);
|
|
546
|
+
log.i("GRAPHSTORAGE", "project_deleted", { path: projectPath });
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Force flush all pending writes to disk.
|
|
550
|
+
* Use after bulk operations to ensure data is persisted immediately.
|
|
551
|
+
*/
|
|
552
|
+
async flush() {
|
|
553
|
+
await this.adapter.flush();
|
|
554
|
+
}
|
|
555
|
+
// ===========================================================================
|
|
556
|
+
// VECTOR STORE ACCESS
|
|
557
|
+
// ===========================================================================
|
|
558
|
+
/**
|
|
559
|
+
* Get the underlying adapter for direct vector operations
|
|
560
|
+
*/
|
|
561
|
+
getAdapter() {
|
|
562
|
+
return this.adapter;
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Get the LibSQL adapter for Prolly Tree operations.
|
|
566
|
+
* Alias for getAdapter() - used by history tools.
|
|
567
|
+
*/
|
|
568
|
+
getLibSQLAdapter() {
|
|
569
|
+
return this.adapter;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Get co-occurrence operations for query expansion.
|
|
573
|
+
* Used by CooccurrenceIndex for term pair storage and retrieval.
|
|
574
|
+
*/
|
|
575
|
+
getCooccurrenceOps() {
|
|
576
|
+
return this.adapter.getCooccurrenceOps();
|
|
577
|
+
}
|
|
578
|
+
// ===========================================================================
|
|
579
|
+
// HELPER METHODS
|
|
580
|
+
// ===========================================================================
|
|
581
|
+
generateId() {
|
|
582
|
+
return nanoid(ID_LENGTH);
|
|
583
|
+
}
|
|
584
|
+
entityKey(e) {
|
|
585
|
+
const s = e.location?.start?.index ?? -1;
|
|
586
|
+
const eIdx = e.location?.end?.index ?? -1;
|
|
587
|
+
return `${e.filePath}|${e.type}|${e.name}|${s}-${eIdx}`;
|
|
588
|
+
}
|
|
589
|
+
stableEntityId(e) {
|
|
590
|
+
if (!this.xxhashInstance) {
|
|
591
|
+
return this.generateId();
|
|
592
|
+
}
|
|
593
|
+
const key = this.entityKey(e);
|
|
594
|
+
const hash = this.xxhashInstance.h64ToString(key);
|
|
595
|
+
return hash.slice(0, ID_LENGTH);
|
|
596
|
+
}
|
|
597
|
+
relationshipKey(r) {
|
|
598
|
+
return `${r.fromId}|${r.toId}|${r.type}`;
|
|
599
|
+
}
|
|
600
|
+
stableRelationshipId(r) {
|
|
601
|
+
if (!this.xxhashInstance) {
|
|
602
|
+
return this.generateId();
|
|
603
|
+
}
|
|
604
|
+
const key = this.relationshipKey(r);
|
|
605
|
+
const hash = this.xxhashInstance.h64ToString(key);
|
|
606
|
+
return hash.slice(0, ID_LENGTH);
|
|
607
|
+
}
|
|
608
|
+
calculateComplexity(entity) {
|
|
609
|
+
let score = 1;
|
|
610
|
+
switch (entity.type) {
|
|
611
|
+
case "function":
|
|
612
|
+
score = 2;
|
|
613
|
+
break;
|
|
614
|
+
case "class":
|
|
615
|
+
score = 3;
|
|
616
|
+
break;
|
|
617
|
+
case "method":
|
|
618
|
+
score = 2;
|
|
619
|
+
break;
|
|
620
|
+
case "interface":
|
|
621
|
+
score = 2;
|
|
622
|
+
break;
|
|
623
|
+
default:
|
|
624
|
+
score = 1;
|
|
625
|
+
}
|
|
626
|
+
if (entity.metadata?.parameters && Array.isArray(entity.metadata.parameters)) {
|
|
627
|
+
score += Math.min(entity.metadata.parameters.length * 0.5, 3);
|
|
628
|
+
}
|
|
629
|
+
if (entity.metadata?.modifiers && Array.isArray(entity.metadata.modifiers)) {
|
|
630
|
+
score += Math.min(entity.metadata.modifiers.length * 0.3, 2);
|
|
631
|
+
}
|
|
632
|
+
return Math.round(score);
|
|
633
|
+
}
|
|
634
|
+
detectLanguage(filePath) {
|
|
635
|
+
const ext = filePath.split(".").pop()?.toLowerCase();
|
|
636
|
+
const langMap = {
|
|
637
|
+
ts: "typescript",
|
|
638
|
+
tsx: "typescript",
|
|
639
|
+
mts: "typescript",
|
|
640
|
+
cts: "typescript",
|
|
641
|
+
js: "javascript",
|
|
642
|
+
jsx: "javascript",
|
|
643
|
+
mjs: "javascript",
|
|
644
|
+
cjs: "javascript",
|
|
645
|
+
py: "python",
|
|
646
|
+
pyi: "python",
|
|
647
|
+
pyw: "python",
|
|
648
|
+
java: "java",
|
|
649
|
+
c: "c",
|
|
650
|
+
h: "c",
|
|
651
|
+
cpp: "cpp",
|
|
652
|
+
cc: "cpp",
|
|
653
|
+
cxx: "cpp",
|
|
654
|
+
hpp: "cpp",
|
|
655
|
+
hxx: "cpp",
|
|
656
|
+
hh: "cpp",
|
|
657
|
+
rs: "rust",
|
|
658
|
+
go: "go",
|
|
659
|
+
kt: "kotlin",
|
|
660
|
+
kts: "kotlin",
|
|
661
|
+
swift: "swift",
|
|
662
|
+
css: "css",
|
|
663
|
+
scss: "css",
|
|
664
|
+
sass: "css",
|
|
665
|
+
less: "css",
|
|
666
|
+
html: "html",
|
|
667
|
+
htm: "html",
|
|
668
|
+
xml: "xml",
|
|
669
|
+
php: "php",
|
|
670
|
+
rb: "ruby"
|
|
671
|
+
};
|
|
672
|
+
return langMap[ext || ""] || "unknown";
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
|
|
676
|
+
// src/storage/libsql-graph-adapter.ts
|
|
677
|
+
init_logging();
|
|
678
|
+
init_storage_paths();
|
|
679
|
+
|
|
680
|
+
// src/storage/libsql/cache-ops.ts
|
|
681
|
+
var CacheOperations = class {
|
|
682
|
+
constructor(getClient, vectorToString) {
|
|
683
|
+
this.getClient = getClient;
|
|
684
|
+
this.vectorToString = vectorToString;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Get cached embedding by content hash.
|
|
688
|
+
* Returns null if not found.
|
|
689
|
+
*/
|
|
690
|
+
async getEmbeddingFromCache(contentHash) {
|
|
691
|
+
const client = this.getClient();
|
|
692
|
+
if (!client) return null;
|
|
693
|
+
try {
|
|
694
|
+
const result = await client.execute({
|
|
695
|
+
sql: `SELECT embedding FROM embedding_cache WHERE content_hash = ?`,
|
|
696
|
+
args: [contentHash]
|
|
697
|
+
});
|
|
698
|
+
if (result.rows.length === 0) return null;
|
|
699
|
+
await client.execute({
|
|
700
|
+
sql: `UPDATE embedding_cache SET last_used_at = ?, hit_count = hit_count + 1 WHERE content_hash = ?`,
|
|
701
|
+
args: [Date.now(), contentHash]
|
|
702
|
+
});
|
|
703
|
+
const row = result.rows[0];
|
|
704
|
+
if (!row?.["embedding"]) return null;
|
|
705
|
+
const embeddingBlob = row["embedding"];
|
|
706
|
+
return new Float32Array(embeddingBlob);
|
|
707
|
+
} catch {
|
|
708
|
+
return null;
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Get multiple cached embeddings by content hashes.
|
|
713
|
+
* Returns Map of contentHash -> Float32Array for found entries.
|
|
714
|
+
*/
|
|
715
|
+
async getEmbeddingsFromCache(contentHashes) {
|
|
716
|
+
const client = this.getClient();
|
|
717
|
+
if (!client || contentHashes.length === 0) return /* @__PURE__ */ new Map();
|
|
718
|
+
const result = /* @__PURE__ */ new Map();
|
|
719
|
+
const now = Date.now();
|
|
720
|
+
try {
|
|
721
|
+
const placeholders = contentHashes.map(() => "?").join(",");
|
|
722
|
+
const queryResult = await client.execute({
|
|
723
|
+
sql: `SELECT content_hash, embedding FROM embedding_cache WHERE content_hash IN (${placeholders})`,
|
|
724
|
+
args: contentHashes
|
|
725
|
+
});
|
|
726
|
+
const foundHashes = [];
|
|
727
|
+
for (const row of queryResult.rows) {
|
|
728
|
+
const hash = row["content_hash"];
|
|
729
|
+
const embeddingBlob = row["embedding"];
|
|
730
|
+
result.set(hash, new Float32Array(embeddingBlob));
|
|
731
|
+
foundHashes.push(hash);
|
|
732
|
+
}
|
|
733
|
+
if (foundHashes.length > 0) {
|
|
734
|
+
const updatePlaceholders = foundHashes.map(() => "?").join(",");
|
|
735
|
+
await client.execute({
|
|
736
|
+
sql: `UPDATE embedding_cache SET last_used_at = ?, hit_count = hit_count + 1 WHERE content_hash IN (${updatePlaceholders})`,
|
|
737
|
+
args: [now, ...foundHashes]
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
} catch {
|
|
741
|
+
}
|
|
742
|
+
return result;
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Store embedding in cache by content hash.
|
|
746
|
+
*/
|
|
747
|
+
async setEmbeddingInCache(contentHash, model, embedding, textPreview) {
|
|
748
|
+
const client = this.getClient();
|
|
749
|
+
if (!client) return;
|
|
750
|
+
const now = Date.now();
|
|
751
|
+
try {
|
|
752
|
+
const vectorStr = this.vectorToString(embedding);
|
|
753
|
+
await client.execute({
|
|
754
|
+
sql: `INSERT OR REPLACE INTO embedding_cache
|
|
755
|
+
(content_hash, model, embedding, text_preview, hit_count, created_at, last_used_at)
|
|
756
|
+
VALUES (?, ?, vector32(?), ?, 0, ?, ?)`,
|
|
757
|
+
args: [contentHash, model, vectorStr, textPreview?.slice(0, 100) ?? null, now, now]
|
|
758
|
+
});
|
|
759
|
+
} catch {
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Store multiple embeddings in cache (batch operation).
|
|
764
|
+
*/
|
|
765
|
+
async setEmbeddingsInCache(entries) {
|
|
766
|
+
const client = this.getClient();
|
|
767
|
+
if (!client || entries.length === 0) return;
|
|
768
|
+
const now = Date.now();
|
|
769
|
+
try {
|
|
770
|
+
const statements = entries.map((entry) => ({
|
|
771
|
+
sql: `INSERT OR REPLACE INTO embedding_cache
|
|
772
|
+
(content_hash, model, embedding, text_preview, hit_count, created_at, last_used_at)
|
|
773
|
+
VALUES (?, ?, vector32(?), ?, 0, ?, ?)`,
|
|
774
|
+
args: [
|
|
775
|
+
entry.contentHash,
|
|
776
|
+
entry.model,
|
|
777
|
+
this.vectorToString(entry.embedding),
|
|
778
|
+
entry.textPreview?.slice(0, 100) ?? null,
|
|
779
|
+
now,
|
|
780
|
+
now
|
|
781
|
+
]
|
|
782
|
+
}));
|
|
783
|
+
await client.batch(statements, "write");
|
|
784
|
+
} catch {
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
};
|
|
788
|
+
|
|
789
|
+
// src/storage/libsql/cooccurrence-ops.ts
|
|
790
|
+
init_logging();
|
|
791
|
+
var CooccurrenceOperations = class {
|
|
792
|
+
constructor(getClient, getContext) {
|
|
793
|
+
this.getClient = getClient;
|
|
794
|
+
this.getContext = getContext;
|
|
795
|
+
}
|
|
796
|
+
// ===========================================================================
|
|
797
|
+
// BATCH UPDATE
|
|
798
|
+
// ===========================================================================
|
|
799
|
+
/**
|
|
800
|
+
* Batch update co-occurrence counts from extracted term pairs.
|
|
801
|
+
* Uses UPSERT (INSERT OR REPLACE) for atomic updates.
|
|
802
|
+
*
|
|
803
|
+
* @param pairs - Map of "term1|term2" → count (terms must be sorted alphabetically)
|
|
804
|
+
*/
|
|
805
|
+
async batchUpdateCooccurrence(pairs) {
|
|
806
|
+
const client = this.getClient();
|
|
807
|
+
if (!client) throw new Error("Client not initialized");
|
|
808
|
+
if (pairs.size === 0) return;
|
|
809
|
+
const { projectHash, branchName } = this.getContext();
|
|
810
|
+
const now = Date.now();
|
|
811
|
+
const BATCH_SIZE = 100;
|
|
812
|
+
const entries = Array.from(pairs.entries());
|
|
813
|
+
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
|
|
814
|
+
const batch = entries.slice(i, i + BATCH_SIZE);
|
|
815
|
+
const values = [];
|
|
816
|
+
const args = [];
|
|
817
|
+
for (const [key, count] of batch) {
|
|
818
|
+
const [term1, term2] = key.split("|");
|
|
819
|
+
if (!term1 || !term2) continue;
|
|
820
|
+
values.push("(?, ?, ?, ?, ?, ?)");
|
|
821
|
+
args.push(term1, term2, count, projectHash, branchName, now);
|
|
822
|
+
}
|
|
823
|
+
if (values.length === 0) continue;
|
|
824
|
+
await client.execute({
|
|
825
|
+
sql: `
|
|
826
|
+
INSERT INTO cooccurrence (term1, term2, count, project_hash, branch_name, updated_at)
|
|
827
|
+
VALUES ${values.join(", ")}
|
|
828
|
+
ON CONFLICT(term1, term2, project_hash, branch_name)
|
|
829
|
+
DO UPDATE SET
|
|
830
|
+
count = cooccurrence.count + excluded.count,
|
|
831
|
+
updated_at = excluded.updated_at
|
|
832
|
+
`,
|
|
833
|
+
args
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
/**
|
|
838
|
+
* Update term frequencies for PMI calculation.
|
|
839
|
+
* Called during indexing alongside co-occurrence updates.
|
|
840
|
+
*
|
|
841
|
+
* @param termCounts - Map of term → count in the document
|
|
842
|
+
* @param isNewDocument - Whether this is a new document (for doc_count increment)
|
|
843
|
+
*/
|
|
844
|
+
async updateTermFrequencies(termCounts, isNewDocument = true) {
|
|
845
|
+
const client = this.getClient();
|
|
846
|
+
if (!client) throw new Error("Client not initialized");
|
|
847
|
+
if (termCounts.size === 0) return;
|
|
848
|
+
const { projectHash, branchName } = this.getContext();
|
|
849
|
+
const BATCH_SIZE = 100;
|
|
850
|
+
const entries = Array.from(termCounts.entries());
|
|
851
|
+
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
|
|
852
|
+
const batch = entries.slice(i, i + BATCH_SIZE);
|
|
853
|
+
const values = [];
|
|
854
|
+
const args = [];
|
|
855
|
+
for (const [term, count] of batch) {
|
|
856
|
+
values.push("(?, ?, ?, ?, ?)");
|
|
857
|
+
args.push(term, isNewDocument ? 1 : 0, count, projectHash, branchName);
|
|
858
|
+
}
|
|
859
|
+
if (values.length === 0) continue;
|
|
860
|
+
await client.execute({
|
|
861
|
+
sql: `
|
|
862
|
+
INSERT INTO term_frequency (term, doc_count, total_count, project_hash, branch_name)
|
|
863
|
+
VALUES ${values.join(", ")}
|
|
864
|
+
ON CONFLICT(term, project_hash, branch_name)
|
|
865
|
+
DO UPDATE SET
|
|
866
|
+
doc_count = term_frequency.doc_count + excluded.doc_count,
|
|
867
|
+
total_count = term_frequency.total_count + excluded.total_count
|
|
868
|
+
`,
|
|
869
|
+
args
|
|
870
|
+
});
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
// ===========================================================================
|
|
874
|
+
// QUERY OPERATIONS
|
|
875
|
+
// ===========================================================================
|
|
876
|
+
/**
|
|
877
|
+
* Get related terms for query expansion.
|
|
878
|
+
* Returns terms that frequently co-occur with the input term,
|
|
879
|
+
* sorted by PMI score (if available) or raw count.
|
|
880
|
+
*
|
|
881
|
+
* @param term - The term to find related terms for
|
|
882
|
+
* @param limit - Maximum number of related terms to return
|
|
883
|
+
*/
|
|
884
|
+
async getRelatedTerms(term, limit = 5) {
|
|
885
|
+
const client = this.getClient();
|
|
886
|
+
if (!client) throw new Error("Client not initialized");
|
|
887
|
+
const { projectHash, branchName } = this.getContext();
|
|
888
|
+
const normalizedTerm = term.toLowerCase();
|
|
889
|
+
const result = await client.execute({
|
|
890
|
+
sql: `
|
|
891
|
+
SELECT
|
|
892
|
+
CASE WHEN term1 = ? THEN term2 ELSE term1 END as related_term,
|
|
893
|
+
count,
|
|
894
|
+
COALESCE(pmi, 0) as pmi_score
|
|
895
|
+
FROM cooccurrence
|
|
896
|
+
WHERE project_hash = ? AND branch_name = ?
|
|
897
|
+
AND (term1 = ? OR term2 = ?)
|
|
898
|
+
ORDER BY
|
|
899
|
+
CASE WHEN pmi IS NOT NULL THEN pmi ELSE count * 0.01 END DESC
|
|
900
|
+
LIMIT ?
|
|
901
|
+
`,
|
|
902
|
+
args: [normalizedTerm, projectHash, branchName, normalizedTerm, normalizedTerm, limit]
|
|
903
|
+
});
|
|
904
|
+
return result.rows.map((row) => ({
|
|
905
|
+
term: row["related_term"],
|
|
906
|
+
score: row["pmi_score"] || row["count"] * 0.01,
|
|
907
|
+
count: row["count"]
|
|
908
|
+
}));
|
|
909
|
+
}
|
|
910
|
+
/**
|
|
911
|
+
* Get multiple related terms for a set of input terms.
|
|
912
|
+
* More efficient than calling getRelatedTerms multiple times.
|
|
913
|
+
*
|
|
914
|
+
* @param terms - Array of terms to find related terms for
|
|
915
|
+
* @param limitPerTerm - Maximum related terms per input term
|
|
916
|
+
*/
|
|
917
|
+
async getRelatedTermsBatch(terms, limitPerTerm = 3) {
|
|
918
|
+
const client = this.getClient();
|
|
919
|
+
if (!client) throw new Error("Client not initialized");
|
|
920
|
+
if (terms.length === 0) return /* @__PURE__ */ new Map();
|
|
921
|
+
const { projectHash, branchName } = this.getContext();
|
|
922
|
+
const normalizedTerms = terms.map((t) => t.toLowerCase());
|
|
923
|
+
const placeholders = normalizedTerms.map(() => "?").join(", ");
|
|
924
|
+
const result = await client.execute({
|
|
925
|
+
sql: `
|
|
926
|
+
SELECT
|
|
927
|
+
CASE WHEN term1 IN (${placeholders}) THEN term1 ELSE term2 END as source_term,
|
|
928
|
+
CASE WHEN term1 IN (${placeholders}) THEN term2 ELSE term1 END as related_term,
|
|
929
|
+
count,
|
|
930
|
+
COALESCE(pmi, 0) as pmi_score
|
|
931
|
+
FROM cooccurrence
|
|
932
|
+
WHERE project_hash = ? AND branch_name = ?
|
|
933
|
+
AND (term1 IN (${placeholders}) OR term2 IN (${placeholders}))
|
|
934
|
+
ORDER BY
|
|
935
|
+
CASE WHEN pmi IS NOT NULL THEN pmi ELSE count * 0.01 END DESC
|
|
936
|
+
`,
|
|
937
|
+
args: [...normalizedTerms, ...normalizedTerms, projectHash, branchName, ...normalizedTerms, ...normalizedTerms]
|
|
938
|
+
});
|
|
939
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
940
|
+
for (const term of normalizedTerms) {
|
|
941
|
+
grouped.set(term, []);
|
|
942
|
+
}
|
|
943
|
+
for (const row of result.rows) {
|
|
944
|
+
const sourceTerm = row["source_term"];
|
|
945
|
+
const relatedTerm = row["related_term"];
|
|
946
|
+
const arr = grouped.get(sourceTerm);
|
|
947
|
+
if (arr && arr.length < limitPerTerm) {
|
|
948
|
+
arr.push({
|
|
949
|
+
term: relatedTerm,
|
|
950
|
+
score: row["pmi_score"] || row["count"] * 0.01,
|
|
951
|
+
count: row["count"]
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return grouped;
|
|
956
|
+
}
|
|
957
|
+
// ===========================================================================
|
|
958
|
+
// PMI CALCULATION
|
|
959
|
+
// ===========================================================================
|
|
960
|
+
/**
|
|
961
|
+
* Recalculate PMI (Pointwise Mutual Information) for all co-occurrence pairs.
|
|
962
|
+
* PMI = log2(P(x,y) / (P(x) * P(y)))
|
|
963
|
+
*
|
|
964
|
+
* Uses batched approach: preloads term frequencies, calculates PMI in JS,
|
|
965
|
+
* then updates in batches with event loop yields to prevent CPU blocking.
|
|
966
|
+
*/
|
|
967
|
+
async recalculatePMI() {
|
|
968
|
+
const client = this.getClient();
|
|
969
|
+
if (!client) throw new Error("Client not initialized");
|
|
970
|
+
const { projectHash, branchName } = this.getContext();
|
|
971
|
+
const startTime = Date.now();
|
|
972
|
+
const tfResult = await client.execute({
|
|
973
|
+
sql: `SELECT term, total_count, doc_count FROM term_frequency WHERE project_hash = ? AND branch_name = ?`,
|
|
974
|
+
args: [projectHash, branchName]
|
|
975
|
+
});
|
|
976
|
+
const termFreqs = /* @__PURE__ */ new Map();
|
|
977
|
+
let maxDocCount = 1;
|
|
978
|
+
for (const row of tfResult.rows) {
|
|
979
|
+
const term = row["term"];
|
|
980
|
+
const totalCount = row["total_count"];
|
|
981
|
+
const docCount = row["doc_count"];
|
|
982
|
+
termFreqs.set(term, totalCount);
|
|
983
|
+
if (docCount > maxDocCount) maxDocCount = docCount;
|
|
984
|
+
}
|
|
985
|
+
const totalDocs = maxDocCount;
|
|
986
|
+
const coocResult = await client.execute({
|
|
987
|
+
sql: `SELECT term1, term2, count FROM cooccurrence WHERE project_hash = ? AND branch_name = ?`,
|
|
988
|
+
args: [projectHash, branchName]
|
|
989
|
+
});
|
|
990
|
+
const totalPairsResult = await client.execute({
|
|
991
|
+
sql: `SELECT SUM(count) as total FROM cooccurrence WHERE project_hash = ? AND branch_name = ?`,
|
|
992
|
+
args: [projectHash, branchName]
|
|
993
|
+
});
|
|
994
|
+
const totalPairs = totalPairsResult.rows[0]?.["total"] || 1;
|
|
995
|
+
if (coocResult.rows.length === 0) {
|
|
996
|
+
log.i("COOCOPS", "pmi_skip", { reason: "no_pairs" });
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
999
|
+
log.i("COOCOPS", "pmi_start", { pairs: coocResult.rows.length, terms: termFreqs.size, totalDocs, totalPairs });
|
|
1000
|
+
const BATCH_SIZE = 500;
|
|
1001
|
+
let updated = 0;
|
|
1002
|
+
for (let i = 0; i < coocResult.rows.length; i += BATCH_SIZE) {
|
|
1003
|
+
const batch = coocResult.rows.slice(i, i + BATCH_SIZE);
|
|
1004
|
+
const statements = [];
|
|
1005
|
+
for (const row of batch) {
|
|
1006
|
+
const term1 = row["term1"];
|
|
1007
|
+
const term2 = row["term2"];
|
|
1008
|
+
const count = row["count"];
|
|
1009
|
+
const tf1 = termFreqs.get(term1) || 0;
|
|
1010
|
+
const tf2 = termFreqs.get(term2) || 0;
|
|
1011
|
+
let pmi = 0;
|
|
1012
|
+
if (tf1 > 0 && tf2 > 0) {
|
|
1013
|
+
const pXY = count / totalPairs;
|
|
1014
|
+
const pX = tf1 / totalDocs;
|
|
1015
|
+
const pY = tf2 / totalDocs;
|
|
1016
|
+
pmi = Math.log2(pXY / (pX * pY));
|
|
1017
|
+
}
|
|
1018
|
+
statements.push({
|
|
1019
|
+
sql: `UPDATE cooccurrence SET pmi = ? WHERE term1 = ? AND term2 = ? AND project_hash = ? AND branch_name = ?`,
|
|
1020
|
+
args: [pmi, term1, term2, projectHash, branchName]
|
|
1021
|
+
});
|
|
1022
|
+
}
|
|
1023
|
+
await client.batch(statements, "write");
|
|
1024
|
+
updated += batch.length;
|
|
1025
|
+
if (i + BATCH_SIZE < coocResult.rows.length) {
|
|
1026
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
const elapsed = Date.now() - startTime;
|
|
1030
|
+
log.i("COOCOPS", "pmi_recalculated", { ms: elapsed, pairs: updated, terms: termFreqs.size, totalDocs });
|
|
1031
|
+
}
|
|
1032
|
+
// ===========================================================================
|
|
1033
|
+
// MAINTENANCE
|
|
1034
|
+
// ===========================================================================
|
|
1035
|
+
/**
|
|
1036
|
+
* Get statistics about the co-occurrence index.
|
|
1037
|
+
*/
|
|
1038
|
+
async getStats() {
|
|
1039
|
+
const client = this.getClient();
|
|
1040
|
+
if (!client) throw new Error("Client not initialized");
|
|
1041
|
+
const { projectHash, branchName } = this.getContext();
|
|
1042
|
+
const [pairsResult, termsResult, avgResult] = await Promise.all([
|
|
1043
|
+
client.execute({
|
|
1044
|
+
sql: `SELECT COUNT(*) as cnt FROM cooccurrence WHERE project_hash = ? AND branch_name = ?`,
|
|
1045
|
+
args: [projectHash, branchName]
|
|
1046
|
+
}),
|
|
1047
|
+
client.execute({
|
|
1048
|
+
sql: `SELECT COUNT(*) as cnt FROM term_frequency WHERE project_hash = ? AND branch_name = ?`,
|
|
1049
|
+
args: [projectHash, branchName]
|
|
1050
|
+
}),
|
|
1051
|
+
client.execute({
|
|
1052
|
+
sql: `SELECT AVG(count) as avg FROM cooccurrence WHERE project_hash = ? AND branch_name = ?`,
|
|
1053
|
+
args: [projectHash, branchName]
|
|
1054
|
+
})
|
|
1055
|
+
]);
|
|
1056
|
+
return {
|
|
1057
|
+
totalPairs: pairsResult.rows[0]?.["cnt"] || 0,
|
|
1058
|
+
totalTerms: termsResult.rows[0]?.["cnt"] || 0,
|
|
1059
|
+
avgPairCount: avgResult.rows[0]?.["avg"] || 0
|
|
1060
|
+
};
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Clear all co-occurrence data for current project/branch.
|
|
1064
|
+
*/
|
|
1065
|
+
async clear() {
|
|
1066
|
+
const client = this.getClient();
|
|
1067
|
+
if (!client) throw new Error("Client not initialized");
|
|
1068
|
+
const { projectHash, branchName } = this.getContext();
|
|
1069
|
+
await client.batch(
|
|
1070
|
+
[
|
|
1071
|
+
{
|
|
1072
|
+
sql: "DELETE FROM cooccurrence WHERE project_hash = ? AND branch_name = ?",
|
|
1073
|
+
args: [projectHash, branchName]
|
|
1074
|
+
},
|
|
1075
|
+
{
|
|
1076
|
+
sql: "DELETE FROM term_frequency WHERE project_hash = ? AND branch_name = ?",
|
|
1077
|
+
args: [projectHash, branchName]
|
|
1078
|
+
}
|
|
1079
|
+
],
|
|
1080
|
+
"write"
|
|
1081
|
+
);
|
|
1082
|
+
log.i("COOCOPS", "cleared", { projectHash, branchName });
|
|
1083
|
+
}
|
|
1084
|
+
/**
|
|
1085
|
+
* Prune low-frequency pairs to reduce index size.
|
|
1086
|
+
* Removes pairs with count below threshold.
|
|
1087
|
+
*
|
|
1088
|
+
* @param minCount - Minimum count to keep (default: 2)
|
|
1089
|
+
*/
|
|
1090
|
+
async pruneRarePairs(minCount = 2) {
|
|
1091
|
+
const client = this.getClient();
|
|
1092
|
+
if (!client) throw new Error("Client not initialized");
|
|
1093
|
+
const { projectHash, branchName } = this.getContext();
|
|
1094
|
+
const result = await client.execute({
|
|
1095
|
+
sql: `
|
|
1096
|
+
DELETE FROM cooccurrence
|
|
1097
|
+
WHERE project_hash = ? AND branch_name = ? AND count < ?
|
|
1098
|
+
`,
|
|
1099
|
+
args: [projectHash, branchName, minCount]
|
|
1100
|
+
});
|
|
1101
|
+
const deleted = result.rowsAffected || 0;
|
|
1102
|
+
if (deleted > 0) {
|
|
1103
|
+
log.i("COOCOPS", "pruned", { deleted, minCount });
|
|
1104
|
+
}
|
|
1105
|
+
return deleted;
|
|
1106
|
+
}
|
|
1107
|
+
};
|
|
1108
|
+
|
|
1109
|
+
// src/storage/libsql/entity-ops.ts
|
|
1110
|
+
init_logging();
|
|
1111
|
+
function splitToTokens(name) {
|
|
1112
|
+
return name.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2").replace(/([a-zA-Z])(\d)/g, "$1 $2").replace(/(\d)([a-zA-Z])/g, "$1 $2").toLowerCase().split(/\s+/).filter((t) => t.length >= 2);
|
|
1113
|
+
}
|
|
1114
|
+
var EntityOperations = class {
|
|
1115
|
+
constructor(getClient, getContext, rowToEntity, genManager) {
|
|
1116
|
+
this.getClient = getClient;
|
|
1117
|
+
this.getContext = getContext;
|
|
1118
|
+
this.rowToEntity = rowToEntity;
|
|
1119
|
+
this.genManager = genManager;
|
|
1120
|
+
}
|
|
1121
|
+
tombstoneAdder;
|
|
1122
|
+
tombstoneGetter;
|
|
1123
|
+
/**
|
|
1124
|
+
* Set tombstone delegates for layered branch support.
|
|
1125
|
+
* Must be called after adapter initialization.
|
|
1126
|
+
*/
|
|
1127
|
+
setTombstoneDelegates(adder, getter) {
|
|
1128
|
+
this.tombstoneAdder = adder;
|
|
1129
|
+
this.tombstoneGetter = getter;
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Insert a single entity
|
|
1133
|
+
*/
|
|
1134
|
+
async insertEntity(entity) {
|
|
1135
|
+
const client = this.getClient();
|
|
1136
|
+
if (!client) throw new Error("Client not initialized");
|
|
1137
|
+
const { projectHash, branchName } = this.getContext();
|
|
1138
|
+
const now = Date.now();
|
|
1139
|
+
if (!this.genManager.isCacheLoaded) {
|
|
1140
|
+
await this.genManager.loadCache();
|
|
1141
|
+
}
|
|
1142
|
+
const newGen = await this.genManager.bumpGeneration(entity.filePath);
|
|
1143
|
+
await client.execute({
|
|
1144
|
+
sql: `
|
|
1145
|
+
INSERT OR REPLACE INTO entities
|
|
1146
|
+
(id, project_hash, branch_name, name, type, file_path, location, metadata, hash,
|
|
1147
|
+
created_at, updated_at, complexity_score, language, size_bytes, embedding_base64, embedding_text, file_gen)
|
|
1148
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1149
|
+
`,
|
|
1150
|
+
args: [
|
|
1151
|
+
entity.id,
|
|
1152
|
+
projectHash,
|
|
1153
|
+
branchName,
|
|
1154
|
+
entity.name,
|
|
1155
|
+
entity.type,
|
|
1156
|
+
entity.filePath,
|
|
1157
|
+
JSON.stringify(entity.location),
|
|
1158
|
+
JSON.stringify(entity.metadata),
|
|
1159
|
+
entity.hash || null,
|
|
1160
|
+
entity.createdAt || now,
|
|
1161
|
+
entity.updatedAt || now,
|
|
1162
|
+
entity.complexityScore || 1,
|
|
1163
|
+
entity.language || null,
|
|
1164
|
+
entity.sizeBytes || 0,
|
|
1165
|
+
entity.embeddingBase64 || null,
|
|
1166
|
+
entity.embeddingText || null,
|
|
1167
|
+
newGen
|
|
1168
|
+
]
|
|
1169
|
+
});
|
|
1170
|
+
const tokens = splitToTokens(entity.name);
|
|
1171
|
+
if (tokens.length > 0 && entity.id) {
|
|
1172
|
+
await client.execute({
|
|
1173
|
+
sql: "DELETE FROM name_tokens WHERE entity_id = ? AND project_hash = ? AND branch_name = ?",
|
|
1174
|
+
args: [entity.id, projectHash, branchName]
|
|
1175
|
+
});
|
|
1176
|
+
const valuePlaceholders = tokens.map(() => "(?, ?, ?, ?)").join(", ");
|
|
1177
|
+
const tokenArgs = [];
|
|
1178
|
+
for (const token of tokens) {
|
|
1179
|
+
tokenArgs.push(token, entity.id, projectHash, branchName);
|
|
1180
|
+
}
|
|
1181
|
+
await client.execute({
|
|
1182
|
+
sql: `INSERT OR IGNORE INTO name_tokens (token, entity_id, project_hash, branch_name) VALUES ${valuePlaceholders}`,
|
|
1183
|
+
args: tokenArgs
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
/**
|
|
1188
|
+
* Insert multiple entities with batch optimization
|
|
1189
|
+
*/
|
|
1190
|
+
async insertEntities(entities) {
|
|
1191
|
+
const client = this.getClient();
|
|
1192
|
+
if (!client) throw new Error("Client not initialized");
|
|
1193
|
+
if (entities.length === 0) return { processed: 0, failed: 0, errors: [], timeMs: 0 };
|
|
1194
|
+
const start = Date.now();
|
|
1195
|
+
const errors = [];
|
|
1196
|
+
const { projectHash, branchName } = this.getContext();
|
|
1197
|
+
const now = Date.now();
|
|
1198
|
+
const withLang = entities.filter((e) => e.language).length;
|
|
1199
|
+
const kotlinCount = entities.filter((e) => e.language === "kotlin").length;
|
|
1200
|
+
const sample = entities.slice(0, 3).map((e) => ({ n: e.name, l: e.language, f: e.filePath?.slice(-30) }));
|
|
1201
|
+
log.w("ENTITY_OPS", "insertEntities", {
|
|
1202
|
+
total: entities.length,
|
|
1203
|
+
withLang,
|
|
1204
|
+
kotlinCount,
|
|
1205
|
+
sample: JSON.stringify(sample)
|
|
1206
|
+
});
|
|
1207
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1208
|
+
const unique = [];
|
|
1209
|
+
for (const e of entities) {
|
|
1210
|
+
if (!seen.has(e.id)) {
|
|
1211
|
+
seen.add(e.id);
|
|
1212
|
+
unique.push(e);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
if (!this.genManager.isCacheLoaded) {
|
|
1216
|
+
await this.genManager.loadCache();
|
|
1217
|
+
}
|
|
1218
|
+
const filePaths = [...new Set(unique.map((e) => e.filePath))];
|
|
1219
|
+
const genMap = await this.genManager.bumpGenerationBatch(filePaths);
|
|
1220
|
+
const batchSize = 900;
|
|
1221
|
+
let processed = 0;
|
|
1222
|
+
for (let i = 0; i < unique.length; i += batchSize) {
|
|
1223
|
+
const batch = unique.slice(i, i + batchSize);
|
|
1224
|
+
const valuePlaceholders = batch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
1225
|
+
const args = [];
|
|
1226
|
+
for (const entity of batch) {
|
|
1227
|
+
const fileGen = genMap.get(entity.filePath) ?? 1;
|
|
1228
|
+
args.push(
|
|
1229
|
+
entity.id,
|
|
1230
|
+
projectHash,
|
|
1231
|
+
branchName,
|
|
1232
|
+
entity.name,
|
|
1233
|
+
entity.type,
|
|
1234
|
+
entity.filePath,
|
|
1235
|
+
JSON.stringify(entity.location),
|
|
1236
|
+
JSON.stringify(entity.metadata),
|
|
1237
|
+
entity.hash || null,
|
|
1238
|
+
entity.createdAt || now,
|
|
1239
|
+
entity.updatedAt || now,
|
|
1240
|
+
entity.complexityScore || 1,
|
|
1241
|
+
entity.language || null,
|
|
1242
|
+
entity.sizeBytes || 0,
|
|
1243
|
+
entity.embeddingBase64 || null,
|
|
1244
|
+
entity.embeddingText || null,
|
|
1245
|
+
fileGen
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
const sql = `
|
|
1249
|
+
INSERT OR REPLACE INTO entities
|
|
1250
|
+
(id, project_hash, branch_name, name, type, file_path, location, metadata, hash,
|
|
1251
|
+
created_at, updated_at, complexity_score, language, size_bytes, embedding_base64, embedding_text, file_gen)
|
|
1252
|
+
VALUES ${valuePlaceholders}
|
|
1253
|
+
`;
|
|
1254
|
+
try {
|
|
1255
|
+
await client.execute({ sql, args });
|
|
1256
|
+
processed += batch.length;
|
|
1257
|
+
} catch (error) {
|
|
1258
|
+
errors.push({
|
|
1259
|
+
item: { batchStart: i, batchEnd: i + batch.length },
|
|
1260
|
+
error: error.message
|
|
1261
|
+
});
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
const tokenRows = [];
|
|
1265
|
+
for (const entity of unique) {
|
|
1266
|
+
if (!entity.id) continue;
|
|
1267
|
+
for (const token of splitToTokens(entity.name)) {
|
|
1268
|
+
tokenRows.push([token, entity.id]);
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
if (tokenRows.length > 0) {
|
|
1272
|
+
const idsToClean = unique.map((e) => e.id).filter((id) => id);
|
|
1273
|
+
const DELETE_CHUNK = 400;
|
|
1274
|
+
for (let i = 0; i < idsToClean.length; i += DELETE_CHUNK) {
|
|
1275
|
+
const chunk = idsToClean.slice(i, i + DELETE_CHUNK);
|
|
1276
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1277
|
+
await client.execute({
|
|
1278
|
+
sql: `DELETE FROM name_tokens WHERE entity_id IN (${placeholders}) AND project_hash = ? AND branch_name = ?`,
|
|
1279
|
+
args: [...chunk, projectHash, branchName]
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
const TOKEN_BATCH = 1e3;
|
|
1283
|
+
for (let i = 0; i < tokenRows.length; i += TOKEN_BATCH) {
|
|
1284
|
+
const chunk = tokenRows.slice(i, i + TOKEN_BATCH);
|
|
1285
|
+
const valuePlaceholders = chunk.map(() => "(?, ?, ?, ?)").join(", ");
|
|
1286
|
+
const tokenArgs = [];
|
|
1287
|
+
for (const [token, entityId] of chunk) {
|
|
1288
|
+
tokenArgs.push(token, entityId, projectHash, branchName);
|
|
1289
|
+
}
|
|
1290
|
+
await client.execute({
|
|
1291
|
+
sql: `INSERT OR IGNORE INTO name_tokens (token, entity_id, project_hash, branch_name) VALUES ${valuePlaceholders}`,
|
|
1292
|
+
args: tokenArgs
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
return {
|
|
1297
|
+
processed,
|
|
1298
|
+
failed: errors.length,
|
|
1299
|
+
errors,
|
|
1300
|
+
timeMs: Date.now() - start
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Get entity by ID (layered: delta → base with tombstone check)
|
|
1305
|
+
*/
|
|
1306
|
+
async getEntity(id) {
|
|
1307
|
+
const client = this.getClient();
|
|
1308
|
+
if (!client) throw new Error("Client not initialized");
|
|
1309
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1310
|
+
log.w("ENTITY_OPS", "getEntity", { branch: branchName, base: baseBranch || "none" });
|
|
1311
|
+
if (baseBranch && this.tombstoneGetter) {
|
|
1312
|
+
const tombstones = await this.tombstoneGetter("entity");
|
|
1313
|
+
if (tombstones.has(id)) {
|
|
1314
|
+
return null;
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
const result = await client.execute({
|
|
1318
|
+
sql: `SELECT e.* FROM entities e
|
|
1319
|
+
JOIN file_generations fg
|
|
1320
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1321
|
+
WHERE e.id = ? AND e.project_hash = ? AND e.branch_name = ?
|
|
1322
|
+
AND e.file_gen = fg.active_gen`,
|
|
1323
|
+
args: [id, projectHash, branchName]
|
|
1324
|
+
});
|
|
1325
|
+
if (result.rows.length > 0) {
|
|
1326
|
+
return this.rowToEntity(result.rows[0]);
|
|
1327
|
+
}
|
|
1328
|
+
if (baseBranch) {
|
|
1329
|
+
const baseResult = await client.execute({
|
|
1330
|
+
sql: `SELECT e.* FROM entities e
|
|
1331
|
+
JOIN file_generations fg
|
|
1332
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1333
|
+
WHERE e.id = ? AND e.project_hash = ? AND e.branch_name = ?
|
|
1334
|
+
AND e.file_gen = fg.active_gen`,
|
|
1335
|
+
args: [id, projectHash, baseBranch]
|
|
1336
|
+
});
|
|
1337
|
+
if (baseResult.rows.length > 0) {
|
|
1338
|
+
return this.rowToEntity(baseResult.rows[0]);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
return null;
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Batch fetch entities by IDs in a single SQL query.
|
|
1345
|
+
* Returns Map<id, Entity> for found entities.
|
|
1346
|
+
* Uses IN clause with chunking for large ID sets.
|
|
1347
|
+
*/
|
|
1348
|
+
async getEntitiesBatch(ids) {
|
|
1349
|
+
const result = /* @__PURE__ */ new Map();
|
|
1350
|
+
if (ids.length === 0) return result;
|
|
1351
|
+
const client = this.getClient();
|
|
1352
|
+
if (!client) throw new Error("Client not initialized");
|
|
1353
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1354
|
+
let tombstones = null;
|
|
1355
|
+
if (baseBranch && this.tombstoneGetter) {
|
|
1356
|
+
tombstones = await this.tombstoneGetter("entity");
|
|
1357
|
+
}
|
|
1358
|
+
const uniqueIds = [...new Set(ids)].filter((id) => !tombstones?.has(id));
|
|
1359
|
+
if (uniqueIds.length === 0) return result;
|
|
1360
|
+
const CHUNK_SIZE = 400;
|
|
1361
|
+
for (let i = 0; i < uniqueIds.length; i += CHUNK_SIZE) {
|
|
1362
|
+
const chunk = uniqueIds.slice(i, i + CHUNK_SIZE);
|
|
1363
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1364
|
+
const rows = await client.execute({
|
|
1365
|
+
sql: `SELECT e.* FROM entities e
|
|
1366
|
+
JOIN file_generations fg
|
|
1367
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1368
|
+
WHERE e.id IN (${placeholders}) AND e.project_hash = ? AND e.branch_name = ?
|
|
1369
|
+
AND e.file_gen = fg.active_gen`,
|
|
1370
|
+
args: [...chunk, projectHash, branchName]
|
|
1371
|
+
});
|
|
1372
|
+
for (const row of rows.rows) {
|
|
1373
|
+
const entity = this.rowToEntity(row);
|
|
1374
|
+
result.set(entity.id, entity);
|
|
1375
|
+
}
|
|
1376
|
+
if (baseBranch) {
|
|
1377
|
+
const missingIds = chunk.filter((id) => !result.has(id));
|
|
1378
|
+
if (missingIds.length > 0) {
|
|
1379
|
+
const missingPlaceholders = missingIds.map(() => "?").join(",");
|
|
1380
|
+
const baseRows = await client.execute({
|
|
1381
|
+
sql: `SELECT e.* FROM entities e
|
|
1382
|
+
JOIN file_generations fg
|
|
1383
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1384
|
+
WHERE e.id IN (${missingPlaceholders}) AND e.project_hash = ? AND e.branch_name = ?
|
|
1385
|
+
AND e.file_gen = fg.active_gen`,
|
|
1386
|
+
args: [...missingIds, projectHash, baseBranch]
|
|
1387
|
+
});
|
|
1388
|
+
for (const row of baseRows.rows) {
|
|
1389
|
+
const entity = this.rowToEntity(row);
|
|
1390
|
+
result.set(entity.id, entity);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
return result;
|
|
1396
|
+
}
|
|
1397
|
+
/**
|
|
1398
|
+
* Build filter SQL clause and args.
|
|
1399
|
+
* When ctx is provided and the name filter is a simple identifier, uses the
|
|
1400
|
+
* name_tokens B-tree index instead of LIKE '%pattern%' for ~60x speedup.
|
|
1401
|
+
*/
|
|
1402
|
+
buildFilterClause(filters, args, ctx) {
|
|
1403
|
+
let sql = "";
|
|
1404
|
+
if (filters) {
|
|
1405
|
+
if (filters.entityType) {
|
|
1406
|
+
const types = Array.isArray(filters.entityType) ? filters.entityType : [filters.entityType];
|
|
1407
|
+
sql += ` AND type IN (${types.map(() => "?").join(",")})`;
|
|
1408
|
+
args.push(...types);
|
|
1409
|
+
}
|
|
1410
|
+
if (filters.filePath) {
|
|
1411
|
+
const paths = Array.isArray(filters.filePath) ? filters.filePath : [filters.filePath];
|
|
1412
|
+
const normalized = [];
|
|
1413
|
+
for (const p of paths) {
|
|
1414
|
+
normalized.push(p);
|
|
1415
|
+
if (p.includes("/")) normalized.push(p.replace(/\//g, "\\"));
|
|
1416
|
+
if (p.includes("\\")) normalized.push(p.replace(/\\/g, "/"));
|
|
1417
|
+
}
|
|
1418
|
+
const unique = [...new Set(normalized)];
|
|
1419
|
+
sql += ` AND e.file_path IN (${unique.map(() => "?").join(",")})`;
|
|
1420
|
+
args.push(...unique);
|
|
1421
|
+
}
|
|
1422
|
+
if (filters.name) {
|
|
1423
|
+
if (filters.name instanceof RegExp) {
|
|
1424
|
+
const source = filters.name.source;
|
|
1425
|
+
if (source.includes("|")) {
|
|
1426
|
+
const alternatives = source.split("|").map((alt) => {
|
|
1427
|
+
let p = alt.replace(/\.\*/g, "%").replace(/\*/g, "%").replace(/\./g, "_");
|
|
1428
|
+
if (!p.includes("%") && !p.includes("_")) p = `%${p}%`;
|
|
1429
|
+
return p;
|
|
1430
|
+
});
|
|
1431
|
+
sql += ` AND (${alternatives.map(() => "name LIKE ?").join(" OR ")})`;
|
|
1432
|
+
args.push(...alternatives);
|
|
1433
|
+
} else if (ctx && /^[a-zA-Z0-9_]+$/.test(source)) {
|
|
1434
|
+
const tokens = splitToTokens(source);
|
|
1435
|
+
if (tokens.length === 1) {
|
|
1436
|
+
sql += " AND id IN (SELECT entity_id FROM name_tokens WHERE token = ? AND project_hash = ? AND branch_name = ?)";
|
|
1437
|
+
args.push(tokens[0], ctx.projectHash, ctx.branchName);
|
|
1438
|
+
} else if (tokens.length > 1) {
|
|
1439
|
+
const placeholders = tokens.map(() => "?").join(",");
|
|
1440
|
+
sql += ` AND id IN (SELECT entity_id FROM name_tokens WHERE token IN (${placeholders}) AND project_hash = ? AND branch_name = ? GROUP BY entity_id HAVING COUNT(DISTINCT token) = ?)`;
|
|
1441
|
+
args.push(...tokens, ctx.projectHash, ctx.branchName, tokens.length);
|
|
1442
|
+
} else {
|
|
1443
|
+
let pattern = source;
|
|
1444
|
+
pattern = pattern.replace(/\.\*/g, "%").replace(/\*/g, "%").replace(/\./g, "_");
|
|
1445
|
+
if (!pattern.includes("%") && !pattern.includes("_")) pattern = `%${pattern}%`;
|
|
1446
|
+
sql += " AND name LIKE ?";
|
|
1447
|
+
args.push(pattern);
|
|
1448
|
+
}
|
|
1449
|
+
} else {
|
|
1450
|
+
let pattern = source;
|
|
1451
|
+
pattern = pattern.replace(/\.\*/g, "%").replace(/\*/g, "%").replace(/\./g, "_");
|
|
1452
|
+
if (!pattern.includes("%") && !pattern.includes("_")) {
|
|
1453
|
+
pattern = `%${pattern}%`;
|
|
1454
|
+
}
|
|
1455
|
+
sql += " AND name LIKE ?";
|
|
1456
|
+
args.push(pattern);
|
|
1457
|
+
}
|
|
1458
|
+
} else {
|
|
1459
|
+
sql += " AND name = ?";
|
|
1460
|
+
args.push(filters.name);
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
return sql;
|
|
1465
|
+
}
|
|
1466
|
+
/**
|
|
1467
|
+
* Find entities with complex filters (layered: delta + base - tombstones)
|
|
1468
|
+
*/
|
|
1469
|
+
async findEntities(query) {
|
|
1470
|
+
const client = this.getClient();
|
|
1471
|
+
if (!client) throw new Error("Client not initialized");
|
|
1472
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1473
|
+
log.w("ENTITY_OPS", "findEntities", { hash: projectHash, branch: branchName, base: baseBranch || "none" });
|
|
1474
|
+
const limit = query.limit || 100;
|
|
1475
|
+
const offset = query.offset || 0;
|
|
1476
|
+
if (!baseBranch) {
|
|
1477
|
+
const args = [projectHash, branchName];
|
|
1478
|
+
let sql = `SELECT e.* FROM entities e
|
|
1479
|
+
JOIN file_generations fg
|
|
1480
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1481
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1482
|
+
sql += this.buildFilterClause(query.filters, args, { projectHash, branchName });
|
|
1483
|
+
sql += " LIMIT ? OFFSET ?";
|
|
1484
|
+
args.push(limit, offset);
|
|
1485
|
+
const result2 = await client.execute({ sql, args });
|
|
1486
|
+
const withLang = result2.rows.filter((r) => r["language"]).length;
|
|
1487
|
+
const sample = result2.rows.slice(0, 3).map((r) => ({ n: r["name"], l: r["language"] }));
|
|
1488
|
+
log.w("ENTITY_OPS", "findEntities_raw", { total: result2.rows.length, withLang, sample: JSON.stringify(sample) });
|
|
1489
|
+
return result2.rows.map((row) => this.rowToEntity(row));
|
|
1490
|
+
}
|
|
1491
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1492
|
+
const deltaArgs = [projectHash, branchName];
|
|
1493
|
+
let deltaSql = `SELECT e.* FROM entities e
|
|
1494
|
+
JOIN file_generations fg
|
|
1495
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1496
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1497
|
+
deltaSql += this.buildFilterClause(query.filters, deltaArgs, { projectHash, branchName });
|
|
1498
|
+
const deltaResult = await client.execute({ sql: deltaSql, args: deltaArgs });
|
|
1499
|
+
const deltaWithLang = deltaResult.rows.filter((r) => r["language"]).length;
|
|
1500
|
+
log.w("ENTITY_OPS", "findEntities_layered", { delta: deltaResult.rows.length, deltaWithLang, branch: branchName });
|
|
1501
|
+
const deltaEntities = deltaResult.rows.map((row) => this.rowToEntity(row));
|
|
1502
|
+
const deltaIds = new Set(deltaEntities.map((e) => e.id));
|
|
1503
|
+
const baseArgs = [projectHash, baseBranch];
|
|
1504
|
+
let baseSql = `SELECT e.* FROM entities e
|
|
1505
|
+
JOIN file_generations fg
|
|
1506
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1507
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1508
|
+
baseSql += this.buildFilterClause(query.filters, baseArgs, { projectHash, branchName: baseBranch });
|
|
1509
|
+
const baseResult = await client.execute({ sql: baseSql, args: baseArgs });
|
|
1510
|
+
log.w("ENTITY_OPS", "findEntities_base", { base: baseResult.rows.length, baseBranch });
|
|
1511
|
+
const baseEntities = baseResult.rows.map((row) => this.rowToEntity(row)).filter((e) => !deltaIds.has(e.id) && !tombstones.has(e.id));
|
|
1512
|
+
const combined = [...deltaEntities, ...baseEntities];
|
|
1513
|
+
const result = combined.slice(offset, offset + limit);
|
|
1514
|
+
const retWithLang = result.filter((e) => e.language).length;
|
|
1515
|
+
const retSample = result.slice(0, 3).map((e) => ({ n: e.name, l: e.language }));
|
|
1516
|
+
log.w("ENTITY_OPS", "findEntities_result", {
|
|
1517
|
+
total: result.length,
|
|
1518
|
+
withLang: retWithLang,
|
|
1519
|
+
sample: JSON.stringify(retSample)
|
|
1520
|
+
});
|
|
1521
|
+
return result;
|
|
1522
|
+
}
|
|
1523
|
+
/**
|
|
1524
|
+
* Build search SQL clause and args.
|
|
1525
|
+
* When ctx is provided, uses name_tokens B-tree index for namePattern instead of LIKE.
|
|
1526
|
+
*/
|
|
1527
|
+
buildSearchClause(options, args, ctx) {
|
|
1528
|
+
let sql = "";
|
|
1529
|
+
if (options.namePattern) {
|
|
1530
|
+
if (ctx) {
|
|
1531
|
+
const tokens = splitToTokens(options.namePattern);
|
|
1532
|
+
if (tokens.length === 1) {
|
|
1533
|
+
sql += " AND id IN (SELECT entity_id FROM name_tokens WHERE token = ? AND project_hash = ? AND branch_name = ?)";
|
|
1534
|
+
args.push(tokens[0], ctx.projectHash, ctx.branchName);
|
|
1535
|
+
} else if (tokens.length > 1) {
|
|
1536
|
+
const placeholders = tokens.map(() => "?").join(",");
|
|
1537
|
+
sql += ` AND id IN (SELECT entity_id FROM name_tokens WHERE token IN (${placeholders}) AND project_hash = ? AND branch_name = ? GROUP BY entity_id HAVING COUNT(DISTINCT token) = ?)`;
|
|
1538
|
+
args.push(...tokens, ctx.projectHash, ctx.branchName, tokens.length);
|
|
1539
|
+
} else {
|
|
1540
|
+
sql += " AND name LIKE ?";
|
|
1541
|
+
args.push(`%${options.namePattern}%`);
|
|
1542
|
+
}
|
|
1543
|
+
} else {
|
|
1544
|
+
sql += " AND name LIKE ?";
|
|
1545
|
+
args.push(`%${options.namePattern}%`);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
if (options.types && options.types.length > 0) {
|
|
1549
|
+
sql += ` AND type IN (${options.types.map(() => "?").join(",")})`;
|
|
1550
|
+
args.push(...options.types);
|
|
1551
|
+
}
|
|
1552
|
+
if (options.filePath) {
|
|
1553
|
+
const normalizedPath = options.filePath.replace(/\\/g, "/");
|
|
1554
|
+
sql += " AND (e.file_path LIKE ? OR e.file_path LIKE ?)";
|
|
1555
|
+
args.push(`%${normalizedPath}`, `%${normalizedPath.replace(/\//g, "\\")}`);
|
|
1556
|
+
}
|
|
1557
|
+
return sql;
|
|
1558
|
+
}
|
|
1559
|
+
/**
|
|
1560
|
+
* Search entities by name pattern and type (layered: delta + base - tombstones)
|
|
1561
|
+
*/
|
|
1562
|
+
async searchEntities(options) {
|
|
1563
|
+
const client = this.getClient();
|
|
1564
|
+
if (!client) throw new Error("Client not initialized");
|
|
1565
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1566
|
+
const limit = options.limit || 100;
|
|
1567
|
+
if (!baseBranch) {
|
|
1568
|
+
const args = [projectHash, branchName];
|
|
1569
|
+
let sql = `SELECT e.* FROM entities e
|
|
1570
|
+
JOIN file_generations fg
|
|
1571
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1572
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1573
|
+
sql += this.buildSearchClause(options, args, { projectHash, branchName });
|
|
1574
|
+
sql += " LIMIT ?";
|
|
1575
|
+
args.push(limit);
|
|
1576
|
+
const result = await client.execute({ sql, args });
|
|
1577
|
+
return result.rows.map((row) => this.rowToEntity(row));
|
|
1578
|
+
}
|
|
1579
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1580
|
+
const deltaArgs = [projectHash, branchName];
|
|
1581
|
+
let deltaSql = `SELECT e.* FROM entities e
|
|
1582
|
+
JOIN file_generations fg
|
|
1583
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1584
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1585
|
+
deltaSql += this.buildSearchClause(options, deltaArgs, { projectHash, branchName });
|
|
1586
|
+
const deltaResult = await client.execute({ sql: deltaSql, args: deltaArgs });
|
|
1587
|
+
const deltaEntities = deltaResult.rows.map((row) => this.rowToEntity(row));
|
|
1588
|
+
const deltaIds = new Set(deltaEntities.map((e) => e.id));
|
|
1589
|
+
const baseArgs = [projectHash, baseBranch];
|
|
1590
|
+
let baseSql = `SELECT e.* FROM entities e
|
|
1591
|
+
JOIN file_generations fg
|
|
1592
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1593
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`;
|
|
1594
|
+
baseSql += this.buildSearchClause(options, baseArgs, { projectHash, branchName: baseBranch });
|
|
1595
|
+
const baseResult = await client.execute({ sql: baseSql, args: baseArgs });
|
|
1596
|
+
const baseEntities = baseResult.rows.map((row) => this.rowToEntity(row)).filter((e) => !deltaIds.has(e.id) && !tombstones.has(e.id));
|
|
1597
|
+
return [...deltaEntities, ...baseEntities].slice(0, limit);
|
|
1598
|
+
}
|
|
1599
|
+
/**
|
|
1600
|
+
* Search entities by directory path (LIKE pattern) (layered: delta + base - tombstones)
|
|
1601
|
+
*/
|
|
1602
|
+
async searchEntitiesInDirectory(directoryPath) {
|
|
1603
|
+
const client = this.getClient();
|
|
1604
|
+
if (!client) throw new Error("Client not initialized");
|
|
1605
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1606
|
+
const forwardPath = directoryPath.replace(/\\/g, "/");
|
|
1607
|
+
const backPath = directoryPath.replace(/\//g, "\\");
|
|
1608
|
+
if (!baseBranch) {
|
|
1609
|
+
const sql = `
|
|
1610
|
+
SELECT e.* FROM entities e
|
|
1611
|
+
JOIN file_generations fg
|
|
1612
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1613
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1614
|
+
AND (e.file_path LIKE ? OR e.file_path LIKE ?)
|
|
1615
|
+
AND e.file_gen = fg.active_gen
|
|
1616
|
+
`;
|
|
1617
|
+
const args = [projectHash, branchName, `${forwardPath}%`, `${backPath}%`];
|
|
1618
|
+
const result = await client.execute({ sql, args });
|
|
1619
|
+
return result.rows.map((row) => this.rowToEntity(row));
|
|
1620
|
+
}
|
|
1621
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1622
|
+
const deltaSql = `
|
|
1623
|
+
SELECT e.* FROM entities e
|
|
1624
|
+
JOIN file_generations fg
|
|
1625
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1626
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1627
|
+
AND (e.file_path LIKE ? OR e.file_path LIKE ?)
|
|
1628
|
+
AND e.file_gen = fg.active_gen
|
|
1629
|
+
`;
|
|
1630
|
+
const deltaResult = await client.execute({
|
|
1631
|
+
sql: deltaSql,
|
|
1632
|
+
args: [projectHash, branchName, `${forwardPath}%`, `${backPath}%`]
|
|
1633
|
+
});
|
|
1634
|
+
const deltaEntities = deltaResult.rows.map((row) => this.rowToEntity(row));
|
|
1635
|
+
const deltaIds = new Set(deltaEntities.map((e) => e.id));
|
|
1636
|
+
const baseSql = `
|
|
1637
|
+
SELECT e.* FROM entities e
|
|
1638
|
+
JOIN file_generations fg
|
|
1639
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1640
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1641
|
+
AND (e.file_path LIKE ? OR e.file_path LIKE ?)
|
|
1642
|
+
AND e.file_gen = fg.active_gen
|
|
1643
|
+
`;
|
|
1644
|
+
const baseResult = await client.execute({
|
|
1645
|
+
sql: baseSql,
|
|
1646
|
+
args: [projectHash, baseBranch, `${forwardPath}%`, `${backPath}%`]
|
|
1647
|
+
});
|
|
1648
|
+
const baseEntities = baseResult.rows.map((row) => this.rowToEntity(row)).filter((e) => !deltaIds.has(e.id) && !tombstones.has(e.id));
|
|
1649
|
+
return [...deltaEntities, ...baseEntities];
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* Delete entity by ID.
|
|
1653
|
+
* On feature branches with baseBranch set, adds tombstone instead of deleting.
|
|
1654
|
+
*/
|
|
1655
|
+
async deleteEntity(id) {
|
|
1656
|
+
const client = this.getClient();
|
|
1657
|
+
if (!client) throw new Error("Client not initialized");
|
|
1658
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1659
|
+
if (baseBranch && this.tombstoneAdder) {
|
|
1660
|
+
await this.tombstoneAdder(id, "entity");
|
|
1661
|
+
}
|
|
1662
|
+
await client.execute({
|
|
1663
|
+
sql: "DELETE FROM entities WHERE id = ? AND project_hash = ? AND branch_name = ?",
|
|
1664
|
+
args: [id, projectHash, branchName]
|
|
1665
|
+
});
|
|
1666
|
+
await client.execute({
|
|
1667
|
+
sql: "DELETE FROM name_tokens WHERE entity_id = ? AND project_hash = ? AND branch_name = ?",
|
|
1668
|
+
args: [id, projectHash, branchName]
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
/**
|
|
1672
|
+
* Get entity IDs by file path (for FAISS cleanup) (layered: delta + base - tombstones)
|
|
1673
|
+
*/
|
|
1674
|
+
async getEntityIdsByFilePath(filePath) {
|
|
1675
|
+
const client = this.getClient();
|
|
1676
|
+
if (!client) throw new Error("Client not initialized");
|
|
1677
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1678
|
+
const forwardPath = filePath.replace(/\\/g, "/");
|
|
1679
|
+
const backPath = filePath.replace(/\//g, "\\");
|
|
1680
|
+
if (!baseBranch) {
|
|
1681
|
+
const result = await client.execute({
|
|
1682
|
+
sql: `
|
|
1683
|
+
SELECT e.id FROM entities e
|
|
1684
|
+
JOIN file_generations fg
|
|
1685
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1686
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1687
|
+
AND (e.file_path = ? OR e.file_path = ?)
|
|
1688
|
+
AND e.file_gen = fg.active_gen
|
|
1689
|
+
`,
|
|
1690
|
+
args: [projectHash, branchName, forwardPath, backPath]
|
|
1691
|
+
});
|
|
1692
|
+
return result.rows.map((row) => row["id"]);
|
|
1693
|
+
}
|
|
1694
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1695
|
+
const deltaResult = await client.execute({
|
|
1696
|
+
sql: `
|
|
1697
|
+
SELECT e.id FROM entities e
|
|
1698
|
+
JOIN file_generations fg
|
|
1699
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1700
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1701
|
+
AND (e.file_path = ? OR e.file_path = ?)
|
|
1702
|
+
AND e.file_gen = fg.active_gen
|
|
1703
|
+
`,
|
|
1704
|
+
args: [projectHash, branchName, forwardPath, backPath]
|
|
1705
|
+
});
|
|
1706
|
+
const deltaIds = new Set(deltaResult.rows.map((row) => row["id"]));
|
|
1707
|
+
const baseResult = await client.execute({
|
|
1708
|
+
sql: `
|
|
1709
|
+
SELECT e.id FROM entities e
|
|
1710
|
+
JOIN file_generations fg
|
|
1711
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1712
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1713
|
+
AND (e.file_path = ? OR e.file_path = ?)
|
|
1714
|
+
AND e.file_gen = fg.active_gen
|
|
1715
|
+
`,
|
|
1716
|
+
args: [projectHash, baseBranch, forwardPath, backPath]
|
|
1717
|
+
});
|
|
1718
|
+
const baseIds = baseResult.rows.map((row) => row["id"]).filter((id) => !deltaIds.has(id) && !tombstones.has(id));
|
|
1719
|
+
return [...deltaIds, ...baseIds];
|
|
1720
|
+
}
|
|
1721
|
+
/**
|
|
1722
|
+
* Delete all entities for a file path
|
|
1723
|
+
* Returns the IDs of deleted entities (for FAISS cleanup)
|
|
1724
|
+
*/
|
|
1725
|
+
async deleteEntitiesByFilePath(filePath) {
|
|
1726
|
+
const ids = await this.getEntityIdsByFilePath(filePath);
|
|
1727
|
+
if (ids.length === 0) return [];
|
|
1728
|
+
await this.genManager.invalidateFileGeneration(filePath);
|
|
1729
|
+
return ids;
|
|
1730
|
+
}
|
|
1731
|
+
/**
|
|
1732
|
+
* Get all entities for current project/branch (layered: delta + base - tombstones)
|
|
1733
|
+
*/
|
|
1734
|
+
async getAllEntities() {
|
|
1735
|
+
const client = this.getClient();
|
|
1736
|
+
if (!client) throw new Error("Client not initialized");
|
|
1737
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1738
|
+
if (!baseBranch) {
|
|
1739
|
+
const result = await client.execute({
|
|
1740
|
+
sql: `SELECT e.* FROM entities e
|
|
1741
|
+
JOIN file_generations fg
|
|
1742
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1743
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`,
|
|
1744
|
+
args: [projectHash, branchName]
|
|
1745
|
+
});
|
|
1746
|
+
return result.rows.map((row) => this.rowToEntity(row));
|
|
1747
|
+
}
|
|
1748
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1749
|
+
const deltaResult = await client.execute({
|
|
1750
|
+
sql: `SELECT e.* FROM entities e
|
|
1751
|
+
JOIN file_generations fg
|
|
1752
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1753
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`,
|
|
1754
|
+
args: [projectHash, branchName]
|
|
1755
|
+
});
|
|
1756
|
+
const deltaEntities = deltaResult.rows.map((row) => this.rowToEntity(row));
|
|
1757
|
+
const deltaIds = new Set(deltaEntities.map((e) => e.id));
|
|
1758
|
+
const baseResult = await client.execute({
|
|
1759
|
+
sql: `SELECT e.* FROM entities e
|
|
1760
|
+
JOIN file_generations fg
|
|
1761
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1762
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_gen = fg.active_gen`,
|
|
1763
|
+
args: [projectHash, baseBranch]
|
|
1764
|
+
});
|
|
1765
|
+
const baseEntities = baseResult.rows.map((row) => this.rowToEntity(row)).filter((e) => !deltaIds.has(e.id) && !tombstones.has(e.id));
|
|
1766
|
+
return [...deltaEntities, ...baseEntities];
|
|
1767
|
+
}
|
|
1768
|
+
/**
|
|
1769
|
+
* Count entities by language (efficient SQL aggregation for TechnologyDetector).
|
|
1770
|
+
* Excludes external placeholder entities (file_path starting with 'external://').
|
|
1771
|
+
* Returns map of language -> { count, fileCount }
|
|
1772
|
+
*/
|
|
1773
|
+
async countByLanguage() {
|
|
1774
|
+
const client = this.getClient();
|
|
1775
|
+
if (!client) throw new Error("Client not initialized");
|
|
1776
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
1777
|
+
if (!baseBranch) {
|
|
1778
|
+
const result2 = await client.execute({
|
|
1779
|
+
sql: `
|
|
1780
|
+
SELECT
|
|
1781
|
+
COALESCE(e.language, 'unknown') as lang,
|
|
1782
|
+
COUNT(*) as cnt,
|
|
1783
|
+
COUNT(DISTINCT e.file_path) as file_cnt
|
|
1784
|
+
FROM entities e
|
|
1785
|
+
JOIN file_generations fg
|
|
1786
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1787
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1788
|
+
AND e.file_path NOT LIKE 'external://%'
|
|
1789
|
+
AND e.file_gen = fg.active_gen
|
|
1790
|
+
GROUP BY COALESCE(e.language, 'unknown')
|
|
1791
|
+
`,
|
|
1792
|
+
args: [projectHash, branchName]
|
|
1793
|
+
});
|
|
1794
|
+
const counts2 = /* @__PURE__ */ new Map();
|
|
1795
|
+
for (const row of result2.rows) {
|
|
1796
|
+
const lang = row["lang"];
|
|
1797
|
+
counts2.set(lang, {
|
|
1798
|
+
count: Number(row["cnt"]),
|
|
1799
|
+
fileCount: Number(row["file_cnt"])
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
return counts2;
|
|
1803
|
+
}
|
|
1804
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("entity") : /* @__PURE__ */ new Set();
|
|
1805
|
+
const deltaResult = await client.execute({
|
|
1806
|
+
sql: `SELECT e.id, e.language, e.file_path FROM entities e
|
|
1807
|
+
JOIN file_generations fg
|
|
1808
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1809
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_path NOT LIKE 'external://%'
|
|
1810
|
+
AND e.file_gen = fg.active_gen`,
|
|
1811
|
+
args: [projectHash, branchName]
|
|
1812
|
+
});
|
|
1813
|
+
const deltaEntities = deltaResult.rows.map((row) => ({
|
|
1814
|
+
id: row["id"],
|
|
1815
|
+
language: row["language"],
|
|
1816
|
+
filePath: row["file_path"]
|
|
1817
|
+
}));
|
|
1818
|
+
const deltaIds = new Set(deltaEntities.map((e) => e.id));
|
|
1819
|
+
const baseResult = await client.execute({
|
|
1820
|
+
sql: `SELECT e.id, e.language, e.file_path FROM entities e
|
|
1821
|
+
JOIN file_generations fg
|
|
1822
|
+
ON e.file_path = fg.file_path AND e.project_hash = fg.project_hash AND e.branch_name = fg.branch_name
|
|
1823
|
+
WHERE e.project_hash = ? AND e.branch_name = ? AND e.file_path NOT LIKE 'external://%'
|
|
1824
|
+
AND e.file_gen = fg.active_gen`,
|
|
1825
|
+
args: [projectHash, baseBranch]
|
|
1826
|
+
});
|
|
1827
|
+
const baseEntities = baseResult.rows.map((row) => ({
|
|
1828
|
+
id: row["id"],
|
|
1829
|
+
language: row["language"],
|
|
1830
|
+
filePath: row["file_path"]
|
|
1831
|
+
})).filter((e) => !deltaIds.has(e.id) && !tombstones.has(e.id));
|
|
1832
|
+
const allEntities = [...deltaEntities, ...baseEntities];
|
|
1833
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1834
|
+
for (const e of allEntities) {
|
|
1835
|
+
const lang = e.language || "unknown";
|
|
1836
|
+
if (!counts.has(lang)) {
|
|
1837
|
+
counts.set(lang, { count: 0, files: /* @__PURE__ */ new Set() });
|
|
1838
|
+
}
|
|
1839
|
+
const entry = counts.get(lang);
|
|
1840
|
+
entry.count++;
|
|
1841
|
+
entry.files.add(e.filePath);
|
|
1842
|
+
}
|
|
1843
|
+
const result = /* @__PURE__ */ new Map();
|
|
1844
|
+
for (const [lang, data] of counts) {
|
|
1845
|
+
result.set(lang, { count: data.count, fileCount: data.files.size });
|
|
1846
|
+
}
|
|
1847
|
+
return result;
|
|
1848
|
+
}
|
|
1849
|
+
};
|
|
1850
|
+
|
|
1851
|
+
// src/storage/libsql/generation-ops.ts
|
|
1852
|
+
init_logging();
|
|
1853
|
+
var GenerationManager = class {
|
|
1854
|
+
constructor(getClient, getContext) {
|
|
1855
|
+
this.getClient = getClient;
|
|
1856
|
+
this.getContext = getContext;
|
|
1857
|
+
}
|
|
1858
|
+
/** In-memory cache: "filePath" → active generation number */
|
|
1859
|
+
cache = /* @__PURE__ */ new Map();
|
|
1860
|
+
cacheLoaded = false;
|
|
1861
|
+
gcRunning = false;
|
|
1862
|
+
/**
|
|
1863
|
+
* Load generation cache from DB for current project/branch.
|
|
1864
|
+
* Call once after setProjectContext().
|
|
1865
|
+
*/
|
|
1866
|
+
async loadCache() {
|
|
1867
|
+
const client = this.getClient();
|
|
1868
|
+
if (!client) return;
|
|
1869
|
+
const { projectHash, branchName } = this.getContext();
|
|
1870
|
+
const result = await client.execute({
|
|
1871
|
+
sql: `SELECT file_path, active_gen FROM file_generations
|
|
1872
|
+
WHERE project_hash = ? AND branch_name = ?`,
|
|
1873
|
+
args: [projectHash, branchName]
|
|
1874
|
+
});
|
|
1875
|
+
this.cache.clear();
|
|
1876
|
+
for (const row of result.rows) {
|
|
1877
|
+
this.cache.set(row["file_path"], Number(row["active_gen"]));
|
|
1878
|
+
}
|
|
1879
|
+
this.cacheLoaded = true;
|
|
1880
|
+
log.d("GEN_OPS", "cache_loaded", { entries: this.cache.size });
|
|
1881
|
+
}
|
|
1882
|
+
/**
|
|
1883
|
+
* Get active generation for a file. Returns from cache or DB.
|
|
1884
|
+
* Returns 0 if file has no generation record (new file).
|
|
1885
|
+
*/
|
|
1886
|
+
getGeneration(filePath) {
|
|
1887
|
+
return this.cache.get(filePath) ?? 0;
|
|
1888
|
+
}
|
|
1889
|
+
/**
|
|
1890
|
+
* Bump generation for a single file: increment + update cache + DB.
|
|
1891
|
+
* Returns the new generation number.
|
|
1892
|
+
*/
|
|
1893
|
+
async bumpGeneration(filePath) {
|
|
1894
|
+
const client = this.getClient();
|
|
1895
|
+
if (!client) throw new Error("Client not initialized");
|
|
1896
|
+
const { projectHash, branchName } = this.getContext();
|
|
1897
|
+
const currentGen = this.getGeneration(filePath);
|
|
1898
|
+
const newGen = currentGen + 1;
|
|
1899
|
+
const now = Date.now();
|
|
1900
|
+
await client.execute({
|
|
1901
|
+
sql: `INSERT OR REPLACE INTO file_generations
|
|
1902
|
+
(file_path, project_hash, branch_name, active_gen, updated_at)
|
|
1903
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
1904
|
+
args: [filePath, projectHash, branchName, newGen, now]
|
|
1905
|
+
});
|
|
1906
|
+
this.cache.set(filePath, newGen);
|
|
1907
|
+
return newGen;
|
|
1908
|
+
}
|
|
1909
|
+
/**
|
|
1910
|
+
* Bump generation for multiple files in a single batch.
|
|
1911
|
+
* Returns map of filePath → newGen.
|
|
1912
|
+
*/
|
|
1913
|
+
async bumpGenerationBatch(filePaths) {
|
|
1914
|
+
const client = this.getClient();
|
|
1915
|
+
if (!client) throw new Error("Client not initialized");
|
|
1916
|
+
if (filePaths.length === 0) return /* @__PURE__ */ new Map();
|
|
1917
|
+
const { projectHash, branchName } = this.getContext();
|
|
1918
|
+
const now = Date.now();
|
|
1919
|
+
const result = /* @__PURE__ */ new Map();
|
|
1920
|
+
const unique = [...new Set(filePaths)];
|
|
1921
|
+
const BATCH_SIZE = 500;
|
|
1922
|
+
for (let i = 0; i < unique.length; i += BATCH_SIZE) {
|
|
1923
|
+
const batch = unique.slice(i, i + BATCH_SIZE);
|
|
1924
|
+
const valuePlaceholders = batch.map(() => "(?, ?, ?, ?, ?)").join(", ");
|
|
1925
|
+
const args = [];
|
|
1926
|
+
for (const filePath of batch) {
|
|
1927
|
+
const currentGen = this.getGeneration(filePath);
|
|
1928
|
+
const newGen = currentGen + 1;
|
|
1929
|
+
args.push(filePath, projectHash, branchName, newGen, now);
|
|
1930
|
+
this.cache.set(filePath, newGen);
|
|
1931
|
+
result.set(filePath, newGen);
|
|
1932
|
+
}
|
|
1933
|
+
await client.execute({
|
|
1934
|
+
sql: `INSERT OR REPLACE INTO file_generations
|
|
1935
|
+
(file_path, project_hash, branch_name, active_gen, updated_at)
|
|
1936
|
+
VALUES ${valuePlaceholders}`,
|
|
1937
|
+
args
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
return result;
|
|
1941
|
+
}
|
|
1942
|
+
/**
|
|
1943
|
+
* Invalidate a file's generation (for deleted files).
|
|
1944
|
+
* Sets active_gen = -1 so all entities for this file become stale.
|
|
1945
|
+
* Returns the old entity IDs for FAISS cleanup.
|
|
1946
|
+
*/
|
|
1947
|
+
async invalidateFileGeneration(filePath) {
|
|
1948
|
+
const client = this.getClient();
|
|
1949
|
+
if (!client) return;
|
|
1950
|
+
const { projectHash, branchName } = this.getContext();
|
|
1951
|
+
await client.execute({
|
|
1952
|
+
sql: `INSERT OR REPLACE INTO file_generations
|
|
1953
|
+
(file_path, project_hash, branch_name, active_gen, updated_at)
|
|
1954
|
+
VALUES (?, ?, ?, -1, ?)`,
|
|
1955
|
+
args: [filePath, projectHash, branchName, Date.now()]
|
|
1956
|
+
});
|
|
1957
|
+
this.cache.set(filePath, -1);
|
|
1958
|
+
}
|
|
1959
|
+
/**
|
|
1960
|
+
* GC: Delete stale entities (file_gen < active_gen or active_gen = -1).
|
|
1961
|
+
* Runs in chunks to avoid long locks. Returns number of deleted rows.
|
|
1962
|
+
*/
|
|
1963
|
+
async gcStaleEntities(limit = 1e4) {
|
|
1964
|
+
const client = this.getClient();
|
|
1965
|
+
if (!client || this.gcRunning) return 0;
|
|
1966
|
+
this.gcRunning = true;
|
|
1967
|
+
const { projectHash, branchName } = this.getContext();
|
|
1968
|
+
try {
|
|
1969
|
+
const result = await client.execute({
|
|
1970
|
+
sql: `DELETE FROM entities WHERE rowid IN (
|
|
1971
|
+
SELECT e.rowid FROM entities e
|
|
1972
|
+
JOIN file_generations fg
|
|
1973
|
+
ON e.file_path = fg.file_path
|
|
1974
|
+
AND e.project_hash = fg.project_hash
|
|
1975
|
+
AND e.branch_name = fg.branch_name
|
|
1976
|
+
WHERE e.project_hash = ? AND e.branch_name = ?
|
|
1977
|
+
AND e.file_gen != fg.active_gen
|
|
1978
|
+
LIMIT ?
|
|
1979
|
+
)`,
|
|
1980
|
+
args: [projectHash, branchName, limit]
|
|
1981
|
+
});
|
|
1982
|
+
const deleted = result.rowsAffected;
|
|
1983
|
+
await client.execute({
|
|
1984
|
+
sql: `DELETE FROM file_generations
|
|
1985
|
+
WHERE project_hash = ? AND branch_name = ? AND active_gen = -1
|
|
1986
|
+
AND file_path NOT IN (
|
|
1987
|
+
SELECT DISTINCT file_path FROM entities
|
|
1988
|
+
WHERE project_hash = ? AND branch_name = ?
|
|
1989
|
+
AND file_path IN (
|
|
1990
|
+
SELECT file_path FROM file_generations
|
|
1991
|
+
WHERE project_hash = ? AND branch_name = ? AND active_gen = -1
|
|
1992
|
+
)
|
|
1993
|
+
)`,
|
|
1994
|
+
args: [projectHash, branchName, projectHash, branchName, projectHash, branchName]
|
|
1995
|
+
});
|
|
1996
|
+
if (deleted > 0) {
|
|
1997
|
+
log.i("GEN_OPS", "gc_stale_entities", { deleted });
|
|
1998
|
+
}
|
|
1999
|
+
return deleted;
|
|
2000
|
+
} finally {
|
|
2001
|
+
this.gcRunning = false;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
/**
|
|
2005
|
+
* GC: Delete orphaned name_tokens whose entities are stale.
|
|
2006
|
+
* Returns number of deleted rows.
|
|
2007
|
+
*/
|
|
2008
|
+
async gcStaleNameTokens(limit = 1e4) {
|
|
2009
|
+
const client = this.getClient();
|
|
2010
|
+
if (!client) return 0;
|
|
2011
|
+
const { projectHash, branchName } = this.getContext();
|
|
2012
|
+
const result = await client.execute({
|
|
2013
|
+
sql: `DELETE FROM name_tokens WHERE rowid IN (
|
|
2014
|
+
SELECT nt.rowid FROM name_tokens nt
|
|
2015
|
+
LEFT JOIN entities e
|
|
2016
|
+
ON nt.entity_id = e.id
|
|
2017
|
+
AND nt.project_hash = e.project_hash
|
|
2018
|
+
AND nt.branch_name = e.branch_name
|
|
2019
|
+
LEFT JOIN file_generations fg
|
|
2020
|
+
ON e.file_path = fg.file_path
|
|
2021
|
+
AND e.project_hash = fg.project_hash
|
|
2022
|
+
AND e.branch_name = fg.branch_name
|
|
2023
|
+
WHERE nt.project_hash = ? AND nt.branch_name = ?
|
|
2024
|
+
AND (e.id IS NULL OR e.file_gen != fg.active_gen)
|
|
2025
|
+
LIMIT ?
|
|
2026
|
+
)`,
|
|
2027
|
+
args: [projectHash, branchName, limit]
|
|
2028
|
+
});
|
|
2029
|
+
const deleted = result.rowsAffected;
|
|
2030
|
+
if (deleted > 0) {
|
|
2031
|
+
log.i("GEN_OPS", "gc_stale_tokens", { deleted });
|
|
2032
|
+
}
|
|
2033
|
+
return deleted;
|
|
2034
|
+
}
|
|
2035
|
+
/**
|
|
2036
|
+
* Run full GC cycle: entities first, then orphan tokens.
|
|
2037
|
+
* Repeats in chunks until nothing left to clean.
|
|
2038
|
+
*/
|
|
2039
|
+
async runFullGC() {
|
|
2040
|
+
let totalEntities = 0;
|
|
2041
|
+
let totalTokens = 0;
|
|
2042
|
+
let deleted;
|
|
2043
|
+
do {
|
|
2044
|
+
deleted = await this.gcStaleEntities();
|
|
2045
|
+
totalEntities += deleted;
|
|
2046
|
+
} while (deleted > 0);
|
|
2047
|
+
do {
|
|
2048
|
+
deleted = await this.gcStaleNameTokens();
|
|
2049
|
+
totalTokens += deleted;
|
|
2050
|
+
} while (deleted > 0);
|
|
2051
|
+
if (totalEntities > 0 || totalTokens > 0) {
|
|
2052
|
+
log.i("GEN_OPS", "full_gc_complete", { entities: totalEntities, tokens: totalTokens });
|
|
2053
|
+
}
|
|
2054
|
+
return { entities: totalEntities, tokens: totalTokens };
|
|
2055
|
+
}
|
|
2056
|
+
/**
|
|
2057
|
+
* Clear cache (e.g., on branch switch).
|
|
2058
|
+
*/
|
|
2059
|
+
clearCache() {
|
|
2060
|
+
this.cache.clear();
|
|
2061
|
+
this.cacheLoaded = false;
|
|
2062
|
+
}
|
|
2063
|
+
get isCacheLoaded() {
|
|
2064
|
+
return this.cacheLoaded;
|
|
2065
|
+
}
|
|
2066
|
+
};
|
|
2067
|
+
|
|
2068
|
+
// src/storage/libsql/metadata-ops.ts
|
|
2069
|
+
init_logging();
|
|
2070
|
+
var MetadataOperations = class {
|
|
2071
|
+
constructor(getClient, getContext) {
|
|
2072
|
+
this.getClient = getClient;
|
|
2073
|
+
this.getContext = getContext;
|
|
2074
|
+
}
|
|
2075
|
+
// ===========================================================================
|
|
2076
|
+
// FILE OPERATIONS
|
|
2077
|
+
// ===========================================================================
|
|
2078
|
+
/**
|
|
2079
|
+
* Update or insert file info
|
|
2080
|
+
*/
|
|
2081
|
+
async updateFileInfo(info) {
|
|
2082
|
+
const client = this.getClient();
|
|
2083
|
+
if (!client) throw new Error("Client not initialized");
|
|
2084
|
+
const { projectHash, branchName } = this.getContext();
|
|
2085
|
+
await client.execute({
|
|
2086
|
+
sql: `
|
|
2087
|
+
INSERT OR REPLACE INTO files
|
|
2088
|
+
(path, project_hash, branch_name, hash, last_indexed, entity_count)
|
|
2089
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2090
|
+
`,
|
|
2091
|
+
args: [info.path, projectHash, branchName, info.hash, info.lastIndexed, info.entityCount]
|
|
2092
|
+
});
|
|
2093
|
+
}
|
|
2094
|
+
/**
|
|
2095
|
+
* Batch update or insert multiple file infos in a single DB round-trip.
|
|
2096
|
+
*/
|
|
2097
|
+
async batchUpdateFileInfo(infos) {
|
|
2098
|
+
if (infos.length === 0) return;
|
|
2099
|
+
const client = this.getClient();
|
|
2100
|
+
if (!client) throw new Error("Client not initialized");
|
|
2101
|
+
const { projectHash, branchName } = this.getContext();
|
|
2102
|
+
const statements = infos.map((info) => ({
|
|
2103
|
+
sql: `
|
|
2104
|
+
INSERT OR REPLACE INTO files
|
|
2105
|
+
(path, project_hash, branch_name, hash, last_indexed, entity_count)
|
|
2106
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
2107
|
+
`,
|
|
2108
|
+
args: [info.path, projectHash, branchName, info.hash, info.lastIndexed, info.entityCount]
|
|
2109
|
+
}));
|
|
2110
|
+
await client.batch(statements, "write");
|
|
2111
|
+
}
|
|
2112
|
+
/**
|
|
2113
|
+
* Get file info by path
|
|
2114
|
+
*/
|
|
2115
|
+
async getFileInfo(path) {
|
|
2116
|
+
const client = this.getClient();
|
|
2117
|
+
if (!client) throw new Error("Client not initialized");
|
|
2118
|
+
const { projectHash, branchName } = this.getContext();
|
|
2119
|
+
const result = await client.execute({
|
|
2120
|
+
sql: "SELECT * FROM files WHERE path = ? AND project_hash = ? AND branch_name = ?",
|
|
2121
|
+
args: [path, projectHash, branchName]
|
|
2122
|
+
});
|
|
2123
|
+
if (result.rows.length === 0 || !result.rows[0]) return null;
|
|
2124
|
+
const row = result.rows[0];
|
|
2125
|
+
return {
|
|
2126
|
+
path: row["path"],
|
|
2127
|
+
hash: row["hash"],
|
|
2128
|
+
lastIndexed: row["last_indexed"],
|
|
2129
|
+
entityCount: row["entity_count"]
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
2132
|
+
/**
|
|
2133
|
+
* Get files that were indexed before the specified timestamp
|
|
2134
|
+
*/
|
|
2135
|
+
async getOutdatedFiles(since) {
|
|
2136
|
+
const client = this.getClient();
|
|
2137
|
+
if (!client) throw new Error("Client not initialized");
|
|
2138
|
+
const { projectHash, branchName } = this.getContext();
|
|
2139
|
+
const result = await client.execute({
|
|
2140
|
+
sql: `
|
|
2141
|
+
SELECT * FROM files
|
|
2142
|
+
WHERE project_hash = ? AND branch_name = ? AND last_indexed < ?
|
|
2143
|
+
`,
|
|
2144
|
+
args: [projectHash, branchName, since]
|
|
2145
|
+
});
|
|
2146
|
+
return result.rows.map((row) => ({
|
|
2147
|
+
path: row["path"],
|
|
2148
|
+
hash: row["hash"],
|
|
2149
|
+
lastIndexed: row["last_indexed"],
|
|
2150
|
+
entityCount: row["entity_count"]
|
|
2151
|
+
}));
|
|
2152
|
+
}
|
|
2153
|
+
/**
|
|
2154
|
+
* Get all indexed files with their lastIndexed timestamps
|
|
2155
|
+
* Used for incremental indexing to compare with file mtime
|
|
2156
|
+
*/
|
|
2157
|
+
async getAllIndexedFiles() {
|
|
2158
|
+
const client = this.getClient();
|
|
2159
|
+
if (!client) throw new Error("Client not initialized");
|
|
2160
|
+
const { projectHash, branchName } = this.getContext();
|
|
2161
|
+
const result = await client.execute({
|
|
2162
|
+
sql: `
|
|
2163
|
+
SELECT path, last_indexed FROM files
|
|
2164
|
+
WHERE project_hash = ? AND branch_name = ?
|
|
2165
|
+
`,
|
|
2166
|
+
args: [projectHash, branchName]
|
|
2167
|
+
});
|
|
2168
|
+
const fileMap = /* @__PURE__ */ new Map();
|
|
2169
|
+
for (const row of result.rows) {
|
|
2170
|
+
const path = row["path"];
|
|
2171
|
+
const lastIndexed = row["last_indexed"];
|
|
2172
|
+
fileMap.set(path.replace(/\\/g, "/"), lastIndexed);
|
|
2173
|
+
}
|
|
2174
|
+
return fileMap;
|
|
2175
|
+
}
|
|
2176
|
+
/**
|
|
2177
|
+
* Delete file info by path
|
|
2178
|
+
*/
|
|
2179
|
+
async deleteFileInfo(path) {
|
|
2180
|
+
const client = this.getClient();
|
|
2181
|
+
if (!client) throw new Error("Client not initialized");
|
|
2182
|
+
const { projectHash, branchName } = this.getContext();
|
|
2183
|
+
const forwardPath = path.replace(/\\/g, "/");
|
|
2184
|
+
const backPath = path.replace(/\//g, "\\");
|
|
2185
|
+
await client.execute({
|
|
2186
|
+
sql: `
|
|
2187
|
+
DELETE FROM files
|
|
2188
|
+
WHERE project_hash = ? AND branch_name = ?
|
|
2189
|
+
AND (path = ? OR path = ?)
|
|
2190
|
+
`,
|
|
2191
|
+
args: [projectHash, branchName, forwardPath, backPath]
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
// ===========================================================================
|
|
2195
|
+
// PROJECT METADATA
|
|
2196
|
+
// ===========================================================================
|
|
2197
|
+
/**
|
|
2198
|
+
* Update project metadata after indexing
|
|
2199
|
+
*/
|
|
2200
|
+
async updateProjectMetadata(projectPath, isFullIndex = false) {
|
|
2201
|
+
const client = this.getClient();
|
|
2202
|
+
if (!client) throw new Error("Client not initialized");
|
|
2203
|
+
const { projectHash, branchName } = this.getContext();
|
|
2204
|
+
const now = Date.now();
|
|
2205
|
+
const entityCount = await client.execute({
|
|
2206
|
+
sql: "SELECT COUNT(*) as count FROM entities WHERE project_hash = ? AND branch_name = ?",
|
|
2207
|
+
args: [projectHash, branchName]
|
|
2208
|
+
});
|
|
2209
|
+
const fileCount = await client.execute({
|
|
2210
|
+
sql: "SELECT COUNT(*) as count FROM files WHERE project_hash = ? AND branch_name = ?",
|
|
2211
|
+
args: [projectHash, branchName]
|
|
2212
|
+
});
|
|
2213
|
+
const existing = await client.execute({
|
|
2214
|
+
sql: `SELECT last_full_index_at, incremental_changes_count, created_at
|
|
2215
|
+
FROM project_metadata WHERE project_hash = ? AND branch_name = ?`,
|
|
2216
|
+
args: [projectHash, branchName]
|
|
2217
|
+
});
|
|
2218
|
+
const existingRow = existing.rows[0];
|
|
2219
|
+
const createdAt = existingRow?.["created_at"] || now;
|
|
2220
|
+
const lastFullIndexAt = isFullIndex ? now : existingRow?.["last_full_index_at"] || 0;
|
|
2221
|
+
const incrementalChangesCount = isFullIndex ? 0 : existingRow?.["incremental_changes_count"] || 0;
|
|
2222
|
+
await client.execute({
|
|
2223
|
+
sql: `
|
|
2224
|
+
INSERT OR REPLACE INTO project_metadata
|
|
2225
|
+
(project_hash, branch_name, project_path, last_indexed_at, entity_count, file_count,
|
|
2226
|
+
created_at, updated_at, last_full_index_at, incremental_changes_count)
|
|
2227
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2228
|
+
`,
|
|
2229
|
+
args: [
|
|
2230
|
+
projectHash,
|
|
2231
|
+
branchName,
|
|
2232
|
+
projectPath,
|
|
2233
|
+
now,
|
|
2234
|
+
entityCount.rows[0]?.["count"] || 0,
|
|
2235
|
+
fileCount.rows[0]?.["count"] || 0,
|
|
2236
|
+
createdAt,
|
|
2237
|
+
now,
|
|
2238
|
+
lastFullIndexAt,
|
|
2239
|
+
incrementalChangesCount
|
|
2240
|
+
]
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2243
|
+
/**
|
|
2244
|
+
* Get incremental tracking info for the current project/branch
|
|
2245
|
+
*/
|
|
2246
|
+
async getIncrementalTrackingInfo() {
|
|
2247
|
+
const client = this.getClient();
|
|
2248
|
+
if (!client) throw new Error("Client not initialized");
|
|
2249
|
+
const { projectHash, branchName } = this.getContext();
|
|
2250
|
+
const result = await client.execute({
|
|
2251
|
+
sql: `SELECT last_full_index_at, incremental_changes_count, file_count
|
|
2252
|
+
FROM project_metadata WHERE project_hash = ? AND branch_name = ?`,
|
|
2253
|
+
args: [projectHash, branchName]
|
|
2254
|
+
});
|
|
2255
|
+
if (result.rows.length === 0) {
|
|
2256
|
+
return { lastFullIndexAt: 0, incrementalChangesCount: 0, totalFiles: 0 };
|
|
2257
|
+
}
|
|
2258
|
+
const row = result.rows[0];
|
|
2259
|
+
return {
|
|
2260
|
+
lastFullIndexAt: row["last_full_index_at"] || 0,
|
|
2261
|
+
incrementalChangesCount: row["incremental_changes_count"] || 0,
|
|
2262
|
+
totalFiles: row["file_count"] || 0
|
|
2263
|
+
};
|
|
2264
|
+
}
|
|
2265
|
+
/**
|
|
2266
|
+
* Record incremental file changes (called after each incremental update)
|
|
2267
|
+
*/
|
|
2268
|
+
async recordIncrementalChanges(changedFileCount) {
|
|
2269
|
+
const client = this.getClient();
|
|
2270
|
+
if (!client) throw new Error("Client not initialized");
|
|
2271
|
+
const { projectHash, branchName } = this.getContext();
|
|
2272
|
+
await client.execute({
|
|
2273
|
+
sql: `UPDATE project_metadata
|
|
2274
|
+
SET incremental_changes_count = incremental_changes_count + ?,
|
|
2275
|
+
updated_at = ?
|
|
2276
|
+
WHERE project_hash = ? AND branch_name = ?`,
|
|
2277
|
+
args: [changedFileCount, Date.now(), projectHash, branchName]
|
|
2278
|
+
});
|
|
2279
|
+
}
|
|
2280
|
+
/**
|
|
2281
|
+
* Reset incremental tracking (called after full index)
|
|
2282
|
+
*/
|
|
2283
|
+
async resetIncrementalTracking() {
|
|
2284
|
+
const client = this.getClient();
|
|
2285
|
+
if (!client) throw new Error("Client not initialized");
|
|
2286
|
+
const { projectHash, branchName } = this.getContext();
|
|
2287
|
+
const now = Date.now();
|
|
2288
|
+
await client.execute({
|
|
2289
|
+
sql: `UPDATE project_metadata
|
|
2290
|
+
SET last_full_index_at = ?,
|
|
2291
|
+
incremental_changes_count = 0,
|
|
2292
|
+
updated_at = ?
|
|
2293
|
+
WHERE project_hash = ? AND branch_name = ?`,
|
|
2294
|
+
args: [now, now, projectHash, branchName]
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
/**
|
|
2298
|
+
* List all projects in the database
|
|
2299
|
+
*/
|
|
2300
|
+
async listProjects() {
|
|
2301
|
+
const client = this.getClient();
|
|
2302
|
+
if (!client) throw new Error("Client not initialized");
|
|
2303
|
+
const result = await client.execute(`
|
|
2304
|
+
SELECT project_hash, branch_name, project_path, last_indexed_at, entity_count, file_count
|
|
2305
|
+
FROM project_metadata
|
|
2306
|
+
ORDER BY updated_at DESC
|
|
2307
|
+
`);
|
|
2308
|
+
return result.rows.map((r) => ({
|
|
2309
|
+
projectHash: r["project_hash"],
|
|
2310
|
+
branchName: r["branch_name"],
|
|
2311
|
+
projectPath: r["project_path"],
|
|
2312
|
+
lastIndexedAt: r["last_indexed_at"],
|
|
2313
|
+
entityCount: r["entity_count"],
|
|
2314
|
+
fileCount: r["file_count"]
|
|
2315
|
+
}));
|
|
2316
|
+
}
|
|
2317
|
+
/**
|
|
2318
|
+
* List all branches for current project
|
|
2319
|
+
*/
|
|
2320
|
+
async listBranches() {
|
|
2321
|
+
const client = this.getClient();
|
|
2322
|
+
if (!client) throw new Error("Client not initialized");
|
|
2323
|
+
const { projectHash } = this.getContext();
|
|
2324
|
+
const result = await client.execute({
|
|
2325
|
+
sql: `
|
|
2326
|
+
SELECT DISTINCT branch_name FROM project_metadata
|
|
2327
|
+
WHERE project_hash = ?
|
|
2328
|
+
ORDER BY branch_name
|
|
2329
|
+
`,
|
|
2330
|
+
args: [projectHash]
|
|
2331
|
+
});
|
|
2332
|
+
return result.rows.map((r) => r["branch_name"]);
|
|
2333
|
+
}
|
|
2334
|
+
// ===========================================================================
|
|
2335
|
+
// STATS
|
|
2336
|
+
// ===========================================================================
|
|
2337
|
+
/**
|
|
2338
|
+
* Get stats for current project/branch
|
|
2339
|
+
* Uses layered approach: if baseBranch is set and different from current, includes both
|
|
2340
|
+
*/
|
|
2341
|
+
async getStats() {
|
|
2342
|
+
const client = this.getClient();
|
|
2343
|
+
if (!client) throw new Error("Client not initialized");
|
|
2344
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
2345
|
+
const useLayered = baseBranch && baseBranch !== branchName;
|
|
2346
|
+
const branchFilter = useLayered ? "branch_name IN (?, ?)" : "branch_name = ?";
|
|
2347
|
+
const branchArgs = useLayered ? [baseBranch, branchName] : [branchName];
|
|
2348
|
+
const [entities, relationships, files] = await Promise.all([
|
|
2349
|
+
client.execute({
|
|
2350
|
+
sql: `SELECT COUNT(*) as cnt FROM entities WHERE project_hash = ? AND ${branchFilter}`,
|
|
2351
|
+
args: [projectHash, ...branchArgs]
|
|
2352
|
+
}),
|
|
2353
|
+
client.execute({
|
|
2354
|
+
sql: `SELECT COUNT(*) as cnt FROM relationships WHERE project_hash = ? AND ${branchFilter}`,
|
|
2355
|
+
args: [projectHash, ...branchArgs]
|
|
2356
|
+
}),
|
|
2357
|
+
client.execute({
|
|
2358
|
+
sql: `SELECT COUNT(*) as cnt FROM files WHERE project_hash = ? AND ${branchFilter}`,
|
|
2359
|
+
args: [projectHash, ...branchArgs]
|
|
2360
|
+
})
|
|
2361
|
+
]);
|
|
2362
|
+
return {
|
|
2363
|
+
totalEntities: entities.rows[0]?.["cnt"] || 0,
|
|
2364
|
+
totalRelationships: relationships.rows[0]?.["cnt"] || 0,
|
|
2365
|
+
totalFiles: files.rows[0]?.["cnt"] || 0,
|
|
2366
|
+
totalEmbeddings: 0
|
|
2367
|
+
// v5: embeddings stored in FAISS, not LibSQL
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2370
|
+
/**
|
|
2371
|
+
* Get stats across all projects
|
|
2372
|
+
*/
|
|
2373
|
+
async getTotalStats() {
|
|
2374
|
+
const client = this.getClient();
|
|
2375
|
+
if (!client) throw new Error("Client not initialized");
|
|
2376
|
+
const [entities, relationships, files] = await Promise.all([
|
|
2377
|
+
client.execute("SELECT COUNT(*) as cnt FROM entities"),
|
|
2378
|
+
client.execute("SELECT COUNT(*) as cnt FROM relationships"),
|
|
2379
|
+
client.execute("SELECT COUNT(*) as cnt FROM files")
|
|
2380
|
+
]);
|
|
2381
|
+
return {
|
|
2382
|
+
totalEntities: entities.rows[0]?.["cnt"] || 0,
|
|
2383
|
+
totalRelationships: relationships.rows[0]?.["cnt"] || 0,
|
|
2384
|
+
totalFiles: files.rows[0]?.["cnt"] || 0,
|
|
2385
|
+
totalEmbeddings: 0
|
|
2386
|
+
// v5: embeddings stored in FAISS, not LibSQL
|
|
2387
|
+
};
|
|
2388
|
+
}
|
|
2389
|
+
// ===========================================================================
|
|
2390
|
+
// CLEAR OPERATIONS
|
|
2391
|
+
// ===========================================================================
|
|
2392
|
+
/**
|
|
2393
|
+
* Clear all data for current project/branch.
|
|
2394
|
+
* Auto-detects single-project DB and uses fast truncation path (clearAll)
|
|
2395
|
+
* to avoid SQLite B-tree fragmentation that causes 56x slower INSERTs.
|
|
2396
|
+
*/
|
|
2397
|
+
async clear() {
|
|
2398
|
+
const client = this.getClient();
|
|
2399
|
+
if (!client) throw new Error("Client not initialized");
|
|
2400
|
+
const { projectHash, branchName } = this.getContext();
|
|
2401
|
+
const otherProjects = await client.execute({
|
|
2402
|
+
sql: "SELECT 1 FROM entities WHERE project_hash != ? LIMIT 1",
|
|
2403
|
+
args: [projectHash]
|
|
2404
|
+
});
|
|
2405
|
+
if (otherProjects.rows.length === 0) {
|
|
2406
|
+
await this.clearAll();
|
|
2407
|
+
try {
|
|
2408
|
+
await client.execute({ sql: "VACUUM", args: [] });
|
|
2409
|
+
} catch {
|
|
2410
|
+
}
|
|
2411
|
+
log.i("METADATAOPS", "data_cleared_fast", { ctx: `${projectHash}/${branchName}`, mode: "truncate+vacuum" });
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
await client.batch(
|
|
2415
|
+
[
|
|
2416
|
+
// NOTE: embeddings table removed in v5 - FAISS handles vector storage
|
|
2417
|
+
{
|
|
2418
|
+
sql: "DELETE FROM relationships WHERE project_hash = ? AND branch_name = ?",
|
|
2419
|
+
args: [projectHash, branchName]
|
|
2420
|
+
},
|
|
2421
|
+
{ sql: "DELETE FROM entities WHERE project_hash = ? AND branch_name = ?", args: [projectHash, branchName] },
|
|
2422
|
+
{ sql: "DELETE FROM files WHERE project_hash = ? AND branch_name = ?", args: [projectHash, branchName] },
|
|
2423
|
+
{ sql: "DELETE FROM query_cache WHERE project_hash = ? AND branch_name = ?", args: [projectHash, branchName] },
|
|
2424
|
+
{
|
|
2425
|
+
sql: "DELETE FROM project_metadata WHERE project_hash = ? AND branch_name = ?",
|
|
2426
|
+
args: [projectHash, branchName]
|
|
2427
|
+
},
|
|
2428
|
+
{
|
|
2429
|
+
sql: "DELETE FROM name_tokens WHERE project_hash = ? AND branch_name = ?",
|
|
2430
|
+
args: [projectHash, branchName]
|
|
2431
|
+
},
|
|
2432
|
+
{
|
|
2433
|
+
sql: "DELETE FROM cooccurrence WHERE project_hash = ? AND branch_name = ?",
|
|
2434
|
+
args: [projectHash, branchName]
|
|
2435
|
+
},
|
|
2436
|
+
{
|
|
2437
|
+
sql: "DELETE FROM term_frequency WHERE project_hash = ? AND branch_name = ?",
|
|
2438
|
+
args: [projectHash, branchName]
|
|
2439
|
+
}
|
|
2440
|
+
],
|
|
2441
|
+
"write"
|
|
2442
|
+
);
|
|
2443
|
+
try {
|
|
2444
|
+
await client.execute({ sql: "VACUUM", args: [] });
|
|
2445
|
+
log.i("METADATAOPS", "data_cleared", { ctx: `${projectHash}/${branchName}`, mode: "delete+vacuum" });
|
|
2446
|
+
} catch (error) {
|
|
2447
|
+
log.w("METADATAOPS", "vacuum_fail", { err: error.message });
|
|
2448
|
+
log.i("METADATAOPS", "data_cleared", { ctx: `${projectHash}/${branchName}`, mode: "delete" });
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
/**
|
|
2452
|
+
* Clear ALL data in the database
|
|
2453
|
+
*/
|
|
2454
|
+
async clearAll() {
|
|
2455
|
+
const client = this.getClient();
|
|
2456
|
+
if (!client) throw new Error("Client not initialized");
|
|
2457
|
+
await client.batch(
|
|
2458
|
+
[
|
|
2459
|
+
// NOTE: embeddings table removed in v5 - FAISS handles vector storage
|
|
2460
|
+
{ sql: "DELETE FROM relationships", args: [] },
|
|
2461
|
+
{ sql: "DELETE FROM entities", args: [] },
|
|
2462
|
+
{ sql: "DELETE FROM files", args: [] },
|
|
2463
|
+
{ sql: "DELETE FROM query_cache", args: [] },
|
|
2464
|
+
{ sql: "DELETE FROM project_metadata", args: [] },
|
|
2465
|
+
{ sql: "DELETE FROM name_tokens", args: [] },
|
|
2466
|
+
{ sql: "DELETE FROM cooccurrence", args: [] },
|
|
2467
|
+
{ sql: "DELETE FROM term_frequency", args: [] }
|
|
2468
|
+
],
|
|
2469
|
+
"write"
|
|
2470
|
+
);
|
|
2471
|
+
try {
|
|
2472
|
+
await client.execute({ sql: "PRAGMA wal_checkpoint(TRUNCATE)", args: [] });
|
|
2473
|
+
} catch {
|
|
2474
|
+
}
|
|
2475
|
+
log.i("METADATAOPS", "all_data_cleared");
|
|
2476
|
+
}
|
|
2477
|
+
};
|
|
2478
|
+
|
|
2479
|
+
// src/storage/libsql/relationship-ops.ts
|
|
2480
|
+
var RelationshipOperations = class {
|
|
2481
|
+
constructor(getClient, getContext, rowToRelationship) {
|
|
2482
|
+
this.getClient = getClient;
|
|
2483
|
+
this.getContext = getContext;
|
|
2484
|
+
this.rowToRelationship = rowToRelationship;
|
|
2485
|
+
}
|
|
2486
|
+
tombstoneAdder;
|
|
2487
|
+
tombstoneGetter;
|
|
2488
|
+
/**
|
|
2489
|
+
* Set tombstone delegates for layered branch support.
|
|
2490
|
+
* Must be called after adapter initialization.
|
|
2491
|
+
*/
|
|
2492
|
+
setTombstoneDelegates(adder, getter) {
|
|
2493
|
+
this.tombstoneAdder = adder;
|
|
2494
|
+
this.tombstoneGetter = getter;
|
|
2495
|
+
}
|
|
2496
|
+
/**
|
|
2497
|
+
* Insert a single relationship
|
|
2498
|
+
*/
|
|
2499
|
+
async insertRelationship(relationship) {
|
|
2500
|
+
const client = this.getClient();
|
|
2501
|
+
if (!client) throw new Error("Client not initialized");
|
|
2502
|
+
const { projectHash, branchName } = this.getContext();
|
|
2503
|
+
const now = Date.now();
|
|
2504
|
+
await client.execute({
|
|
2505
|
+
sql: `
|
|
2506
|
+
INSERT OR REPLACE INTO relationships
|
|
2507
|
+
(id, project_hash, branch_name, from_id, to_id, type, metadata, weight, created_at)
|
|
2508
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2509
|
+
`,
|
|
2510
|
+
args: [
|
|
2511
|
+
relationship.id,
|
|
2512
|
+
projectHash,
|
|
2513
|
+
branchName,
|
|
2514
|
+
relationship.fromId,
|
|
2515
|
+
relationship.toId,
|
|
2516
|
+
relationship.type,
|
|
2517
|
+
relationship.metadata ? JSON.stringify(relationship.metadata) : null,
|
|
2518
|
+
relationship.weight ?? 1,
|
|
2519
|
+
relationship.createdAt ?? now
|
|
2520
|
+
]
|
|
2521
|
+
});
|
|
2522
|
+
}
|
|
2523
|
+
/**
|
|
2524
|
+
* Insert multiple relationships with batch optimization
|
|
2525
|
+
*/
|
|
2526
|
+
async insertRelationships(relationships) {
|
|
2527
|
+
const client = this.getClient();
|
|
2528
|
+
if (!client) throw new Error("Client not initialized");
|
|
2529
|
+
if (relationships.length === 0) return { processed: 0, failed: 0, errors: [], timeMs: 0 };
|
|
2530
|
+
const start = Date.now();
|
|
2531
|
+
const errors = [];
|
|
2532
|
+
const { projectHash, branchName } = this.getContext();
|
|
2533
|
+
const now = Date.now();
|
|
2534
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2535
|
+
const unique = [];
|
|
2536
|
+
for (const r of relationships) {
|
|
2537
|
+
if (!seen.has(r.id)) {
|
|
2538
|
+
seen.add(r.id);
|
|
2539
|
+
unique.push(r);
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
2542
|
+
const batchSize = 1e3;
|
|
2543
|
+
let processed = 0;
|
|
2544
|
+
for (let i = 0; i < unique.length; i += batchSize) {
|
|
2545
|
+
const batch = unique.slice(i, i + batchSize);
|
|
2546
|
+
const valuePlaceholders = batch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
2547
|
+
const args = [];
|
|
2548
|
+
for (const r of batch) {
|
|
2549
|
+
args.push(
|
|
2550
|
+
r.id,
|
|
2551
|
+
projectHash,
|
|
2552
|
+
branchName,
|
|
2553
|
+
r.fromId,
|
|
2554
|
+
r.toId,
|
|
2555
|
+
r.type,
|
|
2556
|
+
r.metadata ? JSON.stringify(r.metadata) : null,
|
|
2557
|
+
r.weight ?? 1,
|
|
2558
|
+
r.createdAt ?? now
|
|
2559
|
+
);
|
|
2560
|
+
}
|
|
2561
|
+
const sql = `
|
|
2562
|
+
INSERT OR REPLACE INTO relationships
|
|
2563
|
+
(id, project_hash, branch_name, from_id, to_id, type, metadata, weight, created_at)
|
|
2564
|
+
VALUES ${valuePlaceholders}
|
|
2565
|
+
`;
|
|
2566
|
+
try {
|
|
2567
|
+
await client.execute({ sql, args });
|
|
2568
|
+
processed += batch.length;
|
|
2569
|
+
} catch (error) {
|
|
2570
|
+
errors.push({
|
|
2571
|
+
item: { batchStart: i, batchEnd: i + batch.length },
|
|
2572
|
+
error: error.message
|
|
2573
|
+
});
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
return {
|
|
2577
|
+
processed,
|
|
2578
|
+
failed: errors.length,
|
|
2579
|
+
errors,
|
|
2580
|
+
timeMs: Date.now() - start
|
|
2581
|
+
};
|
|
2582
|
+
}
|
|
2583
|
+
/**
|
|
2584
|
+
* Get all relationships for an entity (as source or target) (layered: delta + base - tombstones)
|
|
2585
|
+
*/
|
|
2586
|
+
async getRelationshipsForEntity(entityId, type) {
|
|
2587
|
+
const client = this.getClient();
|
|
2588
|
+
if (!client) throw new Error("Client not initialized");
|
|
2589
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
2590
|
+
if (!baseBranch) {
|
|
2591
|
+
let sql = `
|
|
2592
|
+
SELECT * FROM relationships
|
|
2593
|
+
WHERE project_hash = ? AND branch_name = ? AND (from_id = ? OR to_id = ?)
|
|
2594
|
+
`;
|
|
2595
|
+
const args = [projectHash, branchName, entityId, entityId];
|
|
2596
|
+
if (type) {
|
|
2597
|
+
sql += " AND type = ?";
|
|
2598
|
+
args.push(type);
|
|
2599
|
+
}
|
|
2600
|
+
const result = await client.execute({ sql, args });
|
|
2601
|
+
return result.rows.map((row) => this.rowToRelationship(row));
|
|
2602
|
+
}
|
|
2603
|
+
const tombstones = this.tombstoneGetter ? await this.tombstoneGetter("relationship") : /* @__PURE__ */ new Set();
|
|
2604
|
+
let deltaSql = `
|
|
2605
|
+
SELECT * FROM relationships
|
|
2606
|
+
WHERE project_hash = ? AND branch_name = ? AND (from_id = ? OR to_id = ?)
|
|
2607
|
+
`;
|
|
2608
|
+
const deltaArgs = [projectHash, branchName, entityId, entityId];
|
|
2609
|
+
if (type) {
|
|
2610
|
+
deltaSql += " AND type = ?";
|
|
2611
|
+
deltaArgs.push(type);
|
|
2612
|
+
}
|
|
2613
|
+
const deltaResult = await client.execute({ sql: deltaSql, args: deltaArgs });
|
|
2614
|
+
const deltaRels = deltaResult.rows.map((row) => this.rowToRelationship(row));
|
|
2615
|
+
const deltaIds = new Set(deltaRels.map((r) => r.id));
|
|
2616
|
+
let baseSql = `
|
|
2617
|
+
SELECT * FROM relationships
|
|
2618
|
+
WHERE project_hash = ? AND branch_name = ? AND (from_id = ? OR to_id = ?)
|
|
2619
|
+
`;
|
|
2620
|
+
const baseArgs = [projectHash, baseBranch, entityId, entityId];
|
|
2621
|
+
if (type) {
|
|
2622
|
+
baseSql += " AND type = ?";
|
|
2623
|
+
baseArgs.push(type);
|
|
2624
|
+
}
|
|
2625
|
+
const baseResult = await client.execute({ sql: baseSql, args: baseArgs });
|
|
2626
|
+
const baseRels = baseResult.rows.map((row) => this.rowToRelationship(row)).filter((r) => !deltaIds.has(r.id) && !tombstones.has(r.id));
|
|
2627
|
+
return [...deltaRels, ...baseRels];
|
|
2628
|
+
}
|
|
2629
|
+
/**
|
|
2630
|
+
* Build filter clause for relationship type
|
|
2631
|
+
*/
|
|
2632
|
+
buildTypeFilter(types, args) {
|
|
2633
|
+
if (!types) return "";
|
|
2634
|
+
const typeArray = Array.isArray(types) ? types : [types];
|
|
2635
|
+
args.push(...typeArray);
|
|
2636
|
+
return ` AND type IN (${typeArray.map(() => "?").join(",")})`;
|
|
2637
|
+
}
|
|
2638
|
+
/**
|
|
2639
|
+
* Build type filter for CTE (static SQL, no parameter mutation)
|
|
2640
|
+
*/
|
|
2641
|
+
buildTypeFilterForCTE(types) {
|
|
2642
|
+
if (!types) return "";
|
|
2643
|
+
const typeArray = Array.isArray(types) ? types : [types];
|
|
2644
|
+
const quoted = typeArray.map((t) => `'${t}'`).join(",");
|
|
2645
|
+
return `AND type IN (${quoted})`;
|
|
2646
|
+
}
|
|
2647
|
+
/**
|
|
2648
|
+
* Build fromId filter for direct SQL
|
|
2649
|
+
*/
|
|
2650
|
+
buildFromIdFilter(fromIds, args) {
|
|
2651
|
+
if (!fromIds) return "";
|
|
2652
|
+
const idArray = Array.isArray(fromIds) ? fromIds : [fromIds];
|
|
2653
|
+
args.push(...idArray);
|
|
2654
|
+
return ` AND from_id IN (${idArray.map(() => "?").join(",")})`;
|
|
2655
|
+
}
|
|
2656
|
+
/**
|
|
2657
|
+
* Build fromId filter for CTE (static SQL)
|
|
2658
|
+
*/
|
|
2659
|
+
buildFromIdFilterForCTE(fromIds) {
|
|
2660
|
+
if (!fromIds) return "";
|
|
2661
|
+
const idArray = Array.isArray(fromIds) ? fromIds : [fromIds];
|
|
2662
|
+
const quoted = idArray.map((id) => `'${id}'`).join(",");
|
|
2663
|
+
return `AND from_id IN (${quoted})`;
|
|
2664
|
+
}
|
|
2665
|
+
/**
|
|
2666
|
+
* Build toId filter for direct SQL
|
|
2667
|
+
*/
|
|
2668
|
+
buildToIdFilter(toIds, args) {
|
|
2669
|
+
if (!toIds) return "";
|
|
2670
|
+
const idArray = Array.isArray(toIds) ? toIds : [toIds];
|
|
2671
|
+
args.push(...idArray);
|
|
2672
|
+
return ` AND to_id IN (${idArray.map(() => "?").join(",")})`;
|
|
2673
|
+
}
|
|
2674
|
+
/**
|
|
2675
|
+
* Build toId filter for CTE (static SQL)
|
|
2676
|
+
*/
|
|
2677
|
+
buildToIdFilterForCTE(toIds) {
|
|
2678
|
+
if (!toIds) return "";
|
|
2679
|
+
const idArray = Array.isArray(toIds) ? toIds : [toIds];
|
|
2680
|
+
const quoted = idArray.map((id) => `'${id}'`).join(",");
|
|
2681
|
+
return `AND to_id IN (${quoted})`;
|
|
2682
|
+
}
|
|
2683
|
+
/**
|
|
2684
|
+
* Find relationships with complex filters (layered: delta + base - tombstones)
|
|
2685
|
+
* Uses CTE for efficient layered queries with proper LIMIT/OFFSET at SQL level
|
|
2686
|
+
*/
|
|
2687
|
+
async findRelationships(query) {
|
|
2688
|
+
const client = this.getClient();
|
|
2689
|
+
if (!client) throw new Error("Client not initialized");
|
|
2690
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
2691
|
+
const limit = query.limit || 100;
|
|
2692
|
+
const offset = query.offset || 0;
|
|
2693
|
+
if (!baseBranch) {
|
|
2694
|
+
const args = [projectHash, branchName];
|
|
2695
|
+
let sql2 = "SELECT * FROM relationships WHERE project_hash = ? AND branch_name = ?";
|
|
2696
|
+
sql2 += this.buildTypeFilter(query.filters?.relationshipType, args);
|
|
2697
|
+
sql2 += this.buildFromIdFilter(query.filters?.fromId, args);
|
|
2698
|
+
sql2 += this.buildToIdFilter(query.filters?.toId, args);
|
|
2699
|
+
sql2 += " LIMIT ? OFFSET ?";
|
|
2700
|
+
args.push(limit, offset);
|
|
2701
|
+
const result2 = await client.execute({ sql: sql2, args });
|
|
2702
|
+
return result2.rows.map((row) => this.rowToRelationship(row));
|
|
2703
|
+
}
|
|
2704
|
+
const typeFilter = this.buildTypeFilterForCTE(query.filters?.relationshipType);
|
|
2705
|
+
const fromIdFilter = this.buildFromIdFilterForCTE(query.filters?.fromId);
|
|
2706
|
+
const toIdFilter = this.buildToIdFilterForCTE(query.filters?.toId);
|
|
2707
|
+
const combinedFilters = `${typeFilter} ${fromIdFilter} ${toIdFilter}`;
|
|
2708
|
+
const sql = `
|
|
2709
|
+
WITH
|
|
2710
|
+
delta AS (
|
|
2711
|
+
SELECT * FROM relationships
|
|
2712
|
+
WHERE project_hash = ?1 AND branch_name = ?2 ${combinedFilters}
|
|
2713
|
+
),
|
|
2714
|
+
tombstone_ids AS (
|
|
2715
|
+
SELECT entity_id FROM tombstones
|
|
2716
|
+
WHERE project_hash = ?1 AND branch_name = ?2 AND entity_type = 'relationship'
|
|
2717
|
+
),
|
|
2718
|
+
base_filtered AS (
|
|
2719
|
+
SELECT * FROM relationships
|
|
2720
|
+
WHERE project_hash = ?1 AND branch_name = ?3 ${combinedFilters}
|
|
2721
|
+
AND id NOT IN (SELECT id FROM delta)
|
|
2722
|
+
AND id NOT IN (SELECT entity_id FROM tombstone_ids)
|
|
2723
|
+
),
|
|
2724
|
+
layered AS (
|
|
2725
|
+
SELECT * FROM delta
|
|
2726
|
+
UNION ALL
|
|
2727
|
+
SELECT * FROM base_filtered
|
|
2728
|
+
)
|
|
2729
|
+
SELECT * FROM layered
|
|
2730
|
+
LIMIT ?4 OFFSET ?5
|
|
2731
|
+
`;
|
|
2732
|
+
const result = await client.execute({
|
|
2733
|
+
sql,
|
|
2734
|
+
args: [projectHash, branchName, baseBranch, limit, offset]
|
|
2735
|
+
});
|
|
2736
|
+
return result.rows.map((row) => this.rowToRelationship(row));
|
|
2737
|
+
}
|
|
2738
|
+
/**
|
|
2739
|
+
* Delete relationship by ID.
|
|
2740
|
+
* On feature branches with baseBranch set, adds tombstone instead of deleting.
|
|
2741
|
+
*/
|
|
2742
|
+
async deleteRelationship(id) {
|
|
2743
|
+
const client = this.getClient();
|
|
2744
|
+
if (!client) throw new Error("Client not initialized");
|
|
2745
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
2746
|
+
if (baseBranch && this.tombstoneAdder) {
|
|
2747
|
+
await this.tombstoneAdder(id, "relationship");
|
|
2748
|
+
}
|
|
2749
|
+
await client.execute({
|
|
2750
|
+
sql: "DELETE FROM relationships WHERE id = ? AND project_hash = ? AND branch_name = ?",
|
|
2751
|
+
args: [id, projectHash, branchName]
|
|
2752
|
+
});
|
|
2753
|
+
}
|
|
2754
|
+
/**
|
|
2755
|
+
* Get ALL relationships efficiently in a single query.
|
|
2756
|
+
* Layered: returns delta + base - tombstones using CTE.
|
|
2757
|
+
*/
|
|
2758
|
+
async getAllRelationships() {
|
|
2759
|
+
const client = this.getClient();
|
|
2760
|
+
if (!client) throw new Error("Client not initialized");
|
|
2761
|
+
const { projectHash, branchName, baseBranch } = this.getContext();
|
|
2762
|
+
if (!baseBranch) {
|
|
2763
|
+
const result2 = await client.execute({
|
|
2764
|
+
sql: "SELECT * FROM relationships WHERE project_hash = ? AND branch_name = ?",
|
|
2765
|
+
args: [projectHash, branchName]
|
|
2766
|
+
});
|
|
2767
|
+
return result2.rows.map((row) => this.rowToRelationship(row));
|
|
2768
|
+
}
|
|
2769
|
+
const sql = `
|
|
2770
|
+
WITH
|
|
2771
|
+
delta AS (
|
|
2772
|
+
SELECT * FROM relationships
|
|
2773
|
+
WHERE project_hash = ?1 AND branch_name = ?2
|
|
2774
|
+
),
|
|
2775
|
+
tombstone_ids AS (
|
|
2776
|
+
SELECT entity_id FROM tombstones
|
|
2777
|
+
WHERE project_hash = ?1 AND branch_name = ?2 AND entity_type = 'relationship'
|
|
2778
|
+
),
|
|
2779
|
+
base_filtered AS (
|
|
2780
|
+
SELECT * FROM relationships
|
|
2781
|
+
WHERE project_hash = ?1 AND branch_name = ?3
|
|
2782
|
+
AND id NOT IN (SELECT id FROM delta)
|
|
2783
|
+
AND id NOT IN (SELECT entity_id FROM tombstone_ids)
|
|
2784
|
+
)
|
|
2785
|
+
SELECT * FROM delta
|
|
2786
|
+
UNION ALL
|
|
2787
|
+
SELECT * FROM base_filtered
|
|
2788
|
+
`;
|
|
2789
|
+
const result = await client.execute({
|
|
2790
|
+
sql,
|
|
2791
|
+
args: [projectHash, branchName, baseBranch]
|
|
2792
|
+
});
|
|
2793
|
+
return result.rows.map((row) => this.rowToRelationship(row));
|
|
2794
|
+
}
|
|
2795
|
+
};
|
|
2796
|
+
var requestContextStorage = new AsyncLocalStorage();
|
|
2797
|
+
function runWithRequestContext(ctx, fn) {
|
|
2798
|
+
return requestContextStorage.run(ctx, fn);
|
|
2799
|
+
}
|
|
2800
|
+
function getRequestContext() {
|
|
2801
|
+
return requestContextStorage.getStore();
|
|
2802
|
+
}
|
|
2803
|
+
|
|
2804
|
+
// src/storage/libsql/vector-ops.ts
|
|
2805
|
+
init_logging();
|
|
2806
|
+
var VectorOperations = class {
|
|
2807
|
+
constructor(ctx) {
|
|
2808
|
+
this.ctx = ctx;
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Insert a single embedding
|
|
2812
|
+
*/
|
|
2813
|
+
async insertEmbedding(embedding) {
|
|
2814
|
+
const client = this.ctx.getClient();
|
|
2815
|
+
if (!client) throw new Error("Client not initialized");
|
|
2816
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
2817
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
2818
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
2819
|
+
const vectorStr = this.ctx.vectorToString(embedding.vector);
|
|
2820
|
+
const metadataBlob = this.ctx.encodeMetadata(embedding.metadata);
|
|
2821
|
+
await client.execute({
|
|
2822
|
+
sql: `
|
|
2823
|
+
INSERT OR REPLACE INTO embeddings
|
|
2824
|
+
(id, project_hash, branch_name, content, dim_size, ${colName}, metadata, created_at)
|
|
2825
|
+
VALUES (?, ?, ?, ?, ?, vector32(?), ?, ?)
|
|
2826
|
+
`,
|
|
2827
|
+
args: [
|
|
2828
|
+
embedding.id,
|
|
2829
|
+
projectHash,
|
|
2830
|
+
branchName,
|
|
2831
|
+
embedding.content,
|
|
2832
|
+
dims,
|
|
2833
|
+
vectorStr,
|
|
2834
|
+
metadataBlob,
|
|
2835
|
+
embedding.createdAt || Date.now()
|
|
2836
|
+
]
|
|
2837
|
+
});
|
|
2838
|
+
const cacheKey = `${projectHash}:${branchName}:${embedding.id}`;
|
|
2839
|
+
this.ctx.embeddingCache.set(cacheKey, embedding);
|
|
2840
|
+
this.invalidateSearchCache();
|
|
2841
|
+
}
|
|
2842
|
+
/**
|
|
2843
|
+
* @deprecated Use FaissProvider.addBatch() instead. VectorStore v5 uses Faiss directly.
|
|
2844
|
+
*/
|
|
2845
|
+
async insertEmbeddingBatch(embeddings) {
|
|
2846
|
+
const client = this.ctx.getClient();
|
|
2847
|
+
if (!client) throw new Error("Client not initialized");
|
|
2848
|
+
if (embeddings.length === 0) return;
|
|
2849
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
2850
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
2851
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
2852
|
+
const now = Date.now();
|
|
2853
|
+
const statements = embeddings.map((e) => ({
|
|
2854
|
+
sql: `
|
|
2855
|
+
INSERT OR REPLACE INTO embeddings
|
|
2856
|
+
(id, project_hash, branch_name, content, dim_size, ${colName}, metadata, created_at)
|
|
2857
|
+
VALUES (?, ?, ?, ?, ?, vector32(?), ?, ?)
|
|
2858
|
+
`,
|
|
2859
|
+
args: [
|
|
2860
|
+
e.id,
|
|
2861
|
+
projectHash,
|
|
2862
|
+
branchName,
|
|
2863
|
+
e.content,
|
|
2864
|
+
dims,
|
|
2865
|
+
this.ctx.vectorToString(e.vector),
|
|
2866
|
+
this.ctx.encodeMetadata(e.metadata),
|
|
2867
|
+
e.createdAt || now
|
|
2868
|
+
]
|
|
2869
|
+
}));
|
|
2870
|
+
const batchSize = 500;
|
|
2871
|
+
const chunks = [];
|
|
2872
|
+
for (let i = 0; i < statements.length; i += batchSize) {
|
|
2873
|
+
chunks.push(statements.slice(i, i + batchSize));
|
|
2874
|
+
}
|
|
2875
|
+
let totalInserted = 0;
|
|
2876
|
+
const batchPromises = chunks.map(async (batch, index) => {
|
|
2877
|
+
if (!batch || batch.length === 0) return 0;
|
|
2878
|
+
try {
|
|
2879
|
+
await client.batch(batch, "write");
|
|
2880
|
+
return batch.length;
|
|
2881
|
+
} catch (error) {
|
|
2882
|
+
log.e("VECTOROPS", `Batch ${index} failed`, { error: error.message });
|
|
2883
|
+
throw error;
|
|
2884
|
+
}
|
|
2885
|
+
});
|
|
2886
|
+
const results = await Promise.all(batchPromises);
|
|
2887
|
+
totalInserted = results.reduce((sum, count) => sum + count, 0);
|
|
2888
|
+
for (const e of embeddings) {
|
|
2889
|
+
const cacheKey = `${projectHash}:${branchName}:${e.id}`;
|
|
2890
|
+
this.ctx.embeddingCache.set(cacheKey, e);
|
|
2891
|
+
}
|
|
2892
|
+
this.invalidateSearchCache();
|
|
2893
|
+
if (totalInserted >= 500) {
|
|
2894
|
+
try {
|
|
2895
|
+
await client.execute("PRAGMA optimize");
|
|
2896
|
+
} catch {
|
|
2897
|
+
}
|
|
2898
|
+
try {
|
|
2899
|
+
await client.execute("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
2900
|
+
} catch {
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
}
|
|
2904
|
+
/**
|
|
2905
|
+
* @deprecated Faiss HNSW handles live updates, no need to drop/rebuild.
|
|
2906
|
+
*/
|
|
2907
|
+
async dropVectorIndex() {
|
|
2908
|
+
const client = this.ctx.getClient();
|
|
2909
|
+
if (!client) throw new Error("Client not initialized");
|
|
2910
|
+
const { projectHash } = this.ctx.getContext();
|
|
2911
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
2912
|
+
const partialIndexName = `idx_emb_${dims}_${projectHash.substring(0, 8)}`;
|
|
2913
|
+
const start = Date.now();
|
|
2914
|
+
log.t("LIBSQLINDEX", `\u25B6 dropVectorIndex START`, { indexName: partialIndexName, projectHash, dims });
|
|
2915
|
+
try {
|
|
2916
|
+
await client.execute(`DROP INDEX IF EXISTS ${partialIndexName}`);
|
|
2917
|
+
log.t("LIBSQLINDEX", `\u25C0 dropVectorIndex END`, { indexName: partialIndexName, ms: Date.now() - start });
|
|
2918
|
+
log.i("LIBSQLINDEX", `Dropped project vector index`, {
|
|
2919
|
+
indexName: partialIndexName,
|
|
2920
|
+
projectHash,
|
|
2921
|
+
dims,
|
|
2922
|
+
ms: Date.now() - start
|
|
2923
|
+
});
|
|
2924
|
+
} catch (error) {
|
|
2925
|
+
log.w("LIBSQLINDEX", `Failed to drop project vector index`, {
|
|
2926
|
+
indexName: partialIndexName,
|
|
2927
|
+
error: error.message
|
|
2928
|
+
});
|
|
2929
|
+
}
|
|
2930
|
+
}
|
|
2931
|
+
/**
|
|
2932
|
+
* @deprecated Faiss HNSW maintains index automatically, no rebuild needed.
|
|
2933
|
+
*/
|
|
2934
|
+
async rebuildVectorIndex() {
|
|
2935
|
+
const client = this.ctx.getClient();
|
|
2936
|
+
if (!client) throw new Error("Client not initialized");
|
|
2937
|
+
const { projectHash } = this.ctx.getContext();
|
|
2938
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
2939
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
2940
|
+
const indexName = `idx_emb_${dims}_${projectHash.substring(0, 8)}`;
|
|
2941
|
+
const start = Date.now();
|
|
2942
|
+
log.w("LIBSQLINDEX", `[REBUILD] START`, { indexName, projectHash, dims });
|
|
2943
|
+
const indexParams = [
|
|
2944
|
+
`'metric=${this.ctx.config.metric}'`,
|
|
2945
|
+
`'compress_neighbors=${this.ctx.config.compression}'`,
|
|
2946
|
+
`'max_neighbors=${this.ctx.config.maxNeighbors}'`,
|
|
2947
|
+
`'search_l=${this.ctx.config.searchL}'`,
|
|
2948
|
+
`'insert_l=${this.ctx.config.insertL}'`
|
|
2949
|
+
].join(", ");
|
|
2950
|
+
try {
|
|
2951
|
+
await client.execute(`
|
|
2952
|
+
CREATE INDEX IF NOT EXISTS ${indexName}
|
|
2953
|
+
ON embeddings(libsql_vector_idx(${colName}, ${indexParams}))
|
|
2954
|
+
WHERE project_hash = '${projectHash}' AND dim_size = ${dims}
|
|
2955
|
+
`);
|
|
2956
|
+
const elapsed = Date.now() - start;
|
|
2957
|
+
log.w("LIBSQLINDEX", `[REBUILD] END`, { indexName, dims, ms: elapsed });
|
|
2958
|
+
log.i("LIBSQLINDEX", `Rebuilt project vector index`, { indexName, projectHash, dims, ms: elapsed });
|
|
2959
|
+
} catch (error) {
|
|
2960
|
+
log.w("LIBSQLINDEX", `Failed to rebuild project vector index`, {
|
|
2961
|
+
indexName,
|
|
2962
|
+
error: error.message
|
|
2963
|
+
});
|
|
2964
|
+
}
|
|
2965
|
+
}
|
|
2966
|
+
/**
|
|
2967
|
+
* @deprecated Use FaissProvider.addBatch() instead.
|
|
2968
|
+
*/
|
|
2969
|
+
async bulkInsertEmbeddings(embeddings) {
|
|
2970
|
+
const client = this.ctx.getClient();
|
|
2971
|
+
if (!client) throw new Error("Client not initialized");
|
|
2972
|
+
if (embeddings.length === 0) return;
|
|
2973
|
+
const start = Date.now();
|
|
2974
|
+
log.i("LIBSQLBULK", `Starting bulk insert`, { count: embeddings.length });
|
|
2975
|
+
await this.dropVectorIndex();
|
|
2976
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
2977
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
2978
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
2979
|
+
const now = Date.now();
|
|
2980
|
+
const statements = embeddings.map((e) => ({
|
|
2981
|
+
sql: `
|
|
2982
|
+
INSERT OR REPLACE INTO embeddings
|
|
2983
|
+
(id, project_hash, branch_name, content, dim_size, ${colName}, metadata, created_at)
|
|
2984
|
+
VALUES (?, ?, ?, ?, ?, vector32(?), ?, ?)
|
|
2985
|
+
`,
|
|
2986
|
+
args: [
|
|
2987
|
+
e.id,
|
|
2988
|
+
projectHash,
|
|
2989
|
+
branchName,
|
|
2990
|
+
e.content,
|
|
2991
|
+
dims,
|
|
2992
|
+
this.ctx.vectorToString(e.vector),
|
|
2993
|
+
this.ctx.encodeMetadata(e.metadata),
|
|
2994
|
+
e.createdAt || now
|
|
2995
|
+
]
|
|
2996
|
+
}));
|
|
2997
|
+
const batchSize = 500;
|
|
2998
|
+
const totalBatches = Math.ceil(statements.length / batchSize);
|
|
2999
|
+
for (let i = 0; i < statements.length; i += batchSize) {
|
|
3000
|
+
const batch = statements.slice(i, i + batchSize);
|
|
3001
|
+
const batchNum = Math.floor(i / batchSize) + 1;
|
|
3002
|
+
log.t("LIBSQLBULK", ` batch ${batchNum}/${totalBatches}`, { size: batch.length });
|
|
3003
|
+
await client.batch(batch, "write");
|
|
3004
|
+
}
|
|
3005
|
+
for (const e of embeddings) {
|
|
3006
|
+
const cacheKey = `${projectHash}:${branchName}:${e.id}`;
|
|
3007
|
+
this.ctx.embeddingCache.set(cacheKey, e);
|
|
3008
|
+
}
|
|
3009
|
+
await this.rebuildVectorIndex();
|
|
3010
|
+
this.invalidateSearchCache();
|
|
3011
|
+
log.i("LIBSQLBULK", `Bulk insert complete`, { count: embeddings.length, ms: Date.now() - start });
|
|
3012
|
+
}
|
|
3013
|
+
/**
|
|
3014
|
+
* Invalidate search cache
|
|
3015
|
+
*/
|
|
3016
|
+
invalidateSearchCache() {
|
|
3017
|
+
this.ctx.searchCache.clear();
|
|
3018
|
+
}
|
|
3019
|
+
/**
|
|
3020
|
+
* @deprecated Use FaissProvider.search() instead.
|
|
3021
|
+
*/
|
|
3022
|
+
async searchVectors(queryVector, limit) {
|
|
3023
|
+
const client = this.ctx.getClient();
|
|
3024
|
+
if (!client) throw new Error("Client not initialized");
|
|
3025
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
3026
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
3027
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
3028
|
+
const vectorStr = this.ctx.vectorToString(queryVector);
|
|
3029
|
+
const cacheKey = `${projectHash}:${branchName}:${dims}:${limit}:${Array.from(queryVector.slice(0, 16)).join(",")}`;
|
|
3030
|
+
const cached = this.ctx.searchCache.get(cacheKey);
|
|
3031
|
+
if (cached) {
|
|
3032
|
+
return cached;
|
|
3033
|
+
}
|
|
3034
|
+
const countResult = await client.execute({
|
|
3035
|
+
sql: `SELECT COUNT(*) as cnt FROM embeddings WHERE project_hash = ? AND branch_name = ? AND dim_size = ?`,
|
|
3036
|
+
args: [projectHash, branchName, dims]
|
|
3037
|
+
});
|
|
3038
|
+
const embeddingCount = countResult.rows[0]?.["cnt"] || 0;
|
|
3039
|
+
if (embeddingCount === 0) {
|
|
3040
|
+
return [];
|
|
3041
|
+
}
|
|
3042
|
+
let result;
|
|
3043
|
+
if (embeddingCount <= 500) {
|
|
3044
|
+
result = await client.execute({
|
|
3045
|
+
sql: `SELECT id, content, metadata, vector_distance_cos(${colName}, vector32(?)) as distance
|
|
3046
|
+
FROM embeddings WHERE project_hash = ? AND branch_name = ? AND dim_size = ?
|
|
3047
|
+
ORDER BY distance ASC LIMIT ?`,
|
|
3048
|
+
args: [vectorStr, projectHash, branchName, dims, limit]
|
|
3049
|
+
});
|
|
3050
|
+
} else {
|
|
3051
|
+
let partialIndexName = await this.getProjectIndexName();
|
|
3052
|
+
if (!partialIndexName) {
|
|
3053
|
+
await this.ctx.ensureProjectVectorIndex();
|
|
3054
|
+
partialIndexName = await this.getProjectIndexName();
|
|
3055
|
+
}
|
|
3056
|
+
if (!partialIndexName) {
|
|
3057
|
+
throw new Error(
|
|
3058
|
+
`Vector index not available for project ${projectHash}. Ensure embeddings are generated first.`
|
|
3059
|
+
);
|
|
3060
|
+
}
|
|
3061
|
+
result = await client.execute({
|
|
3062
|
+
sql: `SELECT t.id, t.content, t.metadata
|
|
3063
|
+
FROM vector_top_k('${partialIndexName}', vector32(?), ?) AS v
|
|
3064
|
+
JOIN embeddings t ON t.rowid = v.id`,
|
|
3065
|
+
args: [vectorStr, limit]
|
|
3066
|
+
});
|
|
3067
|
+
}
|
|
3068
|
+
const results = this.processVectorResults(result, limit);
|
|
3069
|
+
this.ctx.searchCache.set(cacheKey, results);
|
|
3070
|
+
return results;
|
|
3071
|
+
}
|
|
3072
|
+
/**
|
|
3073
|
+
* Get project-specific index name if it exists
|
|
3074
|
+
*/
|
|
3075
|
+
async getProjectIndexName() {
|
|
3076
|
+
const client = this.ctx.getClient();
|
|
3077
|
+
if (!client) return null;
|
|
3078
|
+
const dims = this.ctx.getEffectiveDimensions();
|
|
3079
|
+
const { projectHash } = this.ctx.getContext();
|
|
3080
|
+
const indexName = `idx_emb_${dims}_${projectHash.substring(0, 8)}`;
|
|
3081
|
+
const r = await client.execute({
|
|
3082
|
+
sql: `SELECT name FROM sqlite_master WHERE type='index' AND name=?`,
|
|
3083
|
+
args: [indexName]
|
|
3084
|
+
});
|
|
3085
|
+
return r.rows.length > 0 ? indexName : null;
|
|
3086
|
+
}
|
|
3087
|
+
/**
|
|
3088
|
+
* Process vector search results
|
|
3089
|
+
*/
|
|
3090
|
+
processVectorResults(result, limit) {
|
|
3091
|
+
const results = [];
|
|
3092
|
+
for (const row of result.rows) {
|
|
3093
|
+
const position = results.length;
|
|
3094
|
+
const estimatedSimilarity = Math.max(0.1, 1 - position * 0.05);
|
|
3095
|
+
const metadata = row["metadata"] ? this.ctx.decodeMetadata(row["metadata"]) : void 0;
|
|
3096
|
+
results.push({
|
|
3097
|
+
id: row["id"],
|
|
3098
|
+
content: row["content"],
|
|
3099
|
+
similarity: estimatedSimilarity,
|
|
3100
|
+
metadata
|
|
3101
|
+
});
|
|
3102
|
+
if (results.length >= limit) break;
|
|
3103
|
+
}
|
|
3104
|
+
return results;
|
|
3105
|
+
}
|
|
3106
|
+
/**
|
|
3107
|
+
* @deprecated Use FaissProvider.getContent() instead.
|
|
3108
|
+
*/
|
|
3109
|
+
async getEmbedding(id) {
|
|
3110
|
+
const client = this.ctx.getClient();
|
|
3111
|
+
if (!client) throw new Error("Client not initialized");
|
|
3112
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
3113
|
+
const colName = this.ctx.getEmbeddingColumnName();
|
|
3114
|
+
const cacheKey = `${projectHash}:${branchName}:${id}`;
|
|
3115
|
+
const cached = this.ctx.embeddingCache.get(cacheKey);
|
|
3116
|
+
if (cached) {
|
|
3117
|
+
return cached;
|
|
3118
|
+
}
|
|
3119
|
+
const result = await client.execute({
|
|
3120
|
+
sql: `
|
|
3121
|
+
SELECT id, content, vector_extract(${colName}) as vector, metadata, created_at
|
|
3122
|
+
FROM embeddings
|
|
3123
|
+
WHERE id = ? AND project_hash = ? AND branch_name = ?
|
|
3124
|
+
`,
|
|
3125
|
+
args: [id, projectHash, branchName]
|
|
3126
|
+
});
|
|
3127
|
+
if (result.rows.length === 0 || !result.rows[0]) return null;
|
|
3128
|
+
const row = result.rows[0];
|
|
3129
|
+
const metadataRaw = row["metadata"];
|
|
3130
|
+
const metadata = metadataRaw ? this.ctx.decodeMetadata(metadataRaw) : void 0;
|
|
3131
|
+
const embedding = {
|
|
3132
|
+
id: row["id"],
|
|
3133
|
+
content: row["content"],
|
|
3134
|
+
vector: this.ctx.stringToVector(row["vector"]),
|
|
3135
|
+
metadata,
|
|
3136
|
+
createdAt: row["created_at"]
|
|
3137
|
+
};
|
|
3138
|
+
this.ctx.embeddingCache.set(cacheKey, embedding);
|
|
3139
|
+
return embedding;
|
|
3140
|
+
}
|
|
3141
|
+
/**
|
|
3142
|
+
* @deprecated Use FaissProvider.remove() instead.
|
|
3143
|
+
*/
|
|
3144
|
+
async deleteEmbedding(id) {
|
|
3145
|
+
const client = this.ctx.getClient();
|
|
3146
|
+
if (!client) throw new Error("Client not initialized");
|
|
3147
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
3148
|
+
await client.execute({
|
|
3149
|
+
sql: "DELETE FROM embeddings WHERE id = ? AND project_hash = ? AND branch_name = ?",
|
|
3150
|
+
args: [id, projectHash, branchName]
|
|
3151
|
+
});
|
|
3152
|
+
const cacheKey = `${projectHash}:${branchName}:${id}`;
|
|
3153
|
+
this.ctx.embeddingCache.delete(cacheKey);
|
|
3154
|
+
this.invalidateSearchCache();
|
|
3155
|
+
}
|
|
3156
|
+
/**
|
|
3157
|
+
* @deprecated Use FaissProvider.getVectorCount() instead.
|
|
3158
|
+
*/
|
|
3159
|
+
async getEmbeddingCount() {
|
|
3160
|
+
const client = this.ctx.getClient();
|
|
3161
|
+
if (!client) throw new Error("Client not initialized");
|
|
3162
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
3163
|
+
log.t("LIBSQL", "[getEmbeddingCount] Executing SQL...", { projectHash, branchName });
|
|
3164
|
+
const result = await client.execute({
|
|
3165
|
+
sql: "SELECT COUNT(*) as cnt FROM embeddings WHERE project_hash = ? AND branch_name = ?",
|
|
3166
|
+
args: [projectHash, branchName]
|
|
3167
|
+
});
|
|
3168
|
+
log.t("LIBSQL", "[getEmbeddingCount] SQL done", { rowCount: result.rows.length });
|
|
3169
|
+
return result.rows[0]?.["cnt"] || 0;
|
|
3170
|
+
}
|
|
3171
|
+
/**
|
|
3172
|
+
* @deprecated Use FaissProvider.getExistingIds() instead.
|
|
3173
|
+
*/
|
|
3174
|
+
async getExistingEmbeddingIds(ids) {
|
|
3175
|
+
const client = this.ctx.getClient();
|
|
3176
|
+
if (!client) throw new Error("Client not initialized");
|
|
3177
|
+
if (ids.length === 0) return /* @__PURE__ */ new Set();
|
|
3178
|
+
const { projectHash, branchName } = this.ctx.getContext();
|
|
3179
|
+
if (ids.length > 0 && ids.length <= 100) {
|
|
3180
|
+
log.t("LIBSQL", `getExistingEmbeddingIds context`, { projectHash, branchName, idsCount: ids.length });
|
|
3181
|
+
}
|
|
3182
|
+
const existingIds = /* @__PURE__ */ new Set();
|
|
3183
|
+
const uncachedIds = [];
|
|
3184
|
+
for (const id of ids) {
|
|
3185
|
+
const cacheKey = `${projectHash}:${branchName}:${id}`;
|
|
3186
|
+
if (this.ctx.embeddingCache.has(cacheKey)) {
|
|
3187
|
+
existingIds.add(id);
|
|
3188
|
+
} else {
|
|
3189
|
+
uncachedIds.push(id);
|
|
3190
|
+
}
|
|
3191
|
+
}
|
|
3192
|
+
if (uncachedIds.length === 0) {
|
|
3193
|
+
return existingIds;
|
|
3194
|
+
}
|
|
3195
|
+
const batchSize = 500;
|
|
3196
|
+
for (let i = 0; i < uncachedIds.length; i += batchSize) {
|
|
3197
|
+
const batch = uncachedIds.slice(i, i + batchSize);
|
|
3198
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
3199
|
+
try {
|
|
3200
|
+
const result = await client.execute({
|
|
3201
|
+
sql: `
|
|
3202
|
+
SELECT id FROM embeddings
|
|
3203
|
+
WHERE id IN (${placeholders})
|
|
3204
|
+
AND project_hash = ? AND branch_name = ?
|
|
3205
|
+
`,
|
|
3206
|
+
args: [...batch, projectHash, branchName]
|
|
3207
|
+
});
|
|
3208
|
+
if (i === 0 && batch.length > 0) {
|
|
3209
|
+
log.t("LIBSQL", `getExistingEmbeddingIds SQL result`, {
|
|
3210
|
+
queriedIds: batch.slice(0, 3),
|
|
3211
|
+
foundCount: result.rows.length,
|
|
3212
|
+
foundIds: result.rows.slice(0, 3).map((r) => r["id"])
|
|
3213
|
+
});
|
|
3214
|
+
}
|
|
3215
|
+
for (const row of result.rows) {
|
|
3216
|
+
existingIds.add(row["id"]);
|
|
3217
|
+
}
|
|
3218
|
+
} catch (error) {
|
|
3219
|
+
if (isCorruptionError(error)) {
|
|
3220
|
+
throw new DatabaseCorruptionError("Database corruption detected during embedding ID check", error);
|
|
3221
|
+
}
|
|
3222
|
+
throw error;
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
return existingIds;
|
|
3226
|
+
}
|
|
3227
|
+
};
|
|
3228
|
+
|
|
3229
|
+
// src/storage/libsql-graph-adapter.ts
|
|
3230
|
+
var LibSQLGraphAdapter = class {
|
|
3231
|
+
client = null;
|
|
3232
|
+
config;
|
|
3233
|
+
isInitialized = false;
|
|
3234
|
+
dbPath = "";
|
|
3235
|
+
// Current project context (will be set via setProject before use)
|
|
3236
|
+
currentContext = {
|
|
3237
|
+
projectHash: "_unset_",
|
|
3238
|
+
branchName: "_unset_"
|
|
3239
|
+
};
|
|
3240
|
+
// Performance caches
|
|
3241
|
+
embeddingCache;
|
|
3242
|
+
searchCache;
|
|
3243
|
+
metadataCache;
|
|
3244
|
+
// Delegated operations (composition pattern)
|
|
3245
|
+
entityOps;
|
|
3246
|
+
relationshipOps;
|
|
3247
|
+
vectorOps;
|
|
3248
|
+
cacheOps;
|
|
3249
|
+
metadataOps;
|
|
3250
|
+
cooccurrenceOps;
|
|
3251
|
+
generationManager;
|
|
3252
|
+
// Prolly Tree components for versioned graph storage
|
|
3253
|
+
prollyNodeStore = null;
|
|
3254
|
+
prollyTree = null;
|
|
3255
|
+
commitManager = null;
|
|
3256
|
+
branchDiffCache = null;
|
|
3257
|
+
constructor(config = {}) {
|
|
3258
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
3259
|
+
this.embeddingCache = new LRUCache(CACHE_CONFIG.embeddingCache);
|
|
3260
|
+
this.searchCache = new LRUCache(CACHE_CONFIG.searchCache);
|
|
3261
|
+
this.metadataCache = new LRUCache(CACHE_CONFIG.metadataCache);
|
|
3262
|
+
const getClient = () => this.client;
|
|
3263
|
+
const getContext = () => getRequestContext() ?? this.currentContext;
|
|
3264
|
+
this.generationManager = new GenerationManager(getClient, getContext);
|
|
3265
|
+
this.entityOps = new EntityOperations(
|
|
3266
|
+
getClient,
|
|
3267
|
+
getContext,
|
|
3268
|
+
(row) => this.rowToEntity(row),
|
|
3269
|
+
this.generationManager
|
|
3270
|
+
);
|
|
3271
|
+
this.relationshipOps = new RelationshipOperations(
|
|
3272
|
+
getClient,
|
|
3273
|
+
getContext,
|
|
3274
|
+
(row) => this.rowToRelationship(row)
|
|
3275
|
+
);
|
|
3276
|
+
const vectorOpsContext = {
|
|
3277
|
+
getClient,
|
|
3278
|
+
getContext,
|
|
3279
|
+
config: this.config,
|
|
3280
|
+
getEffectiveDimensions: () => this.getEffectiveDimensions(),
|
|
3281
|
+
getEmbeddingColumnName: () => this.getEmbeddingColumnName(),
|
|
3282
|
+
vectorToString: (v) => this.vectorToString(v),
|
|
3283
|
+
stringToVector: (s) => this.stringToVector(s),
|
|
3284
|
+
encodeMetadata: (m) => this.encodeMetadata(m),
|
|
3285
|
+
decodeMetadata: (d) => this.decodeMetadata(d),
|
|
3286
|
+
embeddingCache: this.embeddingCache,
|
|
3287
|
+
searchCache: this.searchCache,
|
|
3288
|
+
ensureProjectVectorIndex: () => this.ensureProjectVectorIndex()
|
|
3289
|
+
};
|
|
3290
|
+
this.vectorOps = new VectorOperations(vectorOpsContext);
|
|
3291
|
+
this.cacheOps = new CacheOperations(getClient, (v) => this.vectorToString(v));
|
|
3292
|
+
this.metadataOps = new MetadataOperations(getClient, getContext);
|
|
3293
|
+
this.cooccurrenceOps = new CooccurrenceOperations(getClient, getContext);
|
|
3294
|
+
}
|
|
3295
|
+
// ===========================================================================
|
|
3296
|
+
// CBOR SERIALIZATION (faster than JSON for binary/metadata)
|
|
3297
|
+
// ===========================================================================
|
|
3298
|
+
encodeMetadata(metadata) {
|
|
3299
|
+
if (!metadata) return null;
|
|
3300
|
+
try {
|
|
3301
|
+
return Buffer.from(cbor.encode(metadata));
|
|
3302
|
+
} catch {
|
|
3303
|
+
return Buffer.from(JSON.stringify(metadata));
|
|
3304
|
+
}
|
|
3305
|
+
}
|
|
3306
|
+
decodeMetadata(data) {
|
|
3307
|
+
if (!data) return void 0;
|
|
3308
|
+
let cacheKey;
|
|
3309
|
+
if (typeof data === "string") {
|
|
3310
|
+
cacheKey = data.length <= 64 ? data : data.slice(0, 64);
|
|
3311
|
+
} else {
|
|
3312
|
+
const slice = data.length <= 48 ? data : data.slice(0, 48);
|
|
3313
|
+
cacheKey = Buffer.from(slice).toString("base64");
|
|
3314
|
+
}
|
|
3315
|
+
const cached = this.metadataCache.get(cacheKey);
|
|
3316
|
+
if (cached) return cached;
|
|
3317
|
+
try {
|
|
3318
|
+
let result;
|
|
3319
|
+
if (typeof data === "string") {
|
|
3320
|
+
result = JSON.parse(data);
|
|
3321
|
+
} else {
|
|
3322
|
+
try {
|
|
3323
|
+
result = cbor.decode(data instanceof Uint8Array ? data : Buffer.from(data));
|
|
3324
|
+
} catch {
|
|
3325
|
+
result = JSON.parse(Buffer.from(data).toString("utf8"));
|
|
3326
|
+
}
|
|
3327
|
+
}
|
|
3328
|
+
this.metadataCache.set(cacheKey, result);
|
|
3329
|
+
return result;
|
|
3330
|
+
} catch {
|
|
3331
|
+
return void 0;
|
|
3332
|
+
}
|
|
3333
|
+
}
|
|
3334
|
+
// ===========================================================================
|
|
3335
|
+
// INITIALIZATION
|
|
3336
|
+
// ===========================================================================
|
|
3337
|
+
async initialize(dbPath, retryAfterCorruption = true) {
|
|
3338
|
+
const startTime = Date.now();
|
|
3339
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 initialize() START at ${dbPath}`);
|
|
3340
|
+
try {
|
|
3341
|
+
this.dbPath = dbPath;
|
|
3342
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 cleanupStaleLocks`);
|
|
3343
|
+
await this.cleanupStaleLocks(dbPath);
|
|
3344
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 cleanupStaleLocks (${Date.now() - startTime}ms)`);
|
|
3345
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 import @libsql/client`);
|
|
3346
|
+
const importStart = Date.now();
|
|
3347
|
+
const { createClient } = await import('@libsql/client');
|
|
3348
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 import @libsql/client (${Date.now() - importStart}ms)`);
|
|
3349
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 createClient`);
|
|
3350
|
+
const clientStart = Date.now();
|
|
3351
|
+
this.client = createClient({
|
|
3352
|
+
url: `file:${dbPath}`
|
|
3353
|
+
});
|
|
3354
|
+
await this.client.execute("SELECT 1");
|
|
3355
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 createClient + verify (${Date.now() - clientStart}ms)`);
|
|
3356
|
+
await this.client.execute("PRAGMA busy_timeout = 5000");
|
|
3357
|
+
await this.client.execute("PRAGMA cache_size = -8192");
|
|
3358
|
+
await this.client.execute("PRAGMA temp_store = MEMORY");
|
|
3359
|
+
await this.client.execute("PRAGMA mmap_size = 0");
|
|
3360
|
+
await this.client.execute("PRAGMA journal_mode = OFF");
|
|
3361
|
+
await this.client.execute("PRAGMA synchronous = OFF");
|
|
3362
|
+
this.quickIntegrityCheck().then(() => log.i("LIBSQLADAPT", "integrity_passed")).catch((err) => log.e("LIBSQLADAPT", "integrity_error", { err: err.message }));
|
|
3363
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 createTables`);
|
|
3364
|
+
const tablesStart = Date.now();
|
|
3365
|
+
await this.createTables();
|
|
3366
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 createTables (${Date.now() - tablesStart}ms)`);
|
|
3367
|
+
await this.migrateFileGen();
|
|
3368
|
+
this.entityOps.setTombstoneDelegates(
|
|
3369
|
+
(id, type) => this.addTombstone(id, type),
|
|
3370
|
+
(type) => this.getTombstonedIds(type)
|
|
3371
|
+
);
|
|
3372
|
+
this.relationshipOps.setTombstoneDelegates(
|
|
3373
|
+
(id, type) => this.addTombstone(id, type),
|
|
3374
|
+
(type) => this.getTombstonedIds(type)
|
|
3375
|
+
);
|
|
3376
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25B6 initProllyComponents`);
|
|
3377
|
+
const prollyStart = Date.now();
|
|
3378
|
+
await this.initializeProllyComponents();
|
|
3379
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 initProllyComponents (${Date.now() - prollyStart}ms)`);
|
|
3380
|
+
this.isInitialized = true;
|
|
3381
|
+
log.t("STORAGE", `[LibSQLGraphAdapter] \u25C0 initialize() END (${Date.now() - startTime}ms)`);
|
|
3382
|
+
log.i("LIBSQLADAPT", "init_complete", { path: dbPath });
|
|
3383
|
+
return true;
|
|
3384
|
+
} catch (error) {
|
|
3385
|
+
const errorMessage = error.message || String(error);
|
|
3386
|
+
const isCorrupted = errorMessage.includes("SQLITE_CORRUPT") || errorMessage.includes("database disk image is malformed") || errorMessage.includes("file is not a database") || errorMessage.includes("database or disk is full");
|
|
3387
|
+
if (isCorrupted && retryAfterCorruption) {
|
|
3388
|
+
log.e("LIBSQLADAPT", "corruption_detected", { err: errorMessage });
|
|
3389
|
+
log.i("LIBSQLADAPT", "recreating_db");
|
|
3390
|
+
if (this.client) {
|
|
3391
|
+
try {
|
|
3392
|
+
this.client.close();
|
|
3393
|
+
} catch {
|
|
3394
|
+
}
|
|
3395
|
+
this.client = null;
|
|
3396
|
+
}
|
|
3397
|
+
const deleted = await this.deleteCorruptDatabase(dbPath);
|
|
3398
|
+
if (deleted) {
|
|
3399
|
+
log.i("LIBSQLADAPT", "corrupt_db_deleted");
|
|
3400
|
+
return this.initialize(dbPath, false);
|
|
3401
|
+
} else {
|
|
3402
|
+
log.e("LIBSQLADAPT", "corrupt_db_delete_fail");
|
|
3403
|
+
return false;
|
|
3404
|
+
}
|
|
3405
|
+
}
|
|
3406
|
+
const isBusy = errorMessage.includes("SQLITE_BUSY") || errorMessage.includes("database is locked");
|
|
3407
|
+
if (isBusy && retryAfterCorruption) {
|
|
3408
|
+
log.w("LIBSQLADAPT", "busy_retry", { err: errorMessage });
|
|
3409
|
+
if (this.client) {
|
|
3410
|
+
try {
|
|
3411
|
+
this.client.close();
|
|
3412
|
+
} catch {
|
|
3413
|
+
}
|
|
3414
|
+
this.client = null;
|
|
3415
|
+
}
|
|
3416
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
3417
|
+
const delay = attempt * 2e3;
|
|
3418
|
+
log.i("LIBSQLADAPT", "busy_wait", { attempt, delay });
|
|
3419
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
3420
|
+
try {
|
|
3421
|
+
return await this.initialize(dbPath, false);
|
|
3422
|
+
} catch (retryErr) {
|
|
3423
|
+
const retryMsg = retryErr.message || "";
|
|
3424
|
+
if (!retryMsg.includes("SQLITE_BUSY") && !retryMsg.includes("database is locked")) {
|
|
3425
|
+
throw retryErr;
|
|
3426
|
+
}
|
|
3427
|
+
log.w("LIBSQLADAPT", "busy_retry_fail", { attempt, err: retryMsg });
|
|
3428
|
+
}
|
|
3429
|
+
}
|
|
3430
|
+
log.e("LIBSQLADAPT", "busy_exhausted", { retries: 3 });
|
|
3431
|
+
return false;
|
|
3432
|
+
}
|
|
3433
|
+
log.e("LIBSQLADAPT", "init_fail", { err: String(error) });
|
|
3434
|
+
return false;
|
|
3435
|
+
}
|
|
3436
|
+
}
|
|
3437
|
+
/**
|
|
3438
|
+
* Delete corrupt database and all auxiliary files.
|
|
3439
|
+
* Called automatically when SQLITE_CORRUPT is detected.
|
|
3440
|
+
*/
|
|
3441
|
+
async deleteCorruptDatabase(dbPath) {
|
|
3442
|
+
const { unlink, stat } = await import('fs/promises');
|
|
3443
|
+
const filesToDelete = [dbPath, `${dbPath}-journal`, `${dbPath}-wal`, `${dbPath}-shm`];
|
|
3444
|
+
let anyDeleted = false;
|
|
3445
|
+
for (const file of filesToDelete) {
|
|
3446
|
+
try {
|
|
3447
|
+
const fileStats = await stat(file);
|
|
3448
|
+
const sizeMB = (fileStats.size / 1024 / 1024).toFixed(1);
|
|
3449
|
+
await unlink(file);
|
|
3450
|
+
log.i("LIBSQLADAPT", "file_deleted", { file, sizeMB });
|
|
3451
|
+
anyDeleted = true;
|
|
3452
|
+
} catch (error) {
|
|
3453
|
+
const err = error;
|
|
3454
|
+
if (err.code !== "ENOENT") {
|
|
3455
|
+
log.w("LIBSQLADAPT", "file_delete_fail", { file, err: err.message });
|
|
3456
|
+
}
|
|
3457
|
+
}
|
|
3458
|
+
}
|
|
3459
|
+
return anyDeleted;
|
|
3460
|
+
}
|
|
3461
|
+
/**
|
|
3462
|
+
* Quick integrity check to detect corruption early.
|
|
3463
|
+
* Probes tables AND indexes to catch DiskANN corruption.
|
|
3464
|
+
* Much faster than full PRAGMA integrity_check.
|
|
3465
|
+
*/
|
|
3466
|
+
async quickIntegrityCheck() {
|
|
3467
|
+
if (!this.client) return;
|
|
3468
|
+
const tables = await this.client.execute(`
|
|
3469
|
+
SELECT name FROM sqlite_master WHERE type='table'
|
|
3470
|
+
AND name IN ('entities', 'relationships', 'embeddings', 'files')
|
|
3471
|
+
`);
|
|
3472
|
+
if (tables.rows.length === 0) {
|
|
3473
|
+
return;
|
|
3474
|
+
}
|
|
3475
|
+
const probes = [
|
|
3476
|
+
"SELECT id FROM entities LIMIT 1",
|
|
3477
|
+
"SELECT id FROM relationships LIMIT 1",
|
|
3478
|
+
"SELECT path FROM files LIMIT 1"
|
|
3479
|
+
];
|
|
3480
|
+
for (const probe of probes) {
|
|
3481
|
+
try {
|
|
3482
|
+
await this.client.execute(probe);
|
|
3483
|
+
} catch (error) {
|
|
3484
|
+
const msg = error.message || "";
|
|
3485
|
+
if (msg.includes("no such table")) continue;
|
|
3486
|
+
throw error;
|
|
3487
|
+
}
|
|
3488
|
+
}
|
|
3489
|
+
const integrityCheck = await this.client.execute("PRAGMA quick_check");
|
|
3490
|
+
const firstRow = integrityCheck.rows[0];
|
|
3491
|
+
const result = firstRow ? String(Object.values(firstRow)[0]) : "ok";
|
|
3492
|
+
if (result !== "ok") {
|
|
3493
|
+
throw new Error(`SQLITE_CORRUPT: quick_check failed: ${result}`);
|
|
3494
|
+
}
|
|
3495
|
+
try {
|
|
3496
|
+
const shadowTables = await this.client.execute(`
|
|
3497
|
+
SELECT name FROM sqlite_master
|
|
3498
|
+
WHERE type='table' AND name LIKE '%shadow%'
|
|
3499
|
+
`);
|
|
3500
|
+
for (const row of shadowTables.rows) {
|
|
3501
|
+
const tableName = row["name"];
|
|
3502
|
+
try {
|
|
3503
|
+
await this.client.execute(`SELECT COUNT(*) FROM "${tableName}"`);
|
|
3504
|
+
} catch (shadowError) {
|
|
3505
|
+
const smsg = shadowError.message || "";
|
|
3506
|
+
log.e("LIBSQLADAPT", "shadow_table_corrupt", { table: tableName, err: smsg });
|
|
3507
|
+
throw shadowError;
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
log.i("LIBSQLADAPT", "shadow_tables_ok");
|
|
3511
|
+
} catch (error) {
|
|
3512
|
+
const msg = error.message || "";
|
|
3513
|
+
if (!msg.includes("no such table") && !msg.includes("All shadow")) {
|
|
3514
|
+
throw error;
|
|
3515
|
+
}
|
|
3516
|
+
}
|
|
3517
|
+
log.i("LIBSQLADAPT", "integrity_passed");
|
|
3518
|
+
}
|
|
3519
|
+
/**
|
|
3520
|
+
* Remove stale SQLite lock files before opening database.
|
|
3521
|
+
* Safe in single-user mode where we're the only consumer.
|
|
3522
|
+
*/
|
|
3523
|
+
async cleanupStaleLocks(dbPath) {
|
|
3524
|
+
const { unlink } = await import('fs/promises');
|
|
3525
|
+
const lockFiles = [`${dbPath}-journal`, `${dbPath}-wal`, `${dbPath}-shm`];
|
|
3526
|
+
for (const lockFile of lockFiles) {
|
|
3527
|
+
try {
|
|
3528
|
+
await unlink(lockFile);
|
|
3529
|
+
log.i("LIBSQLADAPT", "stale_lock_removed", { file: lockFile });
|
|
3530
|
+
} catch {
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
async createTables() {
|
|
3535
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3536
|
+
const startTime = Date.now();
|
|
3537
|
+
await this.client.batch(
|
|
3538
|
+
[
|
|
3539
|
+
// Entities table - composite PK ensures isolation between projects/branches
|
|
3540
|
+
`CREATE TABLE IF NOT EXISTS entities (
|
|
3541
|
+
id TEXT NOT NULL,
|
|
3542
|
+
project_hash TEXT NOT NULL DEFAULT 'legacy',
|
|
3543
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3544
|
+
name TEXT NOT NULL,
|
|
3545
|
+
type TEXT NOT NULL,
|
|
3546
|
+
file_path TEXT NOT NULL,
|
|
3547
|
+
location TEXT NOT NULL,
|
|
3548
|
+
metadata TEXT,
|
|
3549
|
+
hash TEXT,
|
|
3550
|
+
created_at INTEGER NOT NULL,
|
|
3551
|
+
updated_at INTEGER NOT NULL,
|
|
3552
|
+
complexity_score INTEGER DEFAULT 1,
|
|
3553
|
+
language TEXT,
|
|
3554
|
+
size_bytes INTEGER DEFAULT 0,
|
|
3555
|
+
embedding_base64 TEXT,
|
|
3556
|
+
embedding_text TEXT,
|
|
3557
|
+
file_gen INTEGER NOT NULL DEFAULT 1,
|
|
3558
|
+
PRIMARY KEY (id, project_hash, branch_name)
|
|
3559
|
+
)`,
|
|
3560
|
+
// Relationships table - composite PK ensures isolation between projects/branches
|
|
3561
|
+
`CREATE TABLE IF NOT EXISTS relationships (
|
|
3562
|
+
id TEXT NOT NULL,
|
|
3563
|
+
project_hash TEXT NOT NULL DEFAULT 'legacy',
|
|
3564
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3565
|
+
from_id TEXT NOT NULL,
|
|
3566
|
+
to_id TEXT NOT NULL,
|
|
3567
|
+
type TEXT NOT NULL,
|
|
3568
|
+
metadata TEXT,
|
|
3569
|
+
weight REAL DEFAULT 1.0,
|
|
3570
|
+
created_at INTEGER NOT NULL,
|
|
3571
|
+
PRIMARY KEY (id, project_hash, branch_name)
|
|
3572
|
+
)`,
|
|
3573
|
+
// Files table
|
|
3574
|
+
`CREATE TABLE IF NOT EXISTS files (
|
|
3575
|
+
path TEXT NOT NULL,
|
|
3576
|
+
project_hash TEXT NOT NULL DEFAULT 'legacy',
|
|
3577
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3578
|
+
hash TEXT,
|
|
3579
|
+
last_indexed INTEGER NOT NULL,
|
|
3580
|
+
entity_count INTEGER DEFAULT 0,
|
|
3581
|
+
PRIMARY KEY (path, project_hash, branch_name)
|
|
3582
|
+
)`,
|
|
3583
|
+
// Project metadata table
|
|
3584
|
+
`CREATE TABLE IF NOT EXISTS project_metadata (
|
|
3585
|
+
project_hash TEXT NOT NULL,
|
|
3586
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3587
|
+
project_path TEXT NOT NULL,
|
|
3588
|
+
last_indexed_at INTEGER NOT NULL,
|
|
3589
|
+
entity_count INTEGER DEFAULT 0,
|
|
3590
|
+
file_count INTEGER DEFAULT 0,
|
|
3591
|
+
created_at INTEGER NOT NULL,
|
|
3592
|
+
updated_at INTEGER NOT NULL,
|
|
3593
|
+
last_full_index_at INTEGER DEFAULT 0,
|
|
3594
|
+
incremental_changes_count INTEGER DEFAULT 0,
|
|
3595
|
+
PRIMARY KEY (project_hash, branch_name)
|
|
3596
|
+
)`,
|
|
3597
|
+
// Query cache table - composite PK for project isolation
|
|
3598
|
+
`CREATE TABLE IF NOT EXISTS query_cache (
|
|
3599
|
+
id TEXT NOT NULL,
|
|
3600
|
+
project_hash TEXT NOT NULL DEFAULT 'legacy',
|
|
3601
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3602
|
+
query_hash TEXT NOT NULL,
|
|
3603
|
+
result TEXT NOT NULL,
|
|
3604
|
+
hit_count INTEGER DEFAULT 0,
|
|
3605
|
+
miss_count INTEGER DEFAULT 0,
|
|
3606
|
+
created_at INTEGER NOT NULL,
|
|
3607
|
+
expires_at INTEGER NOT NULL,
|
|
3608
|
+
PRIMARY KEY (id, project_hash, branch_name)
|
|
3609
|
+
)`,
|
|
3610
|
+
// Performance metrics table
|
|
3611
|
+
`CREATE TABLE IF NOT EXISTS performance_metrics (
|
|
3612
|
+
id TEXT PRIMARY KEY,
|
|
3613
|
+
operation TEXT NOT NULL,
|
|
3614
|
+
duration_ms INTEGER NOT NULL,
|
|
3615
|
+
entity_count INTEGER DEFAULT 0,
|
|
3616
|
+
memory_usage INTEGER DEFAULT 0,
|
|
3617
|
+
created_at INTEGER NOT NULL
|
|
3618
|
+
)`,
|
|
3619
|
+
// Tombstones table - tracks deleted entities/relationships on feature branches
|
|
3620
|
+
// When on feature branch, DELETE adds tombstone instead of removing from base
|
|
3621
|
+
// Layered reads exclude tombstoned IDs from base branch results
|
|
3622
|
+
`CREATE TABLE IF NOT EXISTS tombstones (
|
|
3623
|
+
entity_id TEXT NOT NULL,
|
|
3624
|
+
project_hash TEXT NOT NULL,
|
|
3625
|
+
branch_name TEXT NOT NULL,
|
|
3626
|
+
entity_type TEXT NOT NULL DEFAULT 'entity',
|
|
3627
|
+
deleted_at INTEGER NOT NULL,
|
|
3628
|
+
PRIMARY KEY (entity_id, project_hash, branch_name, entity_type)
|
|
3629
|
+
)`,
|
|
3630
|
+
// File generations table — tracks active generation per file for copy-on-write reindex
|
|
3631
|
+
`CREATE TABLE IF NOT EXISTS file_generations (
|
|
3632
|
+
file_path TEXT NOT NULL,
|
|
3633
|
+
project_hash TEXT NOT NULL,
|
|
3634
|
+
branch_name TEXT NOT NULL,
|
|
3635
|
+
active_gen INTEGER NOT NULL DEFAULT 1,
|
|
3636
|
+
updated_at INTEGER NOT NULL,
|
|
3637
|
+
PRIMARY KEY (file_path, project_hash, branch_name)
|
|
3638
|
+
)`,
|
|
3639
|
+
// Name tokens table — enables fast B-tree token lookup instead of LIKE '%pattern%'
|
|
3640
|
+
// splitToTokens("getAuthToken") → ["get", "auth", "token"]
|
|
3641
|
+
`CREATE TABLE IF NOT EXISTS name_tokens (
|
|
3642
|
+
token TEXT NOT NULL,
|
|
3643
|
+
entity_id TEXT NOT NULL,
|
|
3644
|
+
project_hash TEXT NOT NULL,
|
|
3645
|
+
branch_name TEXT NOT NULL,
|
|
3646
|
+
PRIMARY KEY (token, entity_id, project_hash, branch_name)
|
|
3647
|
+
)`,
|
|
3648
|
+
// NOTE: embeddings table REMOVED in v5 - FAISS is used for all vector operations
|
|
3649
|
+
// See: src/semantic/vector-store.ts (v5: Faiss-only backend)
|
|
3650
|
+
// === INDEXES (batched for speed) ===
|
|
3651
|
+
// Entity indexes
|
|
3652
|
+
`CREATE INDEX IF NOT EXISTS idx_entities_project_branch ON entities(project_hash, branch_name)`,
|
|
3653
|
+
`CREATE INDEX IF NOT EXISTS idx_entities_file_path ON entities(file_path, project_hash, branch_name)`,
|
|
3654
|
+
`CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type, project_hash, branch_name)`,
|
|
3655
|
+
`CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name, project_hash, branch_name)`,
|
|
3656
|
+
// Relationship indexes
|
|
3657
|
+
`CREATE INDEX IF NOT EXISTS idx_relationships_project_branch ON relationships(project_hash, branch_name)`,
|
|
3658
|
+
`CREATE INDEX IF NOT EXISTS idx_relationships_from ON relationships(from_id, project_hash, branch_name)`,
|
|
3659
|
+
`CREATE INDEX IF NOT EXISTS idx_relationships_to ON relationships(to_id, project_hash, branch_name)`,
|
|
3660
|
+
// Files index
|
|
3661
|
+
`CREATE INDEX IF NOT EXISTS idx_files_project_branch ON files(project_hash, branch_name)`,
|
|
3662
|
+
// Tombstones index
|
|
3663
|
+
`CREATE INDEX IF NOT EXISTS idx_tombstones_lookup ON tombstones(project_hash, branch_name, entity_type)`,
|
|
3664
|
+
`CREATE INDEX IF NOT EXISTS idx_name_tokens_lookup ON name_tokens(token, project_hash, branch_name)`,
|
|
3665
|
+
// Generation index for efficient filtering by active generation
|
|
3666
|
+
`CREATE INDEX IF NOT EXISTS idx_entities_file_gen ON entities(file_path, project_hash, branch_name, file_gen)`,
|
|
3667
|
+
// Co-occurrence table for query expansion
|
|
3668
|
+
// Stores term pairs that frequently appear together in comments/docs
|
|
3669
|
+
`CREATE TABLE IF NOT EXISTS cooccurrence (
|
|
3670
|
+
term1 TEXT NOT NULL,
|
|
3671
|
+
term2 TEXT NOT NULL,
|
|
3672
|
+
count INTEGER NOT NULL DEFAULT 1,
|
|
3673
|
+
pmi REAL,
|
|
3674
|
+
project_hash TEXT NOT NULL,
|
|
3675
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3676
|
+
updated_at INTEGER NOT NULL,
|
|
3677
|
+
PRIMARY KEY (term1, term2, project_hash, branch_name)
|
|
3678
|
+
)`,
|
|
3679
|
+
// Term frequency table for PMI calculation
|
|
3680
|
+
`CREATE TABLE IF NOT EXISTS term_frequency (
|
|
3681
|
+
term TEXT NOT NULL,
|
|
3682
|
+
doc_count INTEGER NOT NULL DEFAULT 1,
|
|
3683
|
+
total_count INTEGER NOT NULL DEFAULT 1,
|
|
3684
|
+
project_hash TEXT NOT NULL,
|
|
3685
|
+
branch_name TEXT NOT NULL DEFAULT 'main',
|
|
3686
|
+
PRIMARY KEY (term, project_hash, branch_name)
|
|
3687
|
+
)`,
|
|
3688
|
+
// Co-occurrence indexes
|
|
3689
|
+
`CREATE INDEX IF NOT EXISTS idx_cooc_term1 ON cooccurrence(term1, project_hash, branch_name)`,
|
|
3690
|
+
`CREATE INDEX IF NOT EXISTS idx_cooc_pmi ON cooccurrence(pmi DESC, project_hash, branch_name)`,
|
|
3691
|
+
`CREATE INDEX IF NOT EXISTS idx_term_freq_project ON term_frequency(project_hash, branch_name)`
|
|
3692
|
+
],
|
|
3693
|
+
"write"
|
|
3694
|
+
);
|
|
3695
|
+
const batchElapsed = Date.now() - startTime;
|
|
3696
|
+
log.i("STORAGE", `Tables and basic indexes created`, { ms: batchElapsed });
|
|
3697
|
+
log.i("STORAGE", `Skipping global DiskANN index (using partial indexes per project)`);
|
|
3698
|
+
const totalElapsed = Date.now() - startTime;
|
|
3699
|
+
log.i("STORAGE", `Total initialization complete`, { ms: totalElapsed });
|
|
3700
|
+
await this.logDatabaseStats("after_init");
|
|
3701
|
+
}
|
|
3702
|
+
/**
|
|
3703
|
+
* Log database statistics and memory usage for diagnostics.
|
|
3704
|
+
*/
|
|
3705
|
+
async logDatabaseStats(label) {
|
|
3706
|
+
if (!this.client) return;
|
|
3707
|
+
try {
|
|
3708
|
+
const pageCount = await this.client.execute("PRAGMA page_count");
|
|
3709
|
+
const pageSize = await this.client.execute("PRAGMA page_size");
|
|
3710
|
+
const cacheSize = await this.client.execute("PRAGMA cache_size");
|
|
3711
|
+
const freelistCount = await this.client.execute("PRAGMA freelist_count");
|
|
3712
|
+
const pages = Number(pageCount.rows[0]?.["page_count"] ?? 0);
|
|
3713
|
+
const size = Number(pageSize.rows[0]?.["page_size"] ?? 4096);
|
|
3714
|
+
const cache = Number(cacheSize.rows[0]?.["cache_size"] ?? 0);
|
|
3715
|
+
const freelist = Number(freelistCount.rows[0]?.["freelist_count"] ?? 0);
|
|
3716
|
+
const dbSizeMB = pages * size / 1024 / 1024;
|
|
3717
|
+
const cacheMB = cache < 0 ? -cache / 1024 : cache * size / 1024 / 1024;
|
|
3718
|
+
log.i("STORAGE", label, {
|
|
3719
|
+
dbSizeMB: dbSizeMB.toFixed(1),
|
|
3720
|
+
pages,
|
|
3721
|
+
pageSize: size,
|
|
3722
|
+
cacheSizeMB: cacheMB.toFixed(1),
|
|
3723
|
+
freelistPages: freelist
|
|
3724
|
+
});
|
|
3725
|
+
const mem = process.memoryUsage();
|
|
3726
|
+
log.i("STORAGE", label, {
|
|
3727
|
+
rssMB: Math.round(mem.rss / 1024 / 1024),
|
|
3728
|
+
heapUsedMB: Math.round(mem.heapUsed / 1024 / 1024),
|
|
3729
|
+
externalMB: Math.round(mem.external / 1024 / 1024),
|
|
3730
|
+
arrayBuffersMB: Math.round(mem.arrayBuffers / 1024 / 1024)
|
|
3731
|
+
});
|
|
3732
|
+
} catch (error) {
|
|
3733
|
+
log.d("STORAGE", "Failed to get stats", { error: error.message });
|
|
3734
|
+
}
|
|
3735
|
+
}
|
|
3736
|
+
/**
|
|
3737
|
+
* Migration: Add file_gen column to entities table if missing.
|
|
3738
|
+
* Also backfills file_generations for existing data.
|
|
3739
|
+
*/
|
|
3740
|
+
async migrateFileGen() {
|
|
3741
|
+
if (!this.client) return;
|
|
3742
|
+
try {
|
|
3743
|
+
await this.client.execute("SELECT file_gen FROM entities LIMIT 0");
|
|
3744
|
+
return;
|
|
3745
|
+
} catch {
|
|
3746
|
+
}
|
|
3747
|
+
log.i("LIBSQLADAPT", "migrate_file_gen_start");
|
|
3748
|
+
const start = Date.now();
|
|
3749
|
+
await this.client.execute("ALTER TABLE entities ADD COLUMN file_gen INTEGER NOT NULL DEFAULT 1");
|
|
3750
|
+
await this.client.execute(`
|
|
3751
|
+
INSERT OR IGNORE INTO file_generations (file_path, project_hash, branch_name, active_gen, updated_at)
|
|
3752
|
+
SELECT DISTINCT file_path, project_hash, branch_name, 1, ${Date.now()}
|
|
3753
|
+
FROM entities
|
|
3754
|
+
`);
|
|
3755
|
+
log.i("LIBSQLADAPT", "migrate_file_gen_done", { ms: Date.now() - start });
|
|
3756
|
+
}
|
|
3757
|
+
isReady() {
|
|
3758
|
+
return this.isInitialized && this.client !== null;
|
|
3759
|
+
}
|
|
3760
|
+
getDbPath() {
|
|
3761
|
+
return this.dbPath;
|
|
3762
|
+
}
|
|
3763
|
+
// ===========================================================================
|
|
3764
|
+
// PROJECT CONTEXT
|
|
3765
|
+
// ===========================================================================
|
|
3766
|
+
setProjectContext(context) {
|
|
3767
|
+
this.currentContext = {
|
|
3768
|
+
projectHash: context.projectHash,
|
|
3769
|
+
branchName: normalizeBranchName(context.branchName),
|
|
3770
|
+
baseBranch: context.baseBranch,
|
|
3771
|
+
// For layered reads on feature branches
|
|
3772
|
+
dimensions: context.dimensions
|
|
3773
|
+
};
|
|
3774
|
+
this.generationManager.clearCache();
|
|
3775
|
+
log.d("LIBSQLADAPT", "setProjectContext", {
|
|
3776
|
+
branch: this.currentContext.branchName,
|
|
3777
|
+
base: context.baseBranch || "none"
|
|
3778
|
+
});
|
|
3779
|
+
}
|
|
3780
|
+
getProjectContext() {
|
|
3781
|
+
return { ...this.currentContext };
|
|
3782
|
+
}
|
|
3783
|
+
/**
|
|
3784
|
+
* Get effective dimensions for current project.
|
|
3785
|
+
* Uses project-specific dimensions if set, otherwise global config.
|
|
3786
|
+
*/
|
|
3787
|
+
getEffectiveDimensions() {
|
|
3788
|
+
return this.currentContext.dimensions ?? normalizeToSupportedDimension(this.config.dimensions);
|
|
3789
|
+
}
|
|
3790
|
+
/**
|
|
3791
|
+
* Get embedding column name for current project's dimensions.
|
|
3792
|
+
*/
|
|
3793
|
+
getEmbeddingColumnName() {
|
|
3794
|
+
return getEmbeddingColumn(this.getEffectiveDimensions());
|
|
3795
|
+
}
|
|
3796
|
+
async ensureProjectVectorIndex() {
|
|
3797
|
+
if (!this.client) return;
|
|
3798
|
+
const { projectHash } = this.currentContext;
|
|
3799
|
+
const dims = this.getEffectiveDimensions();
|
|
3800
|
+
const colName = getEmbeddingColumn(dims);
|
|
3801
|
+
const indexName = `idx_emb_${dims}_${projectHash.substring(0, 8)}`;
|
|
3802
|
+
try {
|
|
3803
|
+
const check = await this.client.execute({
|
|
3804
|
+
sql: `SELECT name FROM sqlite_master WHERE type='index' AND name=?`,
|
|
3805
|
+
args: [indexName]
|
|
3806
|
+
});
|
|
3807
|
+
if (check.rows.length > 0) return;
|
|
3808
|
+
const params = [
|
|
3809
|
+
`'metric=${this.config.metric}'`,
|
|
3810
|
+
`'compress_neighbors=${this.config.compression}'`,
|
|
3811
|
+
`'max_neighbors=${this.config.maxNeighbors}'`,
|
|
3812
|
+
`'search_l=${this.config.searchL}'`,
|
|
3813
|
+
`'insert_l=${this.config.insertL}'`
|
|
3814
|
+
].join(", ");
|
|
3815
|
+
const t = Date.now();
|
|
3816
|
+
await this.client.execute(
|
|
3817
|
+
`CREATE INDEX IF NOT EXISTS ${indexName} ON embeddings(libsql_vector_idx(${colName}, ${params})) WHERE project_hash = '${projectHash}' AND dim_size = ${dims}`
|
|
3818
|
+
);
|
|
3819
|
+
log.i("STORAGE", `Created partial index`, { indexName, dims, ms: Date.now() - t });
|
|
3820
|
+
} catch (e) {
|
|
3821
|
+
log.w("STORAGE", `Partial index failed`, { error: e.message });
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
// ===========================================================================
|
|
3825
|
+
// ENTITY OPERATIONS (delegated to EntityOperations)
|
|
3826
|
+
// ===========================================================================
|
|
3827
|
+
insertEntity = (entity) => this.entityOps.insertEntity(entity);
|
|
3828
|
+
insertEntities = (entities) => this.entityOps.insertEntities(entities);
|
|
3829
|
+
getEntity = (id) => this.entityOps.getEntity(id);
|
|
3830
|
+
getEntitiesBatch = (ids) => this.entityOps.getEntitiesBatch(ids);
|
|
3831
|
+
findEntities(query) {
|
|
3832
|
+
return this.entityOps.findEntities(query);
|
|
3833
|
+
}
|
|
3834
|
+
searchEntities = (options) => this.entityOps.searchEntities(options);
|
|
3835
|
+
searchEntitiesInDirectory = (directoryPath) => this.entityOps.searchEntitiesInDirectory(directoryPath);
|
|
3836
|
+
deleteEntity = (id) => this.entityOps.deleteEntity(id);
|
|
3837
|
+
getEntityIdsByFilePath = (filePath) => this.entityOps.getEntityIdsByFilePath(filePath);
|
|
3838
|
+
deleteEntitiesByFilePath = (filePath) => this.entityOps.deleteEntitiesByFilePath(filePath);
|
|
3839
|
+
getAllEntities = () => this.entityOps.getAllEntities();
|
|
3840
|
+
countByLanguage = () => this.entityOps.countByLanguage();
|
|
3841
|
+
/** Get GenerationManager for GC scheduling */
|
|
3842
|
+
getGenerationManager() {
|
|
3843
|
+
return this.generationManager;
|
|
3844
|
+
}
|
|
3845
|
+
/** Load generation cache (call after setProjectContext) */
|
|
3846
|
+
async loadGenerationCache() {
|
|
3847
|
+
await this.generationManager.loadCache();
|
|
3848
|
+
}
|
|
3849
|
+
// ===========================================================================
|
|
3850
|
+
// RELATIONSHIP OPERATIONS (delegated to RelationshipOperations)
|
|
3851
|
+
// ===========================================================================
|
|
3852
|
+
insertRelationship = (relationship) => this.relationshipOps.insertRelationship(relationship);
|
|
3853
|
+
insertRelationships = (relationships) => this.relationshipOps.insertRelationships(relationships);
|
|
3854
|
+
getRelationshipsForEntity = (entityId, type) => this.relationshipOps.getRelationshipsForEntity(entityId, type);
|
|
3855
|
+
findRelationships = (query) => this.relationshipOps.findRelationships(query);
|
|
3856
|
+
deleteRelationship = (id) => this.relationshipOps.deleteRelationship(id);
|
|
3857
|
+
getAllRelationships = () => this.relationshipOps.getAllRelationships();
|
|
3858
|
+
// ===========================================================================
|
|
3859
|
+
// FILE/METADATA OPERATIONS (delegated to MetadataOperations)
|
|
3860
|
+
// ===========================================================================
|
|
3861
|
+
updateFileInfo = (info) => this.metadataOps.updateFileInfo(info);
|
|
3862
|
+
batchUpdateFileInfo = (infos) => this.metadataOps.batchUpdateFileInfo(infos);
|
|
3863
|
+
getFileInfo = (path) => this.metadataOps.getFileInfo(path);
|
|
3864
|
+
getOutdatedFiles = (since) => this.metadataOps.getOutdatedFiles(since);
|
|
3865
|
+
getAllIndexedFiles = () => this.metadataOps.getAllIndexedFiles();
|
|
3866
|
+
deleteFileInfo = (path) => this.metadataOps.deleteFileInfo(path);
|
|
3867
|
+
// ===========================================================================
|
|
3868
|
+
// VECTOR OPERATIONS (delegated to VectorOperations)
|
|
3869
|
+
// ===========================================================================
|
|
3870
|
+
/** @deprecated Use FaissProvider.add() instead */
|
|
3871
|
+
insertEmbedding = (embedding) => this.vectorOps.insertEmbedding(embedding);
|
|
3872
|
+
/** @deprecated Use FaissProvider.addBatch() instead */
|
|
3873
|
+
insertEmbeddingBatch = (embeddings) => this.vectorOps.insertEmbeddingBatch(embeddings);
|
|
3874
|
+
/** @deprecated Faiss HNSW handles live updates, no need to drop/rebuild */
|
|
3875
|
+
dropVectorIndex = () => this.vectorOps.dropVectorIndex();
|
|
3876
|
+
/** @deprecated Faiss HNSW maintains index automatically, no rebuild needed */
|
|
3877
|
+
rebuildVectorIndex = () => this.vectorOps.rebuildVectorIndex();
|
|
3878
|
+
/** @deprecated Use FaissProvider.addBatch() instead */
|
|
3879
|
+
bulkInsertEmbeddings = (embeddings) => this.vectorOps.bulkInsertEmbeddings(embeddings);
|
|
3880
|
+
/** @deprecated Use FaissProvider.search() instead */
|
|
3881
|
+
searchVectors = (queryVector, limit) => this.vectorOps.searchVectors(queryVector, limit);
|
|
3882
|
+
/** @deprecated Use FaissProvider.getContent() instead */
|
|
3883
|
+
getEmbedding = (id) => this.vectorOps.getEmbedding(id);
|
|
3884
|
+
/** @deprecated Use FaissProvider.remove() instead */
|
|
3885
|
+
deleteEmbedding = (id) => this.vectorOps.deleteEmbedding(id);
|
|
3886
|
+
/** @deprecated Use FaissProvider.getVectorCount() instead */
|
|
3887
|
+
getEmbeddingCount = () => this.vectorOps.getEmbeddingCount();
|
|
3888
|
+
/** @deprecated Use FaissProvider.getExistingIds() instead */
|
|
3889
|
+
getExistingEmbeddingIds = (ids) => this.vectorOps.getExistingEmbeddingIds(ids);
|
|
3890
|
+
// ===========================================================================
|
|
3891
|
+
// EMBEDDING CACHE OPERATIONS (delegated to CacheOperations)
|
|
3892
|
+
// ===========================================================================
|
|
3893
|
+
getEmbeddingFromCache = (contentHash) => this.cacheOps.getEmbeddingFromCache(contentHash);
|
|
3894
|
+
getEmbeddingsFromCache = (contentHashes) => this.cacheOps.getEmbeddingsFromCache(contentHashes);
|
|
3895
|
+
setEmbeddingInCache = (contentHash, model, embedding, textPreview) => this.cacheOps.setEmbeddingInCache(contentHash, model, embedding, textPreview);
|
|
3896
|
+
setEmbeddingsInCache = (entries) => this.cacheOps.setEmbeddingsInCache(entries);
|
|
3897
|
+
// ===========================================================================
|
|
3898
|
+
// METADATA OPERATIONS (delegated to MetadataOperations)
|
|
3899
|
+
// ===========================================================================
|
|
3900
|
+
updateProjectMetadata = (projectPath, isFullIndex) => this.metadataOps.updateProjectMetadata(projectPath, isFullIndex);
|
|
3901
|
+
getIncrementalTrackingInfo = () => this.metadataOps.getIncrementalTrackingInfo();
|
|
3902
|
+
recordIncrementalChanges = (changedFileCount) => this.metadataOps.recordIncrementalChanges(changedFileCount);
|
|
3903
|
+
resetIncrementalTracking = () => this.metadataOps.resetIncrementalTracking();
|
|
3904
|
+
listProjects = () => this.metadataOps.listProjects();
|
|
3905
|
+
listBranches = () => this.metadataOps.listBranches();
|
|
3906
|
+
// ===========================================================================
|
|
3907
|
+
// METRICS & STATS (delegated to MetadataOperations)
|
|
3908
|
+
// ===========================================================================
|
|
3909
|
+
getStats = () => this.metadataOps.getStats();
|
|
3910
|
+
getTotalStats = () => this.metadataOps.getTotalStats();
|
|
3911
|
+
// ===========================================================================
|
|
3912
|
+
// CLEAR OPERATIONS (delegated to MetadataOperations)
|
|
3913
|
+
// ===========================================================================
|
|
3914
|
+
clear = () => this.metadataOps.clear();
|
|
3915
|
+
clearAll = () => this.metadataOps.clearAll();
|
|
3916
|
+
// ===========================================================================
|
|
3917
|
+
// COOCCURRENCE OPERATIONS (for query expansion)
|
|
3918
|
+
// ===========================================================================
|
|
3919
|
+
/**
|
|
3920
|
+
* Get the CooccurrenceOperations instance for query expansion.
|
|
3921
|
+
* Used by CooccurrenceIndex to update/query term pairs.
|
|
3922
|
+
*/
|
|
3923
|
+
getCooccurrenceOps() {
|
|
3924
|
+
return this.cooccurrenceOps;
|
|
3925
|
+
}
|
|
3926
|
+
// ===========================================================================
|
|
3927
|
+
// TOMBSTONE OPERATIONS (for layered branch support)
|
|
3928
|
+
// ===========================================================================
|
|
3929
|
+
/**
|
|
3930
|
+
* Add a tombstone for an entity/relationship deleted on feature branch.
|
|
3931
|
+
* This prevents the deleted item from appearing in layered reads from base.
|
|
3932
|
+
*/
|
|
3933
|
+
async addTombstone(entityId, entityType = "entity") {
|
|
3934
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3935
|
+
const { projectHash, branchName } = this.currentContext;
|
|
3936
|
+
await this.client.execute({
|
|
3937
|
+
sql: `INSERT OR REPLACE INTO tombstones (entity_id, project_hash, branch_name, entity_type, deleted_at)
|
|
3938
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
3939
|
+
args: [entityId, projectHash, branchName, entityType, Date.now()]
|
|
3940
|
+
});
|
|
3941
|
+
}
|
|
3942
|
+
/**
|
|
3943
|
+
* Remove a tombstone (when entity is re-added on feature branch).
|
|
3944
|
+
*/
|
|
3945
|
+
async removeTombstone(entityId, entityType = "entity") {
|
|
3946
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3947
|
+
const { projectHash, branchName } = this.currentContext;
|
|
3948
|
+
await this.client.execute({
|
|
3949
|
+
sql: `DELETE FROM tombstones WHERE entity_id = ? AND project_hash = ? AND branch_name = ? AND entity_type = ?`,
|
|
3950
|
+
args: [entityId, projectHash, branchName, entityType]
|
|
3951
|
+
});
|
|
3952
|
+
}
|
|
3953
|
+
/**
|
|
3954
|
+
* Check if an entity is tombstoned on current feature branch.
|
|
3955
|
+
*/
|
|
3956
|
+
async isTombstoned(entityId, entityType = "entity") {
|
|
3957
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3958
|
+
const { projectHash, branchName } = this.currentContext;
|
|
3959
|
+
const result = await this.client.execute({
|
|
3960
|
+
sql: `SELECT 1 FROM tombstones WHERE entity_id = ? AND project_hash = ? AND branch_name = ? AND entity_type = ? LIMIT 1`,
|
|
3961
|
+
args: [entityId, projectHash, branchName, entityType]
|
|
3962
|
+
});
|
|
3963
|
+
return result.rows.length > 0;
|
|
3964
|
+
}
|
|
3965
|
+
/**
|
|
3966
|
+
* Get all tombstoned entity IDs for current branch (for batch operations).
|
|
3967
|
+
*/
|
|
3968
|
+
async getTombstonedIds(entityType = "entity") {
|
|
3969
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3970
|
+
const { projectHash, branchName } = this.currentContext;
|
|
3971
|
+
const result = await this.client.execute({
|
|
3972
|
+
sql: `SELECT entity_id FROM tombstones WHERE project_hash = ? AND branch_name = ? AND entity_type = ?`,
|
|
3973
|
+
args: [projectHash, branchName, entityType]
|
|
3974
|
+
});
|
|
3975
|
+
return new Set(result.rows.map((row) => row["entity_id"]));
|
|
3976
|
+
}
|
|
3977
|
+
/**
|
|
3978
|
+
* Clear all tombstones for current branch (used when merging to base).
|
|
3979
|
+
*/
|
|
3980
|
+
async clearTombstones() {
|
|
3981
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
3982
|
+
const { projectHash, branchName } = this.currentContext;
|
|
3983
|
+
await this.client.execute({
|
|
3984
|
+
sql: `DELETE FROM tombstones WHERE project_hash = ? AND branch_name = ?`,
|
|
3985
|
+
args: [projectHash, branchName]
|
|
3986
|
+
});
|
|
3987
|
+
}
|
|
3988
|
+
/**
|
|
3989
|
+
* Force flush all pending writes to disk.
|
|
3990
|
+
* With journal_mode=OFF and synchronous=OFF, we need to close/reopen
|
|
3991
|
+
* to ensure OS buffers are flushed.
|
|
3992
|
+
*/
|
|
3993
|
+
async flush() {
|
|
3994
|
+
if (!this.client || !this.dbPath) {
|
|
3995
|
+
log.w("LIBSQLADAPT", "flush_skipped", { hasClient: !!this.client, hasDbPath: !!this.dbPath });
|
|
3996
|
+
return;
|
|
3997
|
+
}
|
|
3998
|
+
const startTime = Date.now();
|
|
3999
|
+
log.d("LIBSQLADAPT", "flush_start");
|
|
4000
|
+
this.client.close();
|
|
4001
|
+
this.client = null;
|
|
4002
|
+
const { createClient } = await import('@libsql/client');
|
|
4003
|
+
this.client = createClient({ url: `file:${this.dbPath}` });
|
|
4004
|
+
await this.client.execute("PRAGMA cache_size = -8192");
|
|
4005
|
+
await this.client.execute("PRAGMA temp_store = MEMORY");
|
|
4006
|
+
await this.client.execute("PRAGMA mmap_size = 0");
|
|
4007
|
+
await this.client.execute("PRAGMA journal_mode = OFF");
|
|
4008
|
+
await this.client.execute("PRAGMA synchronous = OFF");
|
|
4009
|
+
if (this.prollyNodeStore) {
|
|
4010
|
+
this.prollyNodeStore.updateClient(this.client);
|
|
4011
|
+
}
|
|
4012
|
+
if (this.commitManager) {
|
|
4013
|
+
this.commitManager.updateClient(this.client);
|
|
4014
|
+
}
|
|
4015
|
+
try {
|
|
4016
|
+
const { statSync: statSync2 } = await import('fs');
|
|
4017
|
+
const stats = statSync2(this.dbPath);
|
|
4018
|
+
log.i("LIBSQLADAPT", "flush_complete", { ms: Date.now() - startTime, sizeBytes: stats.size });
|
|
4019
|
+
} catch {
|
|
4020
|
+
log.i("LIBSQLADAPT", "flush_complete", { ms: Date.now() - startTime, sizeBytes: "unknown" });
|
|
4021
|
+
}
|
|
4022
|
+
}
|
|
4023
|
+
async close() {
|
|
4024
|
+
if (this.client) {
|
|
4025
|
+
this.client.close();
|
|
4026
|
+
this.client = null;
|
|
4027
|
+
this.isInitialized = false;
|
|
4028
|
+
log.i("LIBSQLADAPT", "connection_closed");
|
|
4029
|
+
}
|
|
4030
|
+
}
|
|
4031
|
+
// ===========================================================================
|
|
4032
|
+
// HELPER METHODS
|
|
4033
|
+
// ===========================================================================
|
|
4034
|
+
_langDebugDone = false;
|
|
4035
|
+
rowToEntity(row) {
|
|
4036
|
+
const rawRow = row;
|
|
4037
|
+
if (!this._langDebugDone && rawRow["language"]) {
|
|
4038
|
+
log.w("ADAPTER", "rowToEntity_debug", {
|
|
4039
|
+
hasLang: "language" in row,
|
|
4040
|
+
langVal: row.language,
|
|
4041
|
+
rawLang: String(rawRow["language"])
|
|
4042
|
+
});
|
|
4043
|
+
this._langDebugDone = true;
|
|
4044
|
+
}
|
|
4045
|
+
return {
|
|
4046
|
+
id: row.id,
|
|
4047
|
+
name: row.name,
|
|
4048
|
+
type: row.type,
|
|
4049
|
+
filePath: row.file_path,
|
|
4050
|
+
location: JSON.parse(row.location),
|
|
4051
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : {},
|
|
4052
|
+
hash: row.hash || "",
|
|
4053
|
+
createdAt: row.created_at,
|
|
4054
|
+
updatedAt: row.updated_at,
|
|
4055
|
+
complexityScore: row.complexity_score ?? void 0,
|
|
4056
|
+
language: row.language ?? void 0,
|
|
4057
|
+
sizeBytes: row.size_bytes ?? void 0,
|
|
4058
|
+
embeddingBase64: row.embedding_base64 ?? void 0,
|
|
4059
|
+
embeddingText: row.embedding_text ?? void 0
|
|
4060
|
+
};
|
|
4061
|
+
}
|
|
4062
|
+
rowToRelationship(row) {
|
|
4063
|
+
return {
|
|
4064
|
+
id: row.id,
|
|
4065
|
+
fromId: row.from_id,
|
|
4066
|
+
toId: row.to_id,
|
|
4067
|
+
type: row.type,
|
|
4068
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
|
|
4069
|
+
weight: row.weight,
|
|
4070
|
+
createdAt: row.created_at
|
|
4071
|
+
};
|
|
4072
|
+
}
|
|
4073
|
+
vectorToString(vector) {
|
|
4074
|
+
const values = Array.from(vector).map((v) => v.toFixed(6));
|
|
4075
|
+
return `[${values.join(", ")}]`;
|
|
4076
|
+
}
|
|
4077
|
+
stringToVector(str) {
|
|
4078
|
+
const clean = str.replace(/[[\]]/g, "");
|
|
4079
|
+
const values = clean.split(",").map((s) => parseFloat(s.trim()));
|
|
4080
|
+
return new Float32Array(values);
|
|
4081
|
+
}
|
|
4082
|
+
// ===========================================================================
|
|
4083
|
+
// CACHE MANAGEMENT
|
|
4084
|
+
// ===========================================================================
|
|
4085
|
+
/**
|
|
4086
|
+
* Get cache statistics for monitoring
|
|
4087
|
+
*/
|
|
4088
|
+
getCacheStats() {
|
|
4089
|
+
return {
|
|
4090
|
+
embedding: {
|
|
4091
|
+
size: this.embeddingCache.size,
|
|
4092
|
+
maxSize: CACHE_CONFIG.embeddingCache.max
|
|
4093
|
+
},
|
|
4094
|
+
search: {
|
|
4095
|
+
size: this.searchCache.size,
|
|
4096
|
+
maxSize: CACHE_CONFIG.searchCache.max
|
|
4097
|
+
},
|
|
4098
|
+
metadata: {
|
|
4099
|
+
size: this.metadataCache.size,
|
|
4100
|
+
maxSize: CACHE_CONFIG.metadataCache.max
|
|
4101
|
+
}
|
|
4102
|
+
};
|
|
4103
|
+
}
|
|
4104
|
+
/**
|
|
4105
|
+
* Clear all caches (useful after bulk operations or project switch)
|
|
4106
|
+
*/
|
|
4107
|
+
clearCaches() {
|
|
4108
|
+
this.embeddingCache.clear();
|
|
4109
|
+
this.searchCache.clear();
|
|
4110
|
+
this.metadataCache.clear();
|
|
4111
|
+
log.i("CACHE", `All caches cleared`);
|
|
4112
|
+
}
|
|
4113
|
+
// ===========================================================================
|
|
4114
|
+
// PROLLY TREE OPERATIONS (versioned graph storage)
|
|
4115
|
+
// ===========================================================================
|
|
4116
|
+
/**
|
|
4117
|
+
* Initialize Prolly Tree components for versioned storage.
|
|
4118
|
+
* Called during adapter initialization.
|
|
4119
|
+
*/
|
|
4120
|
+
async initializeProllyComponents() {
|
|
4121
|
+
if (!this.client) throw new Error("Client not initialized");
|
|
4122
|
+
this.prollyNodeStore = new ProllyNodeStore();
|
|
4123
|
+
await this.prollyNodeStore.initialize(this.client);
|
|
4124
|
+
this.commitManager = new CommitManager();
|
|
4125
|
+
await this.commitManager.initialize(this.client);
|
|
4126
|
+
this.prollyTree = new ProllyTree(this.prollyNodeStore);
|
|
4127
|
+
await this.prollyTree.initialize();
|
|
4128
|
+
this.branchDiffCache = new BranchDiffCache(this.prollyNodeStore, this.commitManager);
|
|
4129
|
+
log.i("LIBSQLADAPT", "prolly_components_init", { components: 4 });
|
|
4130
|
+
}
|
|
4131
|
+
/**
|
|
4132
|
+
* Set Prolly Tree context for current project/branch.
|
|
4133
|
+
* Should be called after setProjectContext().
|
|
4134
|
+
*/
|
|
4135
|
+
setProllyContext(projectHash, branchName) {
|
|
4136
|
+
if (this.commitManager) {
|
|
4137
|
+
this.commitManager.setContext(projectHash, branchName);
|
|
4138
|
+
}
|
|
4139
|
+
log.d("LIBSQLADAPT", "prolly_context_set", { project: projectHash.slice(0, 8), branch: branchName });
|
|
4140
|
+
}
|
|
4141
|
+
/**
|
|
4142
|
+
* Get the Prolly Node Store for content-addressed storage operations.
|
|
4143
|
+
*/
|
|
4144
|
+
getProllyNodeStore() {
|
|
4145
|
+
return this.prollyNodeStore;
|
|
4146
|
+
}
|
|
4147
|
+
/**
|
|
4148
|
+
* Get the Prolly Tree for tree operations (build, insert, delete, diff).
|
|
4149
|
+
*/
|
|
4150
|
+
getProllyTree() {
|
|
4151
|
+
return this.prollyTree;
|
|
4152
|
+
}
|
|
4153
|
+
/**
|
|
4154
|
+
* Get the Commit Manager for versioning operations.
|
|
4155
|
+
*/
|
|
4156
|
+
getCommitManager() {
|
|
4157
|
+
return this.commitManager;
|
|
4158
|
+
}
|
|
4159
|
+
/**
|
|
4160
|
+
* Get the Branch Diff Cache for optimized branch reads.
|
|
4161
|
+
*/
|
|
4162
|
+
getBranchDiffCache() {
|
|
4163
|
+
return this.branchDiffCache;
|
|
4164
|
+
}
|
|
4165
|
+
/**
|
|
4166
|
+
* Create a new commit from current graph state.
|
|
4167
|
+
* Builds Prolly tree from entities and creates a versioned snapshot.
|
|
4168
|
+
*/
|
|
4169
|
+
async createGraphCommit(message) {
|
|
4170
|
+
if (!this.prollyTree || !this.commitManager) {
|
|
4171
|
+
log.w("LIBSQLADAPT", "prolly_not_ready");
|
|
4172
|
+
return null;
|
|
4173
|
+
}
|
|
4174
|
+
const entities = await this.entityOps.getAllEntities();
|
|
4175
|
+
const relationships = await this.relationshipOps.getAllRelationships();
|
|
4176
|
+
const entries = entities.map((e) => ({
|
|
4177
|
+
key: e.id,
|
|
4178
|
+
value: serializeEntity(e)
|
|
4179
|
+
}));
|
|
4180
|
+
const rootHash = await this.prollyTree.build(entries);
|
|
4181
|
+
const commit = await this.commitManager.commit(
|
|
4182
|
+
rootHash,
|
|
4183
|
+
null,
|
|
4184
|
+
// fileTreeHash - can be set via MerkleFileTracker
|
|
4185
|
+
{ entityCount: entities.length, relationshipCount: relationships.length },
|
|
4186
|
+
message
|
|
4187
|
+
);
|
|
4188
|
+
log.i("LIBSQLADAPT", "commit_created", {
|
|
4189
|
+
hash: commit.commitHash.slice(0, 8),
|
|
4190
|
+
entities: entities.length,
|
|
4191
|
+
relationships: relationships.length
|
|
4192
|
+
});
|
|
4193
|
+
return commit.commitHash;
|
|
4194
|
+
}
|
|
4195
|
+
/** Timestamp of last GC run — used to rate-limit pruneAndGC() */
|
|
4196
|
+
lastGcRunAt = 0;
|
|
4197
|
+
/**
|
|
4198
|
+
* Prune old commits for the current branch and garbage-collect orphaned
|
|
4199
|
+
* Prolly Tree nodes. Rate-limited to run at most once per 10 minutes
|
|
4200
|
+
* unless `force` is true.
|
|
4201
|
+
*
|
|
4202
|
+
* @param keepCommits Number of most-recent commits to keep per branch.
|
|
4203
|
+
* @param force Skip the rate-limit check.
|
|
4204
|
+
* @returns Counts of pruned commits and GC-deleted nodes (or null if skipped).
|
|
4205
|
+
*/
|
|
4206
|
+
async pruneAndGC(keepCommits = 20, force = false) {
|
|
4207
|
+
if (!this.commitManager || !this.prollyNodeStore) {
|
|
4208
|
+
return null;
|
|
4209
|
+
}
|
|
4210
|
+
const now = Date.now();
|
|
4211
|
+
if (!force && now - this.lastGcRunAt < 10 * 60 * 1e3) {
|
|
4212
|
+
return null;
|
|
4213
|
+
}
|
|
4214
|
+
this.lastGcRunAt = now;
|
|
4215
|
+
const pruned = await this.commitManager.pruneHistory(keepCommits);
|
|
4216
|
+
if (pruned === 0) return { pruned: 0, gcDeleted: 0 };
|
|
4217
|
+
const roots = await this.commitManager.getAllActiveRootHashes();
|
|
4218
|
+
const gcDeleted = await this.prollyNodeStore.collectGarbage([...roots]);
|
|
4219
|
+
if (gcDeleted > 1e3 && this.client) {
|
|
4220
|
+
await this.client.execute("VACUUM");
|
|
4221
|
+
log.i("LIBSQLADAPT", "vacuum_after_gc", { gcDeleted });
|
|
4222
|
+
}
|
|
4223
|
+
log.i("LIBSQLADAPT", "prune_gc_complete", { pruned, gcDeleted });
|
|
4224
|
+
return { pruned, gcDeleted };
|
|
4225
|
+
}
|
|
4226
|
+
/**
|
|
4227
|
+
* Initialize branch diff cache for optimized reads on feature branches.
|
|
4228
|
+
* Call this when switching to a feature branch with a base branch.
|
|
4229
|
+
*/
|
|
4230
|
+
async initBranchDiff(baseBranch) {
|
|
4231
|
+
if (!this.branchDiffCache || !this.prollyTree) {
|
|
4232
|
+
log.w("LIBSQLADAPT", "prolly_not_ready_for_diff");
|
|
4233
|
+
return;
|
|
4234
|
+
}
|
|
4235
|
+
const { branchName } = this.currentContext;
|
|
4236
|
+
await this.branchDiffCache.initForBranch(baseBranch, branchName);
|
|
4237
|
+
log.i("LIBSQLADAPT", "branch_diff_init", { branch: branchName, base: baseBranch });
|
|
4238
|
+
}
|
|
4239
|
+
/**
|
|
4240
|
+
* Check if entity is deleted on current branch (via diff cache).
|
|
4241
|
+
* Returns false if diff cache not initialized.
|
|
4242
|
+
*/
|
|
4243
|
+
isEntityDeletedOnBranch(entityId) {
|
|
4244
|
+
if (!this.branchDiffCache) return false;
|
|
4245
|
+
return this.branchDiffCache.isDeleted(entityId);
|
|
4246
|
+
}
|
|
4247
|
+
};
|
|
4248
|
+
|
|
4249
|
+
// src/storage/graph-storage-factory.ts
|
|
4250
|
+
var graphStorage = null;
|
|
4251
|
+
var libsqlAdapter = null;
|
|
4252
|
+
var initializationPromise = null;
|
|
4253
|
+
var globalConfig = {
|
|
4254
|
+
dimensions: 768,
|
|
4255
|
+
// granite-278m = 768, all-MiniLM-L6-v2 = 384
|
|
4256
|
+
metric: "cosine",
|
|
4257
|
+
compression: "float8",
|
|
4258
|
+
// float8 = 1 byte/dim, float32 = 4 bytes/dim (4x savings!)
|
|
4259
|
+
searchL: 150,
|
|
4260
|
+
insertL: 30,
|
|
4261
|
+
maxNeighbors: 12
|
|
4262
|
+
// DiskANN neighbors (lower = smaller index)
|
|
4263
|
+
};
|
|
4264
|
+
function configureGraphStorage(config) {
|
|
4265
|
+
globalConfig = { ...globalConfig, ...config };
|
|
4266
|
+
log.w("FACTORY", `[CONFIG] DiskANN params`, {
|
|
4267
|
+
dims: globalConfig.dimensions,
|
|
4268
|
+
compression: globalConfig.compression,
|
|
4269
|
+
maxNeighbors: globalConfig.maxNeighbors,
|
|
4270
|
+
insertL: globalConfig.insertL
|
|
4271
|
+
});
|
|
4272
|
+
}
|
|
4273
|
+
async function getGraphStorage() {
|
|
4274
|
+
if (graphStorage && libsqlAdapter?.isReady()) {
|
|
4275
|
+
return graphStorage;
|
|
4276
|
+
}
|
|
4277
|
+
if (initializationPromise) {
|
|
4278
|
+
return initializationPromise;
|
|
4279
|
+
}
|
|
4280
|
+
initializationPromise = (async () => {
|
|
4281
|
+
try {
|
|
4282
|
+
log.i("STORAGEFACT", "creating_singleton");
|
|
4283
|
+
const paths = getGlobalDbPaths();
|
|
4284
|
+
const unifiedDbPath = join(dirname(paths.graphDbPath), "unified-storage.db");
|
|
4285
|
+
const dbDir = dirname(unifiedDbPath);
|
|
4286
|
+
if (!existsSync(dbDir)) {
|
|
4287
|
+
log.i("STORAGEFACT", "creating_dir", { dir: dbDir });
|
|
4288
|
+
mkdirSync(dbDir, { recursive: true });
|
|
4289
|
+
}
|
|
4290
|
+
libsqlAdapter = new LibSQLGraphAdapter(globalConfig);
|
|
4291
|
+
const initialized = await libsqlAdapter.initialize(unifiedDbPath);
|
|
4292
|
+
if (!initialized) {
|
|
4293
|
+
throw new Error("Failed to initialize LibSQL adapter");
|
|
4294
|
+
}
|
|
4295
|
+
graphStorage = new GraphStorageLibSQL(libsqlAdapter);
|
|
4296
|
+
await graphStorage.initialize();
|
|
4297
|
+
log.i("STORAGEFACT", "init_complete", { path: unifiedDbPath });
|
|
4298
|
+
return graphStorage;
|
|
4299
|
+
} catch (error) {
|
|
4300
|
+
initializationPromise = null;
|
|
4301
|
+
const errMsg = error.message || "";
|
|
4302
|
+
const isBusy = errMsg.includes("SQLITE_BUSY") || errMsg.includes("database is locked");
|
|
4303
|
+
if (isBusy) {
|
|
4304
|
+
log.w("STORAGE", `Initialization failed due to database lock (will retry on next access)`, {
|
|
4305
|
+
error: errMsg
|
|
4306
|
+
});
|
|
4307
|
+
throw error;
|
|
4308
|
+
}
|
|
4309
|
+
const paths = getGlobalDbPaths();
|
|
4310
|
+
const unifiedDbPath = join(dirname(paths.graphDbPath), "unified-storage.db");
|
|
4311
|
+
if (existsSync(unifiedDbPath)) {
|
|
4312
|
+
log.w("STORAGE", `Initialization failed, attempting auto-recovery by deleting corrupt DB`, {
|
|
4313
|
+
path: unifiedDbPath,
|
|
4314
|
+
error: errMsg
|
|
4315
|
+
});
|
|
4316
|
+
try {
|
|
4317
|
+
unlinkSync(unifiedDbPath);
|
|
4318
|
+
const walPath = unifiedDbPath + "-wal";
|
|
4319
|
+
const shmPath = unifiedDbPath + "-shm";
|
|
4320
|
+
if (existsSync(walPath)) unlinkSync(walPath);
|
|
4321
|
+
if (existsSync(shmPath)) unlinkSync(shmPath);
|
|
4322
|
+
log.i("STORAGE", `Deleted corrupt DB, will recreate on next access`);
|
|
4323
|
+
} catch (deleteError) {
|
|
4324
|
+
log.e("STORAGE", `Failed to delete corrupt DB`, {
|
|
4325
|
+
error: deleteError.message
|
|
4326
|
+
});
|
|
4327
|
+
}
|
|
4328
|
+
}
|
|
4329
|
+
throw error;
|
|
4330
|
+
}
|
|
4331
|
+
})();
|
|
4332
|
+
return initializationPromise;
|
|
4333
|
+
}
|
|
4334
|
+
async function initializeGraphStorage() {
|
|
4335
|
+
return getGraphStorage();
|
|
4336
|
+
}
|
|
4337
|
+
function getLibSQLAdapter() {
|
|
4338
|
+
return libsqlAdapter;
|
|
4339
|
+
}
|
|
4340
|
+
async function resetGraphStorage() {
|
|
4341
|
+
if (libsqlAdapter) {
|
|
4342
|
+
await libsqlAdapter.close();
|
|
4343
|
+
libsqlAdapter = null;
|
|
4344
|
+
}
|
|
4345
|
+
graphStorage = null;
|
|
4346
|
+
initializationPromise = null;
|
|
4347
|
+
log.i("STORAGEFACT", "storage_reset");
|
|
4348
|
+
}
|
|
4349
|
+
function setGlobalProjectContext(projectPath, branchName) {
|
|
4350
|
+
if (graphStorage) {
|
|
4351
|
+
const resolvedBranch = branchName ?? getCurrentGitBranchOrDefault(projectPath);
|
|
4352
|
+
graphStorage.setProject(projectPath, resolvedBranch);
|
|
4353
|
+
log.i("STORAGEFACT", "context_set", { path: projectPath, branch: resolvedBranch });
|
|
4354
|
+
} else {
|
|
4355
|
+
log.w("STORAGEFACT", "context_set_fail", { reason: "not initialized" });
|
|
4356
|
+
}
|
|
4357
|
+
}
|
|
4358
|
+
function isStorageReady() {
|
|
4359
|
+
return graphStorage !== null && libsqlAdapter?.isReady() === true;
|
|
4360
|
+
}
|
|
4361
|
+
async function handleDatabaseCorruption() {
|
|
4362
|
+
log.e("STORAGEFACT", "corruption_handler_start");
|
|
4363
|
+
const paths = getGlobalDbPaths();
|
|
4364
|
+
const unifiedDbPath = join(dirname(paths.graphDbPath), "unified-storage.db");
|
|
4365
|
+
if (libsqlAdapter) {
|
|
4366
|
+
try {
|
|
4367
|
+
await libsqlAdapter.close();
|
|
4368
|
+
} catch {
|
|
4369
|
+
}
|
|
4370
|
+
libsqlAdapter = null;
|
|
4371
|
+
}
|
|
4372
|
+
graphStorage = null;
|
|
4373
|
+
initializationPromise = null;
|
|
4374
|
+
const filesToDelete = [unifiedDbPath, `${unifiedDbPath}-journal`, `${unifiedDbPath}-wal`, `${unifiedDbPath}-shm`];
|
|
4375
|
+
for (const file of filesToDelete) {
|
|
4376
|
+
try {
|
|
4377
|
+
if (existsSync(file)) {
|
|
4378
|
+
const size = statSync(file).size;
|
|
4379
|
+
const sizeMB = (size / 1024 / 1024).toFixed(1);
|
|
4380
|
+
unlinkSync(file);
|
|
4381
|
+
log.i("STORAGEFACT", "file_deleted", { file, sizeMB });
|
|
4382
|
+
}
|
|
4383
|
+
} catch (error) {
|
|
4384
|
+
log.w("STORAGEFACT", "file_delete_fail", { file, err: error.message });
|
|
4385
|
+
}
|
|
4386
|
+
}
|
|
4387
|
+
try {
|
|
4388
|
+
log.i("STORAGEFACT", "reinit_start");
|
|
4389
|
+
await getGraphStorage();
|
|
4390
|
+
log.i("STORAGEFACT", "reinit_success");
|
|
4391
|
+
return true;
|
|
4392
|
+
} catch (error) {
|
|
4393
|
+
log.e("STORAGEFACT", "reinit_fail", { err: String(error) });
|
|
4394
|
+
return false;
|
|
4395
|
+
}
|
|
4396
|
+
}
|
|
4397
|
+
function isDatabaseCorruptionError(error) {
|
|
4398
|
+
return error instanceof DatabaseCorruptionError;
|
|
4399
|
+
}
|
|
4400
|
+
|
|
4401
|
+
export { configureGraphStorage, createProjectContext, getGraphStorage, getLibSQLAdapter, handleDatabaseCorruption, initializeGraphStorage, isDatabaseCorruptionError, isStorageReady, resetGraphStorage, runWithRequestContext, setGlobalProjectContext };
|
|
4402
|
+
//# sourceMappingURL=chunk-XFGXM4CR.js.map
|
|
4403
|
+
//# sourceMappingURL=chunk-XFGXM4CR.js.map
|