wolbarg 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/LICENSE +21 -0
- package/README.md +121 -0
- package/dist/index.cjs +4547 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1149 -0
- package/dist/index.d.ts +1149 -0
- package/dist/index.js +4470 -0
- package/dist/index.js.map +1 -0
- package/package.json +116 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,4547 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var fs = require('fs/promises');
|
|
4
|
+
var path2 = require("node:path");
|
|
5
|
+
var fs2 = require("node:fs");
|
|
6
|
+
var sqlite$1 = require("node:sqlite");
|
|
7
|
+
var sqliteVec = require('sqlite-vec');
|
|
8
|
+
var async_hooks = require('async_hooks');
|
|
9
|
+
|
|
10
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
11
|
+
|
|
12
|
+
function _interopNamespace(e) {
|
|
13
|
+
if (e && e.__esModule) return e;
|
|
14
|
+
var n = Object.create(null);
|
|
15
|
+
if (e) {
|
|
16
|
+
Object.keys(e).forEach(function (k) {
|
|
17
|
+
if (k !== 'default') {
|
|
18
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
19
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
20
|
+
enumerable: true,
|
|
21
|
+
get: function () { return e[k]; }
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
n.default = e;
|
|
27
|
+
return Object.freeze(n);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
|
31
|
+
var path2__default = /*#__PURE__*/_interopDefault(path2);
|
|
32
|
+
var fs2__default = /*#__PURE__*/_interopDefault(fs2);
|
|
33
|
+
var sqliteVec__namespace = /*#__PURE__*/_interopNamespace(sqliteVec);
|
|
34
|
+
|
|
35
|
+
// src/errors/index.ts
|
|
36
|
+
var WolbargError = class extends Error {
|
|
37
|
+
code;
|
|
38
|
+
constructor(message, code, options) {
|
|
39
|
+
super(message, options);
|
|
40
|
+
this.name = "WolbargError";
|
|
41
|
+
this.code = code;
|
|
42
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
var InitializationError = class extends WolbargError {
|
|
46
|
+
constructor(message, options) {
|
|
47
|
+
super(message, "INITIALIZATION_ERROR", options);
|
|
48
|
+
this.name = "InitializationError";
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var ConfigurationError = class extends WolbargError {
|
|
52
|
+
constructor(message, options) {
|
|
53
|
+
super(message, "CONFIGURATION_ERROR", options);
|
|
54
|
+
this.name = "ConfigurationError";
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
var ValidationError = class extends WolbargError {
|
|
58
|
+
constructor(message, options) {
|
|
59
|
+
super(message, "VALIDATION_ERROR", options);
|
|
60
|
+
this.name = "ValidationError";
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
var DatabaseError = class extends WolbargError {
|
|
64
|
+
constructor(message, options) {
|
|
65
|
+
super(message, "DATABASE_ERROR", options);
|
|
66
|
+
this.name = "DatabaseError";
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
var EmbeddingError = class extends WolbargError {
|
|
70
|
+
constructor(message, options) {
|
|
71
|
+
super(message, "EMBEDDING_ERROR", options);
|
|
72
|
+
this.name = "EmbeddingError";
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
var CompressionError = class extends WolbargError {
|
|
76
|
+
constructor(message, options) {
|
|
77
|
+
super(message, "COMPRESSION_ERROR", options);
|
|
78
|
+
this.name = "CompressionError";
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
var MemoryNotFoundError = class extends WolbargError {
|
|
82
|
+
constructor(message, options) {
|
|
83
|
+
super(message, "MEMORY_NOT_FOUND", options);
|
|
84
|
+
this.name = "MemoryNotFoundError";
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
var ProviderNotConfiguredError = class extends ConfigurationError {
|
|
88
|
+
provider;
|
|
89
|
+
constructor(provider, method, hint) {
|
|
90
|
+
super(
|
|
91
|
+
`${method} requires ${provider} \u2014 ${hint}`
|
|
92
|
+
);
|
|
93
|
+
this.name = "ProviderNotConfiguredError";
|
|
94
|
+
this.provider = provider;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
// src/compression/index.ts
|
|
99
|
+
var SYSTEM_PROMPT = `You are a memory compression engine for multi-agent systems.
|
|
100
|
+
Given related memories from a single agent, produce ONE concise summary that preserves:
|
|
101
|
+
- Key facts and decisions
|
|
102
|
+
- Important entities, numbers, and identifiers
|
|
103
|
+
- Actionable conclusions
|
|
104
|
+
|
|
105
|
+
Rules:
|
|
106
|
+
- Output plain text only (no markdown headings, no bullet lists unless essential)
|
|
107
|
+
- Do not invent facts that are not present
|
|
108
|
+
- Prefer precision over verbosity
|
|
109
|
+
- Keep the summary self-contained`;
|
|
110
|
+
var LlmCompressionProvider = class {
|
|
111
|
+
name = "llm";
|
|
112
|
+
llm;
|
|
113
|
+
constructor(llm) {
|
|
114
|
+
this.llm = llm;
|
|
115
|
+
}
|
|
116
|
+
async compress(memories) {
|
|
117
|
+
return compressMemories(this.llm, memories);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
async function compressMemories(llm, memories) {
|
|
121
|
+
if (memories.length === 0) {
|
|
122
|
+
throw new CompressionError("No memories available to compress");
|
|
123
|
+
}
|
|
124
|
+
const payload = memories.map((memory, index) => {
|
|
125
|
+
const meta2 = Object.keys(memory.metadata).length > 0 ? `
|
|
126
|
+
Metadata: ${JSON.stringify(memory.metadata)}` : "";
|
|
127
|
+
return `[${index + 1}] (id=${memory.id}, created=${memory.createdAt.toISOString()})${meta2}
|
|
128
|
+
${memory.content.text}`;
|
|
129
|
+
}).join("\n\n");
|
|
130
|
+
try {
|
|
131
|
+
return await llm.complete([
|
|
132
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
133
|
+
{
|
|
134
|
+
role: "user",
|
|
135
|
+
content: `Compress the following ${memories.length} memories into a single concise summary:
|
|
136
|
+
|
|
137
|
+
${payload}`
|
|
138
|
+
}
|
|
139
|
+
]);
|
|
140
|
+
} catch (error) {
|
|
141
|
+
if (error instanceof CompressionError) {
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
throw new CompressionError(
|
|
145
|
+
`Compression failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
146
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function createCompressionProvider(llm) {
|
|
151
|
+
return new LlmCompressionProvider(llm);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// src/chunking/index.ts
|
|
155
|
+
function clampOptions(options) {
|
|
156
|
+
const chunkSize = Math.max(32, options?.chunkSize ?? 800);
|
|
157
|
+
const overlap = Math.min(
|
|
158
|
+
Math.max(0, options?.overlap ?? 100),
|
|
159
|
+
Math.floor(chunkSize / 2)
|
|
160
|
+
);
|
|
161
|
+
return { chunkSize, overlap };
|
|
162
|
+
}
|
|
163
|
+
function windowChunks(pieces, chunkSize, overlap, joiner) {
|
|
164
|
+
if (pieces.length === 0) {
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
const chunks = [];
|
|
168
|
+
let buf = "";
|
|
169
|
+
let index = 0;
|
|
170
|
+
const flush = () => {
|
|
171
|
+
const trimmed = buf.trim();
|
|
172
|
+
if (trimmed) {
|
|
173
|
+
chunks.push({ text: trimmed, index });
|
|
174
|
+
index += 1;
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
for (const piece of pieces) {
|
|
178
|
+
const candidate = buf ? `${buf}${joiner}${piece}` : piece;
|
|
179
|
+
if (candidate.length <= chunkSize) {
|
|
180
|
+
buf = candidate;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
flush();
|
|
184
|
+
if (piece.length > chunkSize) {
|
|
185
|
+
let start = 0;
|
|
186
|
+
while (start < piece.length) {
|
|
187
|
+
const end = Math.min(start + chunkSize, piece.length);
|
|
188
|
+
chunks.push({ text: piece.slice(start, end).trim(), index });
|
|
189
|
+
index += 1;
|
|
190
|
+
start = Math.max(end - overlap, end);
|
|
191
|
+
}
|
|
192
|
+
buf = "";
|
|
193
|
+
} else {
|
|
194
|
+
buf = piece;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
flush();
|
|
198
|
+
if (overlap > 0 && chunks.length > 1) {
|
|
199
|
+
const full = pieces.join(joiner);
|
|
200
|
+
return slidingWindow(full, chunkSize, overlap);
|
|
201
|
+
}
|
|
202
|
+
return chunks;
|
|
203
|
+
}
|
|
204
|
+
function slidingWindow(text, chunkSize, overlap) {
|
|
205
|
+
const chunks = [];
|
|
206
|
+
let start = 0;
|
|
207
|
+
let index = 0;
|
|
208
|
+
while (start < text.length) {
|
|
209
|
+
const end = Math.min(start + chunkSize, text.length);
|
|
210
|
+
const slice = text.slice(start, end).trim();
|
|
211
|
+
if (slice) {
|
|
212
|
+
chunks.push({ text: slice, index });
|
|
213
|
+
index += 1;
|
|
214
|
+
}
|
|
215
|
+
if (end >= text.length) {
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
start = Math.max(end - overlap, start + 1);
|
|
219
|
+
}
|
|
220
|
+
return chunks;
|
|
221
|
+
}
|
|
222
|
+
var FixedChunkingStrategy = class {
|
|
223
|
+
name = "fixed";
|
|
224
|
+
chunk(text, options) {
|
|
225
|
+
const { chunkSize, overlap } = clampOptions(options);
|
|
226
|
+
return slidingWindow(text, chunkSize, overlap);
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
var SentenceChunkingStrategy = class {
|
|
230
|
+
name = "sentence";
|
|
231
|
+
chunk(text, options) {
|
|
232
|
+
const { chunkSize, overlap } = clampOptions(options);
|
|
233
|
+
const sentences = text.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
|
|
234
|
+
return windowChunks(sentences, chunkSize, overlap, " ");
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
var ParagraphChunkingStrategy = class {
|
|
238
|
+
name = "paragraph";
|
|
239
|
+
chunk(text, options) {
|
|
240
|
+
const { chunkSize, overlap } = clampOptions(options);
|
|
241
|
+
const paragraphs = text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
|
|
242
|
+
return windowChunks(paragraphs, chunkSize, overlap, "\n\n");
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
var MarkdownChunkingStrategy = class {
|
|
246
|
+
name = "markdown";
|
|
247
|
+
chunk(text, options) {
|
|
248
|
+
const { chunkSize, overlap } = clampOptions(options);
|
|
249
|
+
const sections = text.split(/(?=^#{1,6}\s)/m).map((s) => s.trim()).filter(Boolean);
|
|
250
|
+
if (sections.length <= 1) {
|
|
251
|
+
return new ParagraphChunkingStrategy().chunk(text, options);
|
|
252
|
+
}
|
|
253
|
+
return windowChunks(sections, chunkSize, overlap, "\n\n");
|
|
254
|
+
}
|
|
255
|
+
};
|
|
256
|
+
var HeadingChunkingStrategy = class {
|
|
257
|
+
name = "heading";
|
|
258
|
+
chunk(text, options) {
|
|
259
|
+
return new MarkdownChunkingStrategy().chunk(text, options);
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
function createChunkingStrategy(name = "sentence") {
|
|
263
|
+
switch (name) {
|
|
264
|
+
case "fixed":
|
|
265
|
+
return new FixedChunkingStrategy();
|
|
266
|
+
case "paragraph":
|
|
267
|
+
return new ParagraphChunkingStrategy();
|
|
268
|
+
case "markdown":
|
|
269
|
+
return new MarkdownChunkingStrategy();
|
|
270
|
+
case "heading":
|
|
271
|
+
return new HeadingChunkingStrategy();
|
|
272
|
+
case "sentence":
|
|
273
|
+
default:
|
|
274
|
+
return new SentenceChunkingStrategy();
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
function inferChunkingStrategy(text) {
|
|
278
|
+
if (/^#{1,6}\s/m.test(text)) {
|
|
279
|
+
return new MarkdownChunkingStrategy();
|
|
280
|
+
}
|
|
281
|
+
return new SentenceChunkingStrategy();
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// src/utils/index.ts
|
|
285
|
+
function createId() {
|
|
286
|
+
return crypto.randomUUID();
|
|
287
|
+
}
|
|
288
|
+
function nowIso() {
|
|
289
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
290
|
+
}
|
|
291
|
+
function parseIso(value) {
|
|
292
|
+
return new Date(value);
|
|
293
|
+
}
|
|
294
|
+
function assertNonEmptyString(value, fieldName) {
|
|
295
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
296
|
+
throw new ValidationError(`${fieldName} must be a non-empty string`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
function assertFiniteNumber(value, fieldName, options) {
|
|
300
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
301
|
+
throw new ValidationError(`${fieldName} must be a finite number`);
|
|
302
|
+
}
|
|
303
|
+
if (options?.min !== void 0 && value < options.min) {
|
|
304
|
+
throw new ValidationError(`${fieldName} must be >= ${options.min}`);
|
|
305
|
+
}
|
|
306
|
+
if (options?.max !== void 0 && value > options.max) {
|
|
307
|
+
throw new ValidationError(`${fieldName} must be <= ${options.max}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
var EMPTY_META_JSON = "{}";
|
|
311
|
+
function serializeMetadata(metadata) {
|
|
312
|
+
for (const _key in metadata) {
|
|
313
|
+
return JSON.stringify(metadata);
|
|
314
|
+
}
|
|
315
|
+
return EMPTY_META_JSON;
|
|
316
|
+
}
|
|
317
|
+
function deserializeMetadata(raw) {
|
|
318
|
+
if (!raw || raw === EMPTY_META_JSON) {
|
|
319
|
+
return {};
|
|
320
|
+
}
|
|
321
|
+
try {
|
|
322
|
+
const parsed = JSON.parse(raw);
|
|
323
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
324
|
+
return parsed;
|
|
325
|
+
}
|
|
326
|
+
return {};
|
|
327
|
+
} catch {
|
|
328
|
+
return {};
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
function embeddingToBuffer(embedding) {
|
|
332
|
+
return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
333
|
+
}
|
|
334
|
+
function distanceToSimilarity(distance) {
|
|
335
|
+
return 1 - distance;
|
|
336
|
+
}
|
|
337
|
+
var AsyncMutex = class {
|
|
338
|
+
chain = Promise.resolve();
|
|
339
|
+
async runExclusive(fn) {
|
|
340
|
+
let release;
|
|
341
|
+
const next = new Promise((resolve) => {
|
|
342
|
+
release = resolve;
|
|
343
|
+
});
|
|
344
|
+
const previous = this.chain;
|
|
345
|
+
this.chain = previous.then(() => next);
|
|
346
|
+
await previous;
|
|
347
|
+
try {
|
|
348
|
+
return await fn();
|
|
349
|
+
} finally {
|
|
350
|
+
release();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
};
|
|
354
|
+
function joinUrl(baseUrl, path3) {
|
|
355
|
+
const base = baseUrl.replace(/\/+$/, "");
|
|
356
|
+
const suffix = path3.startsWith("/") ? path3 : `/${path3}`;
|
|
357
|
+
return `${base}${suffix}`;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// src/embedding/index.ts
|
|
361
|
+
var OpenAICompatibleEmbeddingProvider = class {
|
|
362
|
+
model;
|
|
363
|
+
baseUrl;
|
|
364
|
+
apiKey;
|
|
365
|
+
timeoutMs;
|
|
366
|
+
constructor(config) {
|
|
367
|
+
this.baseUrl = config.baseUrl;
|
|
368
|
+
this.apiKey = config.apiKey;
|
|
369
|
+
this.model = config.model;
|
|
370
|
+
this.timeoutMs = config.timeoutMs ?? 3e4;
|
|
371
|
+
}
|
|
372
|
+
async embed(text) {
|
|
373
|
+
const response = await this.request(text);
|
|
374
|
+
const vector = response.data?.[0]?.embedding;
|
|
375
|
+
if (!vector || vector.length === 0) {
|
|
376
|
+
throw new EmbeddingError("Embedding response did not contain a vector");
|
|
377
|
+
}
|
|
378
|
+
return Float32Array.from(vector);
|
|
379
|
+
}
|
|
380
|
+
async embedBatch(texts) {
|
|
381
|
+
if (texts.length === 0) {
|
|
382
|
+
return [];
|
|
383
|
+
}
|
|
384
|
+
if (texts.length === 1) {
|
|
385
|
+
return [await this.embed(texts[0])];
|
|
386
|
+
}
|
|
387
|
+
const response = await this.request(texts);
|
|
388
|
+
const data = response.data ?? [];
|
|
389
|
+
const sorted = [...data].sort(
|
|
390
|
+
(a, b) => (a.index ?? 0) - (b.index ?? 0)
|
|
391
|
+
);
|
|
392
|
+
if (sorted.length !== texts.length) {
|
|
393
|
+
const out = [];
|
|
394
|
+
for (const text of texts) {
|
|
395
|
+
out.push(await this.embed(text));
|
|
396
|
+
}
|
|
397
|
+
return out;
|
|
398
|
+
}
|
|
399
|
+
return sorted.map((item) => {
|
|
400
|
+
if (!item.embedding || item.embedding.length === 0) {
|
|
401
|
+
throw new EmbeddingError("Embedding batch response contained an empty vector");
|
|
402
|
+
}
|
|
403
|
+
return Float32Array.from(item.embedding);
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
async validate() {
|
|
407
|
+
try {
|
|
408
|
+
const embedding = await this.embed("Wolbarg health check");
|
|
409
|
+
return { dimensions: embedding.length };
|
|
410
|
+
} catch (error) {
|
|
411
|
+
if (error instanceof EmbeddingError) {
|
|
412
|
+
throw error;
|
|
413
|
+
}
|
|
414
|
+
throw new EmbeddingError(
|
|
415
|
+
`Failed to validate embedding endpoint: ${this.describe(error)}`,
|
|
416
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
async request(input) {
|
|
421
|
+
const url = joinUrl(this.baseUrl, "/embeddings");
|
|
422
|
+
const controller = new AbortController();
|
|
423
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
424
|
+
try {
|
|
425
|
+
const res = await fetch(url, {
|
|
426
|
+
method: "POST",
|
|
427
|
+
headers: {
|
|
428
|
+
"Content-Type": "application/json",
|
|
429
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
430
|
+
},
|
|
431
|
+
body: JSON.stringify({
|
|
432
|
+
model: this.model,
|
|
433
|
+
input
|
|
434
|
+
}),
|
|
435
|
+
signal: controller.signal
|
|
436
|
+
});
|
|
437
|
+
let body;
|
|
438
|
+
try {
|
|
439
|
+
body = await res.json();
|
|
440
|
+
} catch {
|
|
441
|
+
throw new EmbeddingError(
|
|
442
|
+
`Embedding endpoint returned non-JSON response (HTTP ${res.status})`
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
if (!res.ok) {
|
|
446
|
+
const message = body.error?.message ?? `HTTP ${res.status}`;
|
|
447
|
+
throw new EmbeddingError(`Embedding request failed: ${message}`);
|
|
448
|
+
}
|
|
449
|
+
return body;
|
|
450
|
+
} catch (error) {
|
|
451
|
+
if (error instanceof EmbeddingError) {
|
|
452
|
+
throw error;
|
|
453
|
+
}
|
|
454
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
455
|
+
throw new EmbeddingError(
|
|
456
|
+
`Embedding request timed out after ${this.timeoutMs}ms`
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
throw new EmbeddingError(
|
|
460
|
+
`Embedding request failed: ${this.describe(error)}`,
|
|
461
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
462
|
+
);
|
|
463
|
+
} finally {
|
|
464
|
+
clearTimeout(timer);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
describe(error) {
|
|
468
|
+
if (error instanceof Error) {
|
|
469
|
+
return error.message;
|
|
470
|
+
}
|
|
471
|
+
return String(error);
|
|
472
|
+
}
|
|
473
|
+
};
|
|
474
|
+
function createEmbeddingProvider(config) {
|
|
475
|
+
return new OpenAICompatibleEmbeddingProvider(config);
|
|
476
|
+
}
|
|
477
|
+
function factory(defaults) {
|
|
478
|
+
return (config) => createEmbeddingProvider({
|
|
479
|
+
baseUrl: config.baseUrl ?? defaults.baseUrl,
|
|
480
|
+
apiKey: config.apiKey,
|
|
481
|
+
model: config.model,
|
|
482
|
+
timeoutMs: config.timeoutMs
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
var openaiCompatibleEmbedding = (config) => createEmbeddingProvider(config);
|
|
486
|
+
var openaiEmbedding = factory({
|
|
487
|
+
baseUrl: "https://api.openai.com/v1"
|
|
488
|
+
});
|
|
489
|
+
var ollamaEmbedding = factory({
|
|
490
|
+
baseUrl: "http://127.0.0.1:11434/v1"
|
|
491
|
+
});
|
|
492
|
+
var openRouterEmbedding = factory({
|
|
493
|
+
baseUrl: "https://openrouter.ai/api/v1"
|
|
494
|
+
});
|
|
495
|
+
var lmStudioEmbedding = factory({
|
|
496
|
+
baseUrl: "http://127.0.0.1:1234/v1"
|
|
497
|
+
});
|
|
498
|
+
var geminiEmbedding = factory({
|
|
499
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta/openai"
|
|
500
|
+
});
|
|
501
|
+
var togetherEmbedding = factory({
|
|
502
|
+
baseUrl: "https://api.together.xyz/v1"
|
|
503
|
+
});
|
|
504
|
+
var vllmEmbedding = factory({
|
|
505
|
+
baseUrl: "http://127.0.0.1:8000/v1"
|
|
506
|
+
});
|
|
507
|
+
async function embedMany(provider, texts, concurrency = 8) {
|
|
508
|
+
if (provider.embedBatch) {
|
|
509
|
+
return provider.embedBatch(texts);
|
|
510
|
+
}
|
|
511
|
+
const out = new Array(texts.length);
|
|
512
|
+
let index = 0;
|
|
513
|
+
async function worker() {
|
|
514
|
+
while (index < texts.length) {
|
|
515
|
+
const current = index;
|
|
516
|
+
index += 1;
|
|
517
|
+
out[current] = await provider.embed(texts[current]);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
await Promise.all(
|
|
521
|
+
Array.from({ length: Math.min(concurrency, texts.length) }, () => worker())
|
|
522
|
+
);
|
|
523
|
+
return out;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// src/llm/index.ts
|
|
527
|
+
var OpenAICompatibleLlmProvider = class {
|
|
528
|
+
model;
|
|
529
|
+
baseUrl;
|
|
530
|
+
apiKey;
|
|
531
|
+
temperature;
|
|
532
|
+
maxTokens;
|
|
533
|
+
timeoutMs;
|
|
534
|
+
constructor(config) {
|
|
535
|
+
this.baseUrl = config.baseUrl;
|
|
536
|
+
this.apiKey = config.apiKey;
|
|
537
|
+
this.model = config.model;
|
|
538
|
+
this.temperature = config.temperature ?? 0.2;
|
|
539
|
+
this.maxTokens = config.maxTokens ?? 4096;
|
|
540
|
+
this.timeoutMs = config.timeoutMs ?? 6e4;
|
|
541
|
+
}
|
|
542
|
+
async complete(messages) {
|
|
543
|
+
const response = await this.request(messages);
|
|
544
|
+
const content = response.choices?.[0]?.message?.content;
|
|
545
|
+
if (typeof content !== "string" || content.trim().length === 0) {
|
|
546
|
+
throw new CompressionError("LLM response did not contain text content");
|
|
547
|
+
}
|
|
548
|
+
return content.trim();
|
|
549
|
+
}
|
|
550
|
+
async validate() {
|
|
551
|
+
try {
|
|
552
|
+
await this.complete([
|
|
553
|
+
{
|
|
554
|
+
role: "user",
|
|
555
|
+
content: 'Reply with exactly the word "ok".'
|
|
556
|
+
}
|
|
557
|
+
]);
|
|
558
|
+
} catch (error) {
|
|
559
|
+
if (error instanceof CompressionError) {
|
|
560
|
+
throw new CompressionError(
|
|
561
|
+
`Failed to validate LLM endpoint: ${error.message}`,
|
|
562
|
+
{ cause: error }
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
throw new CompressionError(
|
|
566
|
+
`Failed to validate LLM endpoint: ${this.describe(error)}`,
|
|
567
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
async request(messages) {
|
|
572
|
+
const url = joinUrl(this.baseUrl, "/chat/completions");
|
|
573
|
+
const controller = new AbortController();
|
|
574
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
575
|
+
try {
|
|
576
|
+
const res = await fetch(url, {
|
|
577
|
+
method: "POST",
|
|
578
|
+
headers: {
|
|
579
|
+
"Content-Type": "application/json",
|
|
580
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
581
|
+
},
|
|
582
|
+
body: JSON.stringify({
|
|
583
|
+
model: this.model,
|
|
584
|
+
messages,
|
|
585
|
+
temperature: this.temperature,
|
|
586
|
+
max_tokens: this.maxTokens
|
|
587
|
+
}),
|
|
588
|
+
signal: controller.signal
|
|
589
|
+
});
|
|
590
|
+
let body;
|
|
591
|
+
try {
|
|
592
|
+
body = await res.json();
|
|
593
|
+
} catch {
|
|
594
|
+
throw new CompressionError(
|
|
595
|
+
`LLM endpoint returned non-JSON response (HTTP ${res.status})`
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
if (!res.ok) {
|
|
599
|
+
const message = body.error?.message ?? `HTTP ${res.status}`;
|
|
600
|
+
throw new CompressionError(`LLM request failed: ${message}`);
|
|
601
|
+
}
|
|
602
|
+
return body;
|
|
603
|
+
} catch (error) {
|
|
604
|
+
if (error instanceof CompressionError) {
|
|
605
|
+
throw error;
|
|
606
|
+
}
|
|
607
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
608
|
+
throw new CompressionError(
|
|
609
|
+
`LLM request timed out after ${this.timeoutMs}ms`
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
throw new CompressionError(`LLM request failed: ${this.describe(error)}`, {
|
|
613
|
+
cause: error instanceof Error ? error : void 0
|
|
614
|
+
});
|
|
615
|
+
} finally {
|
|
616
|
+
clearTimeout(timer);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
describe(error) {
|
|
620
|
+
if (error instanceof Error) {
|
|
621
|
+
return error.message;
|
|
622
|
+
}
|
|
623
|
+
return String(error);
|
|
624
|
+
}
|
|
625
|
+
};
|
|
626
|
+
function createLlmProvider(config) {
|
|
627
|
+
return new OpenAICompatibleLlmProvider(config);
|
|
628
|
+
}
|
|
629
|
+
function llmFactory(defaults) {
|
|
630
|
+
return (config) => createLlmProvider({
|
|
631
|
+
baseUrl: config.baseUrl ?? defaults.baseUrl,
|
|
632
|
+
apiKey: config.apiKey,
|
|
633
|
+
model: config.model,
|
|
634
|
+
temperature: config.temperature,
|
|
635
|
+
maxTokens: config.maxTokens,
|
|
636
|
+
timeoutMs: config.timeoutMs
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
var openaiCompatibleLlm = (config) => createLlmProvider(config);
|
|
640
|
+
var openaiLlm = llmFactory({
|
|
641
|
+
baseUrl: "https://api.openai.com/v1"
|
|
642
|
+
});
|
|
643
|
+
var ollamaLlm = llmFactory({
|
|
644
|
+
baseUrl: "http://127.0.0.1:11434/v1"
|
|
645
|
+
});
|
|
646
|
+
var openRouterLlm = llmFactory({
|
|
647
|
+
baseUrl: "https://openrouter.ai/api/v1"
|
|
648
|
+
});
|
|
649
|
+
function extOf(filename) {
|
|
650
|
+
if (!filename) {
|
|
651
|
+
return "";
|
|
652
|
+
}
|
|
653
|
+
return path2__default.default.extname(filename).toLowerCase();
|
|
654
|
+
}
|
|
655
|
+
var TextParser = class {
|
|
656
|
+
name = "text";
|
|
657
|
+
extensions = [".txt", ".md", ".markdown", ".csv", ".json"];
|
|
658
|
+
async parse(input) {
|
|
659
|
+
const ext = extOf(input.filename);
|
|
660
|
+
let text = input.buffer.toString("utf8");
|
|
661
|
+
if (ext === ".json") {
|
|
662
|
+
try {
|
|
663
|
+
text = JSON.stringify(JSON.parse(text), null, 2);
|
|
664
|
+
} catch {
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
return {
|
|
668
|
+
text,
|
|
669
|
+
mimeType: input.mimeType ?? "text/plain",
|
|
670
|
+
filename: input.filename,
|
|
671
|
+
isImage: false
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
var ImageParser = class {
|
|
676
|
+
name = "image";
|
|
677
|
+
extensions = [".png", ".jpg", ".jpeg", ".webp"];
|
|
678
|
+
async parse(input) {
|
|
679
|
+
const ext = extOf(input.filename);
|
|
680
|
+
const mime = input.mimeType ?? (ext === ".png" ? "image/png" : ext === ".webp" ? "image/webp" : "image/jpeg");
|
|
681
|
+
return {
|
|
682
|
+
text: "",
|
|
683
|
+
mimeType: mime,
|
|
684
|
+
filename: input.filename,
|
|
685
|
+
isImage: true,
|
|
686
|
+
imageBuffer: input.buffer
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
};
|
|
690
|
+
var PdfParser = class {
|
|
691
|
+
name = "pdf";
|
|
692
|
+
extensions = [".pdf"];
|
|
693
|
+
async parse(input) {
|
|
694
|
+
try {
|
|
695
|
+
const mod = await import('pdf-parse');
|
|
696
|
+
const maybeClass = mod.PDFParse;
|
|
697
|
+
let text = "";
|
|
698
|
+
if (typeof maybeClass === "function") {
|
|
699
|
+
const parser = new maybeClass({ data: input.buffer });
|
|
700
|
+
try {
|
|
701
|
+
const result = await parser.getText();
|
|
702
|
+
text = result.text ?? "";
|
|
703
|
+
} finally {
|
|
704
|
+
await parser.destroy?.();
|
|
705
|
+
}
|
|
706
|
+
} else {
|
|
707
|
+
const pdfParse = mod.default ?? mod;
|
|
708
|
+
if (typeof pdfParse !== "function") {
|
|
709
|
+
throw new Error("pdf-parse export is not a function");
|
|
710
|
+
}
|
|
711
|
+
const result = await pdfParse(input.buffer);
|
|
712
|
+
text = result.text ?? "";
|
|
713
|
+
}
|
|
714
|
+
return {
|
|
715
|
+
text,
|
|
716
|
+
mimeType: "application/pdf",
|
|
717
|
+
filename: input.filename,
|
|
718
|
+
isImage: false
|
|
719
|
+
};
|
|
720
|
+
} catch (error) {
|
|
721
|
+
if (error instanceof ConfigurationError) {
|
|
722
|
+
throw error;
|
|
723
|
+
}
|
|
724
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
725
|
+
throw new ConfigurationError(
|
|
726
|
+
`PDF ingest failed (${detail}). Install compatible pdf-parse: npm install pdf-parse@1.1.4`,
|
|
727
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
var DocxParser = class {
|
|
733
|
+
name = "docx";
|
|
734
|
+
extensions = [".docx"];
|
|
735
|
+
async parse(input) {
|
|
736
|
+
try {
|
|
737
|
+
const mod = await import('mammoth');
|
|
738
|
+
const extractRawText = mod.extractRawText ?? mod.default?.extractRawText;
|
|
739
|
+
if (!extractRawText) {
|
|
740
|
+
throw new Error("mammoth.extractRawText unavailable");
|
|
741
|
+
}
|
|
742
|
+
const result = await extractRawText({ buffer: input.buffer });
|
|
743
|
+
return {
|
|
744
|
+
text: result.value ?? "",
|
|
745
|
+
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
746
|
+
filename: input.filename,
|
|
747
|
+
isImage: false
|
|
748
|
+
};
|
|
749
|
+
} catch {
|
|
750
|
+
throw new ConfigurationError(
|
|
751
|
+
'DOCX ingest requires the optional "mammoth" package. Install it with: npm install mammoth'
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
};
|
|
756
|
+
var DEFAULT_PARSERS = [
|
|
757
|
+
new TextParser(),
|
|
758
|
+
new ImageParser(),
|
|
759
|
+
new PdfParser(),
|
|
760
|
+
new DocxParser()
|
|
761
|
+
];
|
|
762
|
+
function resolveParser(filename, mimeType, parsers = DEFAULT_PARSERS) {
|
|
763
|
+
const ext = extOf(filename);
|
|
764
|
+
if (ext) {
|
|
765
|
+
const byExt = parsers.find((p) => p.extensions.includes(ext));
|
|
766
|
+
if (byExt) {
|
|
767
|
+
return byExt;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
if (mimeType?.startsWith("image/")) {
|
|
771
|
+
return parsers.find((p) => p.name === "image") ?? new ImageParser();
|
|
772
|
+
}
|
|
773
|
+
if (mimeType === "application/pdf") {
|
|
774
|
+
return parsers.find((p) => p.name === "pdf") ?? new PdfParser();
|
|
775
|
+
}
|
|
776
|
+
return parsers.find((p) => p.name === "text") ?? new TextParser();
|
|
777
|
+
}
|
|
778
|
+
async function loadIngestSource(source) {
|
|
779
|
+
if (source.text !== void 0) {
|
|
780
|
+
return {
|
|
781
|
+
buffer: Buffer.from(source.text, "utf8"),
|
|
782
|
+
filename: source.filename ?? "document.txt",
|
|
783
|
+
mimeType: source.mimeType ?? "text/plain",
|
|
784
|
+
rawText: source.text
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
if (source.buffer) {
|
|
788
|
+
return {
|
|
789
|
+
buffer: source.buffer,
|
|
790
|
+
filename: source.filename,
|
|
791
|
+
mimeType: source.mimeType
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
if (source.path) {
|
|
795
|
+
const buffer = await fs__default.default.readFile(source.path);
|
|
796
|
+
return {
|
|
797
|
+
buffer,
|
|
798
|
+
filename: path2__default.default.basename(source.path),
|
|
799
|
+
mimeType: source.mimeType
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
throw new ConfigurationError("ingest source must include path, buffer, or text");
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// src/filters/match.ts
|
|
806
|
+
function getField(metadata, field) {
|
|
807
|
+
const parts = field.split(".");
|
|
808
|
+
let current = metadata;
|
|
809
|
+
for (const part of parts) {
|
|
810
|
+
if (current === null || typeof current !== "object" || Array.isArray(current)) {
|
|
811
|
+
return void 0;
|
|
812
|
+
}
|
|
813
|
+
current = current[part];
|
|
814
|
+
}
|
|
815
|
+
return current;
|
|
816
|
+
}
|
|
817
|
+
function compare(value, op) {
|
|
818
|
+
if ("eq" in op) {
|
|
819
|
+
return value === op.eq;
|
|
820
|
+
}
|
|
821
|
+
if ("contains" in op) {
|
|
822
|
+
if (typeof value === "string") {
|
|
823
|
+
return value.includes(op.contains);
|
|
824
|
+
}
|
|
825
|
+
if (Array.isArray(value)) {
|
|
826
|
+
return value.some((item) => String(item).includes(op.contains));
|
|
827
|
+
}
|
|
828
|
+
return false;
|
|
829
|
+
}
|
|
830
|
+
if ("gt" in op) {
|
|
831
|
+
return value != null && value > op.gt;
|
|
832
|
+
}
|
|
833
|
+
if ("gte" in op) {
|
|
834
|
+
return value != null && value >= op.gte;
|
|
835
|
+
}
|
|
836
|
+
if ("lt" in op) {
|
|
837
|
+
return value != null && value < op.lt;
|
|
838
|
+
}
|
|
839
|
+
if ("lte" in op) {
|
|
840
|
+
return value != null && value <= op.lte;
|
|
841
|
+
}
|
|
842
|
+
if ("between" in op) {
|
|
843
|
+
const [lo, hi] = op.between;
|
|
844
|
+
return value != null && value >= lo && value <= hi;
|
|
845
|
+
}
|
|
846
|
+
return false;
|
|
847
|
+
}
|
|
848
|
+
function matchesMetadata(metadata, filter) {
|
|
849
|
+
if ("and" in filter) {
|
|
850
|
+
return filter.and.every((f) => matchesMetadata(metadata, f));
|
|
851
|
+
}
|
|
852
|
+
if ("or" in filter) {
|
|
853
|
+
return filter.or.some((f) => matchesMetadata(metadata, f));
|
|
854
|
+
}
|
|
855
|
+
if ("not" in filter) {
|
|
856
|
+
return !matchesMetadata(metadata, filter.not);
|
|
857
|
+
}
|
|
858
|
+
return compare(getField(metadata, filter.field), filter.op);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
// src/schema/index.ts
|
|
862
|
+
var SCHEMA_VERSION = 2;
|
|
863
|
+
var META_KEYS = {
|
|
864
|
+
schemaVersion: "schema_version",
|
|
865
|
+
embeddingDimensions: "embedding_dimensions",
|
|
866
|
+
vectorBackend: "vector_backend"
|
|
867
|
+
};
|
|
868
|
+
var CREATE_META_TABLE = `
|
|
869
|
+
CREATE TABLE IF NOT EXISTS Wolbarg_meta (
|
|
870
|
+
key TEXT PRIMARY KEY NOT NULL,
|
|
871
|
+
value TEXT NOT NULL
|
|
872
|
+
);
|
|
873
|
+
`;
|
|
874
|
+
var CREATE_MEMORIES_TABLE = `
|
|
875
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
876
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
877
|
+
organization TEXT NOT NULL,
|
|
878
|
+
agent TEXT NOT NULL,
|
|
879
|
+
content_text TEXT NOT NULL,
|
|
880
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
881
|
+
archived INTEGER NOT NULL DEFAULT 0 CHECK (archived IN (0, 1)),
|
|
882
|
+
compressed_into TEXT NULL,
|
|
883
|
+
created_at TEXT NOT NULL,
|
|
884
|
+
updated_at TEXT NOT NULL,
|
|
885
|
+
FOREIGN KEY (compressed_into) REFERENCES memories(id) ON DELETE SET NULL
|
|
886
|
+
);
|
|
887
|
+
`;
|
|
888
|
+
var CREATE_HISTORY_TABLE = `
|
|
889
|
+
CREATE TABLE IF NOT EXISTS memory_history (
|
|
890
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
891
|
+
memory_id TEXT NOT NULL,
|
|
892
|
+
event_type TEXT NOT NULL CHECK (event_type IN ('created', 'archived', 'compressed')),
|
|
893
|
+
related_memory_id TEXT NULL,
|
|
894
|
+
created_at TEXT NOT NULL,
|
|
895
|
+
FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
|
|
896
|
+
);
|
|
897
|
+
`;
|
|
898
|
+
var CREATE_BLOB_EMBEDDINGS_TABLE = `
|
|
899
|
+
CREATE TABLE IF NOT EXISTS memory_embeddings_blob (
|
|
900
|
+
memory_rowid INTEGER PRIMARY KEY NOT NULL,
|
|
901
|
+
embedding BLOB NOT NULL
|
|
902
|
+
);
|
|
903
|
+
`;
|
|
904
|
+
var CREATE_FTS_TABLE = `
|
|
905
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
906
|
+
content_text,
|
|
907
|
+
memory_id UNINDEXED,
|
|
908
|
+
organization UNINDEXED,
|
|
909
|
+
agent UNINDEXED,
|
|
910
|
+
tokenize = 'porter unicode61'
|
|
911
|
+
);
|
|
912
|
+
`;
|
|
913
|
+
var CREATE_INDEXES = [
|
|
914
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_agent ON memories(organization, agent);`,
|
|
915
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_archived ON memories(organization, archived);`,
|
|
916
|
+
/** Active-set covering path for org-scoped list / stats / compress. */
|
|
917
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_active_created
|
|
918
|
+
ON memories(organization, created_at) WHERE archived = 0;`,
|
|
919
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_created_at ON memories(created_at);`,
|
|
920
|
+
`CREATE INDEX IF NOT EXISTS idx_history_memory_id ON memory_history(memory_id);`
|
|
921
|
+
];
|
|
922
|
+
function buildVectorTableSql(dimensions) {
|
|
923
|
+
if (!Number.isInteger(dimensions) || dimensions <= 0) {
|
|
924
|
+
throw new Error(`Invalid embedding dimensions: ${dimensions}`);
|
|
925
|
+
}
|
|
926
|
+
return `
|
|
927
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memory_embeddings USING vec0(
|
|
928
|
+
memory_rowid INTEGER PRIMARY KEY,
|
|
929
|
+
embedding float[${dimensions}] distance_metric=cosine
|
|
930
|
+
);
|
|
931
|
+
`;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
// src/sql/index.ts
|
|
935
|
+
var SQL = {
|
|
936
|
+
getMeta: `SELECT value FROM Wolbarg_meta WHERE key = ?`,
|
|
937
|
+
setMeta: `
|
|
938
|
+
INSERT INTO Wolbarg_meta (key, value) VALUES (?, ?)
|
|
939
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
940
|
+
`,
|
|
941
|
+
insertMemory: `
|
|
942
|
+
INSERT INTO memories (
|
|
943
|
+
id, organization, agent, content_text, metadata_json,
|
|
944
|
+
archived, compressed_into, created_at, updated_at
|
|
945
|
+
) VALUES (?, ?, ?, ?, ?, 0, NULL, ?, ?)
|
|
946
|
+
RETURNING rowid, id, organization, agent, content_text, metadata_json,
|
|
947
|
+
archived, compressed_into, created_at, updated_at
|
|
948
|
+
`,
|
|
949
|
+
getMemoryById: `
|
|
950
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
951
|
+
archived, compressed_into, created_at, updated_at
|
|
952
|
+
FROM memories
|
|
953
|
+
WHERE id = ? AND organization = ?
|
|
954
|
+
`,
|
|
955
|
+
getMemoryByRowid: `
|
|
956
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
957
|
+
archived, compressed_into, created_at, updated_at
|
|
958
|
+
FROM memories
|
|
959
|
+
WHERE rowid = ? AND organization = ?
|
|
960
|
+
`,
|
|
961
|
+
getMemoriesByRowidsPrefix: `
|
|
962
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
963
|
+
archived, compressed_into, created_at, updated_at
|
|
964
|
+
FROM memories
|
|
965
|
+
WHERE organization = ? AND rowid IN (
|
|
966
|
+
`,
|
|
967
|
+
listMemoriesBase: `
|
|
968
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
969
|
+
archived, compressed_into, created_at, updated_at
|
|
970
|
+
FROM memories
|
|
971
|
+
WHERE organization = ?
|
|
972
|
+
`,
|
|
973
|
+
insertEmbedding: `
|
|
974
|
+
INSERT INTO memory_embeddings (memory_rowid, embedding) VALUES (?, ?)
|
|
975
|
+
`,
|
|
976
|
+
deleteEmbedding: `
|
|
977
|
+
DELETE FROM memory_embeddings WHERE memory_rowid = ?
|
|
978
|
+
`,
|
|
979
|
+
searchVectors: `
|
|
980
|
+
SELECT memory_rowid, distance
|
|
981
|
+
FROM memory_embeddings
|
|
982
|
+
WHERE embedding MATCH ?
|
|
983
|
+
AND k = ?
|
|
984
|
+
`,
|
|
985
|
+
insertEmbeddingBlob: `
|
|
986
|
+
INSERT INTO memory_embeddings_blob (memory_rowid, embedding) VALUES (?, ?)
|
|
987
|
+
`,
|
|
988
|
+
deleteEmbeddingBlob: `
|
|
989
|
+
DELETE FROM memory_embeddings_blob WHERE memory_rowid = ?
|
|
990
|
+
`,
|
|
991
|
+
listEmbeddingsBlob: `
|
|
992
|
+
SELECT memory_rowid, embedding FROM memory_embeddings_blob
|
|
993
|
+
`,
|
|
994
|
+
archiveMemory: `
|
|
995
|
+
UPDATE memories
|
|
996
|
+
SET archived = 1,
|
|
997
|
+
compressed_into = ?,
|
|
998
|
+
updated_at = ?
|
|
999
|
+
WHERE id = ? AND organization = ? AND archived = 0
|
|
1000
|
+
`,
|
|
1001
|
+
deleteMemoryById: `
|
|
1002
|
+
DELETE FROM memories WHERE id = ? AND organization = ?
|
|
1003
|
+
`,
|
|
1004
|
+
deleteMemoriesByOrg: `
|
|
1005
|
+
DELETE FROM memories WHERE organization = ?
|
|
1006
|
+
`,
|
|
1007
|
+
deleteMemoriesByOrgAgent: `
|
|
1008
|
+
DELETE FROM memories WHERE organization = ? AND agent = ?
|
|
1009
|
+
`,
|
|
1010
|
+
insertHistory: `
|
|
1011
|
+
INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
1012
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1013
|
+
`,
|
|
1014
|
+
getHistory: `
|
|
1015
|
+
SELECT id, memory_id, event_type, related_memory_id, created_at
|
|
1016
|
+
FROM memory_history
|
|
1017
|
+
WHERE memory_id = ?
|
|
1018
|
+
ORDER BY created_at ASC
|
|
1019
|
+
`,
|
|
1020
|
+
countMemories: `
|
|
1021
|
+
SELECT COUNT(*) AS count FROM memories WHERE organization = ?
|
|
1022
|
+
`,
|
|
1023
|
+
countActiveMemories: `
|
|
1024
|
+
SELECT COUNT(*) AS count FROM memories WHERE organization = ? AND archived = 0
|
|
1025
|
+
`,
|
|
1026
|
+
countArchivedMemories: `
|
|
1027
|
+
SELECT COUNT(*) AS count FROM memories WHERE organization = ? AND archived = 1
|
|
1028
|
+
`,
|
|
1029
|
+
countAgents: `
|
|
1030
|
+
SELECT COUNT(DISTINCT agent) AS count FROM memories WHERE organization = ? AND archived = 0
|
|
1031
|
+
`,
|
|
1032
|
+
/** FTS ranked by BM25 (archived rows are deleted from FTS on archive). */
|
|
1033
|
+
searchFts: `
|
|
1034
|
+
SELECT memory_id, bm25(memories_fts) AS rank
|
|
1035
|
+
FROM memories_fts
|
|
1036
|
+
WHERE memories_fts MATCH ?
|
|
1037
|
+
AND organization = ?
|
|
1038
|
+
ORDER BY rank
|
|
1039
|
+
LIMIT ?
|
|
1040
|
+
`,
|
|
1041
|
+
listRowidsForOrg: `
|
|
1042
|
+
SELECT rowid FROM memories WHERE organization = ?
|
|
1043
|
+
`,
|
|
1044
|
+
listRowidsForOrgAgent: `
|
|
1045
|
+
SELECT rowid FROM memories WHERE organization = ? AND agent = ?
|
|
1046
|
+
`,
|
|
1047
|
+
vectorTableExists: `
|
|
1048
|
+
SELECT name FROM sqlite_master
|
|
1049
|
+
WHERE type = 'table' AND name = 'memory_embeddings'
|
|
1050
|
+
`,
|
|
1051
|
+
updateMemoryContent: `
|
|
1052
|
+
UPDATE memories
|
|
1053
|
+
SET content_text = COALESCE(?, content_text),
|
|
1054
|
+
metadata_json = COALESCE(?, metadata_json),
|
|
1055
|
+
updated_at = ?
|
|
1056
|
+
WHERE id = ? AND organization = ?
|
|
1057
|
+
`,
|
|
1058
|
+
insertFts: `
|
|
1059
|
+
INSERT INTO memories_fts (content_text, memory_id, organization, agent)
|
|
1060
|
+
VALUES (?, ?, ?, ?)
|
|
1061
|
+
`,
|
|
1062
|
+
deleteFts: `
|
|
1063
|
+
DELETE FROM memories_fts WHERE memory_id = ?
|
|
1064
|
+
`
|
|
1065
|
+
};
|
|
1066
|
+
|
|
1067
|
+
// src/utils/vector.ts
|
|
1068
|
+
function cosineDistance(a, b) {
|
|
1069
|
+
const len = Math.min(a.length, b.length);
|
|
1070
|
+
let dot = 0;
|
|
1071
|
+
let normA = 0;
|
|
1072
|
+
let normB = 0;
|
|
1073
|
+
for (let i = 0; i < len; i += 1) {
|
|
1074
|
+
const av = a[i] ?? 0;
|
|
1075
|
+
const bv = b[i] ?? 0;
|
|
1076
|
+
dot += av * bv;
|
|
1077
|
+
normA += av * av;
|
|
1078
|
+
normB += bv * bv;
|
|
1079
|
+
}
|
|
1080
|
+
if (normA === 0 || normB === 0) {
|
|
1081
|
+
return 1;
|
|
1082
|
+
}
|
|
1083
|
+
const similarity = dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
1084
|
+
return 1 - similarity;
|
|
1085
|
+
}
|
|
1086
|
+
function bufferToEmbedding(data) {
|
|
1087
|
+
const byteOffset = data.byteOffset ?? 0;
|
|
1088
|
+
const byteLength = data.byteLength;
|
|
1089
|
+
if (byteOffset % Float32Array.BYTES_PER_ELEMENT === 0) {
|
|
1090
|
+
return new Float32Array(
|
|
1091
|
+
data.buffer,
|
|
1092
|
+
byteOffset,
|
|
1093
|
+
byteLength / Float32Array.BYTES_PER_ELEMENT
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
const copy = new Uint8Array(byteLength);
|
|
1097
|
+
copy.set(data);
|
|
1098
|
+
return new Float32Array(copy.buffer);
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
// src/utils/vector-index.ts
|
|
1102
|
+
var InMemoryVectorIndex = class {
|
|
1103
|
+
dims;
|
|
1104
|
+
rowids = [];
|
|
1105
|
+
data;
|
|
1106
|
+
count = 0;
|
|
1107
|
+
rowidToSlot = /* @__PURE__ */ new Map();
|
|
1108
|
+
constructor(dimensions, initialCapacity = 256) {
|
|
1109
|
+
this.dims = dimensions;
|
|
1110
|
+
this.data = new Float32Array(initialCapacity * dimensions);
|
|
1111
|
+
}
|
|
1112
|
+
get size() {
|
|
1113
|
+
return this.count;
|
|
1114
|
+
}
|
|
1115
|
+
clear() {
|
|
1116
|
+
this.rowids.length = 0;
|
|
1117
|
+
this.rowidToSlot.clear();
|
|
1118
|
+
this.count = 0;
|
|
1119
|
+
}
|
|
1120
|
+
upsert(rowid, embedding) {
|
|
1121
|
+
const existing = this.rowidToSlot.get(rowid);
|
|
1122
|
+
const slot = existing !== void 0 ? existing : (() => {
|
|
1123
|
+
this.ensureCapacity(this.count + 1);
|
|
1124
|
+
const s = this.count;
|
|
1125
|
+
this.rowids[s] = rowid;
|
|
1126
|
+
this.rowidToSlot.set(rowid, s);
|
|
1127
|
+
this.count += 1;
|
|
1128
|
+
return s;
|
|
1129
|
+
})();
|
|
1130
|
+
const dims = this.dims;
|
|
1131
|
+
const base = slot * dims;
|
|
1132
|
+
const data = this.data;
|
|
1133
|
+
const len = embedding.length < dims ? embedding.length : dims;
|
|
1134
|
+
let norm = 0;
|
|
1135
|
+
for (let i = 0; i < len; i += 1) {
|
|
1136
|
+
const v = embedding[i];
|
|
1137
|
+
data[base + i] = v;
|
|
1138
|
+
norm += v * v;
|
|
1139
|
+
}
|
|
1140
|
+
for (let i = len; i < dims; i += 1) {
|
|
1141
|
+
data[base + i] = 0;
|
|
1142
|
+
}
|
|
1143
|
+
if (norm > 0) {
|
|
1144
|
+
const inv = 1 / Math.sqrt(norm);
|
|
1145
|
+
for (let i = 0; i < len; i += 1) {
|
|
1146
|
+
data[base + i] *= inv;
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
remove(rowid) {
|
|
1151
|
+
const slot = this.rowidToSlot.get(rowid);
|
|
1152
|
+
if (slot === void 0) {
|
|
1153
|
+
return;
|
|
1154
|
+
}
|
|
1155
|
+
const last = this.count - 1;
|
|
1156
|
+
if (slot !== last) {
|
|
1157
|
+
const lastRowid = this.rowids[last];
|
|
1158
|
+
this.data.copyWithin(
|
|
1159
|
+
slot * this.dims,
|
|
1160
|
+
last * this.dims,
|
|
1161
|
+
(last + 1) * this.dims
|
|
1162
|
+
);
|
|
1163
|
+
this.rowids[slot] = lastRowid;
|
|
1164
|
+
this.rowidToSlot.set(lastRowid, slot);
|
|
1165
|
+
}
|
|
1166
|
+
this.rowidToSlot.delete(rowid);
|
|
1167
|
+
this.count = last;
|
|
1168
|
+
}
|
|
1169
|
+
/** Top-k by cosine distance (1 - dot) assuming query is L2-normalized. */
|
|
1170
|
+
search(queryNormalized, topK) {
|
|
1171
|
+
const n = this.count;
|
|
1172
|
+
if (n === 0 || topK <= 0) {
|
|
1173
|
+
return [];
|
|
1174
|
+
}
|
|
1175
|
+
const dims = this.dims;
|
|
1176
|
+
const data = this.data;
|
|
1177
|
+
const rowids = this.rowids;
|
|
1178
|
+
const k = topK < n ? topK : n;
|
|
1179
|
+
const heapDist = new Float64Array(k);
|
|
1180
|
+
const heapRow = new Int32Array(k);
|
|
1181
|
+
let heapSize = 0;
|
|
1182
|
+
for (let i = 0; i < n; i += 1) {
|
|
1183
|
+
const base = i * dims;
|
|
1184
|
+
let dot = 0;
|
|
1185
|
+
let d = 0;
|
|
1186
|
+
for (; d + 3 < dims; d += 4) {
|
|
1187
|
+
dot += queryNormalized[d] * data[base + d] + queryNormalized[d + 1] * data[base + d + 1] + queryNormalized[d + 2] * data[base + d + 2] + queryNormalized[d + 3] * data[base + d + 3];
|
|
1188
|
+
}
|
|
1189
|
+
for (; d < dims; d += 1) {
|
|
1190
|
+
dot += queryNormalized[d] * data[base + d];
|
|
1191
|
+
}
|
|
1192
|
+
const distance = 1 - dot;
|
|
1193
|
+
if (heapSize < k) {
|
|
1194
|
+
heapDist[heapSize] = distance;
|
|
1195
|
+
heapRow[heapSize] = rowids[i];
|
|
1196
|
+
heapSize += 1;
|
|
1197
|
+
if (heapSize === k) {
|
|
1198
|
+
buildMaxHeap(heapDist, heapRow, k);
|
|
1199
|
+
}
|
|
1200
|
+
} else if (distance < heapDist[0]) {
|
|
1201
|
+
heapDist[0] = distance;
|
|
1202
|
+
heapRow[0] = rowids[i];
|
|
1203
|
+
siftDown(heapDist, heapRow, 0, k);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
const hits = new Array(heapSize);
|
|
1207
|
+
for (let i = 0; i < heapSize; i += 1) {
|
|
1208
|
+
hits[i] = { memoryRowid: heapRow[i], distance: heapDist[i] };
|
|
1209
|
+
}
|
|
1210
|
+
hits.sort((a, b) => a.distance - b.distance);
|
|
1211
|
+
return hits;
|
|
1212
|
+
}
|
|
1213
|
+
ensureCapacity(needed) {
|
|
1214
|
+
if (needed * this.dims <= this.data.length) {
|
|
1215
|
+
return;
|
|
1216
|
+
}
|
|
1217
|
+
let cap = this.data.length / this.dims;
|
|
1218
|
+
if (cap < 1) cap = 1;
|
|
1219
|
+
while (cap < needed) cap *= 2;
|
|
1220
|
+
const next = new Float32Array(cap * this.dims);
|
|
1221
|
+
next.set(this.data.subarray(0, this.count * this.dims));
|
|
1222
|
+
this.data = next;
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
function normalizeEmbedding(embedding, dims = embedding.length) {
|
|
1226
|
+
return normalizeInto(embedding, dims);
|
|
1227
|
+
}
|
|
1228
|
+
function normalizeInto(embedding, dims) {
|
|
1229
|
+
const out = new Float32Array(dims);
|
|
1230
|
+
const len = embedding.length < dims ? embedding.length : dims;
|
|
1231
|
+
let norm = 0;
|
|
1232
|
+
for (let i = 0; i < len; i += 1) {
|
|
1233
|
+
const v = embedding[i];
|
|
1234
|
+
out[i] = v;
|
|
1235
|
+
norm += v * v;
|
|
1236
|
+
}
|
|
1237
|
+
if (norm === 0) {
|
|
1238
|
+
return out;
|
|
1239
|
+
}
|
|
1240
|
+
const inv = 1 / Math.sqrt(norm);
|
|
1241
|
+
for (let i = 0; i < len; i += 1) {
|
|
1242
|
+
out[i] *= inv;
|
|
1243
|
+
}
|
|
1244
|
+
return out;
|
|
1245
|
+
}
|
|
1246
|
+
function buildMaxHeap(dist, rows, n) {
|
|
1247
|
+
for (let i = (n >> 1) - 1; i >= 0; i -= 1) {
|
|
1248
|
+
siftDown(dist, rows, i, n);
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
function siftDown(dist, rows, i, n) {
|
|
1252
|
+
for (; ; ) {
|
|
1253
|
+
let largest = i;
|
|
1254
|
+
const left = (i << 1) + 1;
|
|
1255
|
+
const right = left + 1;
|
|
1256
|
+
if (left < n && dist[left] > dist[largest]) largest = left;
|
|
1257
|
+
if (right < n && dist[right] > dist[largest]) largest = right;
|
|
1258
|
+
if (largest === i) return;
|
|
1259
|
+
const td = dist[i];
|
|
1260
|
+
dist[i] = dist[largest];
|
|
1261
|
+
dist[largest] = td;
|
|
1262
|
+
const tr = rows[i];
|
|
1263
|
+
rows[i] = rows[largest];
|
|
1264
|
+
rows[largest] = tr;
|
|
1265
|
+
i = largest;
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// src/storage/providers/sqlite.ts
|
|
1270
|
+
var SqliteStorageProvider = class {
|
|
1271
|
+
name = "sqlite";
|
|
1272
|
+
connectionString;
|
|
1273
|
+
db = null;
|
|
1274
|
+
statements = null;
|
|
1275
|
+
vectorDimensions = null;
|
|
1276
|
+
vectorBackend = null;
|
|
1277
|
+
sqliteVecLoaded = false;
|
|
1278
|
+
/** Hot in-process ANN for blob backend (sqlite-vec unavailable platforms). */
|
|
1279
|
+
memoryIndex = null;
|
|
1280
|
+
memoryIndexDirty = false;
|
|
1281
|
+
/** Resolved absolute path (or `:memory:`) — avoid re-resolving on size checks. */
|
|
1282
|
+
resolvedPath = null;
|
|
1283
|
+
constructor(options) {
|
|
1284
|
+
this.connectionString = options.connectionString;
|
|
1285
|
+
}
|
|
1286
|
+
async open() {
|
|
1287
|
+
try {
|
|
1288
|
+
const dbPath = this.resolvePath(this.connectionString);
|
|
1289
|
+
this.resolvedPath = dbPath;
|
|
1290
|
+
if (dbPath !== ":memory:") {
|
|
1291
|
+
fs2__default.default.mkdirSync(path2__default.default.dirname(dbPath), { recursive: true });
|
|
1292
|
+
}
|
|
1293
|
+
const db = new sqlite$1.DatabaseSync(dbPath, { allowExtension: true });
|
|
1294
|
+
this.db = db;
|
|
1295
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
1296
|
+
db.exec(`
|
|
1297
|
+
PRAGMA synchronous = NORMAL;
|
|
1298
|
+
PRAGMA foreign_keys = ON;
|
|
1299
|
+
PRAGMA busy_timeout = 5000;
|
|
1300
|
+
PRAGMA temp_store = MEMORY;
|
|
1301
|
+
PRAGMA cache_size = -65536;
|
|
1302
|
+
PRAGMA mmap_size = 268435456;
|
|
1303
|
+
PRAGMA wal_autocheckpoint = 1000;
|
|
1304
|
+
PRAGMA recursive_triggers = OFF;
|
|
1305
|
+
`);
|
|
1306
|
+
this.sqliteVecLoaded = this.tryLoadSqliteVec(db);
|
|
1307
|
+
this.runMigrations(db);
|
|
1308
|
+
this.statements = this.prepareStatements(db);
|
|
1309
|
+
this.ensureFtsConsistency(db);
|
|
1310
|
+
const backend = this.readMetaString(META_KEYS.vectorBackend);
|
|
1311
|
+
const dims = this.readMetaNumber(META_KEYS.embeddingDimensions);
|
|
1312
|
+
if (backend) {
|
|
1313
|
+
this.vectorBackend = backend;
|
|
1314
|
+
} else if (this.sqliteVecLoaded) {
|
|
1315
|
+
this.vectorBackend = "sqlite-vec";
|
|
1316
|
+
} else {
|
|
1317
|
+
this.vectorBackend = "blob";
|
|
1318
|
+
}
|
|
1319
|
+
if (dims !== null) {
|
|
1320
|
+
this.vectorDimensions = dims;
|
|
1321
|
+
this.ensureVectorStorage(dims);
|
|
1322
|
+
this.reprepareVectorStatements();
|
|
1323
|
+
if (this.vectorBackend === "blob") {
|
|
1324
|
+
this.memoryIndexDirty = true;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
} catch (error) {
|
|
1328
|
+
try {
|
|
1329
|
+
this.db?.close();
|
|
1330
|
+
} catch {
|
|
1331
|
+
}
|
|
1332
|
+
this.db = null;
|
|
1333
|
+
this.statements = null;
|
|
1334
|
+
throw new InitializationError(
|
|
1335
|
+
`Failed to open SQLite database: ${this.describe(error)}`,
|
|
1336
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
1337
|
+
);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
async close() {
|
|
1341
|
+
if (!this.db) {
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
try {
|
|
1345
|
+
this.db.close();
|
|
1346
|
+
} catch (error) {
|
|
1347
|
+
throw new DatabaseError(`Failed to close database: ${this.describe(error)}`, {
|
|
1348
|
+
cause: error instanceof Error ? error : void 0
|
|
1349
|
+
});
|
|
1350
|
+
} finally {
|
|
1351
|
+
this.db = null;
|
|
1352
|
+
this.statements = null;
|
|
1353
|
+
this.memoryIndex = null;
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
async ensureVectorSchema(dimensions) {
|
|
1357
|
+
const existing = await this.getEmbeddingDimensions();
|
|
1358
|
+
if (existing !== null && existing !== dimensions) {
|
|
1359
|
+
throw new InitializationError(
|
|
1360
|
+
`Embedding dimensions mismatch: database is configured for ${existing}-d vectors, but the embedding model returned ${dimensions}-d vectors.`
|
|
1361
|
+
);
|
|
1362
|
+
}
|
|
1363
|
+
if (!this.vectorBackend) {
|
|
1364
|
+
this.vectorBackend = this.sqliteVecLoaded ? "sqlite-vec" : "blob";
|
|
1365
|
+
}
|
|
1366
|
+
if (this.sqliteVecLoaded && this.vectorBackend !== "blob") {
|
|
1367
|
+
this.vectorBackend = "sqlite-vec";
|
|
1368
|
+
}
|
|
1369
|
+
this.ensureVectorStorage(dimensions);
|
|
1370
|
+
await this.setMeta(META_KEYS.vectorBackend, this.vectorBackend);
|
|
1371
|
+
if (existing === null) {
|
|
1372
|
+
await this.setEmbeddingDimensions(dimensions);
|
|
1373
|
+
}
|
|
1374
|
+
this.vectorDimensions = dimensions;
|
|
1375
|
+
this.reprepareVectorStatements();
|
|
1376
|
+
this.hydrateMemoryIndex();
|
|
1377
|
+
}
|
|
1378
|
+
async getEmbeddingDimensions() {
|
|
1379
|
+
return this.readMetaNumber(META_KEYS.embeddingDimensions);
|
|
1380
|
+
}
|
|
1381
|
+
async setEmbeddingDimensions(dimensions) {
|
|
1382
|
+
await this.setMeta(META_KEYS.embeddingDimensions, String(dimensions));
|
|
1383
|
+
this.vectorDimensions = dimensions;
|
|
1384
|
+
}
|
|
1385
|
+
async insertMemory(input) {
|
|
1386
|
+
const stmts = this.requireStatements();
|
|
1387
|
+
this.requireVectorReady();
|
|
1388
|
+
return this.withTransaction(() => {
|
|
1389
|
+
const row = stmts.insertMemory.get(
|
|
1390
|
+
input.id,
|
|
1391
|
+
input.organization,
|
|
1392
|
+
input.agent,
|
|
1393
|
+
input.contentText,
|
|
1394
|
+
serializeMetadata(input.metadata),
|
|
1395
|
+
input.createdAt,
|
|
1396
|
+
input.updatedAt
|
|
1397
|
+
);
|
|
1398
|
+
if (!row || row.rowid === void 0) {
|
|
1399
|
+
throw new DatabaseError("Failed to read memory after insert");
|
|
1400
|
+
}
|
|
1401
|
+
this.insertEmbedding(row.rowid, input.embedding);
|
|
1402
|
+
this.insertFtsRow(
|
|
1403
|
+
input.id,
|
|
1404
|
+
input.organization,
|
|
1405
|
+
input.agent,
|
|
1406
|
+
input.contentText
|
|
1407
|
+
);
|
|
1408
|
+
stmts.insertHistory.run(
|
|
1409
|
+
crypto.randomUUID(),
|
|
1410
|
+
input.id,
|
|
1411
|
+
"created",
|
|
1412
|
+
null,
|
|
1413
|
+
input.createdAt
|
|
1414
|
+
);
|
|
1415
|
+
return row;
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
async insertMemoriesBatch(inputs) {
|
|
1419
|
+
if (inputs.length === 0) {
|
|
1420
|
+
return [];
|
|
1421
|
+
}
|
|
1422
|
+
const stmts = this.requireStatements();
|
|
1423
|
+
this.requireVectorReady();
|
|
1424
|
+
return this.withTransaction(() => {
|
|
1425
|
+
const rows = [];
|
|
1426
|
+
for (const input of inputs) {
|
|
1427
|
+
const row = stmts.insertMemory.get(
|
|
1428
|
+
input.id,
|
|
1429
|
+
input.organization,
|
|
1430
|
+
input.agent,
|
|
1431
|
+
input.contentText,
|
|
1432
|
+
serializeMetadata(input.metadata),
|
|
1433
|
+
input.createdAt,
|
|
1434
|
+
input.updatedAt
|
|
1435
|
+
);
|
|
1436
|
+
if (!row || row.rowid === void 0) {
|
|
1437
|
+
throw new DatabaseError("Failed to read memory after batch insert");
|
|
1438
|
+
}
|
|
1439
|
+
this.insertEmbedding(row.rowid, input.embedding);
|
|
1440
|
+
this.insertFtsRow(
|
|
1441
|
+
input.id,
|
|
1442
|
+
input.organization,
|
|
1443
|
+
input.agent,
|
|
1444
|
+
input.contentText
|
|
1445
|
+
);
|
|
1446
|
+
stmts.insertHistory.run(
|
|
1447
|
+
crypto.randomUUID(),
|
|
1448
|
+
input.id,
|
|
1449
|
+
"created",
|
|
1450
|
+
null,
|
|
1451
|
+
input.createdAt
|
|
1452
|
+
);
|
|
1453
|
+
rows.push(row);
|
|
1454
|
+
}
|
|
1455
|
+
return rows;
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
async updateMemory(input) {
|
|
1459
|
+
const stmts = this.requireStatements();
|
|
1460
|
+
return this.withTransaction(() => {
|
|
1461
|
+
const existing = stmts.getMemoryById.get(
|
|
1462
|
+
input.id,
|
|
1463
|
+
input.organization
|
|
1464
|
+
);
|
|
1465
|
+
if (!existing || existing.rowid === void 0) {
|
|
1466
|
+
return null;
|
|
1467
|
+
}
|
|
1468
|
+
const contentText = input.contentText ?? existing.content_text;
|
|
1469
|
+
const metadataJson = input.metadata !== void 0 ? serializeMetadata(input.metadata) : existing.metadata_json;
|
|
1470
|
+
stmts.updateMemoryContent.run(
|
|
1471
|
+
input.contentText ?? null,
|
|
1472
|
+
input.metadata !== void 0 ? metadataJson : null,
|
|
1473
|
+
input.updatedAt,
|
|
1474
|
+
input.id,
|
|
1475
|
+
input.organization
|
|
1476
|
+
);
|
|
1477
|
+
if (input.embedding) {
|
|
1478
|
+
this.deleteEmbedding(existing.rowid);
|
|
1479
|
+
this.insertEmbedding(existing.rowid, input.embedding);
|
|
1480
|
+
}
|
|
1481
|
+
if (input.contentText !== void 0) {
|
|
1482
|
+
this.upsertFts(
|
|
1483
|
+
input.id,
|
|
1484
|
+
input.organization,
|
|
1485
|
+
existing.agent,
|
|
1486
|
+
contentText
|
|
1487
|
+
);
|
|
1488
|
+
}
|
|
1489
|
+
return stmts.getMemoryById.get(input.id, input.organization);
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
async searchByMetadata(filter, limit) {
|
|
1493
|
+
const rows = await this.listMemories(filter, limit);
|
|
1494
|
+
if (!filter.metadata) {
|
|
1495
|
+
return rows;
|
|
1496
|
+
}
|
|
1497
|
+
return rows.filter(
|
|
1498
|
+
(row) => matchesMetadata(deserializeMetadata(row.metadata_json), filter.metadata)
|
|
1499
|
+
);
|
|
1500
|
+
}
|
|
1501
|
+
/** Keyword search via FTS5 BM25. Returns memory IDs ranked by relevance. */
|
|
1502
|
+
async searchKeyword(query, organization, topK) {
|
|
1503
|
+
const stmts = this.requireStatements();
|
|
1504
|
+
if (!stmts.searchFts) {
|
|
1505
|
+
return [];
|
|
1506
|
+
}
|
|
1507
|
+
try {
|
|
1508
|
+
const sanitized = sanitizeFtsQuery(query);
|
|
1509
|
+
if (!sanitized) {
|
|
1510
|
+
return [];
|
|
1511
|
+
}
|
|
1512
|
+
const rows = stmts.searchFts.all(sanitized, organization, topK);
|
|
1513
|
+
return rows.map((row) => ({
|
|
1514
|
+
memoryId: row.memory_id,
|
|
1515
|
+
// bm25 returns lower (more negative) for better matches — invert to [0, ∞)
|
|
1516
|
+
score: 1 / (1 + Math.abs(row.rank))
|
|
1517
|
+
}));
|
|
1518
|
+
} catch {
|
|
1519
|
+
return [];
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
async getMemoryById(id, organization) {
|
|
1523
|
+
const stmts = this.requireStatements();
|
|
1524
|
+
const row = stmts.getMemoryById.get(id, organization);
|
|
1525
|
+
return row ?? null;
|
|
1526
|
+
}
|
|
1527
|
+
async getMemoryByRowid(rowid, organization) {
|
|
1528
|
+
const stmts = this.requireStatements();
|
|
1529
|
+
const row = stmts.getMemoryByRowid.get(rowid, organization);
|
|
1530
|
+
return row ?? null;
|
|
1531
|
+
}
|
|
1532
|
+
async getMemoriesByRowids(rowids, organization) {
|
|
1533
|
+
const out = /* @__PURE__ */ new Map();
|
|
1534
|
+
if (rowids.length === 0) {
|
|
1535
|
+
return out;
|
|
1536
|
+
}
|
|
1537
|
+
const db = this.requireDb();
|
|
1538
|
+
const chunkSize = 400;
|
|
1539
|
+
for (let offset = 0; offset < rowids.length; offset += chunkSize) {
|
|
1540
|
+
const chunk = rowids.slice(offset, offset + chunkSize);
|
|
1541
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1542
|
+
const sql = `${SQL.getMemoriesByRowidsPrefix}${placeholders})`;
|
|
1543
|
+
const rows = db.prepare(sql).all(organization, ...chunk);
|
|
1544
|
+
for (const row of rows) {
|
|
1545
|
+
if (row.rowid !== void 0) {
|
|
1546
|
+
out.set(row.rowid, row);
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
return out;
|
|
1551
|
+
}
|
|
1552
|
+
async listMemories(filter, limit) {
|
|
1553
|
+
const db = this.requireDb();
|
|
1554
|
+
const clauses = [`organization = ?`];
|
|
1555
|
+
const params = [filter.organization];
|
|
1556
|
+
if (filter.agent) {
|
|
1557
|
+
clauses.push(`agent = ?`);
|
|
1558
|
+
params.push(filter.agent);
|
|
1559
|
+
}
|
|
1560
|
+
if (!filter.includeArchived) {
|
|
1561
|
+
clauses.push(`archived = 0`);
|
|
1562
|
+
}
|
|
1563
|
+
let sql = `
|
|
1564
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
1565
|
+
archived, compressed_into, created_at, updated_at
|
|
1566
|
+
FROM memories
|
|
1567
|
+
WHERE ${clauses.join(" AND ")}
|
|
1568
|
+
ORDER BY created_at ASC
|
|
1569
|
+
`;
|
|
1570
|
+
if (limit !== void 0 && !filter.metadata) {
|
|
1571
|
+
sql += ` LIMIT ?`;
|
|
1572
|
+
params.push(limit);
|
|
1573
|
+
}
|
|
1574
|
+
try {
|
|
1575
|
+
let rows = db.prepare(sql).all(...params);
|
|
1576
|
+
if (filter.metadata) {
|
|
1577
|
+
rows = rows.filter(
|
|
1578
|
+
(row) => matchesMetadata(
|
|
1579
|
+
deserializeMetadata(row.metadata_json),
|
|
1580
|
+
filter.metadata
|
|
1581
|
+
)
|
|
1582
|
+
);
|
|
1583
|
+
if (limit !== void 0) {
|
|
1584
|
+
rows = rows.slice(0, limit);
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
return rows;
|
|
1588
|
+
} catch (error) {
|
|
1589
|
+
throw new DatabaseError(`Failed to list memories: ${this.describe(error)}`, {
|
|
1590
|
+
cause: error instanceof Error ? error : void 0
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
async searchVectors(embedding, topK) {
|
|
1595
|
+
this.requireVectorReady();
|
|
1596
|
+
if (this.vectorBackend === "sqlite-vec") {
|
|
1597
|
+
return this.searchWithSqliteVec(embedding, topK);
|
|
1598
|
+
}
|
|
1599
|
+
return this.searchWithBlobFallback(embedding, topK);
|
|
1600
|
+
}
|
|
1601
|
+
/**
|
|
1602
|
+
* Org-scoped KNN + memory rows.
|
|
1603
|
+
* Caller (Wolbarg.recall) already passes an overfetched topK — do not multiply again.
|
|
1604
|
+
*/
|
|
1605
|
+
async searchVectorsWithMemories(embedding, topK, organization, options) {
|
|
1606
|
+
this.requireVectorReady();
|
|
1607
|
+
const hits = await this.searchVectors(embedding, topK);
|
|
1608
|
+
if (hits.length === 0) {
|
|
1609
|
+
return [];
|
|
1610
|
+
}
|
|
1611
|
+
const map = await this.getMemoriesByRowids(
|
|
1612
|
+
hits.map((h) => h.memoryRowid),
|
|
1613
|
+
organization
|
|
1614
|
+
);
|
|
1615
|
+
const out = [];
|
|
1616
|
+
for (const hit of hits) {
|
|
1617
|
+
const row = map.get(hit.memoryRowid);
|
|
1618
|
+
if (!row) continue;
|
|
1619
|
+
if (options?.agent && row.agent !== options.agent) continue;
|
|
1620
|
+
if (!options?.includeArchived && row.archived === 1) continue;
|
|
1621
|
+
out.push({ row, distance: hit.distance });
|
|
1622
|
+
}
|
|
1623
|
+
return out;
|
|
1624
|
+
}
|
|
1625
|
+
async archiveMemories(ids, organization, compressedIntoId, archivedAt) {
|
|
1626
|
+
const stmts = this.requireStatements();
|
|
1627
|
+
const archived = [];
|
|
1628
|
+
return this.withTransaction(() => {
|
|
1629
|
+
for (const id of ids) {
|
|
1630
|
+
const result = stmts.archiveMemory.run(
|
|
1631
|
+
compressedIntoId,
|
|
1632
|
+
archivedAt,
|
|
1633
|
+
id,
|
|
1634
|
+
organization
|
|
1635
|
+
);
|
|
1636
|
+
if (Number(result.changes) > 0) {
|
|
1637
|
+
archived.push(id);
|
|
1638
|
+
this.deleteFts(id);
|
|
1639
|
+
stmts.insertHistory.run(
|
|
1640
|
+
crypto.randomUUID(),
|
|
1641
|
+
id,
|
|
1642
|
+
"archived",
|
|
1643
|
+
compressedIntoId,
|
|
1644
|
+
archivedAt
|
|
1645
|
+
);
|
|
1646
|
+
stmts.insertHistory.run(
|
|
1647
|
+
crypto.randomUUID(),
|
|
1648
|
+
compressedIntoId,
|
|
1649
|
+
"compressed",
|
|
1650
|
+
id,
|
|
1651
|
+
archivedAt
|
|
1652
|
+
);
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
return archived;
|
|
1656
|
+
});
|
|
1657
|
+
}
|
|
1658
|
+
async deleteMemoryById(id, organization) {
|
|
1659
|
+
const stmts = this.requireStatements();
|
|
1660
|
+
return this.withTransaction(() => {
|
|
1661
|
+
const row = stmts.getMemoryById.get(id, organization);
|
|
1662
|
+
if (!row || row.rowid === void 0) {
|
|
1663
|
+
return false;
|
|
1664
|
+
}
|
|
1665
|
+
this.deleteEmbedding(row.rowid);
|
|
1666
|
+
this.deleteFts(id);
|
|
1667
|
+
const result = stmts.deleteMemoryById.run(id, organization);
|
|
1668
|
+
return Number(result.changes) > 0;
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
async deleteMemoriesByFilter(filter) {
|
|
1672
|
+
const stmts = this.requireStatements();
|
|
1673
|
+
const agent = filter.agent;
|
|
1674
|
+
if (!agent) {
|
|
1675
|
+
throw new DatabaseError("deleteMemoriesByFilter requires an agent filter");
|
|
1676
|
+
}
|
|
1677
|
+
return this.withTransaction(() => {
|
|
1678
|
+
const listed = this.listMemoriesSync({
|
|
1679
|
+
organization: filter.organization,
|
|
1680
|
+
agent,
|
|
1681
|
+
includeArchived: true
|
|
1682
|
+
});
|
|
1683
|
+
for (const memory of listed) {
|
|
1684
|
+
if (memory.rowid !== void 0) {
|
|
1685
|
+
this.deleteEmbedding(memory.rowid);
|
|
1686
|
+
}
|
|
1687
|
+
this.deleteFts(memory.id);
|
|
1688
|
+
}
|
|
1689
|
+
const result = stmts.deleteMemoriesByOrgAgent.run(
|
|
1690
|
+
filter.organization,
|
|
1691
|
+
agent
|
|
1692
|
+
);
|
|
1693
|
+
return Number(result.changes);
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
async clearOrganization(organization) {
|
|
1697
|
+
const stmts = this.requireStatements();
|
|
1698
|
+
return this.withTransaction(() => {
|
|
1699
|
+
const listed = this.listMemoriesSync({ organization, includeArchived: true });
|
|
1700
|
+
for (const memory of listed) {
|
|
1701
|
+
if (memory.rowid !== void 0) {
|
|
1702
|
+
this.deleteEmbedding(memory.rowid);
|
|
1703
|
+
}
|
|
1704
|
+
this.deleteFts(memory.id);
|
|
1705
|
+
}
|
|
1706
|
+
const result = stmts.deleteMemoriesByOrg.run(organization);
|
|
1707
|
+
return Number(result.changes);
|
|
1708
|
+
});
|
|
1709
|
+
}
|
|
1710
|
+
async getHistory(memoryId) {
|
|
1711
|
+
const stmts = this.requireStatements();
|
|
1712
|
+
return stmts.getHistory.all(memoryId);
|
|
1713
|
+
}
|
|
1714
|
+
async insertHistoryEvent(event) {
|
|
1715
|
+
const stmts = this.requireStatements();
|
|
1716
|
+
stmts.insertHistory.run(
|
|
1717
|
+
event.id,
|
|
1718
|
+
event.memory_id,
|
|
1719
|
+
event.event_type,
|
|
1720
|
+
event.related_memory_id,
|
|
1721
|
+
event.created_at
|
|
1722
|
+
);
|
|
1723
|
+
}
|
|
1724
|
+
async getStats(organization) {
|
|
1725
|
+
const stmts = this.requireStatements();
|
|
1726
|
+
const memories = stmts.countMemories.get(organization);
|
|
1727
|
+
const active = stmts.countActiveMemories.get(organization);
|
|
1728
|
+
const archived = stmts.countArchivedMemories.get(organization);
|
|
1729
|
+
const agents = stmts.countAgents.get(organization);
|
|
1730
|
+
return {
|
|
1731
|
+
totalMemories: Number(memories.count),
|
|
1732
|
+
activeMemories: Number(active.count),
|
|
1733
|
+
archivedMemories: Number(archived.count),
|
|
1734
|
+
totalAgents: Number(agents.count)
|
|
1735
|
+
};
|
|
1736
|
+
}
|
|
1737
|
+
async getDatabaseSizeBytes() {
|
|
1738
|
+
const db = this.requireDb();
|
|
1739
|
+
if (this.connectionString === ":memory:") {
|
|
1740
|
+
const pageCountRow = db.prepare("PRAGMA page_count").get();
|
|
1741
|
+
const pageSizeRow = db.prepare("PRAGMA page_size").get();
|
|
1742
|
+
const pageCount = Number(
|
|
1743
|
+
pageCountRow?.page_count ?? Object.values(pageCountRow ?? {})[0] ?? 0
|
|
1744
|
+
);
|
|
1745
|
+
const pageSize = Number(
|
|
1746
|
+
pageSizeRow?.page_size ?? Object.values(pageSizeRow ?? {})[0] ?? 0
|
|
1747
|
+
);
|
|
1748
|
+
return pageCount * pageSize;
|
|
1749
|
+
}
|
|
1750
|
+
const dbPath = this.resolvedPath ?? this.resolvePath(this.connectionString);
|
|
1751
|
+
try {
|
|
1752
|
+
let total = fs2__default.default.statSync(dbPath).size;
|
|
1753
|
+
for (const suffix of ["-wal", "-shm"]) {
|
|
1754
|
+
const side = `${dbPath}${suffix}`;
|
|
1755
|
+
if (fs2__default.default.existsSync(side)) {
|
|
1756
|
+
total += fs2__default.default.statSync(side).size;
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
return total;
|
|
1760
|
+
} catch (error) {
|
|
1761
|
+
throw new DatabaseError(
|
|
1762
|
+
`Failed to determine database size: ${this.describe(error)}`,
|
|
1763
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
1764
|
+
);
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
withTransaction(fn) {
|
|
1768
|
+
const db = this.requireDb();
|
|
1769
|
+
db.exec("BEGIN");
|
|
1770
|
+
try {
|
|
1771
|
+
const result = fn();
|
|
1772
|
+
db.exec("COMMIT");
|
|
1773
|
+
return result;
|
|
1774
|
+
} catch (error) {
|
|
1775
|
+
try {
|
|
1776
|
+
db.exec("ROLLBACK");
|
|
1777
|
+
} catch {
|
|
1778
|
+
}
|
|
1779
|
+
if (error instanceof DatabaseError || error instanceof InitializationError) {
|
|
1780
|
+
throw error;
|
|
1781
|
+
}
|
|
1782
|
+
throw new DatabaseError(`Transaction failed: ${this.describe(error)}`, {
|
|
1783
|
+
cause: error instanceof Error ? error : void 0
|
|
1784
|
+
});
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
// ─── internals ───────────────────────────────────────────────────────────
|
|
1788
|
+
tryLoadSqliteVec(db) {
|
|
1789
|
+
try {
|
|
1790
|
+
sqliteVec__namespace.load(db);
|
|
1791
|
+
return true;
|
|
1792
|
+
} catch {
|
|
1793
|
+
return false;
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
runMigrations(db) {
|
|
1797
|
+
db.exec(CREATE_META_TABLE);
|
|
1798
|
+
db.exec(CREATE_MEMORIES_TABLE);
|
|
1799
|
+
db.exec(CREATE_HISTORY_TABLE);
|
|
1800
|
+
db.exec(CREATE_BLOB_EMBEDDINGS_TABLE);
|
|
1801
|
+
for (const indexSql of CREATE_INDEXES) {
|
|
1802
|
+
db.exec(indexSql);
|
|
1803
|
+
}
|
|
1804
|
+
const current = this.readMetaNumberFromDb(db, META_KEYS.schemaVersion);
|
|
1805
|
+
if (current === null) {
|
|
1806
|
+
db.exec(CREATE_FTS_TABLE);
|
|
1807
|
+
this.backfillFts(db);
|
|
1808
|
+
db.prepare(SQL.setMeta).run(META_KEYS.schemaVersion, String(SCHEMA_VERSION));
|
|
1809
|
+
} else if (current > SCHEMA_VERSION) {
|
|
1810
|
+
throw new InitializationError(
|
|
1811
|
+
`Database schema version ${current} is newer than this SDK (supports ${SCHEMA_VERSION}).`
|
|
1812
|
+
);
|
|
1813
|
+
} else if (current < SCHEMA_VERSION) {
|
|
1814
|
+
db.exec(CREATE_FTS_TABLE);
|
|
1815
|
+
this.backfillFts(db);
|
|
1816
|
+
db.prepare(SQL.setMeta).run(META_KEYS.schemaVersion, String(SCHEMA_VERSION));
|
|
1817
|
+
} else {
|
|
1818
|
+
db.exec(CREATE_FTS_TABLE);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
/**
|
|
1822
|
+
* Rebuild FTS from active (non-archived) memories when counts diverge.
|
|
1823
|
+
* Guarantees keyword/hybrid correctness after crashes or interrupted writes.
|
|
1824
|
+
*/
|
|
1825
|
+
ensureFtsConsistency(db) {
|
|
1826
|
+
try {
|
|
1827
|
+
const memCount = db.prepare(`SELECT COUNT(*) AS c FROM memories WHERE archived = 0`).get();
|
|
1828
|
+
const ftsCount = db.prepare(`SELECT COUNT(*) AS c FROM memories_fts`).get();
|
|
1829
|
+
if (Number(memCount.c) !== Number(ftsCount.c)) {
|
|
1830
|
+
this.backfillFts(db);
|
|
1831
|
+
}
|
|
1832
|
+
} catch {
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
backfillFts(db) {
|
|
1836
|
+
try {
|
|
1837
|
+
db.exec(`DELETE FROM memories_fts`);
|
|
1838
|
+
const rows = db.prepare(
|
|
1839
|
+
`SELECT id, organization, agent, content_text FROM memories WHERE archived = 0`
|
|
1840
|
+
).all();
|
|
1841
|
+
const insert = db.prepare(SQL.insertFts);
|
|
1842
|
+
for (const row of rows) {
|
|
1843
|
+
insert.run(row.content_text, row.id, row.organization, row.agent);
|
|
1844
|
+
}
|
|
1845
|
+
} catch {
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
prepareStatements(db) {
|
|
1849
|
+
let insertFts = null;
|
|
1850
|
+
let deleteFts = null;
|
|
1851
|
+
let searchFts = null;
|
|
1852
|
+
try {
|
|
1853
|
+
insertFts = db.prepare(SQL.insertFts);
|
|
1854
|
+
deleteFts = db.prepare(SQL.deleteFts);
|
|
1855
|
+
searchFts = db.prepare(SQL.searchFts);
|
|
1856
|
+
} catch {
|
|
1857
|
+
}
|
|
1858
|
+
return {
|
|
1859
|
+
getMeta: db.prepare(SQL.getMeta),
|
|
1860
|
+
setMeta: db.prepare(SQL.setMeta),
|
|
1861
|
+
insertMemory: db.prepare(SQL.insertMemory),
|
|
1862
|
+
updateMemoryContent: db.prepare(SQL.updateMemoryContent),
|
|
1863
|
+
getMemoryById: db.prepare(SQL.getMemoryById),
|
|
1864
|
+
getMemoryByRowid: db.prepare(SQL.getMemoryByRowid),
|
|
1865
|
+
insertEmbedding: null,
|
|
1866
|
+
deleteEmbedding: null,
|
|
1867
|
+
searchVectors: null,
|
|
1868
|
+
insertEmbeddingBlob: null,
|
|
1869
|
+
deleteEmbeddingBlob: null,
|
|
1870
|
+
listEmbeddingsBlob: null,
|
|
1871
|
+
archiveMemory: db.prepare(SQL.archiveMemory),
|
|
1872
|
+
deleteMemoryById: db.prepare(SQL.deleteMemoryById),
|
|
1873
|
+
deleteMemoriesByOrg: db.prepare(SQL.deleteMemoriesByOrg),
|
|
1874
|
+
deleteMemoriesByOrgAgent: db.prepare(SQL.deleteMemoriesByOrgAgent),
|
|
1875
|
+
insertHistory: db.prepare(SQL.insertHistory),
|
|
1876
|
+
getHistory: db.prepare(SQL.getHistory),
|
|
1877
|
+
countMemories: db.prepare(SQL.countMemories),
|
|
1878
|
+
countActiveMemories: db.prepare(SQL.countActiveMemories),
|
|
1879
|
+
countArchivedMemories: db.prepare(SQL.countArchivedMemories),
|
|
1880
|
+
countAgents: db.prepare(SQL.countAgents),
|
|
1881
|
+
listRowidsForOrg: db.prepare(SQL.listRowidsForOrg),
|
|
1882
|
+
listRowidsForOrgAgent: db.prepare(SQL.listRowidsForOrgAgent),
|
|
1883
|
+
vectorTableExists: db.prepare(SQL.vectorTableExists),
|
|
1884
|
+
blobTableExists: db.prepare(
|
|
1885
|
+
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_embeddings_blob'`
|
|
1886
|
+
),
|
|
1887
|
+
insertFts,
|
|
1888
|
+
deleteFts,
|
|
1889
|
+
searchFts
|
|
1890
|
+
};
|
|
1891
|
+
}
|
|
1892
|
+
listMemoriesSync(filter) {
|
|
1893
|
+
const db = this.requireDb();
|
|
1894
|
+
const clauses = [`organization = ?`];
|
|
1895
|
+
const params = [filter.organization];
|
|
1896
|
+
if (filter.agent) {
|
|
1897
|
+
clauses.push(`agent = ?`);
|
|
1898
|
+
params.push(filter.agent);
|
|
1899
|
+
}
|
|
1900
|
+
if (!filter.includeArchived) {
|
|
1901
|
+
clauses.push(`archived = 0`);
|
|
1902
|
+
}
|
|
1903
|
+
const sql = `
|
|
1904
|
+
SELECT rowid, id, organization, agent, content_text, metadata_json,
|
|
1905
|
+
archived, compressed_into, created_at, updated_at
|
|
1906
|
+
FROM memories
|
|
1907
|
+
WHERE ${clauses.join(" AND ")}
|
|
1908
|
+
`;
|
|
1909
|
+
return db.prepare(sql).all(...params);
|
|
1910
|
+
}
|
|
1911
|
+
/** Insert-only FTS row (new memory IDs never collide). */
|
|
1912
|
+
insertFtsRow(memoryId, organization, agent, contentText) {
|
|
1913
|
+
const stmts = this.requireStatements();
|
|
1914
|
+
if (!stmts.insertFts) {
|
|
1915
|
+
return;
|
|
1916
|
+
}
|
|
1917
|
+
try {
|
|
1918
|
+
stmts.insertFts.run(contentText, memoryId, organization, agent);
|
|
1919
|
+
} catch {
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
upsertFts(memoryId, organization, agent, contentText) {
|
|
1923
|
+
const stmts = this.requireStatements();
|
|
1924
|
+
if (!stmts.insertFts || !stmts.deleteFts) {
|
|
1925
|
+
return;
|
|
1926
|
+
}
|
|
1927
|
+
try {
|
|
1928
|
+
stmts.deleteFts.run(memoryId);
|
|
1929
|
+
stmts.insertFts.run(contentText, memoryId, organization, agent);
|
|
1930
|
+
} catch {
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
deleteFts(memoryId) {
|
|
1934
|
+
const stmts = this.requireStatements();
|
|
1935
|
+
if (!stmts.deleteFts) {
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
try {
|
|
1939
|
+
stmts.deleteFts.run(memoryId);
|
|
1940
|
+
} catch {
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
ensureVectorStorage(dimensions) {
|
|
1944
|
+
const db = this.requireDb();
|
|
1945
|
+
if (this.vectorBackend === "sqlite-vec") {
|
|
1946
|
+
if (!this.sqliteVecLoaded) {
|
|
1947
|
+
throw new InitializationError(
|
|
1948
|
+
"sqlite-vec is required for this database but is unavailable on this platform."
|
|
1949
|
+
);
|
|
1950
|
+
}
|
|
1951
|
+
const stmts = this.requireStatements();
|
|
1952
|
+
const exists = stmts.vectorTableExists.get();
|
|
1953
|
+
if (!exists) {
|
|
1954
|
+
db.exec(buildVectorTableSql(dimensions));
|
|
1955
|
+
}
|
|
1956
|
+
this.memoryIndex = null;
|
|
1957
|
+
return;
|
|
1958
|
+
}
|
|
1959
|
+
db.exec(CREATE_BLOB_EMBEDDINGS_TABLE);
|
|
1960
|
+
if (!this.memoryIndex || this.memoryIndex.size === 0) {
|
|
1961
|
+
this.memoryIndex = new InMemoryVectorIndex(dimensions);
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
hydrateMemoryIndex() {
|
|
1965
|
+
if (this.vectorBackend !== "blob" || this.vectorDimensions === null) {
|
|
1966
|
+
this.memoryIndex = null;
|
|
1967
|
+
return;
|
|
1968
|
+
}
|
|
1969
|
+
const stmts = this.requireStatements();
|
|
1970
|
+
if (!stmts.listEmbeddingsBlob) {
|
|
1971
|
+
stmts.listEmbeddingsBlob = this.requireDb().prepare(SQL.listEmbeddingsBlob);
|
|
1972
|
+
}
|
|
1973
|
+
const index = new InMemoryVectorIndex(this.vectorDimensions);
|
|
1974
|
+
try {
|
|
1975
|
+
const rows = stmts.listEmbeddingsBlob.all();
|
|
1976
|
+
for (const row of rows) {
|
|
1977
|
+
index.upsert(row.memory_rowid, bufferToEmbedding(row.embedding));
|
|
1978
|
+
}
|
|
1979
|
+
} catch {
|
|
1980
|
+
}
|
|
1981
|
+
this.memoryIndex = index;
|
|
1982
|
+
}
|
|
1983
|
+
reprepareVectorStatements() {
|
|
1984
|
+
const db = this.requireDb();
|
|
1985
|
+
const stmts = this.requireStatements();
|
|
1986
|
+
if (this.vectorBackend === "sqlite-vec") {
|
|
1987
|
+
stmts.insertEmbedding = db.prepare(SQL.insertEmbedding);
|
|
1988
|
+
stmts.deleteEmbedding = db.prepare(SQL.deleteEmbedding);
|
|
1989
|
+
stmts.searchVectors = db.prepare(SQL.searchVectors);
|
|
1990
|
+
stmts.insertEmbeddingBlob = null;
|
|
1991
|
+
stmts.deleteEmbeddingBlob = null;
|
|
1992
|
+
stmts.listEmbeddingsBlob = null;
|
|
1993
|
+
return;
|
|
1994
|
+
}
|
|
1995
|
+
stmts.insertEmbeddingBlob = db.prepare(SQL.insertEmbeddingBlob);
|
|
1996
|
+
stmts.deleteEmbeddingBlob = db.prepare(SQL.deleteEmbeddingBlob);
|
|
1997
|
+
stmts.listEmbeddingsBlob = db.prepare(SQL.listEmbeddingsBlob);
|
|
1998
|
+
stmts.insertEmbedding = null;
|
|
1999
|
+
stmts.deleteEmbedding = null;
|
|
2000
|
+
stmts.searchVectors = null;
|
|
2001
|
+
}
|
|
2002
|
+
insertEmbedding(rowid, embedding) {
|
|
2003
|
+
const stmts = this.requireStatements();
|
|
2004
|
+
if (this.vectorBackend === "sqlite-vec") {
|
|
2005
|
+
stmts.insertEmbedding.run(rowid, this.toVectorParam(embedding));
|
|
2006
|
+
return;
|
|
2007
|
+
}
|
|
2008
|
+
stmts.insertEmbeddingBlob.run(rowid, embeddingToBuffer(embedding));
|
|
2009
|
+
this.memoryIndexDirty = true;
|
|
2010
|
+
}
|
|
2011
|
+
deleteEmbedding(rowid) {
|
|
2012
|
+
const stmts = this.requireStatements();
|
|
2013
|
+
try {
|
|
2014
|
+
if (this.vectorBackend === "sqlite-vec" && stmts.deleteEmbedding) {
|
|
2015
|
+
stmts.deleteEmbedding.run(rowid);
|
|
2016
|
+
} else if (stmts.deleteEmbeddingBlob) {
|
|
2017
|
+
stmts.deleteEmbeddingBlob.run(rowid);
|
|
2018
|
+
this.memoryIndex?.remove(rowid);
|
|
2019
|
+
this.memoryIndexDirty = true;
|
|
2020
|
+
}
|
|
2021
|
+
} catch {
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
searchWithSqliteVec(embedding, topK) {
|
|
2025
|
+
const stmts = this.requireStatements();
|
|
2026
|
+
try {
|
|
2027
|
+
const rows = stmts.searchVectors.all(
|
|
2028
|
+
this.toVectorParam(embedding),
|
|
2029
|
+
topK
|
|
2030
|
+
);
|
|
2031
|
+
const hits = new Array(rows.length);
|
|
2032
|
+
for (let i = 0; i < rows.length; i += 1) {
|
|
2033
|
+
const row = rows[i];
|
|
2034
|
+
hits[i] = {
|
|
2035
|
+
memoryRowid: row.memory_rowid,
|
|
2036
|
+
distance: row.distance
|
|
2037
|
+
};
|
|
2038
|
+
}
|
|
2039
|
+
return hits;
|
|
2040
|
+
} catch (error) {
|
|
2041
|
+
throw new DatabaseError(`Vector search failed: ${this.describe(error)}`, {
|
|
2042
|
+
cause: error instanceof Error ? error : void 0
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
searchWithBlobFallback(embedding, topK) {
|
|
2047
|
+
if (this.memoryIndexDirty || !this.memoryIndex) {
|
|
2048
|
+
this.hydrateMemoryIndex();
|
|
2049
|
+
this.memoryIndexDirty = false;
|
|
2050
|
+
}
|
|
2051
|
+
if (this.memoryIndex && this.vectorDimensions !== null) {
|
|
2052
|
+
const query = normalizeEmbedding(embedding, this.vectorDimensions);
|
|
2053
|
+
return this.memoryIndex.search(query, topK);
|
|
2054
|
+
}
|
|
2055
|
+
throw new DatabaseError("Blob vector index is not initialized");
|
|
2056
|
+
}
|
|
2057
|
+
async setMeta(key, value) {
|
|
2058
|
+
const stmts = this.requireStatements();
|
|
2059
|
+
stmts.setMeta.run(key, value);
|
|
2060
|
+
}
|
|
2061
|
+
readMetaNumber(key) {
|
|
2062
|
+
const stmts = this.requireStatements();
|
|
2063
|
+
const row = stmts.getMeta.get(key);
|
|
2064
|
+
if (!row) {
|
|
2065
|
+
return null;
|
|
2066
|
+
}
|
|
2067
|
+
const parsed = Number(row.value);
|
|
2068
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
2069
|
+
}
|
|
2070
|
+
readMetaString(key) {
|
|
2071
|
+
const stmts = this.requireStatements();
|
|
2072
|
+
const row = stmts.getMeta.get(key);
|
|
2073
|
+
return row?.value ?? null;
|
|
2074
|
+
}
|
|
2075
|
+
readMetaNumberFromDb(db, key) {
|
|
2076
|
+
const row = db.prepare(SQL.getMeta).get(key);
|
|
2077
|
+
if (!row) {
|
|
2078
|
+
return null;
|
|
2079
|
+
}
|
|
2080
|
+
const parsed = Number(row.value);
|
|
2081
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
2082
|
+
}
|
|
2083
|
+
resolvePath(connectionString) {
|
|
2084
|
+
if (connectionString === ":memory:") {
|
|
2085
|
+
return ":memory:";
|
|
2086
|
+
}
|
|
2087
|
+
return path2__default.default.isAbsolute(connectionString) ? connectionString : path2__default.default.resolve(process.cwd(), connectionString);
|
|
2088
|
+
}
|
|
2089
|
+
requireDb() {
|
|
2090
|
+
if (!this.db) {
|
|
2091
|
+
throw new DatabaseError("Database is not open. Call init() first.");
|
|
2092
|
+
}
|
|
2093
|
+
return this.db;
|
|
2094
|
+
}
|
|
2095
|
+
requireStatements() {
|
|
2096
|
+
if (!this.statements) {
|
|
2097
|
+
throw new DatabaseError("Database statements are not prepared. Call init() first.");
|
|
2098
|
+
}
|
|
2099
|
+
return this.statements;
|
|
2100
|
+
}
|
|
2101
|
+
requireVectorReady() {
|
|
2102
|
+
if (!this.vectorBackend || this.vectorDimensions === null) {
|
|
2103
|
+
throw new DatabaseError(
|
|
2104
|
+
"Vector index is not ready. Embedding dimensions have not been initialized."
|
|
2105
|
+
);
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
toVectorParam(embedding) {
|
|
2109
|
+
const buffer = embeddingToBuffer(embedding);
|
|
2110
|
+
return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
2111
|
+
}
|
|
2112
|
+
describe(error) {
|
|
2113
|
+
if (error instanceof Error) {
|
|
2114
|
+
return error.message;
|
|
2115
|
+
}
|
|
2116
|
+
return String(error);
|
|
2117
|
+
}
|
|
2118
|
+
};
|
|
2119
|
+
var SqliteDatabaseProvider = SqliteStorageProvider;
|
|
2120
|
+
function sanitizeFtsQuery(query) {
|
|
2121
|
+
const terms = query.split(/\s+/).map((t) => t.replace(/["']/g, "").replace(/[^\p{L}\p{N}_-]/gu, "")).filter((t) => t.length > 0);
|
|
2122
|
+
if (terms.length === 0) {
|
|
2123
|
+
return "";
|
|
2124
|
+
}
|
|
2125
|
+
return terms.map((t) => `"${t}"`).join(" OR ");
|
|
2126
|
+
}
|
|
2127
|
+
var txStore = new async_hooks.AsyncLocalStorage();
|
|
2128
|
+
var STMT = {
|
|
2129
|
+
insertOne: "Wolbarg_insert_one_v1",
|
|
2130
|
+
insertBatch: "Wolbarg_insert_batch_v1"
|
|
2131
|
+
};
|
|
2132
|
+
function toVectorLiteral(embedding) {
|
|
2133
|
+
const n = embedding.length;
|
|
2134
|
+
let s = "[";
|
|
2135
|
+
for (let i = 0; i < n; i += 1) {
|
|
2136
|
+
if (i !== 0) s += ",";
|
|
2137
|
+
s += embedding[i];
|
|
2138
|
+
}
|
|
2139
|
+
return s + "]";
|
|
2140
|
+
}
|
|
2141
|
+
var INSERT_ONE_SQL = `WITH mem AS (
|
|
2142
|
+
INSERT INTO memories (
|
|
2143
|
+
id, organization, agent, content_text, metadata_json,
|
|
2144
|
+
archived, compressed_into, created_at, updated_at
|
|
2145
|
+
) VALUES ($1,$2,$3,$4,$5::jsonb,false,NULL,$6,$7)
|
|
2146
|
+
RETURNING id, organization, agent, content_text, metadata_json,
|
|
2147
|
+
archived::int AS archived, compressed_into, created_at, updated_at
|
|
2148
|
+
),
|
|
2149
|
+
hist AS (
|
|
2150
|
+
INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
2151
|
+
SELECT $8, id, 'created', NULL, $6 FROM mem
|
|
2152
|
+
),
|
|
2153
|
+
mapped AS (
|
|
2154
|
+
INSERT INTO memory_row_map (memory_id)
|
|
2155
|
+
SELECT id FROM mem
|
|
2156
|
+
ON CONFLICT (memory_id) DO NOTHING
|
|
2157
|
+
),
|
|
2158
|
+
emb AS (
|
|
2159
|
+
INSERT INTO memory_embeddings (memory_id, embedding)
|
|
2160
|
+
SELECT id, $9::vector FROM mem
|
|
2161
|
+
ON CONFLICT (memory_id) DO UPDATE SET embedding = EXCLUDED.embedding
|
|
2162
|
+
)
|
|
2163
|
+
SELECT * FROM mem`;
|
|
2164
|
+
var INSERT_BATCH_SQL = `WITH mem AS (
|
|
2165
|
+
INSERT INTO memories (
|
|
2166
|
+
id, organization, agent, content_text, metadata_json,
|
|
2167
|
+
archived, compressed_into, created_at, updated_at
|
|
2168
|
+
)
|
|
2169
|
+
SELECT id, org, agent, txt, meta::jsonb, false, NULL, c, u
|
|
2170
|
+
FROM unnest(
|
|
2171
|
+
$1::text[], $2::text[], $3::text[], $4::text[],
|
|
2172
|
+
$5::text[], $6::timestamptz[], $7::timestamptz[]
|
|
2173
|
+
) AS t(id, org, agent, txt, meta, c, u)
|
|
2174
|
+
RETURNING id, organization, agent, content_text, metadata_json,
|
|
2175
|
+
archived::int AS archived, compressed_into, created_at, updated_at
|
|
2176
|
+
),
|
|
2177
|
+
hist AS (
|
|
2178
|
+
INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
2179
|
+
SELECT h, m, 'created', NULL, c
|
|
2180
|
+
FROM unnest($8::text[], $1::text[], $6::timestamptz[]) AS t(h, m, c)
|
|
2181
|
+
),
|
|
2182
|
+
mapped AS (
|
|
2183
|
+
INSERT INTO memory_row_map (memory_id)
|
|
2184
|
+
SELECT unnest($1::text[])
|
|
2185
|
+
ON CONFLICT (memory_id) DO NOTHING
|
|
2186
|
+
),
|
|
2187
|
+
emb AS (
|
|
2188
|
+
INSERT INTO memory_embeddings (memory_id, embedding)
|
|
2189
|
+
SELECT id, emb::vector
|
|
2190
|
+
FROM unnest($1::text[], $9::text[]) AS t(id, emb)
|
|
2191
|
+
ON CONFLICT (memory_id) DO UPDATE SET embedding = EXCLUDED.embedding
|
|
2192
|
+
)
|
|
2193
|
+
SELECT * FROM mem`;
|
|
2194
|
+
var COPY_BATCH_THRESHOLD = 64;
|
|
2195
|
+
var PostgresStorageProvider = class {
|
|
2196
|
+
name = "postgres";
|
|
2197
|
+
connectionString;
|
|
2198
|
+
maxPoolSize;
|
|
2199
|
+
pool = null;
|
|
2200
|
+
vectorDimensions = null;
|
|
2201
|
+
hasPgvector = false;
|
|
2202
|
+
hnswIndexEnsured = false;
|
|
2203
|
+
hnswCreateFailures = 0;
|
|
2204
|
+
hasContentTsv = false;
|
|
2205
|
+
iterativeScanEnabled = false;
|
|
2206
|
+
/** Coalesce concurrent insertMemory callers into one unnest batch. */
|
|
2207
|
+
insertQueue = [];
|
|
2208
|
+
insertFlushScheduled = false;
|
|
2209
|
+
insertFlushInFlight = false;
|
|
2210
|
+
constructor(options) {
|
|
2211
|
+
this.connectionString = options.connectionString;
|
|
2212
|
+
this.maxPoolSize = options.maxPoolSize ?? 32;
|
|
2213
|
+
}
|
|
2214
|
+
getPoolStats() {
|
|
2215
|
+
const pool = this.pool;
|
|
2216
|
+
return {
|
|
2217
|
+
max: this.maxPoolSize,
|
|
2218
|
+
total: pool?.totalCount ?? 0,
|
|
2219
|
+
idle: pool?.idleCount ?? 0,
|
|
2220
|
+
waiting: pool?.waitingCount ?? 0
|
|
2221
|
+
};
|
|
2222
|
+
}
|
|
2223
|
+
async open() {
|
|
2224
|
+
let PoolCtor;
|
|
2225
|
+
try {
|
|
2226
|
+
const mod = await import('pg');
|
|
2227
|
+
PoolCtor = mod.Pool ?? mod.default.Pool;
|
|
2228
|
+
} catch {
|
|
2229
|
+
throw new ConfigurationError(
|
|
2230
|
+
'PostgreSQL storage requires the optional "pg" package. Install it with: npm install pg'
|
|
2231
|
+
);
|
|
2232
|
+
}
|
|
2233
|
+
try {
|
|
2234
|
+
this.pool = new PoolCtor({
|
|
2235
|
+
connectionString: this.connectionString,
|
|
2236
|
+
max: this.maxPoolSize,
|
|
2237
|
+
idleTimeoutMillis: 3e4,
|
|
2238
|
+
allowExitOnIdle: true,
|
|
2239
|
+
keepAlive: true
|
|
2240
|
+
});
|
|
2241
|
+
await this.runMigrations();
|
|
2242
|
+
this.hasPgvector = await this.tryEnablePgvector();
|
|
2243
|
+
const dims = await this.getEmbeddingDimensions();
|
|
2244
|
+
if (dims !== null) {
|
|
2245
|
+
this.vectorDimensions = dims;
|
|
2246
|
+
await this.ensureVectorTables(dims);
|
|
2247
|
+
}
|
|
2248
|
+
} catch (error) {
|
|
2249
|
+
await this.pool?.end().catch(() => void 0);
|
|
2250
|
+
this.pool = null;
|
|
2251
|
+
if (error instanceof ConfigurationError || error instanceof InitializationError) {
|
|
2252
|
+
throw error;
|
|
2253
|
+
}
|
|
2254
|
+
throw new InitializationError(
|
|
2255
|
+
`Failed to open PostgreSQL database: ${this.describe(error)}`,
|
|
2256
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
2257
|
+
);
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
async close() {
|
|
2261
|
+
if (!this.pool) {
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
await this.pool.end();
|
|
2265
|
+
this.pool = null;
|
|
2266
|
+
}
|
|
2267
|
+
async ensureVectorSchema(dimensions) {
|
|
2268
|
+
const existing = await this.getEmbeddingDimensions();
|
|
2269
|
+
if (existing !== null && existing !== dimensions) {
|
|
2270
|
+
throw new InitializationError(
|
|
2271
|
+
`Embedding dimensions mismatch: database is configured for ${existing}-d vectors, but the embedding model returned ${dimensions}-d vectors.`
|
|
2272
|
+
);
|
|
2273
|
+
}
|
|
2274
|
+
this.hasPgvector = await this.tryEnablePgvector();
|
|
2275
|
+
await this.ensureVectorTables(dimensions);
|
|
2276
|
+
if (existing === null) {
|
|
2277
|
+
await this.setEmbeddingDimensions(dimensions);
|
|
2278
|
+
}
|
|
2279
|
+
this.vectorDimensions = dimensions;
|
|
2280
|
+
}
|
|
2281
|
+
async ensureVectorTables(dimensions) {
|
|
2282
|
+
if (this.hasPgvector) {
|
|
2283
|
+
await this.query(`
|
|
2284
|
+
CREATE TABLE IF NOT EXISTS memory_embeddings (
|
|
2285
|
+
memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE,
|
|
2286
|
+
embedding vector(${dimensions})
|
|
2287
|
+
)
|
|
2288
|
+
`);
|
|
2289
|
+
const idx = await this.query(
|
|
2290
|
+
`SELECT 1 FROM pg_indexes WHERE indexname = 'idx_memory_embeddings_hnsw' LIMIT 1`
|
|
2291
|
+
);
|
|
2292
|
+
this.hnswIndexEnsured = idx.rows.length > 0;
|
|
2293
|
+
} else {
|
|
2294
|
+
await this.query(`
|
|
2295
|
+
CREATE TABLE IF NOT EXISTS memory_embeddings_blob (
|
|
2296
|
+
memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE,
|
|
2297
|
+
embedding BYTEA NOT NULL
|
|
2298
|
+
)
|
|
2299
|
+
`);
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Soft reset for a single organization. Drops HNSW only when the embeddings
|
|
2304
|
+
* table is empty so other corpora on a shared bench DB stay intact.
|
|
2305
|
+
*/
|
|
2306
|
+
async resetOrganization(organization) {
|
|
2307
|
+
await this.query(`DELETE FROM memories WHERE organization = $1`, [
|
|
2308
|
+
organization
|
|
2309
|
+
]);
|
|
2310
|
+
const count = await this.query(
|
|
2311
|
+
`SELECT COUNT(*)::int AS n FROM memory_embeddings`
|
|
2312
|
+
);
|
|
2313
|
+
if (Number(count.rows[0]?.n ?? 0) === 0) {
|
|
2314
|
+
await this.query(
|
|
2315
|
+
`DROP INDEX IF EXISTS idx_memory_embeddings_hnsw`
|
|
2316
|
+
).catch(() => void 0);
|
|
2317
|
+
this.hnswIndexEnsured = false;
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
/**
|
|
2321
|
+
* Wipe all Wolbarg tables (explicit opt-in). Prefer {@link resetOrganization}.
|
|
2322
|
+
*/
|
|
2323
|
+
async wipeAllData() {
|
|
2324
|
+
await this.query(`TRUNCATE TABLE memories CASCADE`).catch(() => void 0);
|
|
2325
|
+
await this.query(
|
|
2326
|
+
`DROP INDEX IF EXISTS idx_memory_embeddings_hnsw`
|
|
2327
|
+
).catch(() => void 0);
|
|
2328
|
+
this.hnswIndexEnsured = false;
|
|
2329
|
+
}
|
|
2330
|
+
/** Build HNSW once before the first KNN query (bulk-friendly inserts). */
|
|
2331
|
+
async ensureHnswIndex() {
|
|
2332
|
+
if (!this.hasPgvector || this.hnswIndexEnsured) {
|
|
2333
|
+
return;
|
|
2334
|
+
}
|
|
2335
|
+
try {
|
|
2336
|
+
await this.query(`
|
|
2337
|
+
CREATE INDEX IF NOT EXISTS idx_memory_embeddings_hnsw
|
|
2338
|
+
ON memory_embeddings USING hnsw (embedding vector_cosine_ops)
|
|
2339
|
+
WITH (m = 16, ef_construction = 64)
|
|
2340
|
+
`);
|
|
2341
|
+
this.hnswIndexEnsured = true;
|
|
2342
|
+
this.hnswCreateFailures = 0;
|
|
2343
|
+
if (!this.iterativeScanEnabled) {
|
|
2344
|
+
try {
|
|
2345
|
+
await this.query(`SET hnsw.iterative_scan = relaxed_order`);
|
|
2346
|
+
this.iterativeScanEnabled = true;
|
|
2347
|
+
} catch {
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
} catch (error) {
|
|
2351
|
+
this.hnswCreateFailures += 1;
|
|
2352
|
+
if (this.hnswCreateFailures >= 3) {
|
|
2353
|
+
throw new DatabaseError(
|
|
2354
|
+
`Failed to create HNSW index after ${this.hnswCreateFailures} attempts: ${this.describe(error)}`,
|
|
2355
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
2356
|
+
);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2360
|
+
async getEmbeddingDimensions() {
|
|
2361
|
+
const result = await this.query(
|
|
2362
|
+
`SELECT value FROM Wolbarg_meta WHERE key = $1`,
|
|
2363
|
+
[META_KEYS.embeddingDimensions]
|
|
2364
|
+
);
|
|
2365
|
+
const value = result.rows[0]?.value;
|
|
2366
|
+
if (typeof value !== "string") {
|
|
2367
|
+
return null;
|
|
2368
|
+
}
|
|
2369
|
+
const parsed = Number(value);
|
|
2370
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
2371
|
+
}
|
|
2372
|
+
async setEmbeddingDimensions(dimensions) {
|
|
2373
|
+
await this.query(
|
|
2374
|
+
`INSERT INTO Wolbarg_meta (key, value) VALUES ($1, $2)
|
|
2375
|
+
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`,
|
|
2376
|
+
[META_KEYS.embeddingDimensions, String(dimensions)]
|
|
2377
|
+
);
|
|
2378
|
+
this.vectorDimensions = dimensions;
|
|
2379
|
+
}
|
|
2380
|
+
async insertMemory(input) {
|
|
2381
|
+
this.requireVectorReady();
|
|
2382
|
+
if (!this.insertFlushInFlight && this.insertQueue.length === 0) {
|
|
2383
|
+
this.insertFlushInFlight = true;
|
|
2384
|
+
try {
|
|
2385
|
+
return await this.insertMemoryImmediate(input);
|
|
2386
|
+
} finally {
|
|
2387
|
+
this.insertFlushInFlight = false;
|
|
2388
|
+
if (this.insertQueue.length > 0) {
|
|
2389
|
+
this.insertFlushScheduled = true;
|
|
2390
|
+
queueMicrotask(() => {
|
|
2391
|
+
void this.flushInsertQueue();
|
|
2392
|
+
});
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
return new Promise((resolve, reject) => {
|
|
2397
|
+
this.insertQueue.push({ input, resolve, reject });
|
|
2398
|
+
if (this.insertQueue.length >= 16) {
|
|
2399
|
+
if (this.insertFlushScheduled) {
|
|
2400
|
+
this.insertFlushScheduled = false;
|
|
2401
|
+
}
|
|
2402
|
+
void this.flushInsertQueue();
|
|
2403
|
+
return;
|
|
2404
|
+
}
|
|
2405
|
+
if (!this.insertFlushScheduled) {
|
|
2406
|
+
this.insertFlushScheduled = true;
|
|
2407
|
+
queueMicrotask(() => {
|
|
2408
|
+
void this.flushInsertQueue();
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
async flushInsertQueue() {
|
|
2414
|
+
if (this.insertFlushInFlight) {
|
|
2415
|
+
return;
|
|
2416
|
+
}
|
|
2417
|
+
const batch = this.insertQueue;
|
|
2418
|
+
this.insertQueue = [];
|
|
2419
|
+
this.insertFlushScheduled = false;
|
|
2420
|
+
if (batch.length === 0) {
|
|
2421
|
+
return;
|
|
2422
|
+
}
|
|
2423
|
+
this.insertFlushInFlight = true;
|
|
2424
|
+
try {
|
|
2425
|
+
if (batch.length === 1) {
|
|
2426
|
+
const row = await this.insertMemoryImmediate(batch[0].input);
|
|
2427
|
+
batch[0].resolve(row);
|
|
2428
|
+
return;
|
|
2429
|
+
}
|
|
2430
|
+
const rows = await this.insertMemoriesBatch(batch.map((b) => b.input));
|
|
2431
|
+
for (let i = 0; i < batch.length; i += 1) {
|
|
2432
|
+
batch[i].resolve(rows[i]);
|
|
2433
|
+
}
|
|
2434
|
+
} catch (error) {
|
|
2435
|
+
for (const item of batch) {
|
|
2436
|
+
item.reject(error);
|
|
2437
|
+
}
|
|
2438
|
+
} finally {
|
|
2439
|
+
this.insertFlushInFlight = false;
|
|
2440
|
+
if (this.insertQueue.length > 0) {
|
|
2441
|
+
this.insertFlushScheduled = true;
|
|
2442
|
+
queueMicrotask(() => {
|
|
2443
|
+
void this.flushInsertQueue();
|
|
2444
|
+
});
|
|
2445
|
+
} else {
|
|
2446
|
+
this.insertFlushScheduled = false;
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
}
|
|
2450
|
+
/** Single-row insert without coalescing (used by flush + batch of 1). */
|
|
2451
|
+
async insertMemoryImmediate(input) {
|
|
2452
|
+
if (this.hasPgvector) {
|
|
2453
|
+
const inserted = await this.queryNamed(STMT.insertOne, INSERT_ONE_SQL, [
|
|
2454
|
+
input.id,
|
|
2455
|
+
input.organization,
|
|
2456
|
+
input.agent,
|
|
2457
|
+
input.contentText,
|
|
2458
|
+
serializeMetadata(input.metadata),
|
|
2459
|
+
input.createdAt,
|
|
2460
|
+
input.updatedAt,
|
|
2461
|
+
crypto.randomUUID(),
|
|
2462
|
+
toVectorLiteral(input.embedding)
|
|
2463
|
+
]);
|
|
2464
|
+
return this.mapRow(inserted.rows[0]);
|
|
2465
|
+
}
|
|
2466
|
+
return this.insertOneBlob(input);
|
|
2467
|
+
}
|
|
2468
|
+
async insertMemoriesBatch(inputs) {
|
|
2469
|
+
if (inputs.length === 0) {
|
|
2470
|
+
return [];
|
|
2471
|
+
}
|
|
2472
|
+
this.requireVectorReady();
|
|
2473
|
+
if (inputs.length === 1 && this.hasPgvector) {
|
|
2474
|
+
return [await this.insertMemoryImmediate(inputs[0])];
|
|
2475
|
+
}
|
|
2476
|
+
if (this.hasPgvector) {
|
|
2477
|
+
if (inputs.length >= COPY_BATCH_THRESHOLD) {
|
|
2478
|
+
return this.insertBatchChunked(inputs);
|
|
2479
|
+
}
|
|
2480
|
+
return this.insertBatchPgvector(inputs);
|
|
2481
|
+
}
|
|
2482
|
+
return this.withTransaction(async () => {
|
|
2483
|
+
const out = [];
|
|
2484
|
+
for (const input of inputs) {
|
|
2485
|
+
out.push(await this.insertOneBlob(input));
|
|
2486
|
+
}
|
|
2487
|
+
return out;
|
|
2488
|
+
});
|
|
2489
|
+
}
|
|
2490
|
+
async insertBatchPgvector(inputs) {
|
|
2491
|
+
const ids = new Array(inputs.length);
|
|
2492
|
+
const orgs = new Array(inputs.length);
|
|
2493
|
+
const agents = new Array(inputs.length);
|
|
2494
|
+
const texts = new Array(inputs.length);
|
|
2495
|
+
const metas = new Array(inputs.length);
|
|
2496
|
+
const created = new Array(inputs.length);
|
|
2497
|
+
const updated = new Array(inputs.length);
|
|
2498
|
+
const histIds = new Array(inputs.length);
|
|
2499
|
+
const vectors = new Array(inputs.length);
|
|
2500
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
2501
|
+
const input = inputs[i];
|
|
2502
|
+
ids[i] = input.id;
|
|
2503
|
+
orgs[i] = input.organization;
|
|
2504
|
+
agents[i] = input.agent;
|
|
2505
|
+
texts[i] = input.contentText;
|
|
2506
|
+
metas[i] = serializeMetadata(input.metadata);
|
|
2507
|
+
created[i] = input.createdAt;
|
|
2508
|
+
updated[i] = input.updatedAt;
|
|
2509
|
+
histIds[i] = crypto.randomUUID();
|
|
2510
|
+
vectors[i] = toVectorLiteral(input.embedding);
|
|
2511
|
+
}
|
|
2512
|
+
const inserted = await this.queryNamed(STMT.insertBatch, INSERT_BATCH_SQL, [
|
|
2513
|
+
ids,
|
|
2514
|
+
orgs,
|
|
2515
|
+
agents,
|
|
2516
|
+
texts,
|
|
2517
|
+
metas,
|
|
2518
|
+
created,
|
|
2519
|
+
updated,
|
|
2520
|
+
histIds,
|
|
2521
|
+
vectors
|
|
2522
|
+
]);
|
|
2523
|
+
const byId = new Map(
|
|
2524
|
+
inserted.rows.map((r) => [String(r.id), this.mapRow(r)])
|
|
2525
|
+
);
|
|
2526
|
+
return ids.map((id) => byId.get(id));
|
|
2527
|
+
}
|
|
2528
|
+
/** Split large ingest batches into parallel unnest chunks (pool pipelining). */
|
|
2529
|
+
async insertBatchChunked(inputs) {
|
|
2530
|
+
const chunkSize = 128;
|
|
2531
|
+
if (inputs.length <= chunkSize) {
|
|
2532
|
+
return this.insertBatchPgvector(inputs);
|
|
2533
|
+
}
|
|
2534
|
+
const chunks = [];
|
|
2535
|
+
for (let i = 0; i < inputs.length; i += chunkSize) {
|
|
2536
|
+
chunks.push(inputs.slice(i, i + chunkSize));
|
|
2537
|
+
}
|
|
2538
|
+
const results = await Promise.all(
|
|
2539
|
+
chunks.map((chunk) => this.insertBatchPgvector(chunk))
|
|
2540
|
+
);
|
|
2541
|
+
const out = new Array(inputs.length);
|
|
2542
|
+
let offset = 0;
|
|
2543
|
+
for (const part of results) {
|
|
2544
|
+
for (let i = 0; i < part.length; i += 1) {
|
|
2545
|
+
out[offset + i] = part[i];
|
|
2546
|
+
}
|
|
2547
|
+
offset += part.length;
|
|
2548
|
+
}
|
|
2549
|
+
return out;
|
|
2550
|
+
}
|
|
2551
|
+
async insertOneBlob(input) {
|
|
2552
|
+
const buf = Buffer.from(
|
|
2553
|
+
input.embedding.buffer,
|
|
2554
|
+
input.embedding.byteOffset,
|
|
2555
|
+
input.embedding.byteLength
|
|
2556
|
+
);
|
|
2557
|
+
const inserted = await this.query(
|
|
2558
|
+
`INSERT INTO memories (
|
|
2559
|
+
id, organization, agent, content_text, metadata_json,
|
|
2560
|
+
archived, compressed_into, created_at, updated_at
|
|
2561
|
+
) VALUES ($1,$2,$3,$4,$5::jsonb,false,NULL,$6,$7)
|
|
2562
|
+
RETURNING id, organization, agent, content_text, metadata_json,
|
|
2563
|
+
archived::int AS archived, compressed_into, created_at, updated_at`,
|
|
2564
|
+
[
|
|
2565
|
+
input.id,
|
|
2566
|
+
input.organization,
|
|
2567
|
+
input.agent,
|
|
2568
|
+
input.contentText,
|
|
2569
|
+
serializeMetadata(input.metadata),
|
|
2570
|
+
input.createdAt,
|
|
2571
|
+
input.updatedAt
|
|
2572
|
+
]
|
|
2573
|
+
);
|
|
2574
|
+
const row = this.mapRow(inserted.rows[0]);
|
|
2575
|
+
await this.query(
|
|
2576
|
+
`WITH mapped AS (
|
|
2577
|
+
INSERT INTO memory_row_map (memory_id) VALUES ($1)
|
|
2578
|
+
ON CONFLICT (memory_id) DO NOTHING
|
|
2579
|
+
)
|
|
2580
|
+
INSERT INTO memory_embeddings_blob (memory_id, embedding)
|
|
2581
|
+
VALUES ($1, $2)
|
|
2582
|
+
ON CONFLICT (memory_id) DO UPDATE SET embedding = EXCLUDED.embedding`,
|
|
2583
|
+
[input.id, buf]
|
|
2584
|
+
);
|
|
2585
|
+
await this.query(
|
|
2586
|
+
`INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
2587
|
+
VALUES ($1,$2,'created',NULL,$3)`,
|
|
2588
|
+
[crypto.randomUUID(), input.id, input.createdAt]
|
|
2589
|
+
);
|
|
2590
|
+
return row;
|
|
2591
|
+
}
|
|
2592
|
+
async updateMemory(input) {
|
|
2593
|
+
const existing = await this.getMemoryById(input.id, input.organization);
|
|
2594
|
+
if (!existing) {
|
|
2595
|
+
return null;
|
|
2596
|
+
}
|
|
2597
|
+
await this.query(
|
|
2598
|
+
`UPDATE memories SET
|
|
2599
|
+
content_text = COALESCE($1, content_text),
|
|
2600
|
+
metadata_json = COALESCE($2::jsonb, metadata_json),
|
|
2601
|
+
updated_at = $3
|
|
2602
|
+
WHERE id = $4 AND organization = $5`,
|
|
2603
|
+
[
|
|
2604
|
+
input.contentText ?? null,
|
|
2605
|
+
input.metadata !== void 0 ? serializeMetadata(input.metadata) : null,
|
|
2606
|
+
input.updatedAt,
|
|
2607
|
+
input.id,
|
|
2608
|
+
input.organization
|
|
2609
|
+
]
|
|
2610
|
+
);
|
|
2611
|
+
if (input.embedding) {
|
|
2612
|
+
await this.deleteEmbedding(input.id);
|
|
2613
|
+
await this.insertEmbedding(input.id, input.embedding);
|
|
2614
|
+
}
|
|
2615
|
+
return this.getMemoryById(input.id, input.organization);
|
|
2616
|
+
}
|
|
2617
|
+
async getMemoryById(id, organization) {
|
|
2618
|
+
const result = await this.query(
|
|
2619
|
+
`SELECT id, organization, agent, content_text, metadata_json,
|
|
2620
|
+
archived::int AS archived, compressed_into, created_at, updated_at
|
|
2621
|
+
FROM memories WHERE id = $1 AND organization = $2`,
|
|
2622
|
+
[id, organization]
|
|
2623
|
+
);
|
|
2624
|
+
const row = result.rows[0];
|
|
2625
|
+
return row ? this.mapRow(row) : null;
|
|
2626
|
+
}
|
|
2627
|
+
async getMemoryByRowid(rowid, organization) {
|
|
2628
|
+
const result = await this.query(
|
|
2629
|
+
`SELECT m.id, m.organization, m.agent, m.content_text, m.metadata_json,
|
|
2630
|
+
m.archived::int AS archived, m.compressed_into, m.created_at, m.updated_at,
|
|
2631
|
+
e.row_num AS rowid
|
|
2632
|
+
FROM memories m
|
|
2633
|
+
JOIN memory_row_map e ON e.memory_id = m.id
|
|
2634
|
+
WHERE e.row_num = $1 AND m.organization = $2`,
|
|
2635
|
+
[rowid, organization]
|
|
2636
|
+
);
|
|
2637
|
+
const row = result.rows[0];
|
|
2638
|
+
return row ? this.mapRow(row) : null;
|
|
2639
|
+
}
|
|
2640
|
+
async getMemoriesByRowids(rowids, organization) {
|
|
2641
|
+
const out = /* @__PURE__ */ new Map();
|
|
2642
|
+
if (rowids.length === 0) {
|
|
2643
|
+
return out;
|
|
2644
|
+
}
|
|
2645
|
+
const result = await this.query(
|
|
2646
|
+
`SELECT m.id, m.organization, m.agent, m.content_text, m.metadata_json,
|
|
2647
|
+
m.archived::int AS archived, m.compressed_into, m.created_at, m.updated_at,
|
|
2648
|
+
e.row_num AS rowid
|
|
2649
|
+
FROM memories m
|
|
2650
|
+
JOIN memory_row_map e ON e.memory_id = m.id
|
|
2651
|
+
WHERE m.organization = $1 AND e.row_num = ANY($2::bigint[])`,
|
|
2652
|
+
[organization, rowids]
|
|
2653
|
+
);
|
|
2654
|
+
for (const row of result.rows) {
|
|
2655
|
+
const mapped = this.mapRow(row);
|
|
2656
|
+
if (mapped.rowid !== void 0) {
|
|
2657
|
+
out.set(mapped.rowid, mapped);
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2660
|
+
return out;
|
|
2661
|
+
}
|
|
2662
|
+
async listMemories(filter, limit) {
|
|
2663
|
+
const clauses = [`organization = $1`];
|
|
2664
|
+
const params = [filter.organization];
|
|
2665
|
+
let idx = 2;
|
|
2666
|
+
if (filter.agent) {
|
|
2667
|
+
clauses.push(`agent = $${idx++}`);
|
|
2668
|
+
params.push(filter.agent);
|
|
2669
|
+
}
|
|
2670
|
+
if (!filter.includeArchived) {
|
|
2671
|
+
clauses.push(`archived = false`);
|
|
2672
|
+
}
|
|
2673
|
+
let sql = `
|
|
2674
|
+
SELECT id, organization, agent, content_text, metadata_json,
|
|
2675
|
+
archived::int AS archived, compressed_into, created_at, updated_at
|
|
2676
|
+
FROM memories
|
|
2677
|
+
WHERE ${clauses.join(" AND ")}
|
|
2678
|
+
ORDER BY created_at ASC
|
|
2679
|
+
`;
|
|
2680
|
+
if (limit !== void 0 && !filter.metadata) {
|
|
2681
|
+
sql += ` LIMIT $${idx}`;
|
|
2682
|
+
params.push(limit);
|
|
2683
|
+
}
|
|
2684
|
+
const result = await this.query(sql, params);
|
|
2685
|
+
let rows = result.rows.map((r) => this.mapRow(r));
|
|
2686
|
+
if (filter.metadata) {
|
|
2687
|
+
rows = rows.filter(
|
|
2688
|
+
(row) => matchesMetadata(deserializeMetadata(row.metadata_json), filter.metadata)
|
|
2689
|
+
);
|
|
2690
|
+
if (limit !== void 0) {
|
|
2691
|
+
rows = rows.slice(0, limit);
|
|
2692
|
+
}
|
|
2693
|
+
}
|
|
2694
|
+
return rows;
|
|
2695
|
+
}
|
|
2696
|
+
async searchByMetadata(filter, limit) {
|
|
2697
|
+
return this.listMemories(filter, limit);
|
|
2698
|
+
}
|
|
2699
|
+
async searchKeyword(query, organization, topK) {
|
|
2700
|
+
const trimmed = query.trim();
|
|
2701
|
+
if (!trimmed || topK <= 0) {
|
|
2702
|
+
return [];
|
|
2703
|
+
}
|
|
2704
|
+
try {
|
|
2705
|
+
const sql = this.hasContentTsv ? `SELECT id AS memory_id,
|
|
2706
|
+
ts_rank(content_tsv, plainto_tsquery('english', $1)) AS rank
|
|
2707
|
+
FROM memories
|
|
2708
|
+
WHERE organization = $2
|
|
2709
|
+
AND archived = false
|
|
2710
|
+
AND content_tsv @@ plainto_tsquery('english', $1)
|
|
2711
|
+
ORDER BY rank DESC
|
|
2712
|
+
LIMIT $3` : `SELECT id AS memory_id,
|
|
2713
|
+
ts_rank(to_tsvector('english', content_text), plainto_tsquery('english', $1)) AS rank
|
|
2714
|
+
FROM memories
|
|
2715
|
+
WHERE organization = $2
|
|
2716
|
+
AND archived = false
|
|
2717
|
+
AND to_tsvector('english', content_text) @@ plainto_tsquery('english', $1)
|
|
2718
|
+
ORDER BY rank DESC
|
|
2719
|
+
LIMIT $3`;
|
|
2720
|
+
const result = await this.query(sql, [trimmed, organization, topK]);
|
|
2721
|
+
return result.rows.map((row) => ({
|
|
2722
|
+
memoryId: String(row.memory_id),
|
|
2723
|
+
score: Number(row.rank)
|
|
2724
|
+
}));
|
|
2725
|
+
} catch {
|
|
2726
|
+
return [];
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
async searchVectors(embedding, topK) {
|
|
2730
|
+
this.requireVectorReady();
|
|
2731
|
+
if (this.hasPgvector) {
|
|
2732
|
+
await this.ensureHnswIndex();
|
|
2733
|
+
const result2 = await this.query(
|
|
2734
|
+
`SELECT r.row_num AS memory_rowid, ann.distance
|
|
2735
|
+
FROM (
|
|
2736
|
+
SELECT e.memory_id, (e.embedding <=> $1::vector) AS distance
|
|
2737
|
+
FROM memory_embeddings e
|
|
2738
|
+
ORDER BY e.embedding <=> $1::vector
|
|
2739
|
+
LIMIT $2
|
|
2740
|
+
) ann
|
|
2741
|
+
JOIN memory_row_map r ON r.memory_id = ann.memory_id
|
|
2742
|
+
ORDER BY ann.distance`,
|
|
2743
|
+
[toVectorLiteral(embedding), topK]
|
|
2744
|
+
);
|
|
2745
|
+
const hits = new Array(result2.rows.length);
|
|
2746
|
+
for (let i = 0; i < result2.rows.length; i += 1) {
|
|
2747
|
+
const row = result2.rows[i];
|
|
2748
|
+
hits[i] = {
|
|
2749
|
+
memoryRowid: Number(row.memory_rowid),
|
|
2750
|
+
distance: Number(row.distance)
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2753
|
+
return hits;
|
|
2754
|
+
}
|
|
2755
|
+
const result = await this.query(
|
|
2756
|
+
`SELECT r.row_num AS memory_rowid, e.embedding
|
|
2757
|
+
FROM memory_embeddings_blob e
|
|
2758
|
+
JOIN memory_row_map r ON r.memory_id = e.memory_id`
|
|
2759
|
+
);
|
|
2760
|
+
const scored = result.rows.map((row) => {
|
|
2761
|
+
const buf = row.embedding;
|
|
2762
|
+
const vec = new Float32Array(
|
|
2763
|
+
buf.buffer,
|
|
2764
|
+
buf.byteOffset,
|
|
2765
|
+
buf.byteLength / Float32Array.BYTES_PER_ELEMENT
|
|
2766
|
+
);
|
|
2767
|
+
return {
|
|
2768
|
+
memoryRowid: Number(row.memory_rowid),
|
|
2769
|
+
distance: cosineDistance(embedding, vec)
|
|
2770
|
+
};
|
|
2771
|
+
});
|
|
2772
|
+
scored.sort((a, b) => a.distance - b.distance);
|
|
2773
|
+
return scored.slice(0, topK);
|
|
2774
|
+
}
|
|
2775
|
+
async searchVectorsWithMemories(embedding, topK, organization, options) {
|
|
2776
|
+
this.requireVectorReady();
|
|
2777
|
+
if (!this.hasPgvector) {
|
|
2778
|
+
const hits = await this.searchVectors(embedding, topK);
|
|
2779
|
+
const map = await this.getMemoriesByRowids(
|
|
2780
|
+
hits.map((h) => h.memoryRowid),
|
|
2781
|
+
organization
|
|
2782
|
+
);
|
|
2783
|
+
const out = [];
|
|
2784
|
+
for (const hit of hits) {
|
|
2785
|
+
const row = map.get(hit.memoryRowid);
|
|
2786
|
+
if (!row) continue;
|
|
2787
|
+
if (options?.agent && row.agent !== options.agent) continue;
|
|
2788
|
+
if (!options?.includeArchived && row.archived === 1) continue;
|
|
2789
|
+
out.push({ row, distance: hit.distance });
|
|
2790
|
+
}
|
|
2791
|
+
return out;
|
|
2792
|
+
}
|
|
2793
|
+
await this.ensureHnswIndex();
|
|
2794
|
+
const vec = toVectorLiteral(embedding);
|
|
2795
|
+
const mapHits = (rows) => rows.map((row) => ({
|
|
2796
|
+
row: this.mapRow(row),
|
|
2797
|
+
distance: Number(row.distance)
|
|
2798
|
+
}));
|
|
2799
|
+
let overfetch = Math.min(Math.max(topK * 8, topK), 512);
|
|
2800
|
+
const maxFetch = Math.min(Math.max(topK * 64, 512), 8192);
|
|
2801
|
+
for (; ; ) {
|
|
2802
|
+
const agentClause = options?.agent ? "AND m.agent = $5" : "";
|
|
2803
|
+
const archivedClause = options?.includeArchived ? "" : "AND m.archived = false";
|
|
2804
|
+
const params = options?.agent ? [vec, overfetch, organization, topK, options.agent] : [vec, overfetch, organization, topK];
|
|
2805
|
+
const ann = await this.query(
|
|
2806
|
+
`WITH ann AS (
|
|
2807
|
+
SELECT e.memory_id, (e.embedding <=> $1::vector) AS distance
|
|
2808
|
+
FROM memory_embeddings e
|
|
2809
|
+
ORDER BY e.embedding <=> $1::vector
|
|
2810
|
+
LIMIT $2
|
|
2811
|
+
)
|
|
2812
|
+
SELECT m.id, m.organization, m.agent, m.content_text, m.metadata_json,
|
|
2813
|
+
m.archived::int AS archived, m.compressed_into, m.created_at, m.updated_at,
|
|
2814
|
+
r.row_num AS rowid,
|
|
2815
|
+
ann.distance,
|
|
2816
|
+
(SELECT COUNT(*)::int FROM ann) AS ann_fetched
|
|
2817
|
+
FROM ann
|
|
2818
|
+
JOIN memory_row_map r ON r.memory_id = ann.memory_id
|
|
2819
|
+
JOIN memories m ON m.id = ann.memory_id
|
|
2820
|
+
WHERE m.organization = $3
|
|
2821
|
+
${agentClause}
|
|
2822
|
+
${archivedClause}
|
|
2823
|
+
ORDER BY ann.distance
|
|
2824
|
+
LIMIT $4`,
|
|
2825
|
+
params
|
|
2826
|
+
);
|
|
2827
|
+
const annFetched = Number(ann.rows[0]?.ann_fetched ?? ann.rows.length);
|
|
2828
|
+
if (ann.rows.length >= topK || annFetched < overfetch || overfetch >= maxFetch) {
|
|
2829
|
+
return mapHits(ann.rows.slice(0, topK));
|
|
2830
|
+
}
|
|
2831
|
+
const next = Math.min(overfetch * 4, maxFetch);
|
|
2832
|
+
if (next === overfetch) {
|
|
2833
|
+
return mapHits(ann.rows.slice(0, topK));
|
|
2834
|
+
}
|
|
2835
|
+
overfetch = next;
|
|
2836
|
+
}
|
|
2837
|
+
}
|
|
2838
|
+
async archiveMemories(ids, organization, compressedIntoId, archivedAt) {
|
|
2839
|
+
if (ids.length === 0) {
|
|
2840
|
+
return [];
|
|
2841
|
+
}
|
|
2842
|
+
const result = await this.query(
|
|
2843
|
+
`UPDATE memories
|
|
2844
|
+
SET archived = true, compressed_into = $1, updated_at = $2
|
|
2845
|
+
WHERE organization = $3 AND archived = false AND id = ANY($4::text[])
|
|
2846
|
+
RETURNING id`,
|
|
2847
|
+
[compressedIntoId, archivedAt, organization, ids]
|
|
2848
|
+
);
|
|
2849
|
+
const archived = result.rows.map((r) => String(r.id));
|
|
2850
|
+
if (archived.length === 0) {
|
|
2851
|
+
return [];
|
|
2852
|
+
}
|
|
2853
|
+
const histIds = [];
|
|
2854
|
+
const memIds = [];
|
|
2855
|
+
const types = [];
|
|
2856
|
+
const related = [];
|
|
2857
|
+
const times = [];
|
|
2858
|
+
for (const id of archived) {
|
|
2859
|
+
histIds.push(crypto.randomUUID(), crypto.randomUUID());
|
|
2860
|
+
memIds.push(id, compressedIntoId);
|
|
2861
|
+
types.push("archived", "compressed");
|
|
2862
|
+
related.push(compressedIntoId, id);
|
|
2863
|
+
times.push(archivedAt, archivedAt);
|
|
2864
|
+
}
|
|
2865
|
+
await this.query(
|
|
2866
|
+
`INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
2867
|
+
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[])`,
|
|
2868
|
+
[histIds, memIds, types, related, times]
|
|
2869
|
+
);
|
|
2870
|
+
return archived;
|
|
2871
|
+
}
|
|
2872
|
+
async deleteMemoryById(id, organization) {
|
|
2873
|
+
const result = await this.query(
|
|
2874
|
+
`DELETE FROM memories WHERE id = $1 AND organization = $2`,
|
|
2875
|
+
[id, organization]
|
|
2876
|
+
);
|
|
2877
|
+
return (result.rowCount ?? 0) > 0;
|
|
2878
|
+
}
|
|
2879
|
+
async deleteMemoriesByFilter(filter) {
|
|
2880
|
+
if (!filter.agent) {
|
|
2881
|
+
throw new DatabaseError("deleteMemoriesByFilter requires an agent filter");
|
|
2882
|
+
}
|
|
2883
|
+
const result = await this.query(
|
|
2884
|
+
`DELETE FROM memories WHERE organization = $1 AND agent = $2`,
|
|
2885
|
+
[filter.organization, filter.agent]
|
|
2886
|
+
);
|
|
2887
|
+
return result.rowCount ?? 0;
|
|
2888
|
+
}
|
|
2889
|
+
async clearOrganization(organization) {
|
|
2890
|
+
const result = await this.query(
|
|
2891
|
+
`DELETE FROM memories WHERE organization = $1`,
|
|
2892
|
+
[organization]
|
|
2893
|
+
);
|
|
2894
|
+
return result.rowCount ?? 0;
|
|
2895
|
+
}
|
|
2896
|
+
async getHistory(memoryId) {
|
|
2897
|
+
const result = await this.query(
|
|
2898
|
+
`SELECT id, memory_id, event_type, related_memory_id, created_at
|
|
2899
|
+
FROM memory_history WHERE memory_id = $1 ORDER BY created_at ASC`,
|
|
2900
|
+
[memoryId]
|
|
2901
|
+
);
|
|
2902
|
+
return result.rows.map((row) => ({
|
|
2903
|
+
id: String(row.id),
|
|
2904
|
+
memory_id: String(row.memory_id),
|
|
2905
|
+
event_type: row.event_type,
|
|
2906
|
+
related_memory_id: row.related_memory_id === null ? null : String(row.related_memory_id),
|
|
2907
|
+
created_at: String(row.created_at)
|
|
2908
|
+
}));
|
|
2909
|
+
}
|
|
2910
|
+
async insertHistoryEvent(event) {
|
|
2911
|
+
await this.query(
|
|
2912
|
+
`INSERT INTO memory_history (id, memory_id, event_type, related_memory_id, created_at)
|
|
2913
|
+
VALUES ($1,$2,$3,$4,$5)`,
|
|
2914
|
+
[
|
|
2915
|
+
event.id,
|
|
2916
|
+
event.memory_id,
|
|
2917
|
+
event.event_type,
|
|
2918
|
+
event.related_memory_id,
|
|
2919
|
+
event.created_at
|
|
2920
|
+
]
|
|
2921
|
+
);
|
|
2922
|
+
}
|
|
2923
|
+
async getStats(organization) {
|
|
2924
|
+
const result = await this.query(
|
|
2925
|
+
`SELECT
|
|
2926
|
+
COUNT(*)::int AS memories,
|
|
2927
|
+
COUNT(*) FILTER (WHERE archived = false)::int AS active,
|
|
2928
|
+
COUNT(*) FILTER (WHERE archived = true)::int AS archived,
|
|
2929
|
+
COUNT(DISTINCT agent) FILTER (WHERE archived = false)::int AS agents
|
|
2930
|
+
FROM memories WHERE organization = $1`,
|
|
2931
|
+
[organization]
|
|
2932
|
+
);
|
|
2933
|
+
return {
|
|
2934
|
+
totalMemories: Number(result.rows[0]?.memories ?? 0),
|
|
2935
|
+
activeMemories: Number(result.rows[0]?.active ?? 0),
|
|
2936
|
+
archivedMemories: Number(result.rows[0]?.archived ?? 0),
|
|
2937
|
+
totalAgents: Number(result.rows[0]?.agents ?? 0)
|
|
2938
|
+
};
|
|
2939
|
+
}
|
|
2940
|
+
async getDatabaseSizeBytes() {
|
|
2941
|
+
const result = await this.query(
|
|
2942
|
+
`SELECT pg_database_size(current_database())::bigint AS size`
|
|
2943
|
+
);
|
|
2944
|
+
return Number(result.rows[0]?.size ?? 0);
|
|
2945
|
+
}
|
|
2946
|
+
async withTransaction(fn) {
|
|
2947
|
+
const existing = txStore.getStore();
|
|
2948
|
+
if (existing) {
|
|
2949
|
+
return fn();
|
|
2950
|
+
}
|
|
2951
|
+
const client = await this.requirePool().connect();
|
|
2952
|
+
try {
|
|
2953
|
+
await client.query("BEGIN");
|
|
2954
|
+
const result = await txStore.run(client, fn);
|
|
2955
|
+
await client.query("COMMIT");
|
|
2956
|
+
return result;
|
|
2957
|
+
} catch (error) {
|
|
2958
|
+
await client.query("ROLLBACK").catch(() => void 0);
|
|
2959
|
+
if (error instanceof DatabaseError || error instanceof InitializationError) {
|
|
2960
|
+
throw error;
|
|
2961
|
+
}
|
|
2962
|
+
throw new DatabaseError(`Transaction failed: ${this.describe(error)}`, {
|
|
2963
|
+
cause: error instanceof Error ? error : void 0
|
|
2964
|
+
});
|
|
2965
|
+
} finally {
|
|
2966
|
+
client.release();
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
async runMigrations() {
|
|
2970
|
+
await this.query(`
|
|
2971
|
+
CREATE TABLE IF NOT EXISTS Wolbarg_meta (
|
|
2972
|
+
key TEXT PRIMARY KEY NOT NULL,
|
|
2973
|
+
value TEXT NOT NULL
|
|
2974
|
+
)
|
|
2975
|
+
`);
|
|
2976
|
+
await this.query(`
|
|
2977
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
2978
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
2979
|
+
organization TEXT NOT NULL,
|
|
2980
|
+
agent TEXT NOT NULL,
|
|
2981
|
+
content_text TEXT NOT NULL,
|
|
2982
|
+
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
2983
|
+
archived BOOLEAN NOT NULL DEFAULT false,
|
|
2984
|
+
compressed_into TEXT NULL REFERENCES memories(id) ON DELETE SET NULL,
|
|
2985
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
2986
|
+
updated_at TIMESTAMPTZ NOT NULL
|
|
2987
|
+
)
|
|
2988
|
+
`);
|
|
2989
|
+
await this.query(`
|
|
2990
|
+
CREATE TABLE IF NOT EXISTS memory_history (
|
|
2991
|
+
id TEXT PRIMARY KEY NOT NULL,
|
|
2992
|
+
memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
|
|
2993
|
+
event_type TEXT NOT NULL CHECK (event_type IN ('created', 'archived', 'compressed')),
|
|
2994
|
+
related_memory_id TEXT NULL,
|
|
2995
|
+
created_at TIMESTAMPTZ NOT NULL
|
|
2996
|
+
)
|
|
2997
|
+
`);
|
|
2998
|
+
await this.query(`
|
|
2999
|
+
CREATE TABLE IF NOT EXISTS memory_row_map (
|
|
3000
|
+
memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE,
|
|
3001
|
+
row_num BIGSERIAL UNIQUE NOT NULL
|
|
3002
|
+
)
|
|
3003
|
+
`);
|
|
3004
|
+
await this.query(
|
|
3005
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_agent ON memories(organization, agent)`
|
|
3006
|
+
);
|
|
3007
|
+
await this.query(
|
|
3008
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_archived ON memories(organization, archived)`
|
|
3009
|
+
);
|
|
3010
|
+
await this.query(
|
|
3011
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_org_active_created
|
|
3012
|
+
ON memories(organization, created_at) WHERE archived = false`
|
|
3013
|
+
).catch(() => void 0);
|
|
3014
|
+
await this.query(
|
|
3015
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_metadata ON memories USING GIN (metadata_json)`
|
|
3016
|
+
);
|
|
3017
|
+
try {
|
|
3018
|
+
await this.query(`
|
|
3019
|
+
ALTER TABLE memories
|
|
3020
|
+
ADD COLUMN IF NOT EXISTS content_tsv tsvector
|
|
3021
|
+
GENERATED ALWAYS AS (to_tsvector('english', content_text)) STORED
|
|
3022
|
+
`);
|
|
3023
|
+
await this.query(
|
|
3024
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_content_tsv ON memories USING GIN (content_tsv)`
|
|
3025
|
+
);
|
|
3026
|
+
this.hasContentTsv = true;
|
|
3027
|
+
} catch {
|
|
3028
|
+
await this.query(
|
|
3029
|
+
`CREATE INDEX IF NOT EXISTS idx_memories_fts
|
|
3030
|
+
ON memories USING GIN (to_tsvector('english', content_text))`
|
|
3031
|
+
).catch(() => void 0);
|
|
3032
|
+
this.hasContentTsv = false;
|
|
3033
|
+
}
|
|
3034
|
+
const versionRow = await this.query(
|
|
3035
|
+
`SELECT value FROM Wolbarg_meta WHERE key = $1`,
|
|
3036
|
+
[META_KEYS.schemaVersion]
|
|
3037
|
+
);
|
|
3038
|
+
if (!versionRow.rows[0]) {
|
|
3039
|
+
await this.query(
|
|
3040
|
+
`INSERT INTO Wolbarg_meta (key, value) VALUES ($1, $2)`,
|
|
3041
|
+
[META_KEYS.schemaVersion, String(SCHEMA_VERSION)]
|
|
3042
|
+
);
|
|
3043
|
+
}
|
|
3044
|
+
}
|
|
3045
|
+
async tryEnablePgvector() {
|
|
3046
|
+
try {
|
|
3047
|
+
await this.query(`CREATE EXTENSION IF NOT EXISTS vector`);
|
|
3048
|
+
return true;
|
|
3049
|
+
} catch {
|
|
3050
|
+
return false;
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
async insertEmbedding(memoryId, embedding) {
|
|
3054
|
+
if (this.hasPgvector) {
|
|
3055
|
+
await this.query(
|
|
3056
|
+
`WITH mapped AS (
|
|
3057
|
+
INSERT INTO memory_row_map (memory_id) VALUES ($1)
|
|
3058
|
+
ON CONFLICT (memory_id) DO NOTHING
|
|
3059
|
+
)
|
|
3060
|
+
INSERT INTO memory_embeddings (memory_id, embedding)
|
|
3061
|
+
VALUES ($1, $2::vector)
|
|
3062
|
+
ON CONFLICT (memory_id) DO UPDATE SET embedding = EXCLUDED.embedding`,
|
|
3063
|
+
[memoryId, toVectorLiteral(embedding)]
|
|
3064
|
+
);
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
const buf = Buffer.from(
|
|
3068
|
+
embedding.buffer,
|
|
3069
|
+
embedding.byteOffset,
|
|
3070
|
+
embedding.byteLength
|
|
3071
|
+
);
|
|
3072
|
+
await this.query(
|
|
3073
|
+
`WITH mapped AS (
|
|
3074
|
+
INSERT INTO memory_row_map (memory_id) VALUES ($1)
|
|
3075
|
+
ON CONFLICT (memory_id) DO NOTHING
|
|
3076
|
+
)
|
|
3077
|
+
INSERT INTO memory_embeddings_blob (memory_id, embedding)
|
|
3078
|
+
VALUES ($1, $2)
|
|
3079
|
+
ON CONFLICT (memory_id) DO UPDATE SET embedding = EXCLUDED.embedding`,
|
|
3080
|
+
[memoryId, buf]
|
|
3081
|
+
);
|
|
3082
|
+
}
|
|
3083
|
+
async deleteEmbedding(memoryId) {
|
|
3084
|
+
await this.query(`DELETE FROM memory_embeddings WHERE memory_id = $1`, [memoryId]).catch(() => void 0);
|
|
3085
|
+
await this.query(`DELETE FROM memory_embeddings_blob WHERE memory_id = $1`, [memoryId]).catch(() => void 0);
|
|
3086
|
+
}
|
|
3087
|
+
async query(text, params) {
|
|
3088
|
+
const tx = txStore.getStore();
|
|
3089
|
+
const target = tx ?? this.requirePool();
|
|
3090
|
+
return target.query(text, params);
|
|
3091
|
+
}
|
|
3092
|
+
/** Named prepared statement — parse/plan cached per pool connection. */
|
|
3093
|
+
async queryNamed(name, text, params) {
|
|
3094
|
+
const tx = txStore.getStore();
|
|
3095
|
+
const target = tx ?? this.requirePool();
|
|
3096
|
+
return target.query({ name, text, values: params });
|
|
3097
|
+
}
|
|
3098
|
+
mapRow(row) {
|
|
3099
|
+
const meta2 = row.metadata_json;
|
|
3100
|
+
let metadata_json;
|
|
3101
|
+
if (typeof meta2 === "string") {
|
|
3102
|
+
metadata_json = meta2;
|
|
3103
|
+
} else if (meta2 && typeof meta2 === "object") {
|
|
3104
|
+
metadata_json = JSON.stringify(meta2);
|
|
3105
|
+
} else {
|
|
3106
|
+
metadata_json = "{}";
|
|
3107
|
+
}
|
|
3108
|
+
const created = row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at);
|
|
3109
|
+
const updated = row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at);
|
|
3110
|
+
return {
|
|
3111
|
+
id: String(row.id),
|
|
3112
|
+
organization: String(row.organization),
|
|
3113
|
+
agent: String(row.agent),
|
|
3114
|
+
content_text: String(row.content_text),
|
|
3115
|
+
metadata_json,
|
|
3116
|
+
archived: Number(row.archived ?? 0),
|
|
3117
|
+
compressed_into: row.compressed_into === null || row.compressed_into === void 0 ? null : String(row.compressed_into),
|
|
3118
|
+
created_at: created,
|
|
3119
|
+
updated_at: updated,
|
|
3120
|
+
rowid: row.rowid !== void 0 ? Number(row.rowid) : void 0
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
requirePool() {
|
|
3124
|
+
if (!this.pool) {
|
|
3125
|
+
throw new DatabaseError("Database is not open. Call open() first.");
|
|
3126
|
+
}
|
|
3127
|
+
return this.pool;
|
|
3128
|
+
}
|
|
3129
|
+
requireVectorReady() {
|
|
3130
|
+
if (this.vectorDimensions === null) {
|
|
3131
|
+
throw new DatabaseError(
|
|
3132
|
+
"Vector index is not ready. Embedding dimensions have not been initialized."
|
|
3133
|
+
);
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
describe(error) {
|
|
3137
|
+
if (error instanceof Error) {
|
|
3138
|
+
const aggregate = error;
|
|
3139
|
+
if (Array.isArray(aggregate.errors) && aggregate.errors.length > 0) {
|
|
3140
|
+
const nested = aggregate.errors.map((item) => item instanceof Error ? item.message : String(item)).join("; ");
|
|
3141
|
+
return `${error.message || error.name}: ${nested}`;
|
|
3142
|
+
}
|
|
3143
|
+
return error.message || error.name;
|
|
3144
|
+
}
|
|
3145
|
+
return String(error);
|
|
3146
|
+
}
|
|
3147
|
+
};
|
|
3148
|
+
|
|
3149
|
+
// src/storage/index.ts
|
|
3150
|
+
function createStorageProvider(config) {
|
|
3151
|
+
if (config.provider === "sqlite") {
|
|
3152
|
+
return new SqliteStorageProvider({
|
|
3153
|
+
connectionString: config.connectionString
|
|
3154
|
+
});
|
|
3155
|
+
}
|
|
3156
|
+
if (config.provider === "postgres") {
|
|
3157
|
+
return new PostgresStorageProvider({
|
|
3158
|
+
connectionString: config.connectionString,
|
|
3159
|
+
maxPoolSize: config.maxPoolSize
|
|
3160
|
+
});
|
|
3161
|
+
}
|
|
3162
|
+
throw new ConfigurationError(
|
|
3163
|
+
`Unsupported storage provider: ${String(config.provider)}`
|
|
3164
|
+
);
|
|
3165
|
+
}
|
|
3166
|
+
function createDatabaseProvider(config) {
|
|
3167
|
+
return createStorageProvider(config);
|
|
3168
|
+
}
|
|
3169
|
+
|
|
3170
|
+
// src/memory/index.ts
|
|
3171
|
+
function toMemoryRecord(row) {
|
|
3172
|
+
return {
|
|
3173
|
+
id: row.id,
|
|
3174
|
+
organization: row.organization,
|
|
3175
|
+
agent: row.agent,
|
|
3176
|
+
content: { text: row.content_text },
|
|
3177
|
+
metadata: deserializeMetadata(row.metadata_json),
|
|
3178
|
+
archived: row.archived === 1,
|
|
3179
|
+
compressedInto: row.compressed_into,
|
|
3180
|
+
createdAt: parseIso(row.created_at),
|
|
3181
|
+
updatedAt: parseIso(row.updated_at)
|
|
3182
|
+
};
|
|
3183
|
+
}
|
|
3184
|
+
function toRecallResult(row, similarity) {
|
|
3185
|
+
return {
|
|
3186
|
+
id: row.id,
|
|
3187
|
+
organization: row.organization,
|
|
3188
|
+
agent: row.agent,
|
|
3189
|
+
content: { text: row.content_text },
|
|
3190
|
+
metadata: deserializeMetadata(row.metadata_json),
|
|
3191
|
+
archived: row.archived === 1,
|
|
3192
|
+
similarity,
|
|
3193
|
+
createdAt: parseIso(row.created_at),
|
|
3194
|
+
updatedAt: parseIso(row.updated_at)
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
function toHistoryEvent(row) {
|
|
3198
|
+
return {
|
|
3199
|
+
id: row.id,
|
|
3200
|
+
memoryId: row.memory_id,
|
|
3201
|
+
eventType: row.event_type,
|
|
3202
|
+
relatedMemoryId: row.related_memory_id,
|
|
3203
|
+
createdAt: parseIso(row.created_at)
|
|
3204
|
+
};
|
|
3205
|
+
}
|
|
3206
|
+
|
|
3207
|
+
// src/retrieval/index.ts
|
|
3208
|
+
function fuseScores(semantic, keyword, weights) {
|
|
3209
|
+
const ids = /* @__PURE__ */ new Set([...semantic.keys(), ...keyword.keys()]);
|
|
3210
|
+
const fused = /* @__PURE__ */ new Map();
|
|
3211
|
+
const semMax = Math.max(...semantic.values(), 1e-9);
|
|
3212
|
+
const kwMax = Math.max(...keyword.values(), 1e-9);
|
|
3213
|
+
for (const id of ids) {
|
|
3214
|
+
const s = (semantic.get(id) ?? 0) / semMax;
|
|
3215
|
+
const k = (keyword.get(id) ?? 0) / kwMax;
|
|
3216
|
+
fused.set(
|
|
3217
|
+
id,
|
|
3218
|
+
weights.semanticWeight * s + weights.keywordWeight * k
|
|
3219
|
+
);
|
|
3220
|
+
}
|
|
3221
|
+
return fused;
|
|
3222
|
+
}
|
|
3223
|
+
function resolveHybridWeights(hybrid) {
|
|
3224
|
+
if (hybrid === false || hybrid === void 0) {
|
|
3225
|
+
return null;
|
|
3226
|
+
}
|
|
3227
|
+
if (hybrid === true) {
|
|
3228
|
+
return { semanticWeight: 0.7, keywordWeight: 0.3 };
|
|
3229
|
+
}
|
|
3230
|
+
return {
|
|
3231
|
+
semanticWeight: hybrid.semanticWeight ?? 0.7,
|
|
3232
|
+
keywordWeight: hybrid.keywordWeight ?? 0.3
|
|
3233
|
+
};
|
|
3234
|
+
}
|
|
3235
|
+
function resolveMmr(mmr) {
|
|
3236
|
+
if (mmr === false || mmr === void 0) {
|
|
3237
|
+
return null;
|
|
3238
|
+
}
|
|
3239
|
+
if (mmr === true) {
|
|
3240
|
+
return 0.5;
|
|
3241
|
+
}
|
|
3242
|
+
return mmr.lambda ?? 0.5;
|
|
3243
|
+
}
|
|
3244
|
+
function applyMmr(candidates, topK, lambda) {
|
|
3245
|
+
if (candidates.length <= topK) {
|
|
3246
|
+
return candidates;
|
|
3247
|
+
}
|
|
3248
|
+
const selected = [];
|
|
3249
|
+
const remaining = [...candidates];
|
|
3250
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
3251
|
+
let bestIdx = 0;
|
|
3252
|
+
let bestScore = -Infinity;
|
|
3253
|
+
for (let i = 0; i < remaining.length; i += 1) {
|
|
3254
|
+
const candidate = remaining[i];
|
|
3255
|
+
const relevance = candidate.similarity;
|
|
3256
|
+
let maxSim = 0;
|
|
3257
|
+
for (const chosen of selected) {
|
|
3258
|
+
maxSim = Math.max(
|
|
3259
|
+
maxSim,
|
|
3260
|
+
jaccard(candidate.content.text, chosen.content.text)
|
|
3261
|
+
);
|
|
3262
|
+
}
|
|
3263
|
+
const score = lambda * relevance - (1 - lambda) * maxSim;
|
|
3264
|
+
if (score > bestScore) {
|
|
3265
|
+
bestScore = score;
|
|
3266
|
+
bestIdx = i;
|
|
3267
|
+
}
|
|
3268
|
+
}
|
|
3269
|
+
selected.push(remaining[bestIdx]);
|
|
3270
|
+
remaining.splice(bestIdx, 1);
|
|
3271
|
+
}
|
|
3272
|
+
return selected;
|
|
3273
|
+
}
|
|
3274
|
+
function jaccard(a, b) {
|
|
3275
|
+
const ta = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));
|
|
3276
|
+
const tb = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));
|
|
3277
|
+
if (ta.size === 0 && tb.size === 0) {
|
|
3278
|
+
return 1;
|
|
3279
|
+
}
|
|
3280
|
+
let inter = 0;
|
|
3281
|
+
for (const t of ta) {
|
|
3282
|
+
if (tb.has(t)) {
|
|
3283
|
+
inter += 1;
|
|
3284
|
+
}
|
|
3285
|
+
}
|
|
3286
|
+
const union = ta.size + tb.size - inter;
|
|
3287
|
+
return union === 0 ? 0 : inter / union;
|
|
3288
|
+
}
|
|
3289
|
+
function adaptiveFetchK(topK, overFetchFactor, hasFilters) {
|
|
3290
|
+
const factor = hasFilters ? Math.max(overFetchFactor, 4) : overFetchFactor;
|
|
3291
|
+
return Math.min(Math.max(Math.ceil(topK * factor), topK), 1e3);
|
|
3292
|
+
}
|
|
3293
|
+
|
|
3294
|
+
// src/core/options.ts
|
|
3295
|
+
function isEmbeddingProvider(value) {
|
|
3296
|
+
return typeof value.embed === "function";
|
|
3297
|
+
}
|
|
3298
|
+
function isLlmProvider(value) {
|
|
3299
|
+
return typeof value.complete === "function";
|
|
3300
|
+
}
|
|
3301
|
+
function isStorageProvider(value) {
|
|
3302
|
+
return typeof value.open === "function";
|
|
3303
|
+
}
|
|
3304
|
+
|
|
3305
|
+
// src/core/validate.ts
|
|
3306
|
+
function assertNonEmpty(value, fieldName) {
|
|
3307
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
3308
|
+
throw new ConfigurationError(`${fieldName} must be a non-empty string`);
|
|
3309
|
+
}
|
|
3310
|
+
}
|
|
3311
|
+
function assertUrl(value, fieldName) {
|
|
3312
|
+
assertNonEmpty(value, fieldName);
|
|
3313
|
+
try {
|
|
3314
|
+
new URL(value);
|
|
3315
|
+
} catch {
|
|
3316
|
+
throw new ConfigurationError(
|
|
3317
|
+
`${fieldName} must be a valid absolute URL (got "${value}")`
|
|
3318
|
+
);
|
|
3319
|
+
}
|
|
3320
|
+
}
|
|
3321
|
+
function validateEmbeddingConfig(config) {
|
|
3322
|
+
assertUrl(config.baseUrl, "embedding.baseUrl");
|
|
3323
|
+
assertNonEmpty(config.apiKey, "embedding.apiKey");
|
|
3324
|
+
assertNonEmpty(config.model, "embedding.model");
|
|
3325
|
+
if (config.timeoutMs !== void 0 && (!Number.isFinite(config.timeoutMs) || config.timeoutMs <= 0)) {
|
|
3326
|
+
throw new ConfigurationError("embedding.timeoutMs must be a positive number");
|
|
3327
|
+
}
|
|
3328
|
+
return {
|
|
3329
|
+
baseUrl: config.baseUrl.trim().replace(/\/+$/, ""),
|
|
3330
|
+
apiKey: config.apiKey,
|
|
3331
|
+
model: config.model.trim(),
|
|
3332
|
+
...config.timeoutMs !== void 0 ? { timeoutMs: config.timeoutMs } : {}
|
|
3333
|
+
};
|
|
3334
|
+
}
|
|
3335
|
+
function validateLlmConfig(config) {
|
|
3336
|
+
assertUrl(config.baseUrl, "llm.baseUrl");
|
|
3337
|
+
assertNonEmpty(config.apiKey, "llm.apiKey");
|
|
3338
|
+
assertNonEmpty(config.model, "llm.model");
|
|
3339
|
+
if (config.temperature !== void 0 && (!Number.isFinite(config.temperature) || config.temperature < 0 || config.temperature > 2)) {
|
|
3340
|
+
throw new ConfigurationError("llm.temperature must be between 0 and 2");
|
|
3341
|
+
}
|
|
3342
|
+
if (config.maxTokens !== void 0 && (!Number.isFinite(config.maxTokens) || config.maxTokens <= 0)) {
|
|
3343
|
+
throw new ConfigurationError("llm.maxTokens must be a positive number");
|
|
3344
|
+
}
|
|
3345
|
+
if (config.timeoutMs !== void 0 && (!Number.isFinite(config.timeoutMs) || config.timeoutMs <= 0)) {
|
|
3346
|
+
throw new ConfigurationError("llm.timeoutMs must be a positive number");
|
|
3347
|
+
}
|
|
3348
|
+
return {
|
|
3349
|
+
baseUrl: config.baseUrl.trim().replace(/\/+$/, ""),
|
|
3350
|
+
apiKey: config.apiKey,
|
|
3351
|
+
model: config.model.trim(),
|
|
3352
|
+
...config.temperature !== void 0 ? { temperature: config.temperature } : {},
|
|
3353
|
+
...config.maxTokens !== void 0 ? { maxTokens: config.maxTokens } : {},
|
|
3354
|
+
...config.timeoutMs !== void 0 ? { timeoutMs: config.timeoutMs } : {}
|
|
3355
|
+
};
|
|
3356
|
+
}
|
|
3357
|
+
function validateInitOptions(options) {
|
|
3358
|
+
if (options === null || typeof options !== "object") {
|
|
3359
|
+
throw new ConfigurationError("init options must be an object");
|
|
3360
|
+
}
|
|
3361
|
+
assertNonEmpty(options.organization, "organization");
|
|
3362
|
+
if (!options.database || typeof options.database !== "object") {
|
|
3363
|
+
throw new ConfigurationError("database configuration is required");
|
|
3364
|
+
}
|
|
3365
|
+
const provider = options.database.provider;
|
|
3366
|
+
if (provider !== "sqlite" && provider !== "postgres") {
|
|
3367
|
+
throw new ConfigurationError(
|
|
3368
|
+
`Unsupported database provider "${String(options.database.provider)}". Supported: "sqlite", "postgres".`
|
|
3369
|
+
);
|
|
3370
|
+
}
|
|
3371
|
+
assertNonEmpty(
|
|
3372
|
+
options.database.connectionString,
|
|
3373
|
+
"database.connectionString"
|
|
3374
|
+
);
|
|
3375
|
+
if (!options.embedding || typeof options.embedding !== "object") {
|
|
3376
|
+
throw new ConfigurationError("embedding configuration is required");
|
|
3377
|
+
}
|
|
3378
|
+
const embedding = validateEmbeddingConfig(options.embedding);
|
|
3379
|
+
const llm = options.llm ? validateLlmConfig(options.llm) : void 0;
|
|
3380
|
+
return {
|
|
3381
|
+
organization: options.organization.trim(),
|
|
3382
|
+
database: provider === "postgres" ? {
|
|
3383
|
+
provider: "postgres",
|
|
3384
|
+
connectionString: options.database.connectionString.trim(),
|
|
3385
|
+
..."maxPoolSize" in options.database && options.database.maxPoolSize !== void 0 ? { maxPoolSize: options.database.maxPoolSize } : {}
|
|
3386
|
+
} : {
|
|
3387
|
+
provider: "sqlite",
|
|
3388
|
+
connectionString: options.database.connectionString.trim()
|
|
3389
|
+
},
|
|
3390
|
+
embedding,
|
|
3391
|
+
...llm ? { llm } : {}
|
|
3392
|
+
};
|
|
3393
|
+
}
|
|
3394
|
+
function validateWolbargOptions(options) {
|
|
3395
|
+
if (options === null || typeof options !== "object") {
|
|
3396
|
+
throw new ConfigurationError("Wolbarg options must be an object");
|
|
3397
|
+
}
|
|
3398
|
+
assertNonEmpty(options.organization, "organization");
|
|
3399
|
+
if (!options.storage) {
|
|
3400
|
+
throw new ConfigurationError("storage is required");
|
|
3401
|
+
}
|
|
3402
|
+
if (!isStorageProvider(options.storage)) {
|
|
3403
|
+
if (typeof options.storage !== "object" || !("provider" in options.storage) || !("connectionString" in options.storage)) {
|
|
3404
|
+
throw new ConfigurationError("storage must be a provider instance or config");
|
|
3405
|
+
}
|
|
3406
|
+
assertNonEmpty(options.storage.connectionString, "storage.connectionString");
|
|
3407
|
+
}
|
|
3408
|
+
if (!options.embedding) {
|
|
3409
|
+
throw new ConfigurationError("embedding is required");
|
|
3410
|
+
}
|
|
3411
|
+
if (!isEmbeddingProvider(options.embedding)) {
|
|
3412
|
+
validateEmbeddingConfig(options.embedding);
|
|
3413
|
+
}
|
|
3414
|
+
if (options.llm !== void 0 && !isLlmProvider(options.llm)) {
|
|
3415
|
+
validateLlmConfig(options.llm);
|
|
3416
|
+
}
|
|
3417
|
+
return {
|
|
3418
|
+
...options,
|
|
3419
|
+
organization: options.organization.trim()
|
|
3420
|
+
};
|
|
3421
|
+
}
|
|
3422
|
+
|
|
3423
|
+
// src/core/wolbarg.ts
|
|
3424
|
+
var Wolbarg = class {
|
|
3425
|
+
initialized = false;
|
|
3426
|
+
booting = null;
|
|
3427
|
+
organization = null;
|
|
3428
|
+
storage = null;
|
|
3429
|
+
embedding = null;
|
|
3430
|
+
llm = null;
|
|
3431
|
+
compression = null;
|
|
3432
|
+
reranker = null;
|
|
3433
|
+
keywordSearch = null;
|
|
3434
|
+
ocr = null;
|
|
3435
|
+
vision = null;
|
|
3436
|
+
chunking = null;
|
|
3437
|
+
retrievalConfig = {};
|
|
3438
|
+
embeddingDimensions = null;
|
|
3439
|
+
writeMutex = new AsyncMutex();
|
|
3440
|
+
constructor(options) {
|
|
3441
|
+
if (!options) {
|
|
3442
|
+
return;
|
|
3443
|
+
}
|
|
3444
|
+
const validated = validateWolbargOptions(options);
|
|
3445
|
+
this.organization = validated.organization;
|
|
3446
|
+
this.storage = isStorageProvider(validated.storage) ? validated.storage : createStorageProvider(validated.storage);
|
|
3447
|
+
this.embedding = isEmbeddingProvider(validated.embedding) ? validated.embedding : createEmbeddingProvider(validated.embedding);
|
|
3448
|
+
if (validated.llm) {
|
|
3449
|
+
this.llm = isLlmProvider(validated.llm) ? validated.llm : createLlmProvider(validated.llm);
|
|
3450
|
+
this.compression = validated.compression ?? createCompressionProvider(this.llm);
|
|
3451
|
+
} else {
|
|
3452
|
+
this.compression = validated.compression ?? null;
|
|
3453
|
+
}
|
|
3454
|
+
this.reranker = validated.reranker ?? null;
|
|
3455
|
+
this.keywordSearch = validated.keywordSearch ?? null;
|
|
3456
|
+
this.ocr = validated.ocr ?? null;
|
|
3457
|
+
this.vision = validated.vision ?? null;
|
|
3458
|
+
this.chunking = validated.chunking ?? null;
|
|
3459
|
+
this.retrievalConfig = validated.retrieval ?? {};
|
|
3460
|
+
}
|
|
3461
|
+
/**
|
|
3462
|
+
* Backwards-compatible initialization (v0.1 API).
|
|
3463
|
+
* Prefer the constructor with `storage` / `embedding` / optional `llm`.
|
|
3464
|
+
*/
|
|
3465
|
+
async init(options) {
|
|
3466
|
+
if (this.initialized || this.storage) {
|
|
3467
|
+
throw new InitializationError("Wolbarg is already initialized");
|
|
3468
|
+
}
|
|
3469
|
+
let validated;
|
|
3470
|
+
try {
|
|
3471
|
+
validated = validateInitOptions(options);
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
if (error instanceof ConfigurationError || error instanceof ValidationError) {
|
|
3474
|
+
throw error;
|
|
3475
|
+
}
|
|
3476
|
+
throw new ConfigurationError(
|
|
3477
|
+
`Invalid configuration: ${error instanceof Error ? error.message : String(error)}`,
|
|
3478
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
3479
|
+
);
|
|
3480
|
+
}
|
|
3481
|
+
this.organization = validated.organization;
|
|
3482
|
+
this.storage = createStorageProvider(validated.database);
|
|
3483
|
+
this.embedding = createEmbeddingProvider(validated.embedding);
|
|
3484
|
+
if (validated.llm) {
|
|
3485
|
+
this.llm = createLlmProvider(validated.llm);
|
|
3486
|
+
this.compression = createCompressionProvider(this.llm);
|
|
3487
|
+
}
|
|
3488
|
+
await this.ready();
|
|
3489
|
+
}
|
|
3490
|
+
/** Ensure storage is open and embedding dimensions are known. */
|
|
3491
|
+
async ready() {
|
|
3492
|
+
if (this.initialized) {
|
|
3493
|
+
return;
|
|
3494
|
+
}
|
|
3495
|
+
if (this.booting) {
|
|
3496
|
+
await this.booting;
|
|
3497
|
+
return;
|
|
3498
|
+
}
|
|
3499
|
+
this.booting = this.boot();
|
|
3500
|
+
try {
|
|
3501
|
+
await this.booting;
|
|
3502
|
+
} finally {
|
|
3503
|
+
this.booting = null;
|
|
3504
|
+
}
|
|
3505
|
+
}
|
|
3506
|
+
async boot() {
|
|
3507
|
+
if (!this.storage || !this.embedding || !this.organization) {
|
|
3508
|
+
throw new InitializationError(
|
|
3509
|
+
"Wolbarg is not configured. Pass options to the constructor or call init()."
|
|
3510
|
+
);
|
|
3511
|
+
}
|
|
3512
|
+
try {
|
|
3513
|
+
await this.storage.open();
|
|
3514
|
+
const probe = await this.embedding.validate();
|
|
3515
|
+
await this.storage.ensureVectorSchema(probe.dimensions);
|
|
3516
|
+
this.embeddingDimensions = probe.dimensions;
|
|
3517
|
+
this.initialized = true;
|
|
3518
|
+
} catch (error) {
|
|
3519
|
+
await this.storage.close().catch(() => void 0);
|
|
3520
|
+
this.initialized = false;
|
|
3521
|
+
if (error instanceof ConfigurationError || error instanceof InitializationError || error instanceof ValidationError) {
|
|
3522
|
+
throw error;
|
|
3523
|
+
}
|
|
3524
|
+
throw new InitializationError(
|
|
3525
|
+
`Initialization failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3526
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
3527
|
+
);
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3530
|
+
/** Store a semantic memory for an agent. */
|
|
3531
|
+
async remember(options) {
|
|
3532
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
3533
|
+
assertNonEmptyString(options.agent, "agent");
|
|
3534
|
+
if (!options.content || typeof options.content.text !== "string") {
|
|
3535
|
+
throw new ValidationError("content.text must be a string");
|
|
3536
|
+
}
|
|
3537
|
+
assertNonEmptyString(options.content.text, "content.text");
|
|
3538
|
+
const metadata = options.metadata ?? {};
|
|
3539
|
+
if (metadata === null || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
3540
|
+
throw new ValidationError("metadata must be a plain object when provided");
|
|
3541
|
+
}
|
|
3542
|
+
const profile = process.env.WOLBARG_PROFILE === "1";
|
|
3543
|
+
const t0 = profile ? performance.now() : 0;
|
|
3544
|
+
const vector = await embedding.embed(options.content.text);
|
|
3545
|
+
const embedMs = profile ? performance.now() - t0 : 0;
|
|
3546
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3547
|
+
const id = createId();
|
|
3548
|
+
const timestamp = nowIso();
|
|
3549
|
+
return this.withWriteLock(async () => {
|
|
3550
|
+
const tStore = profile ? performance.now() : 0;
|
|
3551
|
+
const row = await storage.insertMemory({
|
|
3552
|
+
id,
|
|
3553
|
+
organization,
|
|
3554
|
+
agent: options.agent.trim(),
|
|
3555
|
+
contentText: options.content.text,
|
|
3556
|
+
metadata,
|
|
3557
|
+
embedding: vector,
|
|
3558
|
+
createdAt: timestamp,
|
|
3559
|
+
updatedAt: timestamp
|
|
3560
|
+
});
|
|
3561
|
+
if (profile) {
|
|
3562
|
+
const storeMs = performance.now() - tStore;
|
|
3563
|
+
const totalMs = performance.now() - t0;
|
|
3564
|
+
console.log(
|
|
3565
|
+
`[profile] remember embed=${embedMs.toFixed(2)}ms store=${storeMs.toFixed(2)}ms total=${totalMs.toFixed(2)}ms backend=${storage.name}`
|
|
3566
|
+
);
|
|
3567
|
+
}
|
|
3568
|
+
return toMemoryRecord(row);
|
|
3569
|
+
});
|
|
3570
|
+
}
|
|
3571
|
+
/**
|
|
3572
|
+
* Semantic / hybrid search over stored memories.
|
|
3573
|
+
* Optional keyword, metadata, MMR, and rerank stages degrade gracefully.
|
|
3574
|
+
*/
|
|
3575
|
+
async recall(options) {
|
|
3576
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
3577
|
+
assertNonEmptyString(options.query, "query");
|
|
3578
|
+
const topK = options.topK ?? 5;
|
|
3579
|
+
const threshold = options.threshold ?? 0;
|
|
3580
|
+
assertFiniteNumber(topK, "topK", { min: 1, max: 1e3 });
|
|
3581
|
+
assertFiniteNumber(threshold, "threshold", { min: 0, max: 1 });
|
|
3582
|
+
const query = options.query.trim();
|
|
3583
|
+
const vector = await embedding.embed(query);
|
|
3584
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3585
|
+
const hasFilters = Boolean(
|
|
3586
|
+
options.filter?.agent || options.filter?.metadata
|
|
3587
|
+
);
|
|
3588
|
+
const overFetch = this.retrievalConfig.overFetchFactor ?? (hasFilters ? 4 : 2);
|
|
3589
|
+
const fetchK = adaptiveFetchK(topK, overFetch, hasFilters);
|
|
3590
|
+
const semanticScores = /* @__PURE__ */ new Map();
|
|
3591
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3592
|
+
const joined = storage.searchVectorsWithMemories;
|
|
3593
|
+
if (typeof joined === "function") {
|
|
3594
|
+
const joinedHits = await joined.call(
|
|
3595
|
+
storage,
|
|
3596
|
+
vector,
|
|
3597
|
+
fetchK,
|
|
3598
|
+
organization,
|
|
3599
|
+
{
|
|
3600
|
+
agent: options.filter?.agent,
|
|
3601
|
+
includeArchived: options.filter?.includeArchived
|
|
3602
|
+
}
|
|
3603
|
+
);
|
|
3604
|
+
for (const { row, distance } of joinedHits) {
|
|
3605
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
3606
|
+
deserializeMetadata(row.metadata_json),
|
|
3607
|
+
options.filter.metadata
|
|
3608
|
+
)) {
|
|
3609
|
+
continue;
|
|
3610
|
+
}
|
|
3611
|
+
const similarity = distanceToSimilarity(distance);
|
|
3612
|
+
if (similarity < threshold) {
|
|
3613
|
+
continue;
|
|
3614
|
+
}
|
|
3615
|
+
const result = toRecallResult(row, similarity);
|
|
3616
|
+
semanticScores.set(result.id, similarity);
|
|
3617
|
+
byId.set(result.id, result);
|
|
3618
|
+
}
|
|
3619
|
+
} else {
|
|
3620
|
+
const hits = await storage.searchVectors(vector, fetchK);
|
|
3621
|
+
const rowids = hits.map((h) => h.memoryRowid);
|
|
3622
|
+
let rowMap;
|
|
3623
|
+
if (typeof storage.getMemoriesByRowids === "function") {
|
|
3624
|
+
rowMap = await storage.getMemoriesByRowids(rowids, organization);
|
|
3625
|
+
} else {
|
|
3626
|
+
rowMap = /* @__PURE__ */ new Map();
|
|
3627
|
+
const looked = await Promise.all(
|
|
3628
|
+
hits.map(async (hit) => ({
|
|
3629
|
+
hit,
|
|
3630
|
+
row: await storage.getMemoryByRowid(hit.memoryRowid, organization)
|
|
3631
|
+
}))
|
|
3632
|
+
);
|
|
3633
|
+
for (const { hit, row } of looked) {
|
|
3634
|
+
if (row) rowMap.set(hit.memoryRowid, row);
|
|
3635
|
+
}
|
|
3636
|
+
}
|
|
3637
|
+
for (const hit of hits) {
|
|
3638
|
+
const row = rowMap.get(hit.memoryRowid);
|
|
3639
|
+
if (!row) {
|
|
3640
|
+
continue;
|
|
3641
|
+
}
|
|
3642
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
3643
|
+
continue;
|
|
3644
|
+
}
|
|
3645
|
+
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
3646
|
+
continue;
|
|
3647
|
+
}
|
|
3648
|
+
const metadata = deserializeMetadata(row.metadata_json);
|
|
3649
|
+
if (options.filter?.metadata && !matchesMetadata(metadata, options.filter.metadata)) {
|
|
3650
|
+
continue;
|
|
3651
|
+
}
|
|
3652
|
+
const similarity = distanceToSimilarity(hit.distance);
|
|
3653
|
+
if (similarity < threshold) {
|
|
3654
|
+
continue;
|
|
3655
|
+
}
|
|
3656
|
+
const result = toRecallResult(row, similarity);
|
|
3657
|
+
semanticScores.set(result.id, similarity);
|
|
3658
|
+
byId.set(result.id, result);
|
|
3659
|
+
}
|
|
3660
|
+
}
|
|
3661
|
+
const hybridWeights = resolveHybridWeights(options.hybrid) ?? (options.hybrid === void 0 ? resolveHybridWeights(this.retrievalConfig.hybrid) : null);
|
|
3662
|
+
if (hybridWeights) {
|
|
3663
|
+
let keywordScores = /* @__PURE__ */ new Map();
|
|
3664
|
+
if (typeof storage.searchKeyword === "function") {
|
|
3665
|
+
const ftsHits = await storage.searchKeyword(
|
|
3666
|
+
query,
|
|
3667
|
+
organization,
|
|
3668
|
+
fetchK
|
|
3669
|
+
);
|
|
3670
|
+
keywordScores = new Map(
|
|
3671
|
+
ftsHits.map((h) => [h.memoryId, h.score])
|
|
3672
|
+
);
|
|
3673
|
+
const missingIds = [];
|
|
3674
|
+
for (const id of keywordScores.keys()) {
|
|
3675
|
+
if (!byId.has(id)) missingIds.push(id);
|
|
3676
|
+
}
|
|
3677
|
+
if (missingIds.length > 0) {
|
|
3678
|
+
const fetched = await Promise.all(
|
|
3679
|
+
missingIds.map((id) => storage.getMemoryById(id, organization))
|
|
3680
|
+
);
|
|
3681
|
+
for (const row of fetched) {
|
|
3682
|
+
if (!row) continue;
|
|
3683
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
3684
|
+
continue;
|
|
3685
|
+
}
|
|
3686
|
+
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
3687
|
+
continue;
|
|
3688
|
+
}
|
|
3689
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
3690
|
+
deserializeMetadata(row.metadata_json),
|
|
3691
|
+
options.filter.metadata
|
|
3692
|
+
)) {
|
|
3693
|
+
continue;
|
|
3694
|
+
}
|
|
3695
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
} else if (this.keywordSearch) {
|
|
3699
|
+
const corpus = await storage.listMemories(
|
|
3700
|
+
{
|
|
3701
|
+
organization,
|
|
3702
|
+
agent: options.filter?.agent,
|
|
3703
|
+
includeArchived: options.filter?.includeArchived,
|
|
3704
|
+
metadata: options.filter?.metadata
|
|
3705
|
+
},
|
|
3706
|
+
Math.min(fetchK * 2, 2e3)
|
|
3707
|
+
);
|
|
3708
|
+
const keywordHits = await this.keywordSearch.search(
|
|
3709
|
+
query,
|
|
3710
|
+
corpus.map((row) => ({ id: row.id, text: row.content_text })),
|
|
3711
|
+
fetchK
|
|
3712
|
+
);
|
|
3713
|
+
keywordScores = new Map(
|
|
3714
|
+
keywordHits.map((h) => [h.memoryId, h.score])
|
|
3715
|
+
);
|
|
3716
|
+
for (const row of corpus) {
|
|
3717
|
+
if (byId.has(row.id)) {
|
|
3718
|
+
continue;
|
|
3719
|
+
}
|
|
3720
|
+
if (!keywordScores.has(row.id)) {
|
|
3721
|
+
continue;
|
|
3722
|
+
}
|
|
3723
|
+
const metadata = deserializeMetadata(row.metadata_json);
|
|
3724
|
+
if (options.filter?.metadata && !matchesMetadata(metadata, options.filter.metadata)) {
|
|
3725
|
+
continue;
|
|
3726
|
+
}
|
|
3727
|
+
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
3728
|
+
continue;
|
|
3729
|
+
}
|
|
3730
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
3731
|
+
}
|
|
3732
|
+
}
|
|
3733
|
+
if (keywordScores.size > 0) {
|
|
3734
|
+
const fused = fuseScores(semanticScores, keywordScores, hybridWeights);
|
|
3735
|
+
for (const [id, score] of fused) {
|
|
3736
|
+
const existing = byId.get(id);
|
|
3737
|
+
if (existing) {
|
|
3738
|
+
byId.set(id, { ...existing, similarity: score });
|
|
3739
|
+
}
|
|
3740
|
+
}
|
|
3741
|
+
}
|
|
3742
|
+
}
|
|
3743
|
+
let results = [...byId.values()].sort(
|
|
3744
|
+
(a, b) => b.similarity - a.similarity
|
|
3745
|
+
);
|
|
3746
|
+
const mmrLambda = resolveMmr(options.mmr) ?? resolveMmr(
|
|
3747
|
+
this.retrievalConfig.mmr ? { lambda: this.retrievalConfig.mmr.lambda } : void 0
|
|
3748
|
+
);
|
|
3749
|
+
if (mmrLambda !== null) {
|
|
3750
|
+
results = applyMmr(results, Math.max(topK * 2, topK), mmrLambda);
|
|
3751
|
+
}
|
|
3752
|
+
if (options.rerank && this.reranker) {
|
|
3753
|
+
const reranked = await this.reranker.rerank(
|
|
3754
|
+
query,
|
|
3755
|
+
results.map((r) => ({ id: r.id, text: r.content.text })),
|
|
3756
|
+
topK
|
|
3757
|
+
);
|
|
3758
|
+
const order = new Map(reranked.map((r, i) => [r.id, { score: r.score, i }]));
|
|
3759
|
+
results = results.filter((r) => order.has(r.id)).sort((a, b) => order.get(a.id).i - order.get(b.id).i).map((r) => ({
|
|
3760
|
+
...r,
|
|
3761
|
+
similarity: order.get(r.id)?.score ?? r.similarity
|
|
3762
|
+
}));
|
|
3763
|
+
}
|
|
3764
|
+
return results.slice(0, topK);
|
|
3765
|
+
}
|
|
3766
|
+
/**
|
|
3767
|
+
* Compress related memories for an agent into a single summarized memory.
|
|
3768
|
+
* Only available when `llm` (or `compression`) was configured at construction.
|
|
3769
|
+
*/
|
|
3770
|
+
async compress(options) {
|
|
3771
|
+
return this.runCompress(options);
|
|
3772
|
+
}
|
|
3773
|
+
/** @internal */
|
|
3774
|
+
async runCompress(options) {
|
|
3775
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
3776
|
+
if (!this.compression) {
|
|
3777
|
+
throw new ProviderNotConfiguredError(
|
|
3778
|
+
"llm",
|
|
3779
|
+
"compress",
|
|
3780
|
+
"pass llm: openaiLlm(...) (or compression) in the constructor"
|
|
3781
|
+
);
|
|
3782
|
+
}
|
|
3783
|
+
assertNonEmptyString(options.agent, "agent");
|
|
3784
|
+
const limit = options.limit ?? 50;
|
|
3785
|
+
assertFiniteNumber(limit, "limit", { min: 2, max: 500 });
|
|
3786
|
+
const rows = await storage.listMemories(
|
|
3787
|
+
{
|
|
3788
|
+
organization,
|
|
3789
|
+
agent: options.agent.trim(),
|
|
3790
|
+
includeArchived: false
|
|
3791
|
+
},
|
|
3792
|
+
limit
|
|
3793
|
+
);
|
|
3794
|
+
if (rows.length < 2) {
|
|
3795
|
+
throw new ValidationError(
|
|
3796
|
+
"compress requires at least 2 active memories for the given agent"
|
|
3797
|
+
);
|
|
3798
|
+
}
|
|
3799
|
+
const records = rows.map(toMemoryRecord);
|
|
3800
|
+
const summaryText = await this.compression.compress(records);
|
|
3801
|
+
const vector = await embedding.embed(summaryText);
|
|
3802
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3803
|
+
const summaryId = createId();
|
|
3804
|
+
const timestamp = nowIso();
|
|
3805
|
+
return this.withWriteLock(async () => {
|
|
3806
|
+
const summaryRow = await storage.insertMemory({
|
|
3807
|
+
id: summaryId,
|
|
3808
|
+
organization,
|
|
3809
|
+
agent: options.agent.trim(),
|
|
3810
|
+
contentText: summaryText,
|
|
3811
|
+
metadata: {
|
|
3812
|
+
compressed: true,
|
|
3813
|
+
sourceCount: records.length,
|
|
3814
|
+
sourceIds: records.map((r) => r.id)
|
|
3815
|
+
},
|
|
3816
|
+
embedding: vector,
|
|
3817
|
+
createdAt: timestamp,
|
|
3818
|
+
updatedAt: timestamp
|
|
3819
|
+
});
|
|
3820
|
+
const archivedIds = await storage.archiveMemories(
|
|
3821
|
+
records.map((r) => r.id),
|
|
3822
|
+
organization,
|
|
3823
|
+
summaryId,
|
|
3824
|
+
timestamp
|
|
3825
|
+
);
|
|
3826
|
+
return {
|
|
3827
|
+
summary: toMemoryRecord(summaryRow),
|
|
3828
|
+
archivedIds
|
|
3829
|
+
};
|
|
3830
|
+
});
|
|
3831
|
+
}
|
|
3832
|
+
/** Ingest a document: parse → OCR/vision (optional) → chunk → embed → store. */
|
|
3833
|
+
async ingest(options) {
|
|
3834
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
3835
|
+
assertNonEmptyString(options.agent, "agent");
|
|
3836
|
+
const loaded = await loadIngestSource(options.source);
|
|
3837
|
+
let usedOcr = false;
|
|
3838
|
+
let usedVision = false;
|
|
3839
|
+
let text = loaded.rawText ?? "";
|
|
3840
|
+
if (!loaded.rawText) {
|
|
3841
|
+
const parser = resolveParser(loaded.filename, loaded.mimeType);
|
|
3842
|
+
const parsed = await parser.parse({
|
|
3843
|
+
buffer: loaded.buffer,
|
|
3844
|
+
filename: loaded.filename,
|
|
3845
|
+
mimeType: loaded.mimeType
|
|
3846
|
+
});
|
|
3847
|
+
text = parsed.text;
|
|
3848
|
+
if (parsed.isImage && parsed.imageBuffer) {
|
|
3849
|
+
const parts = [];
|
|
3850
|
+
if (this.ocr) {
|
|
3851
|
+
const ocrResult = await this.ocr.recognize(
|
|
3852
|
+
parsed.imageBuffer,
|
|
3853
|
+
parsed.mimeType
|
|
3854
|
+
);
|
|
3855
|
+
if (ocrResult.text) {
|
|
3856
|
+
parts.push(ocrResult.text);
|
|
3857
|
+
usedOcr = true;
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
if (this.vision) {
|
|
3861
|
+
const visionResult = await this.vision.analyze(
|
|
3862
|
+
parsed.imageBuffer,
|
|
3863
|
+
parsed.mimeType
|
|
3864
|
+
);
|
|
3865
|
+
if (visionResult.caption) {
|
|
3866
|
+
parts.push(`Caption: ${visionResult.caption}`);
|
|
3867
|
+
}
|
|
3868
|
+
if (visionResult.description) {
|
|
3869
|
+
parts.push(visionResult.description);
|
|
3870
|
+
}
|
|
3871
|
+
if (visionResult.entities.length > 0) {
|
|
3872
|
+
parts.push(`Entities: ${visionResult.entities.join(", ")}`);
|
|
3873
|
+
}
|
|
3874
|
+
if (parts.length > 0) {
|
|
3875
|
+
usedVision = true;
|
|
3876
|
+
}
|
|
3877
|
+
}
|
|
3878
|
+
text = parts.join("\n\n").trim();
|
|
3879
|
+
}
|
|
3880
|
+
}
|
|
3881
|
+
if (!text) {
|
|
3882
|
+
throw new ValidationError(
|
|
3883
|
+
"ingest could not extract text from the document (configure ocr/vision for images, or provide text)"
|
|
3884
|
+
);
|
|
3885
|
+
}
|
|
3886
|
+
const strategy = this.chunking ?? (options.chunking?.strategy ? createChunkingStrategy(options.chunking.strategy) : inferChunkingStrategy(text));
|
|
3887
|
+
const chunks = strategy.chunk(text, {
|
|
3888
|
+
chunkSize: options.chunking?.chunkSize,
|
|
3889
|
+
overlap: options.chunking?.overlap
|
|
3890
|
+
});
|
|
3891
|
+
if (chunks.length === 0) {
|
|
3892
|
+
throw new ValidationError("ingest produced zero chunks");
|
|
3893
|
+
}
|
|
3894
|
+
const vectors = await embedMany(
|
|
3895
|
+
embedding,
|
|
3896
|
+
chunks.map((c) => c.text)
|
|
3897
|
+
);
|
|
3898
|
+
for (const vector of vectors) {
|
|
3899
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3900
|
+
}
|
|
3901
|
+
const baseMeta = options.metadata ?? {};
|
|
3902
|
+
const timestamp = nowIso();
|
|
3903
|
+
const inputs = chunks.map((chunk, i) => ({
|
|
3904
|
+
id: createId(),
|
|
3905
|
+
organization,
|
|
3906
|
+
agent: options.agent.trim(),
|
|
3907
|
+
contentText: chunk.text,
|
|
3908
|
+
metadata: {
|
|
3909
|
+
...baseMeta,
|
|
3910
|
+
ingest: true,
|
|
3911
|
+
chunkIndex: chunk.index,
|
|
3912
|
+
chunkCount: chunks.length,
|
|
3913
|
+
sourceFilename: loaded.filename ?? null
|
|
3914
|
+
},
|
|
3915
|
+
embedding: vectors[i],
|
|
3916
|
+
createdAt: timestamp,
|
|
3917
|
+
updatedAt: timestamp
|
|
3918
|
+
}));
|
|
3919
|
+
const rows = await this.withWriteLock(async () => {
|
|
3920
|
+
return storage.insertMemoriesBatch(inputs);
|
|
3921
|
+
});
|
|
3922
|
+
return {
|
|
3923
|
+
memories: rows.map(toMemoryRecord),
|
|
3924
|
+
extractedChars: text.length,
|
|
3925
|
+
chunkCount: chunks.length,
|
|
3926
|
+
usedOcr,
|
|
3927
|
+
usedVision
|
|
3928
|
+
};
|
|
3929
|
+
}
|
|
3930
|
+
/** Delete memories by ID or filter. */
|
|
3931
|
+
async forget(options) {
|
|
3932
|
+
const { storage, organization } = await this.requireReady();
|
|
3933
|
+
return this.withWriteLock(async () => {
|
|
3934
|
+
if ("id" in options && options.id !== void 0) {
|
|
3935
|
+
assertNonEmptyString(options.id, "id");
|
|
3936
|
+
const deleted = await storage.deleteMemoryById(
|
|
3937
|
+
options.id.trim(),
|
|
3938
|
+
organization
|
|
3939
|
+
);
|
|
3940
|
+
return deleted ? 1 : 0;
|
|
3941
|
+
}
|
|
3942
|
+
if ("filter" in options && options.filter?.agent) {
|
|
3943
|
+
assertNonEmptyString(options.filter.agent, "filter.agent");
|
|
3944
|
+
return storage.deleteMemoriesByFilter({
|
|
3945
|
+
organization,
|
|
3946
|
+
agent: options.filter.agent.trim(),
|
|
3947
|
+
includeArchived: true
|
|
3948
|
+
});
|
|
3949
|
+
}
|
|
3950
|
+
throw new ValidationError(
|
|
3951
|
+
"forget requires either { id } or { filter: { agent } }"
|
|
3952
|
+
);
|
|
3953
|
+
});
|
|
3954
|
+
}
|
|
3955
|
+
/** Return the history of a memory. */
|
|
3956
|
+
async history(options) {
|
|
3957
|
+
const { storage, organization } = await this.requireReady();
|
|
3958
|
+
assertNonEmptyString(options.id, "id");
|
|
3959
|
+
const row = await storage.getMemoryById(options.id.trim(), organization);
|
|
3960
|
+
if (!row) {
|
|
3961
|
+
throw new MemoryNotFoundError(`Memory not found: ${options.id}`);
|
|
3962
|
+
}
|
|
3963
|
+
const events = await storage.getHistory(row.id);
|
|
3964
|
+
return {
|
|
3965
|
+
memory: toMemoryRecord(row),
|
|
3966
|
+
events: events.map(toHistoryEvent)
|
|
3967
|
+
};
|
|
3968
|
+
}
|
|
3969
|
+
/** Aggregate statistics for the current organization. */
|
|
3970
|
+
async stats() {
|
|
3971
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
3972
|
+
const counts = await storage.getStats(organization);
|
|
3973
|
+
const databaseSizeBytes = await storage.getDatabaseSizeBytes();
|
|
3974
|
+
return {
|
|
3975
|
+
totalMemories: counts.totalMemories,
|
|
3976
|
+
activeMemories: counts.activeMemories,
|
|
3977
|
+
archivedMemories: counts.archivedMemories,
|
|
3978
|
+
totalAgents: counts.totalAgents,
|
|
3979
|
+
databaseSizeBytes,
|
|
3980
|
+
embeddingModel: embedding.model,
|
|
3981
|
+
llmModel: this.llm?.model ?? null,
|
|
3982
|
+
organization,
|
|
3983
|
+
embeddingDimensions: this.embeddingDimensions ?? 0
|
|
3984
|
+
};
|
|
3985
|
+
}
|
|
3986
|
+
/** Delete every memory in the current organization. */
|
|
3987
|
+
async clear(options) {
|
|
3988
|
+
const { storage, organization } = await this.requireReady();
|
|
3989
|
+
if (!options || options.confirm !== true) {
|
|
3990
|
+
throw new ValidationError(
|
|
3991
|
+
"clear requires { confirm: true } to permanently delete all memories"
|
|
3992
|
+
);
|
|
3993
|
+
}
|
|
3994
|
+
return this.withWriteLock(async () => {
|
|
3995
|
+
return storage.clearOrganization(organization);
|
|
3996
|
+
});
|
|
3997
|
+
}
|
|
3998
|
+
/**
|
|
3999
|
+
* Serialize writes for SQLite (single connection). Postgres uses a pool and
|
|
4000
|
+
* per-statement atomic CTEs — locking here only serializes throughput.
|
|
4001
|
+
*/
|
|
4002
|
+
withWriteLock(fn) {
|
|
4003
|
+
if (this.storage?.name === "sqlite") {
|
|
4004
|
+
return this.writeMutex.runExclusive(fn);
|
|
4005
|
+
}
|
|
4006
|
+
return fn();
|
|
4007
|
+
}
|
|
4008
|
+
/** Close the database connection and release resources. */
|
|
4009
|
+
async close() {
|
|
4010
|
+
if (this.storage) {
|
|
4011
|
+
await this.storage.close();
|
|
4012
|
+
}
|
|
4013
|
+
this.storage = null;
|
|
4014
|
+
this.embedding = null;
|
|
4015
|
+
this.llm = null;
|
|
4016
|
+
this.compression = null;
|
|
4017
|
+
this.organization = null;
|
|
4018
|
+
this.embeddingDimensions = null;
|
|
4019
|
+
this.initialized = false;
|
|
4020
|
+
}
|
|
4021
|
+
/** Whether providers are ready for API calls. */
|
|
4022
|
+
get isInitialized() {
|
|
4023
|
+
return this.initialized;
|
|
4024
|
+
}
|
|
4025
|
+
async requireReady() {
|
|
4026
|
+
await this.ready();
|
|
4027
|
+
if (!this.initialized || !this.storage || !this.embedding || !this.organization) {
|
|
4028
|
+
throw new InitializationError(
|
|
4029
|
+
"Wolbarg is not initialized. Pass options to the constructor or call init()."
|
|
4030
|
+
);
|
|
4031
|
+
}
|
|
4032
|
+
return {
|
|
4033
|
+
storage: this.storage,
|
|
4034
|
+
embedding: this.embedding,
|
|
4035
|
+
organization: this.organization
|
|
4036
|
+
};
|
|
4037
|
+
}
|
|
4038
|
+
assertEmbeddingDimensions(dimensions) {
|
|
4039
|
+
if (this.embeddingDimensions !== null && dimensions !== this.embeddingDimensions) {
|
|
4040
|
+
throw new ValidationError(
|
|
4041
|
+
`Embedding dimension mismatch: expected ${this.embeddingDimensions}, got ${dimensions}`
|
|
4042
|
+
);
|
|
4043
|
+
}
|
|
4044
|
+
}
|
|
4045
|
+
};
|
|
4046
|
+
|
|
4047
|
+
// src/filters/types.ts
|
|
4048
|
+
var meta = {
|
|
4049
|
+
eq: (field, value) => ({
|
|
4050
|
+
field,
|
|
4051
|
+
op: { eq: value }
|
|
4052
|
+
}),
|
|
4053
|
+
contains: (field, value) => ({
|
|
4054
|
+
field,
|
|
4055
|
+
op: { contains: value }
|
|
4056
|
+
}),
|
|
4057
|
+
gt: (field, value) => ({
|
|
4058
|
+
field,
|
|
4059
|
+
op: { gt: value }
|
|
4060
|
+
}),
|
|
4061
|
+
gte: (field, value) => ({
|
|
4062
|
+
field,
|
|
4063
|
+
op: { gte: value }
|
|
4064
|
+
}),
|
|
4065
|
+
lt: (field, value) => ({
|
|
4066
|
+
field,
|
|
4067
|
+
op: { lt: value }
|
|
4068
|
+
}),
|
|
4069
|
+
lte: (field, value) => ({
|
|
4070
|
+
field,
|
|
4071
|
+
op: { lte: value }
|
|
4072
|
+
}),
|
|
4073
|
+
between: (field, lo, hi) => ({
|
|
4074
|
+
field,
|
|
4075
|
+
op: { between: [lo, hi] }
|
|
4076
|
+
}),
|
|
4077
|
+
and: (...filters) => ({ and: filters }),
|
|
4078
|
+
or: (...filters) => ({ or: filters }),
|
|
4079
|
+
not: (filter) => ({ not: filter })
|
|
4080
|
+
};
|
|
4081
|
+
|
|
4082
|
+
// src/factories/index.ts
|
|
4083
|
+
function sqlite(connectionString) {
|
|
4084
|
+
return new SqliteStorageProvider({ connectionString });
|
|
4085
|
+
}
|
|
4086
|
+
function sqliteConfig(connectionString) {
|
|
4087
|
+
return { provider: "sqlite", connectionString };
|
|
4088
|
+
}
|
|
4089
|
+
function postgres(options) {
|
|
4090
|
+
const opts = typeof options === "string" ? { connectionString: options } : options;
|
|
4091
|
+
return new PostgresStorageProvider(opts);
|
|
4092
|
+
}
|
|
4093
|
+
function postgresConfig(connectionString, options) {
|
|
4094
|
+
return {
|
|
4095
|
+
provider: "postgres",
|
|
4096
|
+
connectionString,
|
|
4097
|
+
...options
|
|
4098
|
+
};
|
|
4099
|
+
}
|
|
4100
|
+
|
|
4101
|
+
// src/keyword/index.ts
|
|
4102
|
+
var Bm25KeywordSearchProvider = class {
|
|
4103
|
+
name = "bm25";
|
|
4104
|
+
k1;
|
|
4105
|
+
b;
|
|
4106
|
+
constructor(options) {
|
|
4107
|
+
this.k1 = options?.k1 ?? 1.2;
|
|
4108
|
+
this.b = options?.b ?? 0.75;
|
|
4109
|
+
}
|
|
4110
|
+
async search(query, documents, topK) {
|
|
4111
|
+
if (documents.length === 0 || topK <= 0) {
|
|
4112
|
+
return [];
|
|
4113
|
+
}
|
|
4114
|
+
const docs = documents.map((doc) => ({
|
|
4115
|
+
id: doc.id,
|
|
4116
|
+
tokens: tokenize(doc.text)
|
|
4117
|
+
}));
|
|
4118
|
+
const queryTokens = tokenize(query);
|
|
4119
|
+
if (queryTokens.length === 0) {
|
|
4120
|
+
return [];
|
|
4121
|
+
}
|
|
4122
|
+
const N = docs.length;
|
|
4123
|
+
const avgdl = docs.reduce((sum, d) => sum + d.tokens.length, 0) / Math.max(N, 1);
|
|
4124
|
+
const df = /* @__PURE__ */ new Map();
|
|
4125
|
+
for (const term of new Set(queryTokens)) {
|
|
4126
|
+
let count = 0;
|
|
4127
|
+
for (const doc of docs) {
|
|
4128
|
+
if (doc.tokens.includes(term)) {
|
|
4129
|
+
count += 1;
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
df.set(term, count);
|
|
4133
|
+
}
|
|
4134
|
+
const scored = docs.map((doc) => {
|
|
4135
|
+
const tfMap = /* @__PURE__ */ new Map();
|
|
4136
|
+
for (const token of doc.tokens) {
|
|
4137
|
+
tfMap.set(token, (tfMap.get(token) ?? 0) + 1);
|
|
4138
|
+
}
|
|
4139
|
+
let score = 0;
|
|
4140
|
+
for (const term of queryTokens) {
|
|
4141
|
+
const tf = tfMap.get(term) ?? 0;
|
|
4142
|
+
if (tf === 0) {
|
|
4143
|
+
continue;
|
|
4144
|
+
}
|
|
4145
|
+
const n = df.get(term) ?? 0;
|
|
4146
|
+
const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));
|
|
4147
|
+
const denom = tf + this.k1 * (1 - this.b + this.b * (doc.tokens.length / avgdl));
|
|
4148
|
+
score += idf * (tf * (this.k1 + 1) / denom);
|
|
4149
|
+
}
|
|
4150
|
+
return { memoryId: doc.id, score };
|
|
4151
|
+
});
|
|
4152
|
+
scored.sort((a, b) => b.score - a.score);
|
|
4153
|
+
return scored.filter((s) => s.score > 0).slice(0, topK);
|
|
4154
|
+
}
|
|
4155
|
+
};
|
|
4156
|
+
function tokenize(text) {
|
|
4157
|
+
return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length > 1);
|
|
4158
|
+
}
|
|
4159
|
+
function bm25(options) {
|
|
4160
|
+
return new Bm25KeywordSearchProvider(options);
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
// src/rerank/index.ts
|
|
4164
|
+
var HttpRerankerProvider = class {
|
|
4165
|
+
name;
|
|
4166
|
+
options;
|
|
4167
|
+
constructor(options) {
|
|
4168
|
+
this.name = options.name;
|
|
4169
|
+
this.options = options;
|
|
4170
|
+
}
|
|
4171
|
+
async rerank(query, documents, topK) {
|
|
4172
|
+
if (documents.length === 0) {
|
|
4173
|
+
return [];
|
|
4174
|
+
}
|
|
4175
|
+
const controller = new AbortController();
|
|
4176
|
+
const timeoutMs = this.options.timeoutMs ?? 3e4;
|
|
4177
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
4178
|
+
try {
|
|
4179
|
+
const res = await fetch(this.options.url, {
|
|
4180
|
+
method: "POST",
|
|
4181
|
+
headers: {
|
|
4182
|
+
"Content-Type": "application/json",
|
|
4183
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
4184
|
+
},
|
|
4185
|
+
body: JSON.stringify(
|
|
4186
|
+
this.options.buildBody(
|
|
4187
|
+
query,
|
|
4188
|
+
documents,
|
|
4189
|
+
topK,
|
|
4190
|
+
this.options.model
|
|
4191
|
+
)
|
|
4192
|
+
),
|
|
4193
|
+
signal: controller.signal
|
|
4194
|
+
});
|
|
4195
|
+
const body = await res.json().catch(() => ({}));
|
|
4196
|
+
if (!res.ok) {
|
|
4197
|
+
return documents.slice(0, topK).map((d, i) => ({
|
|
4198
|
+
id: d.id,
|
|
4199
|
+
score: 1 - i / Math.max(documents.length, 1)
|
|
4200
|
+
}));
|
|
4201
|
+
}
|
|
4202
|
+
const parsed = this.options.parseResults(body, documents);
|
|
4203
|
+
return parsed.slice(0, topK);
|
|
4204
|
+
} catch {
|
|
4205
|
+
return documents.slice(0, topK).map((d, i) => ({
|
|
4206
|
+
id: d.id,
|
|
4207
|
+
score: 1 - i / Math.max(documents.length, 1)
|
|
4208
|
+
}));
|
|
4209
|
+
} finally {
|
|
4210
|
+
clearTimeout(timer);
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
};
|
|
4214
|
+
function crossEncoder(options) {
|
|
4215
|
+
const base = options.baseUrl.replace(/\/+$/, "");
|
|
4216
|
+
return new HttpRerankerProvider({
|
|
4217
|
+
name: "cross-encoder",
|
|
4218
|
+
url: `${base}/rerank`,
|
|
4219
|
+
apiKey: options.apiKey,
|
|
4220
|
+
model: options.model,
|
|
4221
|
+
timeoutMs: options.timeoutMs,
|
|
4222
|
+
buildBody: (query, documents, topK, model) => ({
|
|
4223
|
+
model,
|
|
4224
|
+
query,
|
|
4225
|
+
documents: documents.map((d) => d.text),
|
|
4226
|
+
top_n: topK
|
|
4227
|
+
}),
|
|
4228
|
+
parseResults: (body, documents) => {
|
|
4229
|
+
const results = body.results ?? [];
|
|
4230
|
+
return results.map((r) => ({
|
|
4231
|
+
id: documents[r.index]?.id ?? String(r.index),
|
|
4232
|
+
score: r.relevance_score
|
|
4233
|
+
}));
|
|
4234
|
+
}
|
|
4235
|
+
});
|
|
4236
|
+
}
|
|
4237
|
+
function jinaReranker(options) {
|
|
4238
|
+
return new HttpRerankerProvider({
|
|
4239
|
+
name: "jina",
|
|
4240
|
+
url: "https://api.jina.ai/v1/rerank",
|
|
4241
|
+
apiKey: options.apiKey,
|
|
4242
|
+
model: options.model ?? "jina-reranker-v2-base-multilingual",
|
|
4243
|
+
timeoutMs: options.timeoutMs,
|
|
4244
|
+
buildBody: (query, documents, topK, model) => ({
|
|
4245
|
+
model,
|
|
4246
|
+
query,
|
|
4247
|
+
documents: documents.map((d) => d.text),
|
|
4248
|
+
top_n: topK
|
|
4249
|
+
}),
|
|
4250
|
+
parseResults: (body, documents) => {
|
|
4251
|
+
const results = body.results ?? [];
|
|
4252
|
+
return results.map((r) => ({
|
|
4253
|
+
id: documents[r.index]?.id ?? String(r.index),
|
|
4254
|
+
score: r.relevance_score
|
|
4255
|
+
}));
|
|
4256
|
+
}
|
|
4257
|
+
});
|
|
4258
|
+
}
|
|
4259
|
+
function cohereReranker(options) {
|
|
4260
|
+
return new HttpRerankerProvider({
|
|
4261
|
+
name: "cohere",
|
|
4262
|
+
url: "https://api.cohere.com/v2/rerank",
|
|
4263
|
+
apiKey: options.apiKey,
|
|
4264
|
+
model: options.model ?? "rerank-v3.5",
|
|
4265
|
+
timeoutMs: options.timeoutMs,
|
|
4266
|
+
buildBody: (query, documents, topK, model) => ({
|
|
4267
|
+
model,
|
|
4268
|
+
query,
|
|
4269
|
+
documents: documents.map((d) => d.text),
|
|
4270
|
+
top_n: topK
|
|
4271
|
+
}),
|
|
4272
|
+
parseResults: (body, documents) => {
|
|
4273
|
+
const results = body.results ?? [];
|
|
4274
|
+
return results.map((r) => ({
|
|
4275
|
+
id: documents[r.index]?.id ?? String(r.index),
|
|
4276
|
+
score: r.relevance_score
|
|
4277
|
+
}));
|
|
4278
|
+
}
|
|
4279
|
+
});
|
|
4280
|
+
}
|
|
4281
|
+
function bgeReranker(options) {
|
|
4282
|
+
const base = options.baseUrl.replace(/\/+$/, "");
|
|
4283
|
+
return new HttpRerankerProvider({
|
|
4284
|
+
name: "bge",
|
|
4285
|
+
url: `${base}/rerank`,
|
|
4286
|
+
apiKey: options.apiKey,
|
|
4287
|
+
model: options.model,
|
|
4288
|
+
timeoutMs: options.timeoutMs,
|
|
4289
|
+
buildBody: (query, documents, topK, model) => ({
|
|
4290
|
+
model,
|
|
4291
|
+
query,
|
|
4292
|
+
documents: documents.map((d) => d.text),
|
|
4293
|
+
top_n: topK
|
|
4294
|
+
}),
|
|
4295
|
+
parseResults: (body, documents) => {
|
|
4296
|
+
const results = body.results ?? [];
|
|
4297
|
+
return results.map((r) => ({
|
|
4298
|
+
id: documents[r.index]?.id ?? String(r.index),
|
|
4299
|
+
score: r.relevance_score
|
|
4300
|
+
}));
|
|
4301
|
+
}
|
|
4302
|
+
});
|
|
4303
|
+
}
|
|
4304
|
+
function openaiReranker(options) {
|
|
4305
|
+
const base = (options.baseUrl ?? "https://api.openai.com/v1").replace(
|
|
4306
|
+
/\/+$/,
|
|
4307
|
+
""
|
|
4308
|
+
);
|
|
4309
|
+
const model = options.model ?? "gpt-4.1-mini";
|
|
4310
|
+
const timeoutMs = options.timeoutMs ?? 6e4;
|
|
4311
|
+
return {
|
|
4312
|
+
name: "openai",
|
|
4313
|
+
async rerank(query, documents, topK) {
|
|
4314
|
+
if (documents.length === 0) {
|
|
4315
|
+
return [];
|
|
4316
|
+
}
|
|
4317
|
+
const listed = documents.map((d, i) => `[${i}] ${d.text.slice(0, 800)}`).join("\n\n");
|
|
4318
|
+
const controller = new AbortController();
|
|
4319
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
4320
|
+
try {
|
|
4321
|
+
const res = await fetch(`${base}/chat/completions`, {
|
|
4322
|
+
method: "POST",
|
|
4323
|
+
headers: {
|
|
4324
|
+
"Content-Type": "application/json",
|
|
4325
|
+
Authorization: `Bearer ${options.apiKey}`
|
|
4326
|
+
},
|
|
4327
|
+
body: JSON.stringify({
|
|
4328
|
+
model,
|
|
4329
|
+
temperature: 0,
|
|
4330
|
+
response_format: { type: "json_object" },
|
|
4331
|
+
messages: [
|
|
4332
|
+
{
|
|
4333
|
+
role: "system",
|
|
4334
|
+
content: 'You rerank documents for a retrieval system. Reply ONLY with JSON: {"results":[{"index":0,"score":0.0}]} where score is 0-1 relevance.'
|
|
4335
|
+
},
|
|
4336
|
+
{
|
|
4337
|
+
role: "user",
|
|
4338
|
+
content: `Query: ${query}
|
|
4339
|
+
|
|
4340
|
+
Documents:
|
|
4341
|
+
${listed}
|
|
4342
|
+
|
|
4343
|
+
Return the top ${topK} indices sorted by score descending.`
|
|
4344
|
+
}
|
|
4345
|
+
]
|
|
4346
|
+
}),
|
|
4347
|
+
signal: controller.signal
|
|
4348
|
+
});
|
|
4349
|
+
const body = await res.json();
|
|
4350
|
+
const content = body.choices?.[0]?.message?.content ?? "{}";
|
|
4351
|
+
let parsed;
|
|
4352
|
+
try {
|
|
4353
|
+
parsed = JSON.parse(content);
|
|
4354
|
+
} catch {
|
|
4355
|
+
return documents.slice(0, topK).map((d, i) => ({
|
|
4356
|
+
id: d.id,
|
|
4357
|
+
score: 1 - i / Math.max(documents.length, 1)
|
|
4358
|
+
}));
|
|
4359
|
+
}
|
|
4360
|
+
const results = parsed.results ?? [];
|
|
4361
|
+
if (results.length === 0) {
|
|
4362
|
+
return documents.slice(0, topK).map((d, i) => ({
|
|
4363
|
+
id: d.id,
|
|
4364
|
+
score: 1 - i / Math.max(documents.length, 1)
|
|
4365
|
+
}));
|
|
4366
|
+
}
|
|
4367
|
+
return results.filter((r) => documents[r.index]).slice(0, topK).map((r) => ({
|
|
4368
|
+
id: documents[r.index].id,
|
|
4369
|
+
score: Number(r.score) || 0
|
|
4370
|
+
}));
|
|
4371
|
+
} catch {
|
|
4372
|
+
return documents.slice(0, topK).map((d, i) => ({
|
|
4373
|
+
id: d.id,
|
|
4374
|
+
score: 1 - i / Math.max(documents.length, 1)
|
|
4375
|
+
}));
|
|
4376
|
+
} finally {
|
|
4377
|
+
clearTimeout(timer);
|
|
4378
|
+
}
|
|
4379
|
+
}
|
|
4380
|
+
};
|
|
4381
|
+
}
|
|
4382
|
+
|
|
4383
|
+
// src/ocr/index.ts
|
|
4384
|
+
function tesseract() {
|
|
4385
|
+
return {
|
|
4386
|
+
name: "tesseract",
|
|
4387
|
+
async recognize(image) {
|
|
4388
|
+
try {
|
|
4389
|
+
const mod = await import('tesseract.js');
|
|
4390
|
+
const createWorker = mod.createWorker ?? mod.default?.createWorker;
|
|
4391
|
+
if (!createWorker) {
|
|
4392
|
+
return { text: "" };
|
|
4393
|
+
}
|
|
4394
|
+
const worker = await createWorker("eng");
|
|
4395
|
+
try {
|
|
4396
|
+
const result = await worker.recognize(image);
|
|
4397
|
+
return { text: result.data.text.trim() };
|
|
4398
|
+
} finally {
|
|
4399
|
+
await worker.terminate();
|
|
4400
|
+
}
|
|
4401
|
+
} catch {
|
|
4402
|
+
return { text: "" };
|
|
4403
|
+
}
|
|
4404
|
+
}
|
|
4405
|
+
};
|
|
4406
|
+
}
|
|
4407
|
+
|
|
4408
|
+
// src/vision/index.ts
|
|
4409
|
+
var OpenAICompatibleVisionProvider = class {
|
|
4410
|
+
name;
|
|
4411
|
+
options;
|
|
4412
|
+
constructor(options) {
|
|
4413
|
+
this.name = options.name;
|
|
4414
|
+
this.options = options;
|
|
4415
|
+
}
|
|
4416
|
+
async analyze(image, mimeType = "image/png") {
|
|
4417
|
+
const base = this.options.baseUrl.replace(/\/+$/, "");
|
|
4418
|
+
const url = `${base}/chat/completions`;
|
|
4419
|
+
const dataUrl = `data:${mimeType};base64,${image.toString("base64")}`;
|
|
4420
|
+
const controller = new AbortController();
|
|
4421
|
+
const timer = setTimeout(
|
|
4422
|
+
() => controller.abort(),
|
|
4423
|
+
this.options.timeoutMs ?? 6e4
|
|
4424
|
+
);
|
|
4425
|
+
try {
|
|
4426
|
+
const res = await fetch(url, {
|
|
4427
|
+
method: "POST",
|
|
4428
|
+
headers: {
|
|
4429
|
+
"Content-Type": "application/json",
|
|
4430
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
4431
|
+
},
|
|
4432
|
+
body: JSON.stringify({
|
|
4433
|
+
model: this.options.model,
|
|
4434
|
+
messages: [
|
|
4435
|
+
{
|
|
4436
|
+
role: "user",
|
|
4437
|
+
content: [
|
|
4438
|
+
{
|
|
4439
|
+
type: "text",
|
|
4440
|
+
text: 'Describe this image. Reply as JSON: {"caption":"...","description":"...","entities":["..."]}'
|
|
4441
|
+
},
|
|
4442
|
+
{ type: "image_url", image_url: { url: dataUrl } }
|
|
4443
|
+
]
|
|
4444
|
+
}
|
|
4445
|
+
],
|
|
4446
|
+
max_tokens: 512
|
|
4447
|
+
}),
|
|
4448
|
+
signal: controller.signal
|
|
4449
|
+
});
|
|
4450
|
+
const body = await res.json();
|
|
4451
|
+
const content = body.choices?.[0]?.message?.content ?? "";
|
|
4452
|
+
return parseVisionJson(content);
|
|
4453
|
+
} catch {
|
|
4454
|
+
return { caption: "", description: "", entities: [] };
|
|
4455
|
+
} finally {
|
|
4456
|
+
clearTimeout(timer);
|
|
4457
|
+
}
|
|
4458
|
+
}
|
|
4459
|
+
};
|
|
4460
|
+
function parseVisionJson(content) {
|
|
4461
|
+
try {
|
|
4462
|
+
const match = content.match(/\{[\s\S]*\}/);
|
|
4463
|
+
if (!match) {
|
|
4464
|
+
return { caption: content.trim(), description: content.trim(), entities: [] };
|
|
4465
|
+
}
|
|
4466
|
+
const parsed = JSON.parse(match[0]);
|
|
4467
|
+
return {
|
|
4468
|
+
caption: String(parsed.caption ?? ""),
|
|
4469
|
+
description: String(parsed.description ?? ""),
|
|
4470
|
+
entities: Array.isArray(parsed.entities) ? parsed.entities.map(String) : []
|
|
4471
|
+
};
|
|
4472
|
+
} catch {
|
|
4473
|
+
return { caption: content.trim(), description: content.trim(), entities: [] };
|
|
4474
|
+
}
|
|
4475
|
+
}
|
|
4476
|
+
function geminiVision(options) {
|
|
4477
|
+
return new OpenAICompatibleVisionProvider({
|
|
4478
|
+
name: "gemini",
|
|
4479
|
+
baseUrl: options.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
4480
|
+
apiKey: options.apiKey,
|
|
4481
|
+
model: options.model ?? "gemini-2.0-flash",
|
|
4482
|
+
timeoutMs: options.timeoutMs
|
|
4483
|
+
});
|
|
4484
|
+
}
|
|
4485
|
+
function openaiVision(options) {
|
|
4486
|
+
return new OpenAICompatibleVisionProvider({
|
|
4487
|
+
name: "openai-vision",
|
|
4488
|
+
baseUrl: options.baseUrl ?? "https://api.openai.com/v1",
|
|
4489
|
+
apiKey: options.apiKey,
|
|
4490
|
+
model: options.model ?? "gpt-4o-mini",
|
|
4491
|
+
timeoutMs: options.timeoutMs
|
|
4492
|
+
});
|
|
4493
|
+
}
|
|
4494
|
+
|
|
4495
|
+
exports.CompressionError = CompressionError;
|
|
4496
|
+
exports.ConfigurationError = ConfigurationError;
|
|
4497
|
+
exports.DatabaseError = DatabaseError;
|
|
4498
|
+
exports.EmbeddingError = EmbeddingError;
|
|
4499
|
+
exports.FixedChunkingStrategy = FixedChunkingStrategy;
|
|
4500
|
+
exports.HeadingChunkingStrategy = HeadingChunkingStrategy;
|
|
4501
|
+
exports.InitializationError = InitializationError;
|
|
4502
|
+
exports.LlmCompressionProvider = LlmCompressionProvider;
|
|
4503
|
+
exports.MarkdownChunkingStrategy = MarkdownChunkingStrategy;
|
|
4504
|
+
exports.MemoryNotFoundError = MemoryNotFoundError;
|
|
4505
|
+
exports.ParagraphChunkingStrategy = ParagraphChunkingStrategy;
|
|
4506
|
+
exports.PostgresStorageProvider = PostgresStorageProvider;
|
|
4507
|
+
exports.ProviderNotConfiguredError = ProviderNotConfiguredError;
|
|
4508
|
+
exports.SentenceChunkingStrategy = SentenceChunkingStrategy;
|
|
4509
|
+
exports.SqliteDatabaseProvider = SqliteDatabaseProvider;
|
|
4510
|
+
exports.SqliteStorageProvider = SqliteStorageProvider;
|
|
4511
|
+
exports.ValidationError = ValidationError;
|
|
4512
|
+
exports.Wolbarg = Wolbarg;
|
|
4513
|
+
exports.WolbargError = WolbargError;
|
|
4514
|
+
exports.bgeReranker = bgeReranker;
|
|
4515
|
+
exports.bm25 = bm25;
|
|
4516
|
+
exports.cohereReranker = cohereReranker;
|
|
4517
|
+
exports.createChunkingStrategy = createChunkingStrategy;
|
|
4518
|
+
exports.createCompressionProvider = createCompressionProvider;
|
|
4519
|
+
exports.createDatabaseProvider = createDatabaseProvider;
|
|
4520
|
+
exports.createEmbeddingProvider = createEmbeddingProvider;
|
|
4521
|
+
exports.createLlmProvider = createLlmProvider;
|
|
4522
|
+
exports.createStorageProvider = createStorageProvider;
|
|
4523
|
+
exports.crossEncoder = crossEncoder;
|
|
4524
|
+
exports.geminiEmbedding = geminiEmbedding;
|
|
4525
|
+
exports.geminiVision = geminiVision;
|
|
4526
|
+
exports.jinaReranker = jinaReranker;
|
|
4527
|
+
exports.lmStudioEmbedding = lmStudioEmbedding;
|
|
4528
|
+
exports.meta = meta;
|
|
4529
|
+
exports.ollamaEmbedding = ollamaEmbedding;
|
|
4530
|
+
exports.ollamaLlm = ollamaLlm;
|
|
4531
|
+
exports.openRouterEmbedding = openRouterEmbedding;
|
|
4532
|
+
exports.openRouterLlm = openRouterLlm;
|
|
4533
|
+
exports.openaiCompatibleEmbedding = openaiCompatibleEmbedding;
|
|
4534
|
+
exports.openaiCompatibleLlm = openaiCompatibleLlm;
|
|
4535
|
+
exports.openaiEmbedding = openaiEmbedding;
|
|
4536
|
+
exports.openaiLlm = openaiLlm;
|
|
4537
|
+
exports.openaiReranker = openaiReranker;
|
|
4538
|
+
exports.openaiVision = openaiVision;
|
|
4539
|
+
exports.postgres = postgres;
|
|
4540
|
+
exports.postgresConfig = postgresConfig;
|
|
4541
|
+
exports.sqlite = sqlite;
|
|
4542
|
+
exports.sqliteConfig = sqliteConfig;
|
|
4543
|
+
exports.tesseract = tesseract;
|
|
4544
|
+
exports.togetherEmbedding = togetherEmbedding;
|
|
4545
|
+
exports.vllmEmbedding = vllmEmbedding;
|
|
4546
|
+
//# sourceMappingURL=index.cjs.map
|
|
4547
|
+
//# sourceMappingURL=index.cjs.map
|