wolbarg 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -6
- package/README.md +73 -11
- package/dist/index.cjs +2298 -523
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +598 -54
- package/dist/index.d.ts +598 -54
- package/dist/index.js +2283 -522
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/index.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var path5 = require("node:path");
|
|
3
4
|
var fs = require('fs/promises');
|
|
4
|
-
var
|
|
5
|
-
var fs2 = require("node:fs");
|
|
5
|
+
var fs5 = require("node:fs");
|
|
6
6
|
var sqlite$1 = require("node:sqlite");
|
|
7
7
|
var sqliteVec = require('sqlite-vec');
|
|
8
8
|
var async_hooks = require('async_hooks');
|
|
@@ -27,18 +27,26 @@ function _interopNamespace(e) {
|
|
|
27
27
|
return Object.freeze(n);
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
var path5__default = /*#__PURE__*/_interopDefault(path5);
|
|
30
31
|
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
|
31
|
-
var
|
|
32
|
-
var fs2__default = /*#__PURE__*/_interopDefault(fs2);
|
|
32
|
+
var fs5__default = /*#__PURE__*/_interopDefault(fs5);
|
|
33
33
|
var sqliteVec__namespace = /*#__PURE__*/_interopNamespace(sqliteVec);
|
|
34
34
|
|
|
35
|
+
// src/core/wolbarg.ts
|
|
36
|
+
|
|
35
37
|
// src/errors/index.ts
|
|
36
38
|
var WolbargError = class extends Error {
|
|
37
39
|
code;
|
|
40
|
+
reason;
|
|
41
|
+
suggestion;
|
|
42
|
+
operation;
|
|
38
43
|
constructor(message, code, options) {
|
|
39
44
|
super(message, options);
|
|
40
45
|
this.name = "WolbargError";
|
|
41
46
|
this.code = code;
|
|
47
|
+
this.reason = options?.reason;
|
|
48
|
+
this.suggestion = options?.suggestion;
|
|
49
|
+
this.operation = options?.operation;
|
|
42
50
|
Object.setPrototypeOf(this, new.target.prototype);
|
|
43
51
|
}
|
|
44
52
|
};
|
|
@@ -87,13 +95,58 @@ var MemoryNotFoundError = class extends WolbargError {
|
|
|
87
95
|
var ProviderNotConfiguredError = class extends ConfigurationError {
|
|
88
96
|
provider;
|
|
89
97
|
constructor(provider, method, hint) {
|
|
90
|
-
super(
|
|
91
|
-
|
|
92
|
-
|
|
98
|
+
super(`${method} requires ${provider} \u2014 ${hint}`, {
|
|
99
|
+
operation: method,
|
|
100
|
+
reason: `${provider} was not configured`,
|
|
101
|
+
suggestion: hint
|
|
102
|
+
});
|
|
93
103
|
this.name = "ProviderNotConfiguredError";
|
|
94
104
|
this.provider = provider;
|
|
95
105
|
}
|
|
96
106
|
};
|
|
107
|
+
function wrapOperationError(operation, error) {
|
|
108
|
+
if (error instanceof WolbargError) {
|
|
109
|
+
return error;
|
|
110
|
+
}
|
|
111
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
112
|
+
const lower = raw.toLowerCase();
|
|
113
|
+
if (lower.includes("database is locked") || lower.includes("sqlite_busy")) {
|
|
114
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
115
|
+
cause: error instanceof Error ? error : void 0,
|
|
116
|
+
operation,
|
|
117
|
+
reason: "SQLite database locked",
|
|
118
|
+
suggestion: "Increase timeout or reduce concurrent writes."
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
if (lower.includes("no such file") || lower.includes("enoent")) {
|
|
122
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
123
|
+
cause: error instanceof Error ? error : void 0,
|
|
124
|
+
operation,
|
|
125
|
+
reason: "Database file not found",
|
|
126
|
+
suggestion: "Check the database path and ensure the directory exists."
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
if (lower.includes("readonly") || lower.includes("read-only")) {
|
|
130
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
131
|
+
cause: error instanceof Error ? error : void 0,
|
|
132
|
+
operation,
|
|
133
|
+
reason: "Database opened as read-only",
|
|
134
|
+
suggestion: "Open the database with write permissions or choose another path."
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
138
|
+
cause: error instanceof Error ? error : void 0,
|
|
139
|
+
operation,
|
|
140
|
+
reason: raw,
|
|
141
|
+
suggestion: "Inspect the underlying cause and retry the operation."
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
function formatOperationMessage(operation, reason) {
|
|
145
|
+
const reasonText = typeof reason === "string" ? reason : reason.reason ?? reason.message;
|
|
146
|
+
return `Failed to execute ${operation}()
|
|
147
|
+
Reason:
|
|
148
|
+
${reasonText}`;
|
|
149
|
+
}
|
|
97
150
|
|
|
98
151
|
// src/compression/index.ts
|
|
99
152
|
var SYSTEM_PROMPT = `You are a memory compression engine for multi-agent systems.
|
|
@@ -351,9 +404,9 @@ var AsyncMutex = class {
|
|
|
351
404
|
}
|
|
352
405
|
}
|
|
353
406
|
};
|
|
354
|
-
function joinUrl(baseUrl,
|
|
407
|
+
function joinUrl(baseUrl, path7) {
|
|
355
408
|
const base = baseUrl.replace(/\/+$/, "");
|
|
356
|
-
const suffix =
|
|
409
|
+
const suffix = path7.startsWith("/") ? path7 : `/${path7}`;
|
|
357
410
|
return `${base}${suffix}`;
|
|
358
411
|
}
|
|
359
412
|
|
|
@@ -650,7 +703,7 @@ function extOf(filename) {
|
|
|
650
703
|
if (!filename) {
|
|
651
704
|
return "";
|
|
652
705
|
}
|
|
653
|
-
return
|
|
706
|
+
return path5__default.default.extname(filename).toLowerCase();
|
|
654
707
|
}
|
|
655
708
|
var TextParser = class {
|
|
656
709
|
name = "text";
|
|
@@ -795,13 +848,31 @@ async function loadIngestSource(source) {
|
|
|
795
848
|
const buffer = await fs__default.default.readFile(source.path);
|
|
796
849
|
return {
|
|
797
850
|
buffer,
|
|
798
|
-
filename:
|
|
851
|
+
filename: path5__default.default.basename(source.path),
|
|
799
852
|
mimeType: source.mimeType
|
|
800
853
|
};
|
|
801
854
|
}
|
|
802
855
|
throw new ConfigurationError("ingest source must include path, buffer, or text");
|
|
803
856
|
}
|
|
804
857
|
|
|
858
|
+
// src/core/options.ts
|
|
859
|
+
function isEmbeddingProvider(value) {
|
|
860
|
+
return typeof value.embed === "function";
|
|
861
|
+
}
|
|
862
|
+
function isLlmProvider(value) {
|
|
863
|
+
return typeof value.complete === "function";
|
|
864
|
+
}
|
|
865
|
+
function isStorageProvider(value) {
|
|
866
|
+
return typeof value.open === "function";
|
|
867
|
+
}
|
|
868
|
+
function isTelemetryProvider(value) {
|
|
869
|
+
return typeof value.emit === "function";
|
|
870
|
+
}
|
|
871
|
+
function resolveDatabaseUrl(config) {
|
|
872
|
+
const url = "url" in config && config.url || "connectionString" in config && config.connectionString || "";
|
|
873
|
+
return typeof url === "string" ? url : "";
|
|
874
|
+
}
|
|
875
|
+
|
|
805
876
|
// src/filters/match.ts
|
|
806
877
|
function getField(metadata, field) {
|
|
807
878
|
const parts = field.split(".");
|
|
@@ -1283,12 +1354,16 @@ var SqliteStorageProvider = class {
|
|
|
1283
1354
|
constructor(options) {
|
|
1284
1355
|
this.connectionString = options.connectionString;
|
|
1285
1356
|
}
|
|
1357
|
+
/** Absolute or relative path / `:memory:` used by this provider. */
|
|
1358
|
+
get path() {
|
|
1359
|
+
return this.connectionString;
|
|
1360
|
+
}
|
|
1286
1361
|
async open() {
|
|
1287
1362
|
try {
|
|
1288
1363
|
const dbPath = this.resolvePath(this.connectionString);
|
|
1289
1364
|
this.resolvedPath = dbPath;
|
|
1290
1365
|
if (dbPath !== ":memory:") {
|
|
1291
|
-
|
|
1366
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(dbPath), { recursive: true });
|
|
1292
1367
|
}
|
|
1293
1368
|
const db = new sqlite$1.DatabaseSync(dbPath, { allowExtension: true });
|
|
1294
1369
|
this.db = db;
|
|
@@ -1749,11 +1824,11 @@ var SqliteStorageProvider = class {
|
|
|
1749
1824
|
}
|
|
1750
1825
|
const dbPath = this.resolvedPath ?? this.resolvePath(this.connectionString);
|
|
1751
1826
|
try {
|
|
1752
|
-
let total =
|
|
1827
|
+
let total = fs5__default.default.statSync(dbPath).size;
|
|
1753
1828
|
for (const suffix of ["-wal", "-shm"]) {
|
|
1754
1829
|
const side = `${dbPath}${suffix}`;
|
|
1755
|
-
if (
|
|
1756
|
-
total +=
|
|
1830
|
+
if (fs5__default.default.existsSync(side)) {
|
|
1831
|
+
total += fs5__default.default.statSync(side).size;
|
|
1757
1832
|
}
|
|
1758
1833
|
}
|
|
1759
1834
|
return total;
|
|
@@ -2084,7 +2159,7 @@ var SqliteStorageProvider = class {
|
|
|
2084
2159
|
if (connectionString === ":memory:") {
|
|
2085
2160
|
return ":memory:";
|
|
2086
2161
|
}
|
|
2087
|
-
return
|
|
2162
|
+
return path5__default.default.isAbsolute(connectionString) ? connectionString : path5__default.default.resolve(process.cwd(), connectionString);
|
|
2088
2163
|
}
|
|
2089
2164
|
requireDb() {
|
|
2090
2165
|
if (!this.db) {
|
|
@@ -3148,14 +3223,20 @@ var PostgresStorageProvider = class {
|
|
|
3148
3223
|
|
|
3149
3224
|
// src/storage/index.ts
|
|
3150
3225
|
function createStorageProvider(config) {
|
|
3226
|
+
const connectionString = resolveDatabaseUrl(config);
|
|
3227
|
+
if (!connectionString) {
|
|
3228
|
+
throw new ConfigurationError(
|
|
3229
|
+
"database.url or database.connectionString is required"
|
|
3230
|
+
);
|
|
3231
|
+
}
|
|
3151
3232
|
if (config.provider === "sqlite") {
|
|
3152
3233
|
return new SqliteStorageProvider({
|
|
3153
|
-
connectionString
|
|
3234
|
+
connectionString
|
|
3154
3235
|
});
|
|
3155
3236
|
}
|
|
3156
3237
|
if (config.provider === "postgres") {
|
|
3157
3238
|
return new PostgresStorageProvider({
|
|
3158
|
-
connectionString
|
|
3239
|
+
connectionString,
|
|
3159
3240
|
maxPoolSize: config.maxPoolSize
|
|
3160
3241
|
});
|
|
3161
3242
|
}
|
|
@@ -3166,6 +3247,129 @@ function createStorageProvider(config) {
|
|
|
3166
3247
|
function createDatabaseProvider(config) {
|
|
3167
3248
|
return createStorageProvider(config);
|
|
3168
3249
|
}
|
|
3250
|
+
var SDK_VERSION = "0.3.0";
|
|
3251
|
+
var SqliteMemoryTransferProvider = class {
|
|
3252
|
+
async exportTo(exportPath, sourcePath, organization) {
|
|
3253
|
+
const resolvedSource = resolvePath(sourcePath);
|
|
3254
|
+
if (resolvedSource === ":memory:") {
|
|
3255
|
+
throw new ConfigurationError(
|
|
3256
|
+
"Cannot export an in-memory database. Use a file-backed SQLite database."
|
|
3257
|
+
);
|
|
3258
|
+
}
|
|
3259
|
+
if (!fs5__default.default.existsSync(resolvedSource)) {
|
|
3260
|
+
throw new DatabaseError(
|
|
3261
|
+
`Failed to execute export()
|
|
3262
|
+
Reason:
|
|
3263
|
+
Source database not found at ${resolvedSource}
|
|
3264
|
+
Suggestion:
|
|
3265
|
+
Initialize Wolbarg and verify the database path.`
|
|
3266
|
+
);
|
|
3267
|
+
}
|
|
3268
|
+
const resolvedExport = resolvePath(exportPath);
|
|
3269
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(resolvedExport), { recursive: true });
|
|
3270
|
+
const dbExportPath = resolvedExport.endsWith(".db") ? resolvedExport : `${resolvedExport}.db`;
|
|
3271
|
+
const manifestPath = `${dbExportPath}.manifest.json`;
|
|
3272
|
+
await copySqlite(resolvedSource, dbExportPath);
|
|
3273
|
+
const manifest = {
|
|
3274
|
+
format: "wolbarg-export-v1",
|
|
3275
|
+
exportedAt: nowIso(),
|
|
3276
|
+
sdkVersion: SDK_VERSION,
|
|
3277
|
+
provider: "sqlite",
|
|
3278
|
+
sourcePath: resolvedSource,
|
|
3279
|
+
organization
|
|
3280
|
+
};
|
|
3281
|
+
fs5__default.default.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), "utf8");
|
|
3282
|
+
return {
|
|
3283
|
+
path: dbExportPath,
|
|
3284
|
+
manifest,
|
|
3285
|
+
sizeBytes: fs5__default.default.statSync(dbExportPath).size
|
|
3286
|
+
};
|
|
3287
|
+
}
|
|
3288
|
+
async importFrom(exportPath, targetPath) {
|
|
3289
|
+
const resolvedExport = resolvePath(exportPath);
|
|
3290
|
+
const dbExportPath = resolvedExport.endsWith(".db") ? resolvedExport : fs5__default.default.existsSync(`${resolvedExport}.db`) ? `${resolvedExport}.db` : resolvedExport;
|
|
3291
|
+
if (!fs5__default.default.existsSync(dbExportPath)) {
|
|
3292
|
+
throw new ValidationError(
|
|
3293
|
+
`Failed to execute import()
|
|
3294
|
+
Reason:
|
|
3295
|
+
Export file not found at ${dbExportPath}
|
|
3296
|
+
Suggestion:
|
|
3297
|
+
Pass the path returned by export().`
|
|
3298
|
+
);
|
|
3299
|
+
}
|
|
3300
|
+
const manifestPath = `${dbExportPath}.manifest.json`;
|
|
3301
|
+
let manifest;
|
|
3302
|
+
if (fs5__default.default.existsSync(manifestPath)) {
|
|
3303
|
+
manifest = JSON.parse(fs5__default.default.readFileSync(manifestPath, "utf8"));
|
|
3304
|
+
if (manifest.format !== "wolbarg-export-v1") {
|
|
3305
|
+
throw new ValidationError(
|
|
3306
|
+
`Unsupported export format: ${String(manifest.format)}`
|
|
3307
|
+
);
|
|
3308
|
+
}
|
|
3309
|
+
} else {
|
|
3310
|
+
manifest = {
|
|
3311
|
+
format: "wolbarg-export-v1",
|
|
3312
|
+
exportedAt: nowIso(),
|
|
3313
|
+
sdkVersion: SDK_VERSION,
|
|
3314
|
+
provider: "sqlite",
|
|
3315
|
+
sourcePath: dbExportPath
|
|
3316
|
+
};
|
|
3317
|
+
}
|
|
3318
|
+
const resolvedTarget = resolvePath(targetPath);
|
|
3319
|
+
if (resolvedTarget === ":memory:") {
|
|
3320
|
+
throw new ConfigurationError("Cannot import into an in-memory database.");
|
|
3321
|
+
}
|
|
3322
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
3323
|
+
const side = `${resolvedTarget}${suffix}`;
|
|
3324
|
+
if (fs5__default.default.existsSync(side)) {
|
|
3325
|
+
fs5__default.default.rmSync(side, { force: true });
|
|
3326
|
+
}
|
|
3327
|
+
}
|
|
3328
|
+
await copySqlite(dbExportPath, resolvedTarget);
|
|
3329
|
+
return { path: resolvedTarget, manifest };
|
|
3330
|
+
}
|
|
3331
|
+
};
|
|
3332
|
+
async function copySqlite(sourcePath, destPath) {
|
|
3333
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(destPath), { recursive: true });
|
|
3334
|
+
let source = null;
|
|
3335
|
+
let dest = null;
|
|
3336
|
+
try {
|
|
3337
|
+
source = new sqlite$1.DatabaseSync(sourcePath);
|
|
3338
|
+
try {
|
|
3339
|
+
source.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
3340
|
+
} catch {
|
|
3341
|
+
}
|
|
3342
|
+
dest = new sqlite$1.DatabaseSync(destPath);
|
|
3343
|
+
const backupFn = source.backup;
|
|
3344
|
+
if (typeof backupFn === "function") {
|
|
3345
|
+
await backupFn.call(source, dest);
|
|
3346
|
+
} else {
|
|
3347
|
+
source.close();
|
|
3348
|
+
source = null;
|
|
3349
|
+
dest.close();
|
|
3350
|
+
dest = null;
|
|
3351
|
+
fs5__default.default.copyFileSync(sourcePath, destPath);
|
|
3352
|
+
}
|
|
3353
|
+
} catch (error) {
|
|
3354
|
+
throw new DatabaseError(
|
|
3355
|
+
`SQLite transfer failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3356
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
3357
|
+
);
|
|
3358
|
+
} finally {
|
|
3359
|
+
try {
|
|
3360
|
+
source?.close();
|
|
3361
|
+
} catch {
|
|
3362
|
+
}
|
|
3363
|
+
try {
|
|
3364
|
+
dest?.close();
|
|
3365
|
+
} catch {
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
function resolvePath(p) {
|
|
3370
|
+
if (p === ":memory:") return ":memory:";
|
|
3371
|
+
return path5__default.default.isAbsolute(p) ? p : path5__default.default.resolve(process.cwd(), p);
|
|
3372
|
+
}
|
|
3169
3373
|
|
|
3170
3374
|
// src/memory/index.ts
|
|
3171
3375
|
function toMemoryRecord(row) {
|
|
@@ -3291,17 +3495,6 @@ function adaptiveFetchK(topK, overFetchFactor, hasFilters) {
|
|
|
3291
3495
|
return Math.min(Math.max(Math.ceil(topK * factor), topK), 1e3);
|
|
3292
3496
|
}
|
|
3293
3497
|
|
|
3294
|
-
// src/core/options.ts
|
|
3295
|
-
function isEmbeddingProvider(value) {
|
|
3296
|
-
return typeof value.embed === "function";
|
|
3297
|
-
}
|
|
3298
|
-
function isLlmProvider(value) {
|
|
3299
|
-
return typeof value.complete === "function";
|
|
3300
|
-
}
|
|
3301
|
-
function isStorageProvider(value) {
|
|
3302
|
-
return typeof value.open === "function";
|
|
3303
|
-
}
|
|
3304
|
-
|
|
3305
3498
|
// src/core/validate.ts
|
|
3306
3499
|
function assertNonEmpty(value, fieldName) {
|
|
3307
3500
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
@@ -3354,6 +3547,64 @@ function validateLlmConfig(config) {
|
|
|
3354
3547
|
...config.timeoutMs !== void 0 ? { timeoutMs: config.timeoutMs } : {}
|
|
3355
3548
|
};
|
|
3356
3549
|
}
|
|
3550
|
+
function normalizeDatabaseConfig(config) {
|
|
3551
|
+
const provider = config.provider;
|
|
3552
|
+
if (provider !== "sqlite" && provider !== "postgres") {
|
|
3553
|
+
throw new ConfigurationError(
|
|
3554
|
+
`Unsupported database provider "${String(config.provider)}". Supported: "sqlite", "postgres".`
|
|
3555
|
+
);
|
|
3556
|
+
}
|
|
3557
|
+
const connectionString = resolveDatabaseUrl(config).trim();
|
|
3558
|
+
assertNonEmpty(connectionString, "database.url / database.connectionString");
|
|
3559
|
+
if (provider === "postgres") {
|
|
3560
|
+
return {
|
|
3561
|
+
provider: "postgres",
|
|
3562
|
+
connectionString,
|
|
3563
|
+
url: connectionString,
|
|
3564
|
+
..."maxPoolSize" in config && config.maxPoolSize !== void 0 ? { maxPoolSize: config.maxPoolSize } : {}
|
|
3565
|
+
};
|
|
3566
|
+
}
|
|
3567
|
+
return {
|
|
3568
|
+
provider: "sqlite",
|
|
3569
|
+
connectionString,
|
|
3570
|
+
url: connectionString
|
|
3571
|
+
};
|
|
3572
|
+
}
|
|
3573
|
+
function validateTelemetryConfig(config) {
|
|
3574
|
+
if (!config.database || typeof config.database !== "object") {
|
|
3575
|
+
throw new ConfigurationError("telemetry.database is required when telemetry is enabled");
|
|
3576
|
+
}
|
|
3577
|
+
if (config.database.provider !== "sqlite") {
|
|
3578
|
+
throw new ConfigurationError(
|
|
3579
|
+
`Unsupported telemetry provider "${config.database.provider}". Only "sqlite" is implemented in v0.3.0; PostgreSQL will be added later without changing application code.`,
|
|
3580
|
+
{
|
|
3581
|
+
reason: `provider=${config.database.provider}`,
|
|
3582
|
+
suggestion: 'Use telemetry: { database: { provider: "sqlite", url: "./telemetry.db" } }'
|
|
3583
|
+
}
|
|
3584
|
+
);
|
|
3585
|
+
}
|
|
3586
|
+
const url = config.database.url?.trim() || config.database.connectionString?.trim() || "";
|
|
3587
|
+
assertNonEmpty(url, "telemetry.database.url");
|
|
3588
|
+
const level = config.level ?? "info";
|
|
3589
|
+
const allowed = /* @__PURE__ */ new Set(["off", "error", "warn", "info", "debug", "trace"]);
|
|
3590
|
+
if (!allowed.has(level)) {
|
|
3591
|
+
throw new ConfigurationError(`Invalid telemetry.level "${level}"`);
|
|
3592
|
+
}
|
|
3593
|
+
return {
|
|
3594
|
+
enabled: config.enabled ?? true,
|
|
3595
|
+
database: {
|
|
3596
|
+
provider: "sqlite",
|
|
3597
|
+
url,
|
|
3598
|
+
connectionString: url
|
|
3599
|
+
},
|
|
3600
|
+
level,
|
|
3601
|
+
captureQueries: config.captureQueries ?? true,
|
|
3602
|
+
captureLatency: config.captureLatency ?? true,
|
|
3603
|
+
captureErrors: config.captureErrors ?? true,
|
|
3604
|
+
captureSimilarity: config.captureSimilarity ?? true,
|
|
3605
|
+
captureEmbeddings: config.captureEmbeddings ?? false
|
|
3606
|
+
};
|
|
3607
|
+
}
|
|
3357
3608
|
function validateInitOptions(options) {
|
|
3358
3609
|
if (options === null || typeof options !== "object") {
|
|
3359
3610
|
throw new ConfigurationError("init options must be an object");
|
|
@@ -3362,16 +3613,7 @@ function validateInitOptions(options) {
|
|
|
3362
3613
|
if (!options.database || typeof options.database !== "object") {
|
|
3363
3614
|
throw new ConfigurationError("database configuration is required");
|
|
3364
3615
|
}
|
|
3365
|
-
const
|
|
3366
|
-
if (provider !== "sqlite" && provider !== "postgres") {
|
|
3367
|
-
throw new ConfigurationError(
|
|
3368
|
-
`Unsupported database provider "${String(options.database.provider)}". Supported: "sqlite", "postgres".`
|
|
3369
|
-
);
|
|
3370
|
-
}
|
|
3371
|
-
assertNonEmpty(
|
|
3372
|
-
options.database.connectionString,
|
|
3373
|
-
"database.connectionString"
|
|
3374
|
-
);
|
|
3616
|
+
const database = normalizeDatabaseConfig(options.database);
|
|
3375
3617
|
if (!options.embedding || typeof options.embedding !== "object") {
|
|
3376
3618
|
throw new ConfigurationError("embedding configuration is required");
|
|
3377
3619
|
}
|
|
@@ -3379,31 +3621,32 @@ function validateInitOptions(options) {
|
|
|
3379
3621
|
const llm = options.llm ? validateLlmConfig(options.llm) : void 0;
|
|
3380
3622
|
return {
|
|
3381
3623
|
organization: options.organization.trim(),
|
|
3382
|
-
database
|
|
3383
|
-
provider: "postgres",
|
|
3384
|
-
connectionString: options.database.connectionString.trim(),
|
|
3385
|
-
..."maxPoolSize" in options.database && options.database.maxPoolSize !== void 0 ? { maxPoolSize: options.database.maxPoolSize } : {}
|
|
3386
|
-
} : {
|
|
3387
|
-
provider: "sqlite",
|
|
3388
|
-
connectionString: options.database.connectionString.trim()
|
|
3389
|
-
},
|
|
3624
|
+
database,
|
|
3390
3625
|
embedding,
|
|
3391
3626
|
...llm ? { llm } : {}
|
|
3392
3627
|
};
|
|
3393
3628
|
}
|
|
3629
|
+
function resolveStorageInput(options) {
|
|
3630
|
+
if (options.storage && options.database) {
|
|
3631
|
+
throw new ConfigurationError(
|
|
3632
|
+
"Pass either storage or database, not both"
|
|
3633
|
+
);
|
|
3634
|
+
}
|
|
3635
|
+
const input = options.storage ?? options.database;
|
|
3636
|
+
if (!input) {
|
|
3637
|
+
throw new ConfigurationError("storage or database is required");
|
|
3638
|
+
}
|
|
3639
|
+
return input;
|
|
3640
|
+
}
|
|
3394
3641
|
function validateWolbargOptions(options) {
|
|
3395
3642
|
if (options === null || typeof options !== "object") {
|
|
3396
3643
|
throw new ConfigurationError("Wolbarg options must be an object");
|
|
3397
3644
|
}
|
|
3398
3645
|
assertNonEmpty(options.organization, "organization");
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
if (typeof options.storage !== "object" || !("provider" in options.storage) || !("connectionString" in options.storage)) {
|
|
3404
|
-
throw new ConfigurationError("storage must be a provider instance or config");
|
|
3405
|
-
}
|
|
3406
|
-
assertNonEmpty(options.storage.connectionString, "storage.connectionString");
|
|
3646
|
+
const storageInput = resolveStorageInput(options);
|
|
3647
|
+
let storage = storageInput;
|
|
3648
|
+
if (!isStorageProvider(storageInput)) {
|
|
3649
|
+
storage = normalizeDatabaseConfig(storageInput);
|
|
3407
3650
|
}
|
|
3408
3651
|
if (!options.embedding) {
|
|
3409
3652
|
throw new ConfigurationError("embedding is required");
|
|
@@ -3414,82 +3657,1010 @@ function validateWolbargOptions(options) {
|
|
|
3414
3657
|
if (options.llm !== void 0 && !isLlmProvider(options.llm)) {
|
|
3415
3658
|
validateLlmConfig(options.llm);
|
|
3416
3659
|
}
|
|
3660
|
+
let telemetry = options.telemetry;
|
|
3661
|
+
if (telemetry && !isTelemetryProvider(telemetry)) {
|
|
3662
|
+
telemetry = validateTelemetryConfig(telemetry);
|
|
3663
|
+
}
|
|
3417
3664
|
return {
|
|
3418
3665
|
...options,
|
|
3419
|
-
organization: options.organization.trim()
|
|
3666
|
+
organization: options.organization.trim(),
|
|
3667
|
+
storage,
|
|
3668
|
+
database: void 0,
|
|
3669
|
+
...telemetry ? { telemetry } : {}
|
|
3420
3670
|
};
|
|
3421
3671
|
}
|
|
3422
3672
|
|
|
3423
|
-
// src/
|
|
3424
|
-
var
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3673
|
+
// src/telemetry/logger.ts
|
|
3674
|
+
var LEVEL_ORDER = {
|
|
3675
|
+
off: 100,
|
|
3676
|
+
error: 50,
|
|
3677
|
+
warn: 40,
|
|
3678
|
+
info: 30,
|
|
3679
|
+
debug: 20,
|
|
3680
|
+
trace: 10
|
|
3681
|
+
};
|
|
3682
|
+
var WolbargLogger = class {
|
|
3683
|
+
constructor(level = "info") {
|
|
3684
|
+
this.level = level;
|
|
3685
|
+
}
|
|
3686
|
+
level;
|
|
3687
|
+
setLevel(level) {
|
|
3688
|
+
this.level = level;
|
|
3689
|
+
}
|
|
3690
|
+
getLevel() {
|
|
3691
|
+
return this.level;
|
|
3692
|
+
}
|
|
3693
|
+
error(message, extra) {
|
|
3694
|
+
this.write("error", message, extra);
|
|
3695
|
+
}
|
|
3696
|
+
warn(message, extra) {
|
|
3697
|
+
this.write("warn", message, extra);
|
|
3698
|
+
}
|
|
3699
|
+
info(message, extra) {
|
|
3700
|
+
this.write("info", message, extra);
|
|
3701
|
+
}
|
|
3702
|
+
debug(message, extra) {
|
|
3703
|
+
this.write("debug", message, extra);
|
|
3704
|
+
}
|
|
3705
|
+
trace(message, extra) {
|
|
3706
|
+
this.write("trace", message, extra);
|
|
3707
|
+
}
|
|
3708
|
+
write(level, message, extra) {
|
|
3709
|
+
if (LEVEL_ORDER[level] < LEVEL_ORDER[this.level]) {
|
|
3442
3710
|
return;
|
|
3443
3711
|
}
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
if (
|
|
3449
|
-
|
|
3450
|
-
|
|
3712
|
+
if (this.level === "off") {
|
|
3713
|
+
return;
|
|
3714
|
+
}
|
|
3715
|
+
const line = `[wolbarg:${level}] ${message}`;
|
|
3716
|
+
if (level === "error") {
|
|
3717
|
+
console.error(line, extra ?? "");
|
|
3718
|
+
} else if (level === "warn") {
|
|
3719
|
+
console.warn(line, extra ?? "");
|
|
3451
3720
|
} else {
|
|
3452
|
-
|
|
3721
|
+
console.log(line, extra ?? "");
|
|
3453
3722
|
}
|
|
3454
|
-
this.reranker = validated.reranker ?? null;
|
|
3455
|
-
this.keywordSearch = validated.keywordSearch ?? null;
|
|
3456
|
-
this.ocr = validated.ocr ?? null;
|
|
3457
|
-
this.vision = validated.vision ?? null;
|
|
3458
|
-
this.chunking = validated.chunking ?? null;
|
|
3459
|
-
this.retrievalConfig = validated.retrieval ?? {};
|
|
3460
3723
|
}
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3724
|
+
};
|
|
3725
|
+
|
|
3726
|
+
// src/telemetry/trace.ts
|
|
3727
|
+
function createSessionId() {
|
|
3728
|
+
return createId();
|
|
3729
|
+
}
|
|
3730
|
+
function createTraceId() {
|
|
3731
|
+
return createId();
|
|
3732
|
+
}
|
|
3733
|
+
function createRootTrace(sessionId) {
|
|
3734
|
+
return {
|
|
3735
|
+
sessionId,
|
|
3736
|
+
traceId: createTraceId(),
|
|
3737
|
+
parentTraceId: null
|
|
3738
|
+
};
|
|
3739
|
+
}
|
|
3740
|
+
function createChildTrace(parent) {
|
|
3741
|
+
return {
|
|
3742
|
+
sessionId: parent.sessionId,
|
|
3743
|
+
traceId: createTraceId(),
|
|
3744
|
+
parentTraceId: parent.traceId
|
|
3745
|
+
};
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
// src/telemetry/emitter.ts
|
|
3749
|
+
var NoopTelemetryProvider = class {
|
|
3750
|
+
name = "noop";
|
|
3751
|
+
async open() {
|
|
3752
|
+
}
|
|
3753
|
+
async close() {
|
|
3754
|
+
}
|
|
3755
|
+
emit(_event) {
|
|
3756
|
+
}
|
|
3757
|
+
async flush() {
|
|
3758
|
+
}
|
|
3759
|
+
};
|
|
3760
|
+
var TelemetryEmitter = class {
|
|
3761
|
+
sessionId;
|
|
3762
|
+
logger;
|
|
3763
|
+
provider;
|
|
3764
|
+
config;
|
|
3765
|
+
context;
|
|
3766
|
+
constructor(provider, config, context = {}) {
|
|
3767
|
+
this.sessionId = createSessionId();
|
|
3768
|
+
this.provider = provider ?? new NoopTelemetryProvider();
|
|
3769
|
+
this.config = {
|
|
3770
|
+
enabled: config?.enabled ?? Boolean(provider),
|
|
3771
|
+
level: config?.level ?? "info",
|
|
3772
|
+
captureQueries: config?.captureQueries ?? true,
|
|
3773
|
+
captureLatency: config?.captureLatency ?? true,
|
|
3774
|
+
captureErrors: config?.captureErrors ?? true,
|
|
3775
|
+
captureSimilarity: config?.captureSimilarity ?? true,
|
|
3776
|
+
captureEmbeddings: config?.captureEmbeddings ?? false
|
|
3777
|
+
};
|
|
3778
|
+
this.context = context;
|
|
3779
|
+
this.logger = new WolbargLogger(this.config.level);
|
|
3780
|
+
}
|
|
3781
|
+
get enabled() {
|
|
3782
|
+
return this.config.enabled && this.provider.name !== "noop";
|
|
3783
|
+
}
|
|
3784
|
+
setProvider(provider) {
|
|
3785
|
+
this.provider = provider;
|
|
3786
|
+
}
|
|
3787
|
+
async open() {
|
|
3788
|
+
if (!this.config.enabled) {
|
|
3789
|
+
return;
|
|
3468
3790
|
}
|
|
3469
|
-
|
|
3791
|
+
await this.provider.open();
|
|
3792
|
+
}
|
|
3793
|
+
async close() {
|
|
3794
|
+
await this.provider.flush();
|
|
3795
|
+
await this.provider.close();
|
|
3796
|
+
}
|
|
3797
|
+
async flush() {
|
|
3798
|
+
await this.provider.flush();
|
|
3799
|
+
}
|
|
3800
|
+
start(operation, parent) {
|
|
3801
|
+
const context = parent ? createChildTrace(parent) : createRootTrace(this.sessionId);
|
|
3802
|
+
const startedAt = performance.now();
|
|
3803
|
+
const latency = {};
|
|
3804
|
+
const spans = [];
|
|
3805
|
+
const self = this;
|
|
3806
|
+
const finish = (status, fields, error) => {
|
|
3807
|
+
const totalMs = performance.now() - startedAt;
|
|
3808
|
+
const errMessage = error instanceof Error ? error.message : error ? String(error) : fields?.error ?? null;
|
|
3809
|
+
const errStack = error instanceof Error ? error.stack ?? null : fields?.errorStack ?? null;
|
|
3810
|
+
if (status === "error") {
|
|
3811
|
+
self.logger.error(`${operation} failed: ${errMessage ?? "unknown"}`);
|
|
3812
|
+
} else {
|
|
3813
|
+
self.logger.debug(`${operation} completed in ${totalMs.toFixed(2)}ms`);
|
|
3814
|
+
}
|
|
3815
|
+
if (!self.config.enabled) {
|
|
3816
|
+
return;
|
|
3817
|
+
}
|
|
3818
|
+
const event = {
|
|
3819
|
+
id: createId(),
|
|
3820
|
+
timestamp: nowIso(),
|
|
3821
|
+
operation,
|
|
3822
|
+
status,
|
|
3823
|
+
sessionId: context.sessionId,
|
|
3824
|
+
traceId: context.traceId,
|
|
3825
|
+
parentTraceId: context.parentTraceId,
|
|
3826
|
+
organization: fields?.organization ?? self.context.organization ?? null,
|
|
3827
|
+
agentId: fields?.agentId ?? null,
|
|
3828
|
+
tags: fields?.tags ?? null,
|
|
3829
|
+
checkpointId: fields?.checkpointId ?? null,
|
|
3830
|
+
durationMs: totalMs,
|
|
3831
|
+
provider: fields?.provider ?? null,
|
|
3832
|
+
query: self.config.captureQueries ? fields?.query ?? null : null,
|
|
3833
|
+
filters: fields?.filters ?? null,
|
|
3834
|
+
returnedCount: fields?.returnedCount ?? null,
|
|
3835
|
+
memoryIds: fields?.memoryIds ?? null,
|
|
3836
|
+
similarityScores: self.config.captureSimilarity ? fields?.similarityScores ?? null : null,
|
|
3837
|
+
metadata: fields?.metadata ?? null,
|
|
3838
|
+
embeddingProvider: fields?.embeddingProvider ?? null,
|
|
3839
|
+
model: fields?.model ?? null,
|
|
3840
|
+
error: self.config.captureErrors ? errMessage : null,
|
|
3841
|
+
errorStack: self.config.captureErrors ? errStack : null,
|
|
3842
|
+
userMetadata: fields?.userMetadata ?? null,
|
|
3843
|
+
extra: fields?.extra ?? null,
|
|
3844
|
+
latency: self.config.captureLatency ? { ...latency, totalMs, ...fields?.latency ?? {} } : { totalMs },
|
|
3845
|
+
explain: fields?.explain ?? null,
|
|
3846
|
+
spans: self.config.captureLatency ? [...spans, ...fields?.spans ?? []] : null
|
|
3847
|
+
};
|
|
3848
|
+
self.provider.emit(event);
|
|
3849
|
+
};
|
|
3850
|
+
return {
|
|
3851
|
+
context,
|
|
3852
|
+
startedAt,
|
|
3853
|
+
latency,
|
|
3854
|
+
child(childOp) {
|
|
3855
|
+
return self.start(childOp, context);
|
|
3856
|
+
},
|
|
3857
|
+
mark(stage, ms) {
|
|
3858
|
+
latency[stage] = ms;
|
|
3859
|
+
const elapsed = performance.now() - startedAt;
|
|
3860
|
+
spans.push({
|
|
3861
|
+
name: stage,
|
|
3862
|
+
startMs: Math.max(0, elapsed - ms),
|
|
3863
|
+
durationMs: ms
|
|
3864
|
+
});
|
|
3865
|
+
},
|
|
3866
|
+
success(fields) {
|
|
3867
|
+
finish("ok", fields);
|
|
3868
|
+
},
|
|
3869
|
+
failure(error, fields) {
|
|
3870
|
+
finish("error", fields, error);
|
|
3871
|
+
}
|
|
3872
|
+
};
|
|
3873
|
+
}
|
|
3874
|
+
emitStartup(provider) {
|
|
3875
|
+
const trace = this.start("startup");
|
|
3876
|
+
trace.success({ provider });
|
|
3877
|
+
}
|
|
3878
|
+
emitShutdown(provider) {
|
|
3879
|
+
const trace = this.start("shutdown");
|
|
3880
|
+
trace.success({ provider });
|
|
3881
|
+
}
|
|
3882
|
+
};
|
|
3883
|
+
var SCHEMA_SQL = `
|
|
3884
|
+
CREATE TABLE IF NOT EXISTS telemetry_events (
|
|
3885
|
+
id TEXT PRIMARY KEY,
|
|
3886
|
+
timestamp TEXT NOT NULL,
|
|
3887
|
+
operation TEXT NOT NULL,
|
|
3888
|
+
provider TEXT,
|
|
3889
|
+
duration_ms REAL,
|
|
3890
|
+
status TEXT NOT NULL,
|
|
3891
|
+
query TEXT,
|
|
3892
|
+
filters_json TEXT,
|
|
3893
|
+
returned_count INTEGER,
|
|
3894
|
+
memory_ids_json TEXT,
|
|
3895
|
+
similarity_scores_json TEXT,
|
|
3896
|
+
metadata_json TEXT,
|
|
3897
|
+
embedding_provider TEXT,
|
|
3898
|
+
model TEXT,
|
|
3899
|
+
error TEXT,
|
|
3900
|
+
error_stack TEXT,
|
|
3901
|
+
session_id TEXT NOT NULL,
|
|
3902
|
+
trace_id TEXT NOT NULL,
|
|
3903
|
+
parent_trace_id TEXT,
|
|
3904
|
+
user_metadata_json TEXT,
|
|
3905
|
+
extra_json TEXT,
|
|
3906
|
+
latency_json TEXT,
|
|
3907
|
+
organization TEXT,
|
|
3908
|
+
agent_id TEXT,
|
|
3909
|
+
tags_json TEXT,
|
|
3910
|
+
checkpoint_id TEXT,
|
|
3911
|
+
explain_json TEXT,
|
|
3912
|
+
spans_json TEXT
|
|
3913
|
+
);
|
|
3914
|
+
|
|
3915
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_timestamp ON telemetry_events(timestamp);
|
|
3916
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_operation ON telemetry_events(operation);
|
|
3917
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_trace ON telemetry_events(trace_id);
|
|
3918
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_session ON telemetry_events(session_id);
|
|
3919
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_status ON telemetry_events(status);
|
|
3920
|
+
`;
|
|
3921
|
+
var V2_COLUMNS = {
|
|
3922
|
+
organization: "TEXT",
|
|
3923
|
+
agent_id: "TEXT",
|
|
3924
|
+
tags_json: "TEXT",
|
|
3925
|
+
checkpoint_id: "TEXT",
|
|
3926
|
+
explain_json: "TEXT",
|
|
3927
|
+
spans_json: "TEXT"
|
|
3928
|
+
};
|
|
3929
|
+
var V2_INDEXES_SQL = `
|
|
3930
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_agent ON telemetry_events(agent_id);
|
|
3931
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_org ON telemetry_events(organization);
|
|
3932
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_checkpoint ON telemetry_events(checkpoint_id);
|
|
3933
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_operation_time ON telemetry_events(operation, timestamp);
|
|
3934
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_status_time ON telemetry_events(status, timestamp);
|
|
3935
|
+
`;
|
|
3936
|
+
var SqliteEventDatabase = class {
|
|
3937
|
+
name = "sqlite";
|
|
3938
|
+
url;
|
|
3939
|
+
readonly;
|
|
3940
|
+
db = null;
|
|
3941
|
+
insertStmt = null;
|
|
3942
|
+
columns = /* @__PURE__ */ new Set();
|
|
3943
|
+
constructor(options) {
|
|
3944
|
+
this.url = options.url;
|
|
3945
|
+
this.readonly = options.readonly ?? false;
|
|
3946
|
+
}
|
|
3947
|
+
async open() {
|
|
3470
3948
|
try {
|
|
3471
|
-
|
|
3949
|
+
const dbPath = this.resolvePath(this.url);
|
|
3950
|
+
if (dbPath !== ":memory:" && !this.readonly) {
|
|
3951
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(dbPath), { recursive: true });
|
|
3952
|
+
}
|
|
3953
|
+
const db = new sqlite$1.DatabaseSync(dbPath, {
|
|
3954
|
+
allowExtension: false,
|
|
3955
|
+
readOnly: this.readonly
|
|
3956
|
+
});
|
|
3957
|
+
this.db = db;
|
|
3958
|
+
if (!this.readonly) {
|
|
3959
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
3960
|
+
db.exec(`
|
|
3961
|
+
PRAGMA synchronous = NORMAL;
|
|
3962
|
+
PRAGMA busy_timeout = 5000;
|
|
3963
|
+
PRAGMA temp_store = MEMORY;
|
|
3964
|
+
`);
|
|
3965
|
+
db.exec(SCHEMA_SQL);
|
|
3966
|
+
this.migrateToV2(db);
|
|
3967
|
+
}
|
|
3968
|
+
this.columns = readColumns(db);
|
|
3969
|
+
if (!this.readonly) {
|
|
3970
|
+
this.insertStmt = db.prepare(`
|
|
3971
|
+
INSERT INTO telemetry_events (
|
|
3972
|
+
id, timestamp, operation, provider, duration_ms, status,
|
|
3973
|
+
query, filters_json, returned_count, memory_ids_json,
|
|
3974
|
+
similarity_scores_json, metadata_json, embedding_provider, model,
|
|
3975
|
+
error, error_stack, session_id, trace_id, parent_trace_id,
|
|
3976
|
+
user_metadata_json, extra_json, latency_json, organization,
|
|
3977
|
+
agent_id, tags_json, checkpoint_id, explain_json, spans_json
|
|
3978
|
+
) VALUES (
|
|
3979
|
+
?, ?, ?, ?, ?, ?,
|
|
3980
|
+
?, ?, ?, ?,
|
|
3981
|
+
?, ?, ?, ?,
|
|
3982
|
+
?, ?, ?, ?, ?,
|
|
3983
|
+
?, ?, ?, ?,
|
|
3984
|
+
?, ?, ?, ?, ?
|
|
3985
|
+
)
|
|
3986
|
+
`);
|
|
3987
|
+
}
|
|
3472
3988
|
} catch (error) {
|
|
3473
|
-
|
|
3474
|
-
|
|
3989
|
+
try {
|
|
3990
|
+
this.db?.close();
|
|
3991
|
+
} catch {
|
|
3475
3992
|
}
|
|
3476
|
-
|
|
3477
|
-
|
|
3993
|
+
this.db = null;
|
|
3994
|
+
this.columns.clear();
|
|
3995
|
+
throw new InitializationError(
|
|
3996
|
+
`Failed to open telemetry EventDatabase: ${describe(error)}`,
|
|
3478
3997
|
{ cause: error instanceof Error ? error : void 0 }
|
|
3479
3998
|
);
|
|
3480
3999
|
}
|
|
3481
|
-
this.organization = validated.organization;
|
|
3482
|
-
this.storage = createStorageProvider(validated.database);
|
|
3483
|
-
this.embedding = createEmbeddingProvider(validated.embedding);
|
|
3484
|
-
if (validated.llm) {
|
|
3485
|
-
this.llm = createLlmProvider(validated.llm);
|
|
3486
|
-
this.compression = createCompressionProvider(this.llm);
|
|
3487
|
-
}
|
|
3488
|
-
await this.ready();
|
|
3489
4000
|
}
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
4001
|
+
async close() {
|
|
4002
|
+
if (!this.db) return;
|
|
4003
|
+
try {
|
|
4004
|
+
this.db.close();
|
|
4005
|
+
} finally {
|
|
4006
|
+
this.db = null;
|
|
4007
|
+
this.insertStmt = null;
|
|
4008
|
+
this.columns.clear();
|
|
4009
|
+
}
|
|
4010
|
+
}
|
|
4011
|
+
async insertEvent(input) {
|
|
4012
|
+
const event = normalizeEvent(input);
|
|
4013
|
+
const stmt = this.insertStmt;
|
|
4014
|
+
if (!stmt) {
|
|
4015
|
+
throw new DatabaseError("Telemetry EventDatabase is not open for writes");
|
|
4016
|
+
}
|
|
4017
|
+
try {
|
|
4018
|
+
stmt.run(
|
|
4019
|
+
event.id,
|
|
4020
|
+
event.timestamp,
|
|
4021
|
+
event.operation,
|
|
4022
|
+
event.provider,
|
|
4023
|
+
event.durationMs,
|
|
4024
|
+
event.status,
|
|
4025
|
+
event.query,
|
|
4026
|
+
jsonOrNull(event.filters),
|
|
4027
|
+
event.returnedCount,
|
|
4028
|
+
jsonOrNull(event.memoryIds),
|
|
4029
|
+
jsonOrNull(event.similarityScores),
|
|
4030
|
+
jsonOrNull(event.metadata),
|
|
4031
|
+
event.embeddingProvider,
|
|
4032
|
+
event.model,
|
|
4033
|
+
event.error,
|
|
4034
|
+
event.errorStack,
|
|
4035
|
+
event.sessionId,
|
|
4036
|
+
event.traceId,
|
|
4037
|
+
event.parentTraceId,
|
|
4038
|
+
jsonOrNull(event.userMetadata),
|
|
4039
|
+
jsonOrNull(event.extra),
|
|
4040
|
+
jsonOrNull(event.latency),
|
|
4041
|
+
event.organization,
|
|
4042
|
+
event.agentId,
|
|
4043
|
+
jsonOrNull(event.tags),
|
|
4044
|
+
event.checkpointId,
|
|
4045
|
+
jsonOrNull(event.explain),
|
|
4046
|
+
jsonOrNull(event.spans)
|
|
4047
|
+
);
|
|
4048
|
+
return event;
|
|
4049
|
+
} catch (error) {
|
|
4050
|
+
throw new DatabaseError(
|
|
4051
|
+
`Failed to write telemetry event: ${describe(error)}`,
|
|
4052
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4053
|
+
);
|
|
4054
|
+
}
|
|
4055
|
+
}
|
|
4056
|
+
async insertEvents(inputs) {
|
|
4057
|
+
const db = this.requireDb();
|
|
4058
|
+
const out = [];
|
|
4059
|
+
db.exec("BEGIN");
|
|
4060
|
+
try {
|
|
4061
|
+
for (const input of inputs) {
|
|
4062
|
+
out.push(await this.insertEvent(input));
|
|
4063
|
+
}
|
|
4064
|
+
db.exec("COMMIT");
|
|
4065
|
+
return out;
|
|
4066
|
+
} catch (error) {
|
|
4067
|
+
try {
|
|
4068
|
+
db.exec("ROLLBACK");
|
|
4069
|
+
} catch {
|
|
4070
|
+
}
|
|
4071
|
+
throw error;
|
|
4072
|
+
}
|
|
4073
|
+
}
|
|
4074
|
+
async query(options) {
|
|
4075
|
+
const db = this.requireDb();
|
|
4076
|
+
const clauses = [];
|
|
4077
|
+
const params = [];
|
|
4078
|
+
if (options.operation) {
|
|
4079
|
+
const ops = Array.isArray(options.operation) ? options.operation : [options.operation];
|
|
4080
|
+
clauses.push(`operation IN (${ops.map(() => "?").join(",")})`);
|
|
4081
|
+
params.push(...ops);
|
|
4082
|
+
}
|
|
4083
|
+
if (options.status) {
|
|
4084
|
+
clauses.push(`status = ?`);
|
|
4085
|
+
params.push(options.status);
|
|
4086
|
+
}
|
|
4087
|
+
if (options.traceId) {
|
|
4088
|
+
clauses.push(`(trace_id = ? OR parent_trace_id = ?)`);
|
|
4089
|
+
params.push(options.traceId, options.traceId);
|
|
4090
|
+
}
|
|
4091
|
+
if (options.sessionId) {
|
|
4092
|
+
clauses.push(`session_id = ?`);
|
|
4093
|
+
params.push(options.sessionId);
|
|
4094
|
+
}
|
|
4095
|
+
this.addOptionalColumnFilter(
|
|
4096
|
+
clauses,
|
|
4097
|
+
params,
|
|
4098
|
+
"organization",
|
|
4099
|
+
options.organization
|
|
4100
|
+
);
|
|
4101
|
+
this.addOptionalColumnFilter(clauses, params, "agent_id", options.agentId);
|
|
4102
|
+
this.addOptionalColumnFilter(
|
|
4103
|
+
clauses,
|
|
4104
|
+
params,
|
|
4105
|
+
"checkpoint_id",
|
|
4106
|
+
options.checkpointId
|
|
4107
|
+
);
|
|
4108
|
+
if (options.tag) {
|
|
4109
|
+
if (this.columns.has("tags_json")) {
|
|
4110
|
+
clauses.push(
|
|
4111
|
+
`EXISTS (SELECT 1 FROM json_each(tags_json) WHERE json_each.value = ?)`
|
|
4112
|
+
);
|
|
4113
|
+
params.push(options.tag);
|
|
4114
|
+
} else {
|
|
4115
|
+
clauses.push("0 = 1");
|
|
4116
|
+
}
|
|
4117
|
+
}
|
|
4118
|
+
if (options.memoryId) {
|
|
4119
|
+
clauses.push(`memory_ids_json LIKE ?`);
|
|
4120
|
+
params.push(`%${options.memoryId}%`);
|
|
4121
|
+
}
|
|
4122
|
+
if (options.queryText) {
|
|
4123
|
+
clauses.push(`query LIKE ?`);
|
|
4124
|
+
params.push(`%${options.queryText}%`);
|
|
4125
|
+
}
|
|
4126
|
+
if (options.since) {
|
|
4127
|
+
clauses.push(`timestamp >= ?`);
|
|
4128
|
+
params.push(options.since);
|
|
4129
|
+
}
|
|
4130
|
+
if (options.until) {
|
|
4131
|
+
clauses.push(`timestamp <= ?`);
|
|
4132
|
+
params.push(options.until);
|
|
4133
|
+
}
|
|
4134
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
4135
|
+
const sortCol = options.sortBy === "duration_ms" ? "duration_ms" : "timestamp";
|
|
4136
|
+
const sortDir = options.sortDir === "asc" ? "ASC" : "DESC";
|
|
4137
|
+
const limit = options.limit ?? 50;
|
|
4138
|
+
const offset = options.offset ?? 0;
|
|
4139
|
+
const totalRow = db.prepare(`SELECT COUNT(*) AS c FROM telemetry_events ${where}`).get(...params);
|
|
4140
|
+
const rows = db.prepare(
|
|
4141
|
+
`SELECT * FROM telemetry_events ${where}
|
|
4142
|
+
ORDER BY ${sortCol} ${sortDir}
|
|
4143
|
+
LIMIT ? OFFSET ?`
|
|
4144
|
+
).all(...params, limit, offset);
|
|
4145
|
+
return {
|
|
4146
|
+
events: rows.map(rowToEvent),
|
|
4147
|
+
total: Number(totalRow.c),
|
|
4148
|
+
limit,
|
|
4149
|
+
offset
|
|
4150
|
+
};
|
|
4151
|
+
}
|
|
4152
|
+
async getEvent(id) {
|
|
4153
|
+
const db = this.requireDb();
|
|
4154
|
+
const row = db.prepare(`SELECT * FROM telemetry_events WHERE id = ?`).get(id);
|
|
4155
|
+
return row ? rowToEvent(row) : null;
|
|
4156
|
+
}
|
|
4157
|
+
async countEvents(filter) {
|
|
4158
|
+
const db = this.requireDb();
|
|
4159
|
+
const clauses = [];
|
|
4160
|
+
const params = [];
|
|
4161
|
+
if (filter?.since) {
|
|
4162
|
+
clauses.push(`timestamp >= ?`);
|
|
4163
|
+
params.push(filter.since);
|
|
4164
|
+
}
|
|
4165
|
+
if (filter?.operation) {
|
|
4166
|
+
clauses.push(`operation = ?`);
|
|
4167
|
+
params.push(filter.operation);
|
|
4168
|
+
}
|
|
4169
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
4170
|
+
const row = db.prepare(`SELECT COUNT(*) AS c FROM telemetry_events ${where}`).get(...params);
|
|
4171
|
+
return Number(row.c);
|
|
4172
|
+
}
|
|
4173
|
+
requireDb() {
|
|
4174
|
+
if (!this.db) {
|
|
4175
|
+
throw new DatabaseError("Telemetry EventDatabase is not open");
|
|
4176
|
+
}
|
|
4177
|
+
return this.db;
|
|
4178
|
+
}
|
|
4179
|
+
migrateToV2(db) {
|
|
4180
|
+
const columns = readColumns(db);
|
|
4181
|
+
for (const [name, type] of Object.entries(V2_COLUMNS)) {
|
|
4182
|
+
if (!columns.has(name)) {
|
|
4183
|
+
db.exec(`ALTER TABLE telemetry_events ADD COLUMN ${name} ${type};`);
|
|
4184
|
+
}
|
|
4185
|
+
}
|
|
4186
|
+
db.exec(`
|
|
4187
|
+
CREATE TABLE IF NOT EXISTS telemetry_meta (
|
|
4188
|
+
key TEXT PRIMARY KEY,
|
|
4189
|
+
value TEXT NOT NULL
|
|
4190
|
+
);
|
|
4191
|
+
INSERT INTO telemetry_meta(key, value) VALUES ('schema_version', '2')
|
|
4192
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value;
|
|
4193
|
+
${V2_INDEXES_SQL}
|
|
4194
|
+
`);
|
|
4195
|
+
}
|
|
4196
|
+
addOptionalColumnFilter(clauses, params, column, value) {
|
|
4197
|
+
if (!value) return;
|
|
4198
|
+
if (!this.columns.has(column)) {
|
|
4199
|
+
clauses.push("0 = 1");
|
|
4200
|
+
return;
|
|
4201
|
+
}
|
|
4202
|
+
clauses.push(`${column} = ?`);
|
|
4203
|
+
params.push(value);
|
|
4204
|
+
}
|
|
4205
|
+
resolvePath(url) {
|
|
4206
|
+
if (url === ":memory:") return ":memory:";
|
|
4207
|
+
return path5__default.default.isAbsolute(url) ? url : path5__default.default.resolve(process.cwd(), url);
|
|
4208
|
+
}
|
|
4209
|
+
};
|
|
4210
|
+
function normalizeEvent(input) {
|
|
4211
|
+
return {
|
|
4212
|
+
id: input.id ?? createId(),
|
|
4213
|
+
timestamp: input.timestamp ?? nowIso(),
|
|
4214
|
+
operation: input.operation,
|
|
4215
|
+
provider: input.provider ?? null,
|
|
4216
|
+
durationMs: input.durationMs ?? null,
|
|
4217
|
+
status: input.status,
|
|
4218
|
+
query: input.query ?? null,
|
|
4219
|
+
filters: input.filters ?? null,
|
|
4220
|
+
returnedCount: input.returnedCount ?? null,
|
|
4221
|
+
memoryIds: input.memoryIds ?? null,
|
|
4222
|
+
similarityScores: input.similarityScores ?? null,
|
|
4223
|
+
metadata: input.metadata ?? null,
|
|
4224
|
+
embeddingProvider: input.embeddingProvider ?? null,
|
|
4225
|
+
model: input.model ?? null,
|
|
4226
|
+
error: input.error ?? null,
|
|
4227
|
+
errorStack: input.errorStack ?? null,
|
|
4228
|
+
sessionId: input.sessionId,
|
|
4229
|
+
traceId: input.traceId,
|
|
4230
|
+
parentTraceId: input.parentTraceId ?? null,
|
|
4231
|
+
organization: input.organization ?? null,
|
|
4232
|
+
agentId: input.agentId ?? null,
|
|
4233
|
+
tags: input.tags ?? null,
|
|
4234
|
+
checkpointId: input.checkpointId ?? null,
|
|
4235
|
+
userMetadata: input.userMetadata ?? null,
|
|
4236
|
+
extra: input.extra ?? null,
|
|
4237
|
+
latency: input.latency ?? null,
|
|
4238
|
+
explain: input.explain ?? null,
|
|
4239
|
+
spans: input.spans ?? null
|
|
4240
|
+
};
|
|
4241
|
+
}
|
|
4242
|
+
function rowToEvent(row) {
|
|
4243
|
+
return {
|
|
4244
|
+
id: row.id,
|
|
4245
|
+
timestamp: row.timestamp,
|
|
4246
|
+
operation: row.operation,
|
|
4247
|
+
provider: row.provider,
|
|
4248
|
+
durationMs: row.duration_ms,
|
|
4249
|
+
status: row.status,
|
|
4250
|
+
query: row.query,
|
|
4251
|
+
filters: parseJson(row.filters_json),
|
|
4252
|
+
returnedCount: row.returned_count,
|
|
4253
|
+
memoryIds: parseJson(row.memory_ids_json),
|
|
4254
|
+
similarityScores: parseJson(row.similarity_scores_json),
|
|
4255
|
+
metadata: parseJson(row.metadata_json),
|
|
4256
|
+
embeddingProvider: row.embedding_provider,
|
|
4257
|
+
model: row.model,
|
|
4258
|
+
error: row.error,
|
|
4259
|
+
errorStack: row.error_stack,
|
|
4260
|
+
sessionId: row.session_id,
|
|
4261
|
+
traceId: row.trace_id,
|
|
4262
|
+
parentTraceId: row.parent_trace_id,
|
|
4263
|
+
organization: row.organization ?? null,
|
|
4264
|
+
agentId: row.agent_id ?? null,
|
|
4265
|
+
tags: parseJson(row.tags_json ?? null),
|
|
4266
|
+
checkpointId: row.checkpoint_id ?? null,
|
|
4267
|
+
userMetadata: parseJson(row.user_metadata_json),
|
|
4268
|
+
extra: parseJson(row.extra_json),
|
|
4269
|
+
latency: parseJson(row.latency_json),
|
|
4270
|
+
explain: parseJson(
|
|
4271
|
+
row.explain_json ?? null
|
|
4272
|
+
),
|
|
4273
|
+
spans: parseJson(row.spans_json ?? null)
|
|
4274
|
+
};
|
|
4275
|
+
}
|
|
4276
|
+
function readColumns(db) {
|
|
4277
|
+
const rows = db.prepare("PRAGMA table_info(telemetry_events)").all();
|
|
4278
|
+
return new Set(rows.map((row) => row.name));
|
|
4279
|
+
}
|
|
4280
|
+
function jsonOrNull(value) {
|
|
4281
|
+
if (value === void 0 || value === null) return null;
|
|
4282
|
+
return JSON.stringify(value);
|
|
4283
|
+
}
|
|
4284
|
+
function parseJson(value) {
|
|
4285
|
+
if (!value) return null;
|
|
4286
|
+
try {
|
|
4287
|
+
return JSON.parse(value);
|
|
4288
|
+
} catch {
|
|
4289
|
+
return null;
|
|
4290
|
+
}
|
|
4291
|
+
}
|
|
4292
|
+
function describe(error) {
|
|
4293
|
+
return error instanceof Error ? error.message : String(error);
|
|
4294
|
+
}
|
|
4295
|
+
|
|
4296
|
+
// src/providers/sqlite/sqliteTelemetryProvider.ts
|
|
4297
|
+
var SqliteTelemetryProvider = class {
|
|
4298
|
+
name = "sqlite";
|
|
4299
|
+
db;
|
|
4300
|
+
queue = [];
|
|
4301
|
+
flushing = null;
|
|
4302
|
+
closed = false;
|
|
4303
|
+
openPromise = null;
|
|
4304
|
+
constructor(options) {
|
|
4305
|
+
this.db = new SqliteEventDatabase({ url: options.url });
|
|
4306
|
+
}
|
|
4307
|
+
async open() {
|
|
4308
|
+
if (!this.openPromise) {
|
|
4309
|
+
this.openPromise = this.db.open();
|
|
4310
|
+
}
|
|
4311
|
+
await this.openPromise;
|
|
4312
|
+
this.closed = false;
|
|
4313
|
+
}
|
|
4314
|
+
async close() {
|
|
4315
|
+
this.closed = true;
|
|
4316
|
+
await this.flush();
|
|
4317
|
+
await this.db.close();
|
|
4318
|
+
this.openPromise = null;
|
|
4319
|
+
}
|
|
4320
|
+
emit(event) {
|
|
4321
|
+
if (this.closed) {
|
|
4322
|
+
return;
|
|
4323
|
+
}
|
|
4324
|
+
this.queue.push(event);
|
|
4325
|
+
void this.scheduleFlush();
|
|
4326
|
+
}
|
|
4327
|
+
async flush() {
|
|
4328
|
+
while (this.queue.length > 0 || this.flushing) {
|
|
4329
|
+
await this.scheduleFlush();
|
|
4330
|
+
if (this.flushing) {
|
|
4331
|
+
await this.flushing;
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
}
|
|
4335
|
+
async query(options) {
|
|
4336
|
+
await this.open();
|
|
4337
|
+
return this.db.query(options);
|
|
4338
|
+
}
|
|
4339
|
+
async getEvent(id) {
|
|
4340
|
+
await this.open();
|
|
4341
|
+
return this.db.getEvent(id);
|
|
4342
|
+
}
|
|
4343
|
+
scheduleFlush() {
|
|
4344
|
+
if (this.flushing) {
|
|
4345
|
+
return this.flushing;
|
|
4346
|
+
}
|
|
4347
|
+
this.flushing = this.runFlush().finally(() => {
|
|
4348
|
+
this.flushing = null;
|
|
4349
|
+
});
|
|
4350
|
+
return this.flushing;
|
|
4351
|
+
}
|
|
4352
|
+
async runFlush() {
|
|
4353
|
+
if (this.queue.length === 0) {
|
|
4354
|
+
return;
|
|
4355
|
+
}
|
|
4356
|
+
await this.open();
|
|
4357
|
+
const batch = this.queue.splice(0, this.queue.length);
|
|
4358
|
+
try {
|
|
4359
|
+
if (typeof this.db.insertEvents === "function") {
|
|
4360
|
+
await this.db.insertEvents(batch);
|
|
4361
|
+
} else {
|
|
4362
|
+
for (const event of batch) {
|
|
4363
|
+
await this.db.insertEvent(event);
|
|
4364
|
+
}
|
|
4365
|
+
}
|
|
4366
|
+
} catch {
|
|
4367
|
+
}
|
|
4368
|
+
}
|
|
4369
|
+
};
|
|
4370
|
+
var SDK_VERSION2 = "0.3.0";
|
|
4371
|
+
var SqliteCheckpointProvider = class {
|
|
4372
|
+
name = "sqlite";
|
|
4373
|
+
directory;
|
|
4374
|
+
ready = false;
|
|
4375
|
+
constructor(options) {
|
|
4376
|
+
this.directory = options?.directory ?? path5__default.default.resolve(process.cwd(), ".wolbarg", "checkpoints");
|
|
4377
|
+
}
|
|
4378
|
+
async open() {
|
|
4379
|
+
fs5__default.default.mkdirSync(this.directory, { recursive: true });
|
|
4380
|
+
this.ready = true;
|
|
4381
|
+
}
|
|
4382
|
+
async close() {
|
|
4383
|
+
this.ready = false;
|
|
4384
|
+
}
|
|
4385
|
+
async checkpoint(name, sourcePath, options) {
|
|
4386
|
+
this.requireReady();
|
|
4387
|
+
assertCheckpointName(name);
|
|
4388
|
+
const metaPath = this.metaPath(name);
|
|
4389
|
+
if (fs5__default.default.existsSync(metaPath)) {
|
|
4390
|
+
throw new ValidationError(
|
|
4391
|
+
`Checkpoint "${name}" already exists. Choose a different name \u2014 checkpoints are never overwritten.`
|
|
4392
|
+
);
|
|
4393
|
+
}
|
|
4394
|
+
const resolvedSource = resolveDbPath(sourcePath);
|
|
4395
|
+
if (resolvedSource === ":memory:") {
|
|
4396
|
+
throw new ConfigurationError(
|
|
4397
|
+
"Cannot checkpoint an in-memory database. Use a file-backed SQLite database."
|
|
4398
|
+
);
|
|
4399
|
+
}
|
|
4400
|
+
if (!fs5__default.default.existsSync(resolvedSource)) {
|
|
4401
|
+
throw new DatabaseError(
|
|
4402
|
+
`Failed to execute checkpoint()
|
|
4403
|
+
Reason:
|
|
4404
|
+
Memory database not found at ${resolvedSource}
|
|
4405
|
+
Suggestion:
|
|
4406
|
+
Ensure Wolbarg has been initialized and the database path is correct.`
|
|
4407
|
+
);
|
|
4408
|
+
}
|
|
4409
|
+
const snapshotPath = this.snapshotPath(name);
|
|
4410
|
+
await safeSqliteBackup(resolvedSource, snapshotPath);
|
|
4411
|
+
const stats = fs5__default.default.statSync(snapshotPath);
|
|
4412
|
+
const meta2 = {
|
|
4413
|
+
name,
|
|
4414
|
+
description: options?.description ?? null,
|
|
4415
|
+
createdAt: nowIso(),
|
|
4416
|
+
sdkVersion: SDK_VERSION2,
|
|
4417
|
+
provider: this.name,
|
|
4418
|
+
snapshotPath,
|
|
4419
|
+
sourcePath: resolvedSource,
|
|
4420
|
+
sizeBytes: stats.size
|
|
4421
|
+
};
|
|
4422
|
+
fs5__default.default.writeFileSync(metaPath, JSON.stringify(meta2, null, 2), "utf8");
|
|
4423
|
+
return meta2;
|
|
4424
|
+
}
|
|
4425
|
+
async rollback(name, targetPath) {
|
|
4426
|
+
this.requireReady();
|
|
4427
|
+
const meta2 = await this.getCheckpoint(name);
|
|
4428
|
+
if (!meta2) {
|
|
4429
|
+
throw new ValidationError(`Checkpoint "${name}" was not found`);
|
|
4430
|
+
}
|
|
4431
|
+
const resolvedTarget = resolveDbPath(targetPath);
|
|
4432
|
+
if (resolvedTarget === ":memory:") {
|
|
4433
|
+
throw new ConfigurationError(
|
|
4434
|
+
"Cannot rollback into an in-memory database."
|
|
4435
|
+
);
|
|
4436
|
+
}
|
|
4437
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(resolvedTarget), { recursive: true });
|
|
4438
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
4439
|
+
const side = `${resolvedTarget}${suffix}`;
|
|
4440
|
+
if (fs5__default.default.existsSync(side)) {
|
|
4441
|
+
fs5__default.default.rmSync(side, { force: true });
|
|
4442
|
+
}
|
|
4443
|
+
}
|
|
4444
|
+
await safeSqliteBackup(meta2.snapshotPath, resolvedTarget);
|
|
4445
|
+
return meta2;
|
|
4446
|
+
}
|
|
4447
|
+
async deleteCheckpoint(name) {
|
|
4448
|
+
this.requireReady();
|
|
4449
|
+
const metaPath = this.metaPath(name);
|
|
4450
|
+
const snapshotPath = this.snapshotPath(name);
|
|
4451
|
+
let removed = false;
|
|
4452
|
+
if (fs5__default.default.existsSync(metaPath)) {
|
|
4453
|
+
fs5__default.default.rmSync(metaPath, { force: true });
|
|
4454
|
+
removed = true;
|
|
4455
|
+
}
|
|
4456
|
+
if (fs5__default.default.existsSync(snapshotPath)) {
|
|
4457
|
+
fs5__default.default.rmSync(snapshotPath, { force: true });
|
|
4458
|
+
removed = true;
|
|
4459
|
+
}
|
|
4460
|
+
return removed;
|
|
4461
|
+
}
|
|
4462
|
+
async listCheckpoints() {
|
|
4463
|
+
this.requireReady();
|
|
4464
|
+
if (!fs5__default.default.existsSync(this.directory)) {
|
|
4465
|
+
return [];
|
|
4466
|
+
}
|
|
4467
|
+
const files = fs5__default.default.readdirSync(this.directory).filter((f) => f.endsWith(".json"));
|
|
4468
|
+
const out = [];
|
|
4469
|
+
for (const file of files) {
|
|
4470
|
+
try {
|
|
4471
|
+
const raw = fs5__default.default.readFileSync(path5__default.default.join(this.directory, file), "utf8");
|
|
4472
|
+
out.push(JSON.parse(raw));
|
|
4473
|
+
} catch {
|
|
4474
|
+
}
|
|
4475
|
+
}
|
|
4476
|
+
return out.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
4477
|
+
}
|
|
4478
|
+
async getCheckpoint(name) {
|
|
4479
|
+
this.requireReady();
|
|
4480
|
+
const metaPath = this.metaPath(name);
|
|
4481
|
+
if (!fs5__default.default.existsSync(metaPath)) {
|
|
4482
|
+
return null;
|
|
4483
|
+
}
|
|
4484
|
+
try {
|
|
4485
|
+
return JSON.parse(fs5__default.default.readFileSync(metaPath, "utf8"));
|
|
4486
|
+
} catch {
|
|
4487
|
+
return null;
|
|
4488
|
+
}
|
|
4489
|
+
}
|
|
4490
|
+
metaPath(name) {
|
|
4491
|
+
return path5__default.default.join(this.directory, `${sanitize(name)}.json`);
|
|
4492
|
+
}
|
|
4493
|
+
snapshotPath(name) {
|
|
4494
|
+
return path5__default.default.join(this.directory, `${sanitize(name)}.db`);
|
|
4495
|
+
}
|
|
4496
|
+
requireReady() {
|
|
4497
|
+
if (!this.ready) {
|
|
4498
|
+
throw new DatabaseError(
|
|
4499
|
+
"Checkpoint provider is not open. Call ready() / open first."
|
|
4500
|
+
);
|
|
4501
|
+
}
|
|
4502
|
+
}
|
|
4503
|
+
};
|
|
4504
|
+
function sanitize(name) {
|
|
4505
|
+
return name.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
4506
|
+
}
|
|
4507
|
+
function assertCheckpointName(name) {
|
|
4508
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
4509
|
+
throw new ValidationError("checkpoint name must be a non-empty string");
|
|
4510
|
+
}
|
|
4511
|
+
if (name.length > 128) {
|
|
4512
|
+
throw new ValidationError("checkpoint name must be <= 128 characters");
|
|
4513
|
+
}
|
|
4514
|
+
}
|
|
4515
|
+
function resolveDbPath(connectionString) {
|
|
4516
|
+
if (connectionString === ":memory:") return ":memory:";
|
|
4517
|
+
return path5__default.default.isAbsolute(connectionString) ? connectionString : path5__default.default.resolve(process.cwd(), connectionString);
|
|
4518
|
+
}
|
|
4519
|
+
async function safeSqliteBackup(sourcePath, destPath) {
|
|
4520
|
+
fs5__default.default.mkdirSync(path5__default.default.dirname(destPath), { recursive: true });
|
|
4521
|
+
if (fs5__default.default.existsSync(destPath)) {
|
|
4522
|
+
fs5__default.default.rmSync(destPath, { force: true });
|
|
4523
|
+
}
|
|
4524
|
+
let source = null;
|
|
4525
|
+
let dest = null;
|
|
4526
|
+
try {
|
|
4527
|
+
source = new sqlite$1.DatabaseSync(sourcePath);
|
|
4528
|
+
try {
|
|
4529
|
+
source.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
4530
|
+
} catch {
|
|
4531
|
+
}
|
|
4532
|
+
dest = new sqlite$1.DatabaseSync(destPath);
|
|
4533
|
+
const backupFn = source.backup;
|
|
4534
|
+
if (typeof backupFn === "function") {
|
|
4535
|
+
await backupFn.call(source, dest);
|
|
4536
|
+
} else {
|
|
4537
|
+
source.close();
|
|
4538
|
+
source = null;
|
|
4539
|
+
dest.close();
|
|
4540
|
+
dest = null;
|
|
4541
|
+
fs5__default.default.copyFileSync(sourcePath, destPath);
|
|
4542
|
+
}
|
|
4543
|
+
} catch (error) {
|
|
4544
|
+
throw new DatabaseError(
|
|
4545
|
+
`Failed to execute checkpoint()
|
|
4546
|
+
Reason:
|
|
4547
|
+
${error instanceof Error ? error.message : String(error)}
|
|
4548
|
+
Suggestion:
|
|
4549
|
+
Ensure no exclusive lock is held and the destination directory is writable.`,
|
|
4550
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4551
|
+
);
|
|
4552
|
+
} finally {
|
|
4553
|
+
try {
|
|
4554
|
+
source?.close();
|
|
4555
|
+
} catch {
|
|
4556
|
+
}
|
|
4557
|
+
try {
|
|
4558
|
+
dest?.close();
|
|
4559
|
+
} catch {
|
|
4560
|
+
}
|
|
4561
|
+
}
|
|
4562
|
+
}
|
|
4563
|
+
|
|
4564
|
+
// src/core/wolbarg.ts
|
|
4565
|
+
var Wolbarg = class {
|
|
4566
|
+
initialized = false;
|
|
4567
|
+
booting = null;
|
|
4568
|
+
organization = null;
|
|
4569
|
+
storage = null;
|
|
4570
|
+
embedding = null;
|
|
4571
|
+
llm = null;
|
|
4572
|
+
compression = null;
|
|
4573
|
+
reranker = null;
|
|
4574
|
+
keywordSearch = null;
|
|
4575
|
+
ocr = null;
|
|
4576
|
+
vision = null;
|
|
4577
|
+
chunking = null;
|
|
4578
|
+
retrievalConfig = {};
|
|
4579
|
+
embeddingDimensions = null;
|
|
4580
|
+
writeMutex = new AsyncMutex();
|
|
4581
|
+
telemetry;
|
|
4582
|
+
checkpointProvider = null;
|
|
4583
|
+
memoryDbPath = null;
|
|
4584
|
+
transfer = new SqliteMemoryTransferProvider();
|
|
4585
|
+
constructor(options) {
|
|
4586
|
+
this.telemetry = new TelemetryEmitter(null, { enabled: false, level: "off" });
|
|
4587
|
+
if (!options) {
|
|
4588
|
+
return;
|
|
4589
|
+
}
|
|
4590
|
+
const validated = validateWolbargOptions(options);
|
|
4591
|
+
this.organization = validated.organization;
|
|
4592
|
+
const storageInput = validated.storage;
|
|
4593
|
+
this.storage = isStorageProvider(storageInput) ? storageInput : createStorageProvider(storageInput);
|
|
4594
|
+
if (!isStorageProvider(storageInput)) {
|
|
4595
|
+
this.memoryDbPath = resolveDatabaseUrl(storageInput);
|
|
4596
|
+
} else if (storageInput instanceof SqliteStorageProvider) {
|
|
4597
|
+
this.memoryDbPath = storageInput.path;
|
|
4598
|
+
}
|
|
4599
|
+
this.embedding = isEmbeddingProvider(validated.embedding) ? validated.embedding : createEmbeddingProvider(validated.embedding);
|
|
4600
|
+
if (validated.llm) {
|
|
4601
|
+
this.llm = isLlmProvider(validated.llm) ? validated.llm : createLlmProvider(validated.llm);
|
|
4602
|
+
this.compression = validated.compression ?? createCompressionProvider(this.llm);
|
|
4603
|
+
} else {
|
|
4604
|
+
this.compression = validated.compression ?? null;
|
|
4605
|
+
}
|
|
4606
|
+
this.reranker = validated.reranker ?? null;
|
|
4607
|
+
this.keywordSearch = validated.keywordSearch ?? null;
|
|
4608
|
+
this.ocr = validated.ocr ?? null;
|
|
4609
|
+
this.vision = validated.vision ?? null;
|
|
4610
|
+
this.chunking = validated.chunking ?? null;
|
|
4611
|
+
this.retrievalConfig = validated.retrieval ?? {};
|
|
4612
|
+
if (validated.telemetry) {
|
|
4613
|
+
if (isTelemetryProvider(validated.telemetry)) {
|
|
4614
|
+
this.telemetry = new TelemetryEmitter(validated.telemetry, {
|
|
4615
|
+
enabled: true
|
|
4616
|
+
}, { organization: validated.organization });
|
|
4617
|
+
} else {
|
|
4618
|
+
const url = validated.telemetry.database.url ?? validated.telemetry.database.connectionString ?? "";
|
|
4619
|
+
const provider = new SqliteTelemetryProvider({ url });
|
|
4620
|
+
this.telemetry = new TelemetryEmitter(provider, validated.telemetry, {
|
|
4621
|
+
organization: validated.organization
|
|
4622
|
+
});
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4625
|
+
this.checkpointProvider = validated.checkpoint ?? new SqliteCheckpointProvider({
|
|
4626
|
+
directory: validated.checkpointDirectory
|
|
4627
|
+
});
|
|
4628
|
+
}
|
|
4629
|
+
/**
|
|
4630
|
+
* Backwards-compatible initialization (v0.1 API).
|
|
4631
|
+
*/
|
|
4632
|
+
async init(options) {
|
|
4633
|
+
if (this.initialized || this.storage) {
|
|
4634
|
+
throw new InitializationError("Wolbarg is already initialized");
|
|
4635
|
+
}
|
|
4636
|
+
let validated;
|
|
4637
|
+
try {
|
|
4638
|
+
validated = validateInitOptions(options);
|
|
4639
|
+
} catch (error) {
|
|
4640
|
+
if (error instanceof ConfigurationError || error instanceof ValidationError) {
|
|
4641
|
+
throw error;
|
|
4642
|
+
}
|
|
4643
|
+
throw new ConfigurationError(
|
|
4644
|
+
`Invalid configuration: ${error instanceof Error ? error.message : String(error)}`,
|
|
4645
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4646
|
+
);
|
|
4647
|
+
}
|
|
4648
|
+
this.organization = validated.organization;
|
|
4649
|
+
this.storage = createStorageProvider(validated.database);
|
|
4650
|
+
this.memoryDbPath = resolveDatabaseUrl(validated.database);
|
|
4651
|
+
this.embedding = createEmbeddingProvider(validated.embedding);
|
|
4652
|
+
if (validated.llm) {
|
|
4653
|
+
this.llm = createLlmProvider(validated.llm);
|
|
4654
|
+
this.compression = createCompressionProvider(this.llm);
|
|
4655
|
+
}
|
|
4656
|
+
if (!this.checkpointProvider) {
|
|
4657
|
+
this.checkpointProvider = new SqliteCheckpointProvider();
|
|
4658
|
+
}
|
|
4659
|
+
await this.ready();
|
|
4660
|
+
}
|
|
4661
|
+
/** Ensure storage (and optional telemetry) are open. */
|
|
4662
|
+
async ready() {
|
|
4663
|
+
if (this.initialized) {
|
|
3493
4664
|
return;
|
|
3494
4665
|
}
|
|
3495
4666
|
if (this.booting) {
|
|
@@ -3511,12 +4682,16 @@ var Wolbarg = class {
|
|
|
3511
4682
|
}
|
|
3512
4683
|
try {
|
|
3513
4684
|
await this.storage.open();
|
|
4685
|
+
await this.telemetry.open();
|
|
4686
|
+
await this.checkpointProvider?.open();
|
|
3514
4687
|
const probe = await this.embedding.validate();
|
|
3515
4688
|
await this.storage.ensureVectorSchema(probe.dimensions);
|
|
3516
4689
|
this.embeddingDimensions = probe.dimensions;
|
|
3517
4690
|
this.initialized = true;
|
|
4691
|
+
this.telemetry.emitStartup(this.storage.name);
|
|
3518
4692
|
} catch (error) {
|
|
3519
4693
|
await this.storage.close().catch(() => void 0);
|
|
4694
|
+
await this.telemetry.close().catch(() => void 0);
|
|
3520
4695
|
this.initialized = false;
|
|
3521
4696
|
if (error instanceof ConfigurationError || error instanceof InitializationError || error instanceof ValidationError) {
|
|
3522
4697
|
throw error;
|
|
@@ -3529,484 +4704,946 @@ var Wolbarg = class {
|
|
|
3529
4704
|
}
|
|
3530
4705
|
/** Store a semantic memory for an agent. */
|
|
3531
4706
|
async remember(options) {
|
|
3532
|
-
const
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
const
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
4707
|
+
const trace = this.telemetry.start("remember");
|
|
4708
|
+
try {
|
|
4709
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4710
|
+
assertNonEmptyString(options.agent, "agent");
|
|
4711
|
+
if (!options.content || typeof options.content.text !== "string") {
|
|
4712
|
+
throw new ValidationError("content.text must be a string");
|
|
4713
|
+
}
|
|
4714
|
+
assertNonEmptyString(options.content.text, "content.text");
|
|
4715
|
+
const metadata = options.metadata ?? {};
|
|
4716
|
+
if (metadata === null || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
4717
|
+
throw new ValidationError("metadata must be a plain object when provided");
|
|
4718
|
+
}
|
|
4719
|
+
const tEmbed = performance.now();
|
|
4720
|
+
const vector = await embedding.embed(options.content.text);
|
|
4721
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
4722
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
4723
|
+
const id = createId();
|
|
4724
|
+
const timestamp = nowIso();
|
|
4725
|
+
const record = await this.withWriteLock(async () => {
|
|
4726
|
+
const tStore = performance.now();
|
|
4727
|
+
const row = await storage.insertMemory({
|
|
4728
|
+
id,
|
|
4729
|
+
organization,
|
|
4730
|
+
agent: options.agent.trim(),
|
|
4731
|
+
contentText: options.content.text,
|
|
4732
|
+
metadata,
|
|
4733
|
+
embedding: vector,
|
|
4734
|
+
createdAt: timestamp,
|
|
4735
|
+
updatedAt: timestamp
|
|
4736
|
+
});
|
|
4737
|
+
trace.mark("databaseWriteMs", performance.now() - tStore);
|
|
4738
|
+
return toMemoryRecord(row);
|
|
4739
|
+
});
|
|
4740
|
+
trace.success({
|
|
4741
|
+
provider: storage.name,
|
|
4742
|
+
memoryIds: [record.id],
|
|
4743
|
+
returnedCount: 1,
|
|
4744
|
+
embeddingProvider: embedding.model,
|
|
4745
|
+
model: embedding.model,
|
|
3556
4746
|
metadata,
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
updatedAt: timestamp
|
|
4747
|
+
agentId: options.agent.trim(),
|
|
4748
|
+
tags: telemetryTags(metadata)
|
|
3560
4749
|
});
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
);
|
|
3567
|
-
}
|
|
3568
|
-
return toMemoryRecord(row);
|
|
3569
|
-
});
|
|
4750
|
+
return record;
|
|
4751
|
+
} catch (error) {
|
|
4752
|
+
trace.failure(error, { agentId: options.agent });
|
|
4753
|
+
throw wrapOperationError("remember", error);
|
|
4754
|
+
}
|
|
3570
4755
|
}
|
|
3571
|
-
/**
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
assertNonEmptyString(options.query, "query");
|
|
3578
|
-
const topK = options.topK ?? 5;
|
|
3579
|
-
const threshold = options.threshold ?? 0;
|
|
3580
|
-
assertFiniteNumber(topK, "topK", { min: 1, max: 1e3 });
|
|
3581
|
-
assertFiniteNumber(threshold, "threshold", { min: 0, max: 1 });
|
|
3582
|
-
const query = options.query.trim();
|
|
3583
|
-
const vector = await embedding.embed(query);
|
|
3584
|
-
this.assertEmbeddingDimensions(vector.length);
|
|
3585
|
-
const hasFilters = Boolean(
|
|
3586
|
-
options.filter?.agent || options.filter?.metadata
|
|
3587
|
-
);
|
|
3588
|
-
const overFetch = this.retrievalConfig.overFetchFactor ?? (hasFilters ? 4 : 2);
|
|
3589
|
-
const fetchK = adaptiveFetchK(topK, overFetch, hasFilters);
|
|
3590
|
-
const semanticScores = /* @__PURE__ */ new Map();
|
|
3591
|
-
const byId = /* @__PURE__ */ new Map();
|
|
3592
|
-
const joined = storage.searchVectorsWithMemories;
|
|
3593
|
-
if (typeof joined === "function") {
|
|
3594
|
-
const joinedHits = await joined.call(
|
|
3595
|
-
storage,
|
|
3596
|
-
vector,
|
|
3597
|
-
fetchK,
|
|
3598
|
-
organization,
|
|
3599
|
-
{
|
|
3600
|
-
agent: options.filter?.agent,
|
|
3601
|
-
includeArchived: options.filter?.includeArchived
|
|
3602
|
-
}
|
|
3603
|
-
);
|
|
3604
|
-
for (const { row, distance } of joinedHits) {
|
|
3605
|
-
if (options.filter?.metadata && !matchesMetadata(
|
|
3606
|
-
deserializeMetadata(row.metadata_json),
|
|
3607
|
-
options.filter.metadata
|
|
3608
|
-
)) {
|
|
3609
|
-
continue;
|
|
3610
|
-
}
|
|
3611
|
-
const similarity = distanceToSimilarity(distance);
|
|
3612
|
-
if (similarity < threshold) {
|
|
3613
|
-
continue;
|
|
3614
|
-
}
|
|
3615
|
-
const result = toRecallResult(row, similarity);
|
|
3616
|
-
semanticScores.set(result.id, similarity);
|
|
3617
|
-
byId.set(result.id, result);
|
|
4756
|
+
/** Batch remember — one parent event + child traces per item. */
|
|
4757
|
+
async rememberBatch(items) {
|
|
4758
|
+
const parent = this.telemetry.start("rememberBatch");
|
|
4759
|
+
try {
|
|
4760
|
+
if (!Array.isArray(items) || items.length === 0) {
|
|
4761
|
+
throw new ValidationError("rememberBatch requires a non-empty array");
|
|
3618
4762
|
}
|
|
3619
|
-
|
|
3620
|
-
const
|
|
3621
|
-
const
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
} else {
|
|
3626
|
-
rowMap = /* @__PURE__ */ new Map();
|
|
3627
|
-
const looked = await Promise.all(
|
|
3628
|
-
hits.map(async (hit) => ({
|
|
3629
|
-
hit,
|
|
3630
|
-
row: await storage.getMemoryByRowid(hit.memoryRowid, organization)
|
|
3631
|
-
}))
|
|
3632
|
-
);
|
|
3633
|
-
for (const { hit, row } of looked) {
|
|
3634
|
-
if (row) rowMap.set(hit.memoryRowid, row);
|
|
4763
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4764
|
+
const tEmbed = performance.now();
|
|
4765
|
+
const texts = items.map((item, i) => {
|
|
4766
|
+
assertNonEmptyString(item.agent, `items[${i}].agent`);
|
|
4767
|
+
if (!item.content || typeof item.content.text !== "string") {
|
|
4768
|
+
throw new ValidationError(`items[${i}].content.text must be a string`);
|
|
3635
4769
|
}
|
|
4770
|
+
assertNonEmptyString(item.content.text, `items[${i}].content.text`);
|
|
4771
|
+
return item.content.text;
|
|
4772
|
+
});
|
|
4773
|
+
const vectors = await embedMany(embedding, texts);
|
|
4774
|
+
parent.mark("embeddingMs", performance.now() - tEmbed);
|
|
4775
|
+
for (const vector of vectors) {
|
|
4776
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3636
4777
|
}
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
byId.set(result.id, result);
|
|
4778
|
+
const timestamp = nowIso();
|
|
4779
|
+
const inputs = items.map((item, i) => ({
|
|
4780
|
+
id: createId(),
|
|
4781
|
+
organization,
|
|
4782
|
+
agent: item.agent.trim(),
|
|
4783
|
+
contentText: item.content.text,
|
|
4784
|
+
metadata: item.metadata ?? {},
|
|
4785
|
+
embedding: vectors[i],
|
|
4786
|
+
createdAt: timestamp,
|
|
4787
|
+
updatedAt: timestamp
|
|
4788
|
+
}));
|
|
4789
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
4790
|
+
const child = parent.child("remember");
|
|
4791
|
+
child.success({
|
|
4792
|
+
provider: storage.name,
|
|
4793
|
+
memoryIds: [inputs[i].id],
|
|
4794
|
+
returnedCount: 1,
|
|
4795
|
+
metadata: inputs[i].metadata,
|
|
4796
|
+
agentId: inputs[i].agent,
|
|
4797
|
+
tags: telemetryTags(inputs[i].metadata)
|
|
4798
|
+
});
|
|
3659
4799
|
}
|
|
4800
|
+
const rows = await this.withWriteLock(async () => {
|
|
4801
|
+
const tStore = performance.now();
|
|
4802
|
+
const result = await storage.insertMemoriesBatch(inputs);
|
|
4803
|
+
parent.mark("databaseWriteMs", performance.now() - tStore);
|
|
4804
|
+
return result;
|
|
4805
|
+
});
|
|
4806
|
+
const records = rows.map(toMemoryRecord);
|
|
4807
|
+
parent.success({
|
|
4808
|
+
provider: storage.name,
|
|
4809
|
+
memoryIds: records.map((r) => r.id),
|
|
4810
|
+
returnedCount: records.length,
|
|
4811
|
+
embeddingProvider: embedding.model,
|
|
4812
|
+
model: embedding.model,
|
|
4813
|
+
agentId: commonAgent(items.map((item) => item.agent.trim()))
|
|
4814
|
+
});
|
|
4815
|
+
return records;
|
|
4816
|
+
} catch (error) {
|
|
4817
|
+
parent.failure(error);
|
|
4818
|
+
throw wrapOperationError("rememberBatch", error);
|
|
3660
4819
|
}
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
4820
|
+
}
|
|
4821
|
+
async recall(options) {
|
|
4822
|
+
const trace = this.telemetry.start("recall");
|
|
4823
|
+
const explain = options.explain === true;
|
|
4824
|
+
try {
|
|
4825
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4826
|
+
assertNonEmptyString(options.query, "query");
|
|
4827
|
+
const topK = options.topK ?? 5;
|
|
4828
|
+
const threshold = options.threshold ?? 0;
|
|
4829
|
+
assertFiniteNumber(topK, "topK", { min: 1, max: 1e3 });
|
|
4830
|
+
assertFiniteNumber(threshold, "threshold", { min: 0, max: 1 });
|
|
4831
|
+
const query = options.query.trim();
|
|
4832
|
+
const tEmbed = performance.now();
|
|
4833
|
+
const vector = await embedding.embed(query);
|
|
4834
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
4835
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
4836
|
+
const hasFilters = Boolean(
|
|
4837
|
+
options.filter?.agent || options.filter?.metadata
|
|
4838
|
+
);
|
|
4839
|
+
const overFetch = this.retrievalConfig.overFetchFactor ?? (hasFilters ? 4 : 2);
|
|
4840
|
+
const fetchK = adaptiveFetchK(topK, overFetch, hasFilters);
|
|
4841
|
+
const semanticScores = /* @__PURE__ */ new Map();
|
|
4842
|
+
const distances = /* @__PURE__ */ new Map();
|
|
4843
|
+
const byId = /* @__PURE__ */ new Map();
|
|
4844
|
+
const metadataMatched = /* @__PURE__ */ new Map();
|
|
4845
|
+
const tSearch = performance.now();
|
|
4846
|
+
const joined = storage.searchVectorsWithMemories;
|
|
4847
|
+
if (typeof joined === "function") {
|
|
4848
|
+
const joinedHits = await joined.call(
|
|
4849
|
+
storage,
|
|
4850
|
+
vector,
|
|
4851
|
+
fetchK,
|
|
3667
4852
|
organization,
|
|
3668
|
-
fetchK
|
|
3669
|
-
);
|
|
3670
|
-
keywordScores = new Map(
|
|
3671
|
-
ftsHits.map((h) => [h.memoryId, h.score])
|
|
3672
|
-
);
|
|
3673
|
-
const missingIds = [];
|
|
3674
|
-
for (const id of keywordScores.keys()) {
|
|
3675
|
-
if (!byId.has(id)) missingIds.push(id);
|
|
3676
|
-
}
|
|
3677
|
-
if (missingIds.length > 0) {
|
|
3678
|
-
const fetched = await Promise.all(
|
|
3679
|
-
missingIds.map((id) => storage.getMemoryById(id, organization))
|
|
3680
|
-
);
|
|
3681
|
-
for (const row of fetched) {
|
|
3682
|
-
if (!row) continue;
|
|
3683
|
-
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
3684
|
-
continue;
|
|
3685
|
-
}
|
|
3686
|
-
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
3687
|
-
continue;
|
|
3688
|
-
}
|
|
3689
|
-
if (options.filter?.metadata && !matchesMetadata(
|
|
3690
|
-
deserializeMetadata(row.metadata_json),
|
|
3691
|
-
options.filter.metadata
|
|
3692
|
-
)) {
|
|
3693
|
-
continue;
|
|
3694
|
-
}
|
|
3695
|
-
byId.set(row.id, toRecallResult(row, 0));
|
|
3696
|
-
}
|
|
3697
|
-
}
|
|
3698
|
-
} else if (this.keywordSearch) {
|
|
3699
|
-
const corpus = await storage.listMemories(
|
|
3700
4853
|
{
|
|
3701
|
-
organization,
|
|
3702
4854
|
agent: options.filter?.agent,
|
|
3703
|
-
includeArchived: options.filter?.includeArchived
|
|
3704
|
-
|
|
3705
|
-
},
|
|
3706
|
-
Math.min(fetchK * 2, 2e3)
|
|
3707
|
-
);
|
|
3708
|
-
const keywordHits = await this.keywordSearch.search(
|
|
3709
|
-
query,
|
|
3710
|
-
corpus.map((row) => ({ id: row.id, text: row.content_text })),
|
|
3711
|
-
fetchK
|
|
3712
|
-
);
|
|
3713
|
-
keywordScores = new Map(
|
|
3714
|
-
keywordHits.map((h) => [h.memoryId, h.score])
|
|
4855
|
+
includeArchived: options.filter?.includeArchived
|
|
4856
|
+
}
|
|
3715
4857
|
);
|
|
3716
|
-
|
|
3717
|
-
|
|
4858
|
+
const tFilter = performance.now();
|
|
4859
|
+
for (const { row, distance } of joinedHits) {
|
|
4860
|
+
let metaOk = true;
|
|
4861
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
4862
|
+
deserializeMetadata(row.metadata_json),
|
|
4863
|
+
options.filter.metadata
|
|
4864
|
+
)) {
|
|
4865
|
+
metaOk = false;
|
|
3718
4866
|
continue;
|
|
3719
4867
|
}
|
|
3720
|
-
|
|
4868
|
+
const similarity = distanceToSimilarity(distance);
|
|
4869
|
+
if (similarity < threshold) {
|
|
3721
4870
|
continue;
|
|
3722
4871
|
}
|
|
4872
|
+
const result = toRecallResult(row, similarity);
|
|
4873
|
+
semanticScores.set(result.id, similarity);
|
|
4874
|
+
distances.set(result.id, distance);
|
|
4875
|
+
metadataMatched.set(result.id, metaOk);
|
|
4876
|
+
byId.set(result.id, result);
|
|
4877
|
+
}
|
|
4878
|
+
trace.mark("metadataFilteringMs", performance.now() - tFilter);
|
|
4879
|
+
} else {
|
|
4880
|
+
const hits = await storage.searchVectors(vector, fetchK);
|
|
4881
|
+
const rowids = hits.map((h) => h.memoryRowid);
|
|
4882
|
+
let rowMap;
|
|
4883
|
+
if (typeof storage.getMemoriesByRowids === "function") {
|
|
4884
|
+
rowMap = await storage.getMemoriesByRowids(rowids, organization);
|
|
4885
|
+
} else {
|
|
4886
|
+
rowMap = /* @__PURE__ */ new Map();
|
|
4887
|
+
const looked = await Promise.all(
|
|
4888
|
+
hits.map(async (hit) => ({
|
|
4889
|
+
hit,
|
|
4890
|
+
row: await storage.getMemoryByRowid(hit.memoryRowid, organization)
|
|
4891
|
+
}))
|
|
4892
|
+
);
|
|
4893
|
+
for (const { hit, row } of looked) {
|
|
4894
|
+
if (row) rowMap.set(hit.memoryRowid, row);
|
|
4895
|
+
}
|
|
4896
|
+
}
|
|
4897
|
+
const tFilter = performance.now();
|
|
4898
|
+
for (const hit of hits) {
|
|
4899
|
+
const row = rowMap.get(hit.memoryRowid);
|
|
4900
|
+
if (!row) continue;
|
|
4901
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) continue;
|
|
4902
|
+
if (!options.filter?.includeArchived && row.archived === 1) continue;
|
|
3723
4903
|
const metadata = deserializeMetadata(row.metadata_json);
|
|
3724
4904
|
if (options.filter?.metadata && !matchesMetadata(metadata, options.filter.metadata)) {
|
|
3725
4905
|
continue;
|
|
3726
4906
|
}
|
|
3727
|
-
|
|
3728
|
-
|
|
4907
|
+
const similarity = distanceToSimilarity(hit.distance);
|
|
4908
|
+
if (similarity < threshold) continue;
|
|
4909
|
+
const result = toRecallResult(row, similarity);
|
|
4910
|
+
semanticScores.set(result.id, similarity);
|
|
4911
|
+
distances.set(result.id, hit.distance);
|
|
4912
|
+
metadataMatched.set(result.id, true);
|
|
4913
|
+
byId.set(result.id, result);
|
|
4914
|
+
}
|
|
4915
|
+
trace.mark("metadataFilteringMs", performance.now() - tFilter);
|
|
4916
|
+
}
|
|
4917
|
+
const searchTime = performance.now() - tSearch;
|
|
4918
|
+
trace.mark("vectorSearchMs", searchTime);
|
|
4919
|
+
trace.mark("databaseReadMs", searchTime);
|
|
4920
|
+
const hybridWeights = resolveHybridWeights(options.hybrid) ?? (options.hybrid === void 0 ? resolveHybridWeights(this.retrievalConfig.hybrid) : null);
|
|
4921
|
+
let keywordSignal = "disabled";
|
|
4922
|
+
if (hybridWeights) {
|
|
4923
|
+
let keywordScores = /* @__PURE__ */ new Map();
|
|
4924
|
+
if (typeof storage.searchKeyword === "function") {
|
|
4925
|
+
keywordSignal = "enabled";
|
|
4926
|
+
const ftsHits = await storage.searchKeyword(
|
|
4927
|
+
query,
|
|
4928
|
+
organization,
|
|
4929
|
+
fetchK
|
|
4930
|
+
);
|
|
4931
|
+
keywordScores = new Map(ftsHits.map((h) => [h.memoryId, h.score]));
|
|
4932
|
+
const missingIds = [];
|
|
4933
|
+
for (const id of keywordScores.keys()) {
|
|
4934
|
+
if (!byId.has(id)) missingIds.push(id);
|
|
4935
|
+
}
|
|
4936
|
+
if (missingIds.length > 0) {
|
|
4937
|
+
const fetched = await Promise.all(
|
|
4938
|
+
missingIds.map((id) => storage.getMemoryById(id, organization))
|
|
4939
|
+
);
|
|
4940
|
+
for (const row of fetched) {
|
|
4941
|
+
if (!row) continue;
|
|
4942
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
4943
|
+
continue;
|
|
4944
|
+
}
|
|
4945
|
+
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
4946
|
+
continue;
|
|
4947
|
+
}
|
|
4948
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
4949
|
+
deserializeMetadata(row.metadata_json),
|
|
4950
|
+
options.filter.metadata
|
|
4951
|
+
)) {
|
|
4952
|
+
continue;
|
|
4953
|
+
}
|
|
4954
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
4955
|
+
metadataMatched.set(row.id, true);
|
|
4956
|
+
}
|
|
4957
|
+
}
|
|
4958
|
+
} else if (this.keywordSearch) {
|
|
4959
|
+
keywordSignal = "enabled";
|
|
4960
|
+
const corpus = await storage.listMemories(
|
|
4961
|
+
{
|
|
4962
|
+
organization,
|
|
4963
|
+
agent: options.filter?.agent,
|
|
4964
|
+
includeArchived: options.filter?.includeArchived,
|
|
4965
|
+
metadata: options.filter?.metadata
|
|
4966
|
+
},
|
|
4967
|
+
Math.min(fetchK * 2, 2e3)
|
|
4968
|
+
);
|
|
4969
|
+
const keywordHits = await this.keywordSearch.search(
|
|
4970
|
+
query,
|
|
4971
|
+
corpus.map((row) => ({ id: row.id, text: row.content_text })),
|
|
4972
|
+
fetchK
|
|
4973
|
+
);
|
|
4974
|
+
keywordScores = new Map(
|
|
4975
|
+
keywordHits.map((h) => [h.memoryId, h.score])
|
|
4976
|
+
);
|
|
4977
|
+
for (const row of corpus) {
|
|
4978
|
+
if (byId.has(row.id) || !keywordScores.has(row.id)) continue;
|
|
4979
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
4980
|
+
metadataMatched.set(row.id, true);
|
|
3729
4981
|
}
|
|
3730
|
-
byId.set(row.id, toRecallResult(row, 0));
|
|
3731
4982
|
}
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
4983
|
+
if (keywordScores.size > 0) {
|
|
4984
|
+
const fused = fuseScores(semanticScores, keywordScores, hybridWeights);
|
|
4985
|
+
for (const [id, score] of fused) {
|
|
4986
|
+
const existing = byId.get(id);
|
|
4987
|
+
if (existing) {
|
|
4988
|
+
byId.set(id, { ...existing, similarity: score });
|
|
4989
|
+
}
|
|
3739
4990
|
}
|
|
3740
4991
|
}
|
|
3741
4992
|
}
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
);
|
|
3746
|
-
const mmrLambda = resolveMmr(options.mmr) ?? resolveMmr(
|
|
3747
|
-
this.retrievalConfig.mmr ? { lambda: this.retrievalConfig.mmr.lambda } : void 0
|
|
3748
|
-
);
|
|
3749
|
-
if (mmrLambda !== null) {
|
|
3750
|
-
results = applyMmr(results, Math.max(topK * 2, topK), mmrLambda);
|
|
3751
|
-
}
|
|
3752
|
-
if (options.rerank && this.reranker) {
|
|
3753
|
-
const reranked = await this.reranker.rerank(
|
|
3754
|
-
query,
|
|
3755
|
-
results.map((r) => ({ id: r.id, text: r.content.text })),
|
|
3756
|
-
topK
|
|
4993
|
+
const tRank = performance.now();
|
|
4994
|
+
let results = [...byId.values()].sort(
|
|
4995
|
+
(a, b) => b.similarity - a.similarity
|
|
3757
4996
|
);
|
|
3758
|
-
const
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
4997
|
+
const mmrLambda = resolveMmr(options.mmr) ?? resolveMmr(
|
|
4998
|
+
this.retrievalConfig.mmr ? { lambda: this.retrievalConfig.mmr.lambda } : void 0
|
|
4999
|
+
);
|
|
5000
|
+
let rankingReason = "cosine similarity";
|
|
5001
|
+
if (mmrLambda !== null) {
|
|
5002
|
+
results = applyMmr(results, Math.max(topK * 2, topK), mmrLambda);
|
|
5003
|
+
rankingReason = `MMR(lambda=${mmrLambda})`;
|
|
5004
|
+
}
|
|
5005
|
+
if (options.rerank && this.reranker) {
|
|
5006
|
+
const reranked = await this.reranker.rerank(
|
|
5007
|
+
query,
|
|
5008
|
+
results.map((r) => ({ id: r.id, text: r.content.text })),
|
|
5009
|
+
topK
|
|
5010
|
+
);
|
|
5011
|
+
const order = new Map(
|
|
5012
|
+
reranked.map((r, i) => [r.id, { score: r.score, i }])
|
|
5013
|
+
);
|
|
5014
|
+
results = results.filter((r) => order.has(r.id)).sort((a, b) => order.get(a.id).i - order.get(b.id).i).map((r) => ({
|
|
5015
|
+
...r,
|
|
5016
|
+
similarity: order.get(r.id)?.score ?? r.similarity
|
|
5017
|
+
}));
|
|
5018
|
+
rankingReason = `reranker:${this.reranker.name ?? "custom"}`;
|
|
5019
|
+
}
|
|
5020
|
+
const rankingTime = performance.now() - tRank;
|
|
5021
|
+
trace.mark("rankingMs", rankingTime);
|
|
5022
|
+
results = results.slice(0, topK);
|
|
5023
|
+
const tSer = performance.now();
|
|
5024
|
+
const serialized = results.map((r) => ({ ...r }));
|
|
5025
|
+
trace.mark("serializationMs", performance.now() - tSer);
|
|
5026
|
+
const telemetryFields = {
|
|
5027
|
+
provider: storage.name,
|
|
5028
|
+
query,
|
|
5029
|
+
filters: options.filter ?? null,
|
|
5030
|
+
returnedCount: serialized.length,
|
|
5031
|
+
memoryIds: serialized.map((r) => r.id),
|
|
5032
|
+
similarityScores: serialized.map((r) => r.similarity),
|
|
5033
|
+
embeddingProvider: embedding.model,
|
|
5034
|
+
model: embedding.model,
|
|
5035
|
+
agentId: options.filter?.agent ?? commonAgent(serialized.map((result) => result.agent)),
|
|
5036
|
+
tags: commonTags(serialized.map((result) => result.metadata))
|
|
5037
|
+
};
|
|
5038
|
+
if (!explain) {
|
|
5039
|
+
trace.success(telemetryFields);
|
|
5040
|
+
return serialized;
|
|
5041
|
+
}
|
|
5042
|
+
const explanations = serialized.map((memory) => {
|
|
5043
|
+
const matchedFields = ["content"];
|
|
5044
|
+
if (options.filter?.agent) matchedFields.push("agent");
|
|
5045
|
+
if (options.filter?.metadata) matchedFields.push("metadata");
|
|
5046
|
+
return {
|
|
5047
|
+
memory,
|
|
5048
|
+
score: memory.similarity,
|
|
5049
|
+
distance: distances.get(memory.id) ?? 1 - memory.similarity,
|
|
5050
|
+
rankingReason,
|
|
5051
|
+
matchedFields,
|
|
5052
|
+
metadataMatch: metadataMatched.get(memory.id) ?? true,
|
|
5053
|
+
providerUsed: storage.name,
|
|
5054
|
+
searchTime,
|
|
5055
|
+
rankingTime
|
|
5056
|
+
};
|
|
5057
|
+
});
|
|
5058
|
+
const totalTime = performance.now() - trace.startedAt;
|
|
5059
|
+
const persistedExplain = {
|
|
5060
|
+
enabled: true,
|
|
5061
|
+
providerUsed: storage.name,
|
|
5062
|
+
rankingStrategy: rankingReason,
|
|
5063
|
+
signals: {
|
|
5064
|
+
semantic: "enabled",
|
|
5065
|
+
keyword: keywordSignal,
|
|
5066
|
+
reranker: options.rerank === true && this.reranker ? "enabled" : "disabled",
|
|
5067
|
+
mmr: mmrLambda === null ? "disabled" : "enabled",
|
|
5068
|
+
recency: "disabled"
|
|
5069
|
+
},
|
|
5070
|
+
results: explanations.map((hit) => ({
|
|
5071
|
+
memoryId: hit.memory.id,
|
|
5072
|
+
score: hit.score,
|
|
5073
|
+
distance: hit.distance,
|
|
5074
|
+
rankingReason: hit.rankingReason,
|
|
5075
|
+
matchedFields: hit.matchedFields,
|
|
5076
|
+
metadataMatch: hit.metadataMatch
|
|
5077
|
+
})),
|
|
5078
|
+
searchTimeMs: searchTime,
|
|
5079
|
+
rankingTimeMs: rankingTime,
|
|
5080
|
+
totalTimeMs: totalTime
|
|
5081
|
+
};
|
|
5082
|
+
trace.success({ ...telemetryFields, explain: persistedExplain });
|
|
5083
|
+
return {
|
|
5084
|
+
results: explanations,
|
|
5085
|
+
providerUsed: storage.name,
|
|
5086
|
+
searchTime,
|
|
5087
|
+
rankingTime,
|
|
5088
|
+
totalTime,
|
|
5089
|
+
traceId: trace.context.traceId
|
|
5090
|
+
};
|
|
5091
|
+
} catch (error) {
|
|
5092
|
+
trace.failure(error, {
|
|
5093
|
+
query: options.query,
|
|
5094
|
+
agentId: options.filter?.agent ?? null
|
|
5095
|
+
});
|
|
5096
|
+
throw wrapOperationError("recall", error);
|
|
5097
|
+
}
|
|
5098
|
+
}
|
|
5099
|
+
/** Batch recall — parent event + child traces. */
|
|
5100
|
+
async recallBatch(queries) {
|
|
5101
|
+
const parent = this.telemetry.start("recallBatch");
|
|
5102
|
+
try {
|
|
5103
|
+
if (!Array.isArray(queries) || queries.length === 0) {
|
|
5104
|
+
throw new ValidationError("recallBatch requires a non-empty array");
|
|
5105
|
+
}
|
|
5106
|
+
const out = [];
|
|
5107
|
+
for (const q of queries) {
|
|
5108
|
+
const child = parent.child("recall");
|
|
5109
|
+
try {
|
|
5110
|
+
const hits = await this.recall({ ...q, explain: false });
|
|
5111
|
+
child.success({
|
|
5112
|
+
query: q.query,
|
|
5113
|
+
returnedCount: hits.length,
|
|
5114
|
+
memoryIds: hits.map((h) => h.id),
|
|
5115
|
+
similarityScores: hits.map((h) => h.similarity)
|
|
5116
|
+
});
|
|
5117
|
+
out.push(hits);
|
|
5118
|
+
} catch (error) {
|
|
5119
|
+
child.failure(error, { query: q.query });
|
|
5120
|
+
throw error;
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
5123
|
+
parent.success({
|
|
5124
|
+
returnedCount: out.reduce((n, r) => n + r.length, 0),
|
|
5125
|
+
extra: { queryCount: queries.length }
|
|
5126
|
+
});
|
|
5127
|
+
return out;
|
|
5128
|
+
} catch (error) {
|
|
5129
|
+
parent.failure(error);
|
|
5130
|
+
throw wrapOperationError("recallBatch", error);
|
|
3763
5131
|
}
|
|
3764
|
-
return results.slice(0, topK);
|
|
3765
5132
|
}
|
|
3766
|
-
/**
|
|
3767
|
-
* Compress related memories for an agent into a single summarized memory.
|
|
3768
|
-
* Only available when `llm` (or `compression`) was configured at construction.
|
|
3769
|
-
*/
|
|
3770
5133
|
async compress(options) {
|
|
3771
5134
|
return this.runCompress(options);
|
|
3772
5135
|
}
|
|
3773
5136
|
/** @internal */
|
|
3774
5137
|
async runCompress(options) {
|
|
3775
|
-
const
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
5138
|
+
const trace = this.telemetry.start("compress");
|
|
5139
|
+
try {
|
|
5140
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5141
|
+
if (!this.compression) {
|
|
5142
|
+
throw new ProviderNotConfiguredError(
|
|
5143
|
+
"llm",
|
|
5144
|
+
"compress",
|
|
5145
|
+
"pass llm: openaiLlm(...) (or compression) in the constructor"
|
|
5146
|
+
);
|
|
5147
|
+
}
|
|
5148
|
+
assertNonEmptyString(options.agent, "agent");
|
|
5149
|
+
const limit = options.limit ?? 50;
|
|
5150
|
+
assertFiniteNumber(limit, "limit", { min: 2, max: 500 });
|
|
5151
|
+
const tRead = performance.now();
|
|
5152
|
+
const rows = await storage.listMemories(
|
|
5153
|
+
{
|
|
5154
|
+
organization,
|
|
5155
|
+
agent: options.agent.trim(),
|
|
5156
|
+
includeArchived: false
|
|
5157
|
+
},
|
|
5158
|
+
limit
|
|
3781
5159
|
);
|
|
5160
|
+
trace.mark("databaseReadMs", performance.now() - tRead);
|
|
5161
|
+
if (rows.length < 2) {
|
|
5162
|
+
throw new ValidationError(
|
|
5163
|
+
"compress requires at least 2 active memories for the given agent"
|
|
5164
|
+
);
|
|
5165
|
+
}
|
|
5166
|
+
const records = rows.map(toMemoryRecord);
|
|
5167
|
+
const summaryText = await this.compression.compress(records);
|
|
5168
|
+
const tEmbed = performance.now();
|
|
5169
|
+
const vector = await embedding.embed(summaryText);
|
|
5170
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
5171
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
5172
|
+
const summaryId = createId();
|
|
5173
|
+
const timestamp = nowIso();
|
|
5174
|
+
const result = await this.withWriteLock(async () => {
|
|
5175
|
+
const tWrite = performance.now();
|
|
5176
|
+
const summaryRow = await storage.insertMemory({
|
|
5177
|
+
id: summaryId,
|
|
5178
|
+
organization,
|
|
5179
|
+
agent: options.agent.trim(),
|
|
5180
|
+
contentText: summaryText,
|
|
5181
|
+
metadata: {
|
|
5182
|
+
compressed: true,
|
|
5183
|
+
sourceCount: records.length,
|
|
5184
|
+
sourceIds: records.map((r) => r.id)
|
|
5185
|
+
},
|
|
5186
|
+
embedding: vector,
|
|
5187
|
+
createdAt: timestamp,
|
|
5188
|
+
updatedAt: timestamp
|
|
5189
|
+
});
|
|
5190
|
+
const archivedIds = await storage.archiveMemories(
|
|
5191
|
+
records.map((r) => r.id),
|
|
5192
|
+
organization,
|
|
5193
|
+
summaryId,
|
|
5194
|
+
timestamp
|
|
5195
|
+
);
|
|
5196
|
+
trace.mark("databaseWriteMs", performance.now() - tWrite);
|
|
5197
|
+
return {
|
|
5198
|
+
summary: toMemoryRecord(summaryRow),
|
|
5199
|
+
archivedIds
|
|
5200
|
+
};
|
|
5201
|
+
});
|
|
5202
|
+
trace.success({
|
|
5203
|
+
provider: storage.name,
|
|
5204
|
+
memoryIds: [result.summary.id, ...result.archivedIds],
|
|
5205
|
+
returnedCount: 1,
|
|
5206
|
+
agentId: options.agent.trim()
|
|
5207
|
+
});
|
|
5208
|
+
return result;
|
|
5209
|
+
} catch (error) {
|
|
5210
|
+
trace.failure(error, { agentId: options.agent });
|
|
5211
|
+
throw wrapOperationError("compress", error);
|
|
3782
5212
|
}
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
{
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
5213
|
+
}
|
|
5214
|
+
async ingest(options) {
|
|
5215
|
+
const trace = this.telemetry.start("ingest");
|
|
5216
|
+
try {
|
|
5217
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5218
|
+
assertNonEmptyString(options.agent, "agent");
|
|
5219
|
+
const loaded = await loadIngestSource(options.source);
|
|
5220
|
+
let usedOcr = false;
|
|
5221
|
+
let usedVision = false;
|
|
5222
|
+
let text = loaded.rawText ?? "";
|
|
5223
|
+
if (!loaded.rawText) {
|
|
5224
|
+
const parser = resolveParser(loaded.filename, loaded.mimeType);
|
|
5225
|
+
const parsed = await parser.parse({
|
|
5226
|
+
buffer: loaded.buffer,
|
|
5227
|
+
filename: loaded.filename,
|
|
5228
|
+
mimeType: loaded.mimeType
|
|
5229
|
+
});
|
|
5230
|
+
text = parsed.text;
|
|
5231
|
+
if (parsed.isImage && parsed.imageBuffer) {
|
|
5232
|
+
const parts = [];
|
|
5233
|
+
if (this.ocr) {
|
|
5234
|
+
const ocrResult = await this.ocr.recognize(
|
|
5235
|
+
parsed.imageBuffer,
|
|
5236
|
+
parsed.mimeType
|
|
5237
|
+
);
|
|
5238
|
+
if (ocrResult.text) {
|
|
5239
|
+
parts.push(ocrResult.text);
|
|
5240
|
+
usedOcr = true;
|
|
5241
|
+
}
|
|
5242
|
+
}
|
|
5243
|
+
if (this.vision) {
|
|
5244
|
+
const visionResult = await this.vision.analyze(
|
|
5245
|
+
parsed.imageBuffer,
|
|
5246
|
+
parsed.mimeType
|
|
5247
|
+
);
|
|
5248
|
+
if (visionResult.caption) {
|
|
5249
|
+
parts.push(`Caption: ${visionResult.caption}`);
|
|
5250
|
+
}
|
|
5251
|
+
if (visionResult.description) {
|
|
5252
|
+
parts.push(visionResult.description);
|
|
5253
|
+
}
|
|
5254
|
+
if (visionResult.entities.length > 0) {
|
|
5255
|
+
parts.push(`Entities: ${visionResult.entities.join(", ")}`);
|
|
5256
|
+
}
|
|
5257
|
+
if (parts.length > 0) {
|
|
5258
|
+
usedVision = true;
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
text = parts.join("\n\n").trim();
|
|
5262
|
+
}
|
|
5263
|
+
}
|
|
5264
|
+
if (!text) {
|
|
5265
|
+
throw new ValidationError(
|
|
5266
|
+
"ingest could not extract text from the document (configure ocr/vision for images, or provide text)"
|
|
5267
|
+
);
|
|
5268
|
+
}
|
|
5269
|
+
const strategy = this.chunking ?? (options.chunking?.strategy ? createChunkingStrategy(options.chunking.strategy) : inferChunkingStrategy(text));
|
|
5270
|
+
const chunks = strategy.chunk(text, {
|
|
5271
|
+
chunkSize: options.chunking?.chunkSize,
|
|
5272
|
+
overlap: options.chunking?.overlap
|
|
5273
|
+
});
|
|
5274
|
+
if (chunks.length === 0) {
|
|
5275
|
+
throw new ValidationError("ingest produced zero chunks");
|
|
5276
|
+
}
|
|
5277
|
+
const tEmbed = performance.now();
|
|
5278
|
+
const vectors = await embedMany(
|
|
5279
|
+
embedding,
|
|
5280
|
+
chunks.map((c) => c.text)
|
|
3797
5281
|
);
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
const summaryRow = await storage.insertMemory({
|
|
3807
|
-
id: summaryId,
|
|
5282
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
5283
|
+
for (const vector of vectors) {
|
|
5284
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
5285
|
+
}
|
|
5286
|
+
const baseMeta = options.metadata ?? {};
|
|
5287
|
+
const timestamp = nowIso();
|
|
5288
|
+
const inputs = chunks.map((chunk, i) => ({
|
|
5289
|
+
id: createId(),
|
|
3808
5290
|
organization,
|
|
3809
5291
|
agent: options.agent.trim(),
|
|
3810
|
-
contentText:
|
|
5292
|
+
contentText: chunk.text,
|
|
3811
5293
|
metadata: {
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
5294
|
+
...baseMeta,
|
|
5295
|
+
ingest: true,
|
|
5296
|
+
chunkIndex: chunk.index,
|
|
5297
|
+
chunkCount: chunks.length,
|
|
5298
|
+
sourceFilename: loaded.filename ?? null
|
|
3815
5299
|
},
|
|
3816
|
-
embedding:
|
|
5300
|
+
embedding: vectors[i],
|
|
3817
5301
|
createdAt: timestamp,
|
|
3818
5302
|
updatedAt: timestamp
|
|
5303
|
+
}));
|
|
5304
|
+
const rows = await this.withWriteLock(async () => {
|
|
5305
|
+
const tWrite = performance.now();
|
|
5306
|
+
const result2 = await storage.insertMemoriesBatch(inputs);
|
|
5307
|
+
trace.mark("databaseWriteMs", performance.now() - tWrite);
|
|
5308
|
+
return result2;
|
|
3819
5309
|
});
|
|
3820
|
-
const
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
return {
|
|
3827
|
-
summary: toMemoryRecord(summaryRow),
|
|
3828
|
-
archivedIds
|
|
5310
|
+
const result = {
|
|
5311
|
+
memories: rows.map(toMemoryRecord),
|
|
5312
|
+
extractedChars: text.length,
|
|
5313
|
+
chunkCount: chunks.length,
|
|
5314
|
+
usedOcr,
|
|
5315
|
+
usedVision
|
|
3829
5316
|
};
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
const loaded = await loadIngestSource(options.source);
|
|
3837
|
-
let usedOcr = false;
|
|
3838
|
-
let usedVision = false;
|
|
3839
|
-
let text = loaded.rawText ?? "";
|
|
3840
|
-
if (!loaded.rawText) {
|
|
3841
|
-
const parser = resolveParser(loaded.filename, loaded.mimeType);
|
|
3842
|
-
const parsed = await parser.parse({
|
|
3843
|
-
buffer: loaded.buffer,
|
|
3844
|
-
filename: loaded.filename,
|
|
3845
|
-
mimeType: loaded.mimeType
|
|
5317
|
+
trace.success({
|
|
5318
|
+
provider: storage.name,
|
|
5319
|
+
memoryIds: result.memories.map((m) => m.id),
|
|
5320
|
+
returnedCount: result.memories.length,
|
|
5321
|
+
agentId: options.agent.trim(),
|
|
5322
|
+
tags: telemetryTags(baseMeta)
|
|
3846
5323
|
});
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
5324
|
+
return result;
|
|
5325
|
+
} catch (error) {
|
|
5326
|
+
trace.failure(error, { agentId: options.agent });
|
|
5327
|
+
throw wrapOperationError("ingest", error);
|
|
5328
|
+
}
|
|
5329
|
+
}
|
|
5330
|
+
async forget(options) {
|
|
5331
|
+
const trace = this.telemetry.start("forget");
|
|
5332
|
+
try {
|
|
5333
|
+
const { storage, organization } = await this.requireReady();
|
|
5334
|
+
const deleted = await this.withWriteLock(async () => {
|
|
5335
|
+
if ("id" in options && options.id !== void 0) {
|
|
5336
|
+
assertNonEmptyString(options.id, "id");
|
|
5337
|
+
const ok = await storage.deleteMemoryById(
|
|
5338
|
+
options.id.trim(),
|
|
5339
|
+
organization
|
|
3854
5340
|
);
|
|
3855
|
-
|
|
3856
|
-
parts.push(ocrResult.text);
|
|
3857
|
-
usedOcr = true;
|
|
3858
|
-
}
|
|
5341
|
+
return ok ? 1 : 0;
|
|
3859
5342
|
}
|
|
3860
|
-
if (
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
}
|
|
3868
|
-
if (visionResult.description) {
|
|
3869
|
-
parts.push(visionResult.description);
|
|
3870
|
-
}
|
|
3871
|
-
if (visionResult.entities.length > 0) {
|
|
3872
|
-
parts.push(`Entities: ${visionResult.entities.join(", ")}`);
|
|
3873
|
-
}
|
|
3874
|
-
if (parts.length > 0) {
|
|
3875
|
-
usedVision = true;
|
|
3876
|
-
}
|
|
5343
|
+
if ("filter" in options && options.filter?.agent) {
|
|
5344
|
+
assertNonEmptyString(options.filter.agent, "filter.agent");
|
|
5345
|
+
return storage.deleteMemoriesByFilter({
|
|
5346
|
+
organization,
|
|
5347
|
+
agent: options.filter.agent.trim(),
|
|
5348
|
+
includeArchived: true
|
|
5349
|
+
});
|
|
3877
5350
|
}
|
|
3878
|
-
|
|
5351
|
+
throw new ValidationError(
|
|
5352
|
+
"forget requires either { id } or { filter: { agent } }"
|
|
5353
|
+
);
|
|
5354
|
+
});
|
|
5355
|
+
trace.success({
|
|
5356
|
+
provider: storage.name,
|
|
5357
|
+
returnedCount: deleted,
|
|
5358
|
+
memoryIds: "id" in options && options.id ? [options.id.trim()] : null,
|
|
5359
|
+
filters: "filter" in options ? options.filter : null,
|
|
5360
|
+
agentId: "filter" in options ? options.filter?.agent?.trim() ?? null : null
|
|
5361
|
+
});
|
|
5362
|
+
return deleted;
|
|
5363
|
+
} catch (error) {
|
|
5364
|
+
trace.failure(error, {
|
|
5365
|
+
agentId: "filter" in options ? options.filter?.agent ?? null : null
|
|
5366
|
+
});
|
|
5367
|
+
throw wrapOperationError("forget", error);
|
|
5368
|
+
}
|
|
5369
|
+
}
|
|
5370
|
+
async history(options) {
|
|
5371
|
+
const trace = this.telemetry.start("history");
|
|
5372
|
+
try {
|
|
5373
|
+
const { storage, organization } = await this.requireReady();
|
|
5374
|
+
assertNonEmptyString(options.id, "id");
|
|
5375
|
+
const row = await storage.getMemoryById(options.id.trim(), organization);
|
|
5376
|
+
if (!row) {
|
|
5377
|
+
throw new MemoryNotFoundError(`Memory not found: ${options.id}`);
|
|
3879
5378
|
}
|
|
5379
|
+
const events = await storage.getHistory(row.id);
|
|
5380
|
+
const result = {
|
|
5381
|
+
memory: toMemoryRecord(row),
|
|
5382
|
+
events: events.map(toHistoryEvent)
|
|
5383
|
+
};
|
|
5384
|
+
trace.success({
|
|
5385
|
+
provider: storage.name,
|
|
5386
|
+
memoryIds: [row.id],
|
|
5387
|
+
returnedCount: events.length,
|
|
5388
|
+
agentId: row.agent,
|
|
5389
|
+
tags: telemetryTags(deserializeMetadata(row.metadata_json))
|
|
5390
|
+
});
|
|
5391
|
+
return result;
|
|
5392
|
+
} catch (error) {
|
|
5393
|
+
trace.failure(error);
|
|
5394
|
+
throw wrapOperationError("history", error);
|
|
3880
5395
|
}
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
5396
|
+
}
|
|
5397
|
+
async stats() {
|
|
5398
|
+
const trace = this.telemetry.start("stats");
|
|
5399
|
+
try {
|
|
5400
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5401
|
+
const counts = await storage.getStats(organization);
|
|
5402
|
+
const databaseSizeBytes = await storage.getDatabaseSizeBytes();
|
|
5403
|
+
const result = {
|
|
5404
|
+
totalMemories: counts.totalMemories,
|
|
5405
|
+
activeMemories: counts.activeMemories,
|
|
5406
|
+
archivedMemories: counts.archivedMemories,
|
|
5407
|
+
totalAgents: counts.totalAgents,
|
|
5408
|
+
databaseSizeBytes,
|
|
5409
|
+
embeddingModel: embedding.model,
|
|
5410
|
+
llmModel: this.llm?.model ?? null,
|
|
5411
|
+
organization,
|
|
5412
|
+
embeddingDimensions: this.embeddingDimensions ?? 0
|
|
5413
|
+
};
|
|
5414
|
+
trace.success({
|
|
5415
|
+
provider: storage.name,
|
|
5416
|
+
returnedCount: result.totalMemories
|
|
5417
|
+
});
|
|
5418
|
+
return result;
|
|
5419
|
+
} catch (error) {
|
|
5420
|
+
trace.failure(error);
|
|
5421
|
+
throw wrapOperationError("stats", error);
|
|
3885
5422
|
}
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
5423
|
+
}
|
|
5424
|
+
async clear(options) {
|
|
5425
|
+
const trace = this.telemetry.start("clear");
|
|
5426
|
+
try {
|
|
5427
|
+
const { storage, organization } = await this.requireReady();
|
|
5428
|
+
if (!options || options.confirm !== true) {
|
|
5429
|
+
throw new ValidationError(
|
|
5430
|
+
"clear requires { confirm: true } to permanently delete all memories"
|
|
5431
|
+
);
|
|
5432
|
+
}
|
|
5433
|
+
const deleted = await this.withWriteLock(async () => {
|
|
5434
|
+
return storage.clearOrganization(organization);
|
|
5435
|
+
});
|
|
5436
|
+
trace.success({ provider: storage.name, returnedCount: deleted });
|
|
5437
|
+
return deleted;
|
|
5438
|
+
} catch (error) {
|
|
5439
|
+
trace.failure(error);
|
|
5440
|
+
throw wrapOperationError("clear", error);
|
|
3893
5441
|
}
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
);
|
|
3898
|
-
|
|
3899
|
-
this.
|
|
5442
|
+
}
|
|
5443
|
+
/** Create an immutable named checkpoint of the memory database. */
|
|
5444
|
+
async checkpoint(name, options) {
|
|
5445
|
+
const trace = this.telemetry.start("checkpoint");
|
|
5446
|
+
try {
|
|
5447
|
+
await this.requireReady();
|
|
5448
|
+
const provider = this.requireCheckpointProvider();
|
|
5449
|
+
const source = this.requireMemoryDbPath();
|
|
5450
|
+
const tCheckpoint = performance.now();
|
|
5451
|
+
const meta2 = await provider.checkpoint(name, source, options);
|
|
5452
|
+
trace.mark("databaseWriteMs", performance.now() - tCheckpoint);
|
|
5453
|
+
trace.success({
|
|
5454
|
+
provider: provider.name,
|
|
5455
|
+
checkpointId: meta2.name,
|
|
5456
|
+
extra: { checkpoint: meta2.name }
|
|
5457
|
+
});
|
|
5458
|
+
return meta2;
|
|
5459
|
+
} catch (error) {
|
|
5460
|
+
trace.failure(error, { checkpointId: name });
|
|
5461
|
+
throw wrapOperationError("checkpoint", error);
|
|
3900
5462
|
}
|
|
3901
|
-
const baseMeta = options.metadata ?? {};
|
|
3902
|
-
const timestamp = nowIso();
|
|
3903
|
-
const inputs = chunks.map((chunk, i) => ({
|
|
3904
|
-
id: createId(),
|
|
3905
|
-
organization,
|
|
3906
|
-
agent: options.agent.trim(),
|
|
3907
|
-
contentText: chunk.text,
|
|
3908
|
-
metadata: {
|
|
3909
|
-
...baseMeta,
|
|
3910
|
-
ingest: true,
|
|
3911
|
-
chunkIndex: chunk.index,
|
|
3912
|
-
chunkCount: chunks.length,
|
|
3913
|
-
sourceFilename: loaded.filename ?? null
|
|
3914
|
-
},
|
|
3915
|
-
embedding: vectors[i],
|
|
3916
|
-
createdAt: timestamp,
|
|
3917
|
-
updatedAt: timestamp
|
|
3918
|
-
}));
|
|
3919
|
-
const rows = await this.withWriteLock(async () => {
|
|
3920
|
-
return storage.insertMemoriesBatch(inputs);
|
|
3921
|
-
});
|
|
3922
|
-
return {
|
|
3923
|
-
memories: rows.map(toMemoryRecord),
|
|
3924
|
-
extractedChars: text.length,
|
|
3925
|
-
chunkCount: chunks.length,
|
|
3926
|
-
usedOcr,
|
|
3927
|
-
usedVision
|
|
3928
|
-
};
|
|
3929
5463
|
}
|
|
3930
|
-
/**
|
|
3931
|
-
async
|
|
3932
|
-
const
|
|
3933
|
-
|
|
3934
|
-
|
|
3935
|
-
|
|
3936
|
-
|
|
3937
|
-
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
5464
|
+
/** Restore the memory database from a named checkpoint. */
|
|
5465
|
+
async rollback(name) {
|
|
5466
|
+
const trace = this.telemetry.start("rollback");
|
|
5467
|
+
let storageClosed = false;
|
|
5468
|
+
try {
|
|
5469
|
+
await this.requireReady();
|
|
5470
|
+
const provider = this.requireCheckpointProvider();
|
|
5471
|
+
const target = this.requireMemoryDbPath();
|
|
5472
|
+
const existing = await provider.getCheckpoint(name);
|
|
5473
|
+
if (!existing) {
|
|
5474
|
+
throw new ValidationError(`Checkpoint "${name}" was not found`);
|
|
3941
5475
|
}
|
|
3942
|
-
if (
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
organization,
|
|
3946
|
-
agent: options.filter.agent.trim(),
|
|
3947
|
-
includeArchived: true
|
|
3948
|
-
});
|
|
5476
|
+
if (this.storage) {
|
|
5477
|
+
await this.storage.close();
|
|
5478
|
+
storageClosed = true;
|
|
3949
5479
|
}
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
);
|
|
3953
|
-
|
|
5480
|
+
const tRollback = performance.now();
|
|
5481
|
+
const meta2 = await provider.rollback(name, target);
|
|
5482
|
+
trace.mark("databaseWriteMs", performance.now() - tRollback);
|
|
5483
|
+
await this.storage?.open();
|
|
5484
|
+
storageClosed = false;
|
|
5485
|
+
if (this.embeddingDimensions !== null) {
|
|
5486
|
+
await this.storage?.ensureVectorSchema(this.embeddingDimensions);
|
|
5487
|
+
}
|
|
5488
|
+
trace.success({
|
|
5489
|
+
provider: provider.name,
|
|
5490
|
+
checkpointId: meta2.name,
|
|
5491
|
+
extra: { checkpoint: meta2.name }
|
|
5492
|
+
});
|
|
5493
|
+
return meta2;
|
|
5494
|
+
} catch (error) {
|
|
5495
|
+
if (storageClosed && this.storage) {
|
|
5496
|
+
await this.storage.open().catch(() => void 0);
|
|
5497
|
+
if (this.embeddingDimensions !== null) {
|
|
5498
|
+
await this.storage.ensureVectorSchema(this.embeddingDimensions).catch(() => void 0);
|
|
5499
|
+
}
|
|
5500
|
+
}
|
|
5501
|
+
trace.failure(error, { checkpointId: name });
|
|
5502
|
+
throw wrapOperationError("rollback", error);
|
|
5503
|
+
}
|
|
3954
5504
|
}
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
5505
|
+
async deleteCheckpoint(name) {
|
|
5506
|
+
const trace = this.telemetry.start("deleteCheckpoint");
|
|
5507
|
+
try {
|
|
5508
|
+
await this.requireReady();
|
|
5509
|
+
const provider = this.requireCheckpointProvider();
|
|
5510
|
+
const tDelete = performance.now();
|
|
5511
|
+
const removed = await provider.deleteCheckpoint(name);
|
|
5512
|
+
trace.mark("databaseWriteMs", performance.now() - tDelete);
|
|
5513
|
+
trace.success({
|
|
5514
|
+
provider: provider.name,
|
|
5515
|
+
checkpointId: name,
|
|
5516
|
+
returnedCount: removed ? 1 : 0
|
|
5517
|
+
});
|
|
5518
|
+
return removed;
|
|
5519
|
+
} catch (error) {
|
|
5520
|
+
trace.failure(error, { checkpointId: name });
|
|
5521
|
+
throw wrapOperationError("deleteCheckpoint", error);
|
|
3962
5522
|
}
|
|
3963
|
-
const events = await storage.getHistory(row.id);
|
|
3964
|
-
return {
|
|
3965
|
-
memory: toMemoryRecord(row),
|
|
3966
|
-
events: events.map(toHistoryEvent)
|
|
3967
|
-
};
|
|
3968
5523
|
}
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
|
|
3972
|
-
|
|
3973
|
-
|
|
3974
|
-
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
5524
|
+
async listCheckpoints() {
|
|
5525
|
+
const trace = this.telemetry.start("listCheckpoints");
|
|
5526
|
+
try {
|
|
5527
|
+
await this.requireReady();
|
|
5528
|
+
const provider = this.requireCheckpointProvider();
|
|
5529
|
+
const tList = performance.now();
|
|
5530
|
+
const list = await provider.listCheckpoints();
|
|
5531
|
+
trace.mark("databaseReadMs", performance.now() - tList);
|
|
5532
|
+
trace.success({
|
|
5533
|
+
provider: provider.name,
|
|
5534
|
+
returnedCount: list.length
|
|
5535
|
+
});
|
|
5536
|
+
return list;
|
|
5537
|
+
} catch (error) {
|
|
5538
|
+
trace.failure(error);
|
|
5539
|
+
throw wrapOperationError("listCheckpoints", error);
|
|
5540
|
+
}
|
|
3985
5541
|
}
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
5542
|
+
async getCheckpoint(name) {
|
|
5543
|
+
const trace = this.telemetry.start("getCheckpoint");
|
|
5544
|
+
try {
|
|
5545
|
+
await this.requireReady();
|
|
5546
|
+
const provider = this.requireCheckpointProvider();
|
|
5547
|
+
const tGet = performance.now();
|
|
5548
|
+
const meta2 = await provider.getCheckpoint(name);
|
|
5549
|
+
trace.mark("databaseReadMs", performance.now() - tGet);
|
|
5550
|
+
trace.success({
|
|
5551
|
+
provider: provider.name,
|
|
5552
|
+
checkpointId: name,
|
|
5553
|
+
returnedCount: meta2 ? 1 : 0
|
|
5554
|
+
});
|
|
5555
|
+
return meta2;
|
|
5556
|
+
} catch (error) {
|
|
5557
|
+
trace.failure(error, { checkpointId: name });
|
|
5558
|
+
throw wrapOperationError("getCheckpoint", error);
|
|
5559
|
+
}
|
|
5560
|
+
}
|
|
5561
|
+
/** Export the memory database to a portable SQLite + manifest bundle. */
|
|
5562
|
+
async export(exportPath) {
|
|
5563
|
+
const trace = this.telemetry.start("export");
|
|
5564
|
+
try {
|
|
5565
|
+
await this.requireReady();
|
|
5566
|
+
const source = this.requireMemoryDbPath();
|
|
5567
|
+
if (this.storage?.name === "sqlite") {
|
|
5568
|
+
}
|
|
5569
|
+
const result = await this.transfer.exportTo(
|
|
5570
|
+
exportPath,
|
|
5571
|
+
source,
|
|
5572
|
+
this.organization ?? void 0
|
|
5573
|
+
);
|
|
5574
|
+
trace.success({
|
|
5575
|
+
provider: "sqlite",
|
|
5576
|
+
extra: { path: result.path, sizeBytes: result.sizeBytes }
|
|
5577
|
+
});
|
|
5578
|
+
return {
|
|
5579
|
+
path: result.path,
|
|
5580
|
+
sizeBytes: result.sizeBytes,
|
|
5581
|
+
exportedAt: result.manifest.exportedAt
|
|
5582
|
+
};
|
|
5583
|
+
} catch (error) {
|
|
5584
|
+
trace.failure(error);
|
|
5585
|
+
throw wrapOperationError("export", error);
|
|
5586
|
+
}
|
|
5587
|
+
}
|
|
5588
|
+
/** Import a previously exported memory database, replacing the current file. */
|
|
5589
|
+
async import(exportPath) {
|
|
5590
|
+
const trace = this.telemetry.start("import");
|
|
5591
|
+
let storageClosed = false;
|
|
5592
|
+
try {
|
|
5593
|
+
await this.requireReady();
|
|
5594
|
+
const target = this.requireMemoryDbPath();
|
|
5595
|
+
if (this.storage) {
|
|
5596
|
+
await this.storage.close();
|
|
5597
|
+
storageClosed = true;
|
|
5598
|
+
}
|
|
5599
|
+
const result = await this.transfer.importFrom(
|
|
5600
|
+
exportPath,
|
|
5601
|
+
target
|
|
3992
5602
|
);
|
|
5603
|
+
await this.storage?.open();
|
|
5604
|
+
storageClosed = false;
|
|
5605
|
+
if (this.embeddingDimensions !== null) {
|
|
5606
|
+
await this.storage?.ensureVectorSchema(this.embeddingDimensions);
|
|
5607
|
+
}
|
|
5608
|
+
trace.success({
|
|
5609
|
+
provider: "sqlite",
|
|
5610
|
+
extra: { path: result.path }
|
|
5611
|
+
});
|
|
5612
|
+
return {
|
|
5613
|
+
path: result.path,
|
|
5614
|
+
importedAt: nowIso()
|
|
5615
|
+
};
|
|
5616
|
+
} catch (error) {
|
|
5617
|
+
if (storageClosed && this.storage) {
|
|
5618
|
+
await this.storage.open().catch(() => void 0);
|
|
5619
|
+
if (this.embeddingDimensions !== null) {
|
|
5620
|
+
await this.storage.ensureVectorSchema(this.embeddingDimensions).catch(() => void 0);
|
|
5621
|
+
}
|
|
5622
|
+
}
|
|
5623
|
+
trace.failure(error);
|
|
5624
|
+
throw wrapOperationError("import", error);
|
|
3993
5625
|
}
|
|
3994
|
-
return this.withWriteLock(async () => {
|
|
3995
|
-
return storage.clearOrganization(organization);
|
|
3996
|
-
});
|
|
3997
5626
|
}
|
|
3998
|
-
/**
|
|
3999
|
-
|
|
4000
|
-
|
|
4001
|
-
|
|
5627
|
+
/** Flush pending telemetry (useful in tests). */
|
|
5628
|
+
async flushTelemetry() {
|
|
5629
|
+
await this.telemetry.flush();
|
|
5630
|
+
}
|
|
5631
|
+
/** Session id for this SDK instance (telemetry traces). */
|
|
5632
|
+
get sessionId() {
|
|
5633
|
+
return this.telemetry.sessionId;
|
|
5634
|
+
}
|
|
4002
5635
|
withWriteLock(fn) {
|
|
4003
5636
|
if (this.storage?.name === "sqlite") {
|
|
4004
5637
|
return this.writeMutex.runExclusive(fn);
|
|
4005
5638
|
}
|
|
4006
5639
|
return fn();
|
|
4007
5640
|
}
|
|
4008
|
-
/** Close the database connection and release resources. */
|
|
4009
5641
|
async close() {
|
|
5642
|
+
if (this.storage) {
|
|
5643
|
+
this.telemetry.emitShutdown(this.storage.name);
|
|
5644
|
+
}
|
|
5645
|
+
await this.telemetry.close().catch(() => void 0);
|
|
5646
|
+
await this.checkpointProvider?.close().catch(() => void 0);
|
|
4010
5647
|
if (this.storage) {
|
|
4011
5648
|
await this.storage.close();
|
|
4012
5649
|
}
|
|
@@ -4018,7 +5655,6 @@ var Wolbarg = class {
|
|
|
4018
5655
|
this.embeddingDimensions = null;
|
|
4019
5656
|
this.initialized = false;
|
|
4020
5657
|
}
|
|
4021
|
-
/** Whether providers are ready for API calls. */
|
|
4022
5658
|
get isInitialized() {
|
|
4023
5659
|
return this.initialized;
|
|
4024
5660
|
}
|
|
@@ -4042,7 +5678,56 @@ var Wolbarg = class {
|
|
|
4042
5678
|
);
|
|
4043
5679
|
}
|
|
4044
5680
|
}
|
|
5681
|
+
requireCheckpointProvider() {
|
|
5682
|
+
if (!this.checkpointProvider) {
|
|
5683
|
+
throw new ProviderNotConfiguredError(
|
|
5684
|
+
"checkpoint",
|
|
5685
|
+
"checkpoint",
|
|
5686
|
+
"construct Wolbarg with a file-backed database"
|
|
5687
|
+
);
|
|
5688
|
+
}
|
|
5689
|
+
return this.checkpointProvider;
|
|
5690
|
+
}
|
|
5691
|
+
requireMemoryDbPath() {
|
|
5692
|
+
if (!this.memoryDbPath || this.memoryDbPath === ":memory:") {
|
|
5693
|
+
throw new ConfigurationError(
|
|
5694
|
+
"This operation requires a file-backed SQLite memory database (not :memory:).",
|
|
5695
|
+
{
|
|
5696
|
+
suggestion: 'Use database: { provider: "sqlite", url: "./memory.db" }'
|
|
5697
|
+
}
|
|
5698
|
+
);
|
|
5699
|
+
}
|
|
5700
|
+
return path5__default.default.isAbsolute(this.memoryDbPath) ? this.memoryDbPath : path5__default.default.resolve(process.cwd(), this.memoryDbPath);
|
|
5701
|
+
}
|
|
4045
5702
|
};
|
|
5703
|
+
function wolbarg(options) {
|
|
5704
|
+
return new Wolbarg(options);
|
|
5705
|
+
}
|
|
5706
|
+
function telemetryTags(metadata) {
|
|
5707
|
+
const value = metadata.tags;
|
|
5708
|
+
if (typeof value === "string") {
|
|
5709
|
+
const tag = value.trim();
|
|
5710
|
+
return tag ? [tag] : null;
|
|
5711
|
+
}
|
|
5712
|
+
if (Array.isArray(value)) {
|
|
5713
|
+
const tags = value.filter(
|
|
5714
|
+
(tag) => typeof tag === "string" && tag.trim().length > 0
|
|
5715
|
+
);
|
|
5716
|
+
return tags.length > 0 ? tags.map((tag) => tag.trim()) : null;
|
|
5717
|
+
}
|
|
5718
|
+
return null;
|
|
5719
|
+
}
|
|
5720
|
+
function commonAgent(agents) {
|
|
5721
|
+
const first = agents[0];
|
|
5722
|
+
return first && agents.every((agent) => agent === first) ? first : null;
|
|
5723
|
+
}
|
|
5724
|
+
function commonTags(metadata) {
|
|
5725
|
+
const tags = /* @__PURE__ */ new Set();
|
|
5726
|
+
for (const item of metadata) {
|
|
5727
|
+
for (const tag of telemetryTags(item) ?? []) tags.add(tag);
|
|
5728
|
+
}
|
|
5729
|
+
return tags.size > 0 ? [...tags] : null;
|
|
5730
|
+
}
|
|
4046
5731
|
|
|
4047
5732
|
// src/filters/types.ts
|
|
4048
5733
|
var meta = {
|
|
@@ -4084,7 +5769,11 @@ function sqlite(connectionString) {
|
|
|
4084
5769
|
return new SqliteStorageProvider({ connectionString });
|
|
4085
5770
|
}
|
|
4086
5771
|
function sqliteConfig(connectionString) {
|
|
4087
|
-
return {
|
|
5772
|
+
return {
|
|
5773
|
+
provider: "sqlite",
|
|
5774
|
+
connectionString,
|
|
5775
|
+
url: connectionString
|
|
5776
|
+
};
|
|
4088
5777
|
}
|
|
4089
5778
|
function postgres(options) {
|
|
4090
5779
|
const opts = typeof options === "string" ? { connectionString: options } : options;
|
|
@@ -4094,9 +5783,31 @@ function postgresConfig(connectionString, options) {
|
|
|
4094
5783
|
return {
|
|
4095
5784
|
provider: "postgres",
|
|
4096
5785
|
connectionString,
|
|
5786
|
+
url: connectionString,
|
|
4097
5787
|
...options
|
|
4098
5788
|
};
|
|
4099
5789
|
}
|
|
5790
|
+
function sqliteTelemetry(url) {
|
|
5791
|
+
return new SqliteTelemetryProvider({ url });
|
|
5792
|
+
}
|
|
5793
|
+
function sqliteCheckpoint(directory) {
|
|
5794
|
+
return new SqliteCheckpointProvider({ directory });
|
|
5795
|
+
}
|
|
5796
|
+
function createTelemetryProvider(config) {
|
|
5797
|
+
if (config.database.provider !== "sqlite") {
|
|
5798
|
+
throw new ConfigurationError(
|
|
5799
|
+
`Unsupported telemetry provider "${config.database.provider}". Only "sqlite" is implemented in v0.3.0.`
|
|
5800
|
+
);
|
|
5801
|
+
}
|
|
5802
|
+
const url = config.database.url ?? config.database.connectionString ?? "";
|
|
5803
|
+
if (!url) {
|
|
5804
|
+
throw new ConfigurationError("telemetry.database.url is required");
|
|
5805
|
+
}
|
|
5806
|
+
return new SqliteTelemetryProvider({ url });
|
|
5807
|
+
}
|
|
5808
|
+
function wolbarg2(options) {
|
|
5809
|
+
return new Wolbarg(options);
|
|
5810
|
+
}
|
|
4100
5811
|
|
|
4101
5812
|
// src/keyword/index.ts
|
|
4102
5813
|
var Bm25KeywordSearchProvider = class {
|
|
@@ -4492,6 +6203,56 @@ function openaiVision(options) {
|
|
|
4492
6203
|
});
|
|
4493
6204
|
}
|
|
4494
6205
|
|
|
6206
|
+
// src/benchmark/index.ts
|
|
6207
|
+
async function runBenchmark(name, iterations, fn) {
|
|
6208
|
+
const samples = [];
|
|
6209
|
+
for (let i = 0; i < iterations; i += 1) {
|
|
6210
|
+
const t0 = performance.now();
|
|
6211
|
+
try {
|
|
6212
|
+
await fn();
|
|
6213
|
+
samples.push({
|
|
6214
|
+
name,
|
|
6215
|
+
durationMs: performance.now() - t0,
|
|
6216
|
+
ok: true
|
|
6217
|
+
});
|
|
6218
|
+
} catch (error) {
|
|
6219
|
+
samples.push({
|
|
6220
|
+
name,
|
|
6221
|
+
durationMs: performance.now() - t0,
|
|
6222
|
+
ok: false,
|
|
6223
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6224
|
+
});
|
|
6225
|
+
}
|
|
6226
|
+
}
|
|
6227
|
+
return samples;
|
|
6228
|
+
}
|
|
6229
|
+
function summarizeBenchmark(samples, startedAt, finishedAt) {
|
|
6230
|
+
const durations = samples.filter((s) => s.ok).map((s) => s.durationMs).sort((a, b) => a - b);
|
|
6231
|
+
const avg = durations.length === 0 ? 0 : durations.reduce((a, b) => a + b, 0) / durations.length;
|
|
6232
|
+
return {
|
|
6233
|
+
startedAt,
|
|
6234
|
+
finishedAt,
|
|
6235
|
+
samples,
|
|
6236
|
+
summary: {
|
|
6237
|
+
count: samples.length,
|
|
6238
|
+
ok: samples.filter((s) => s.ok).length,
|
|
6239
|
+
failed: samples.filter((s) => !s.ok).length,
|
|
6240
|
+
avgMs: avg,
|
|
6241
|
+
p50Ms: percentile(durations, 0.5),
|
|
6242
|
+
p95Ms: percentile(durations, 0.95),
|
|
6243
|
+
p99Ms: percentile(durations, 0.99)
|
|
6244
|
+
}
|
|
6245
|
+
};
|
|
6246
|
+
}
|
|
6247
|
+
function percentile(sorted, p) {
|
|
6248
|
+
if (sorted.length === 0) return 0;
|
|
6249
|
+
const idx = Math.min(
|
|
6250
|
+
sorted.length - 1,
|
|
6251
|
+
Math.max(0, Math.ceil(p * sorted.length) - 1)
|
|
6252
|
+
);
|
|
6253
|
+
return sorted[idx];
|
|
6254
|
+
}
|
|
6255
|
+
|
|
4495
6256
|
exports.CompressionError = CompressionError;
|
|
4496
6257
|
exports.ConfigurationError = ConfigurationError;
|
|
4497
6258
|
exports.DatabaseError = DatabaseError;
|
|
@@ -4502,15 +6263,21 @@ exports.InitializationError = InitializationError;
|
|
|
4502
6263
|
exports.LlmCompressionProvider = LlmCompressionProvider;
|
|
4503
6264
|
exports.MarkdownChunkingStrategy = MarkdownChunkingStrategy;
|
|
4504
6265
|
exports.MemoryNotFoundError = MemoryNotFoundError;
|
|
6266
|
+
exports.NoopTelemetryProvider = NoopTelemetryProvider;
|
|
4505
6267
|
exports.ParagraphChunkingStrategy = ParagraphChunkingStrategy;
|
|
4506
6268
|
exports.PostgresStorageProvider = PostgresStorageProvider;
|
|
4507
6269
|
exports.ProviderNotConfiguredError = ProviderNotConfiguredError;
|
|
4508
6270
|
exports.SentenceChunkingStrategy = SentenceChunkingStrategy;
|
|
6271
|
+
exports.SqliteCheckpointProvider = SqliteCheckpointProvider;
|
|
4509
6272
|
exports.SqliteDatabaseProvider = SqliteDatabaseProvider;
|
|
6273
|
+
exports.SqliteEventDatabase = SqliteEventDatabase;
|
|
4510
6274
|
exports.SqliteStorageProvider = SqliteStorageProvider;
|
|
6275
|
+
exports.SqliteTelemetryProvider = SqliteTelemetryProvider;
|
|
6276
|
+
exports.TelemetryEmitter = TelemetryEmitter;
|
|
4511
6277
|
exports.ValidationError = ValidationError;
|
|
4512
6278
|
exports.Wolbarg = Wolbarg;
|
|
4513
6279
|
exports.WolbargError = WolbargError;
|
|
6280
|
+
exports.WolbargLogger = WolbargLogger;
|
|
4514
6281
|
exports.bgeReranker = bgeReranker;
|
|
4515
6282
|
exports.bm25 = bm25;
|
|
4516
6283
|
exports.cohereReranker = cohereReranker;
|
|
@@ -4520,6 +6287,8 @@ exports.createDatabaseProvider = createDatabaseProvider;
|
|
|
4520
6287
|
exports.createEmbeddingProvider = createEmbeddingProvider;
|
|
4521
6288
|
exports.createLlmProvider = createLlmProvider;
|
|
4522
6289
|
exports.createStorageProvider = createStorageProvider;
|
|
6290
|
+
exports.createTelemetryProvider = createTelemetryProvider;
|
|
6291
|
+
exports.createWolbarg = wolbarg2;
|
|
4523
6292
|
exports.crossEncoder = crossEncoder;
|
|
4524
6293
|
exports.geminiEmbedding = geminiEmbedding;
|
|
4525
6294
|
exports.geminiVision = geminiVision;
|
|
@@ -4538,10 +6307,16 @@ exports.openaiReranker = openaiReranker;
|
|
|
4538
6307
|
exports.openaiVision = openaiVision;
|
|
4539
6308
|
exports.postgres = postgres;
|
|
4540
6309
|
exports.postgresConfig = postgresConfig;
|
|
6310
|
+
exports.runBenchmark = runBenchmark;
|
|
4541
6311
|
exports.sqlite = sqlite;
|
|
6312
|
+
exports.sqliteCheckpoint = sqliteCheckpoint;
|
|
4542
6313
|
exports.sqliteConfig = sqliteConfig;
|
|
6314
|
+
exports.sqliteTelemetry = sqliteTelemetry;
|
|
6315
|
+
exports.summarizeBenchmark = summarizeBenchmark;
|
|
4543
6316
|
exports.tesseract = tesseract;
|
|
4544
6317
|
exports.togetherEmbedding = togetherEmbedding;
|
|
4545
6318
|
exports.vllmEmbedding = vllmEmbedding;
|
|
6319
|
+
exports.wolbarg = wolbarg;
|
|
6320
|
+
exports.wrapOperationError = wrapOperationError;
|
|
4546
6321
|
//# sourceMappingURL=index.cjs.map
|
|
4547
6322
|
//# sourceMappingURL=index.cjs.map
|