wolbarg 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -6
- package/README.md +73 -11
- package/dist/index.cjs +2298 -523
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +598 -54
- package/dist/index.d.ts +598 -54
- package/dist/index.js +2283 -522
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -1,17 +1,25 @@
|
|
|
1
|
+
import path5 from "node:path";
|
|
1
2
|
import fs from 'fs/promises';
|
|
2
|
-
import
|
|
3
|
-
import fs2 from "node:fs";
|
|
3
|
+
import fs5 from "node:fs";
|
|
4
4
|
import { DatabaseSync } from "node:sqlite";
|
|
5
5
|
import * as sqliteVec from 'sqlite-vec';
|
|
6
6
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
7
7
|
|
|
8
|
+
// src/core/wolbarg.ts
|
|
9
|
+
|
|
8
10
|
// src/errors/index.ts
|
|
9
11
|
var WolbargError = class extends Error {
|
|
10
12
|
code;
|
|
13
|
+
reason;
|
|
14
|
+
suggestion;
|
|
15
|
+
operation;
|
|
11
16
|
constructor(message, code, options) {
|
|
12
17
|
super(message, options);
|
|
13
18
|
this.name = "WolbargError";
|
|
14
19
|
this.code = code;
|
|
20
|
+
this.reason = options?.reason;
|
|
21
|
+
this.suggestion = options?.suggestion;
|
|
22
|
+
this.operation = options?.operation;
|
|
15
23
|
Object.setPrototypeOf(this, new.target.prototype);
|
|
16
24
|
}
|
|
17
25
|
};
|
|
@@ -60,13 +68,58 @@ var MemoryNotFoundError = class extends WolbargError {
|
|
|
60
68
|
var ProviderNotConfiguredError = class extends ConfigurationError {
|
|
61
69
|
provider;
|
|
62
70
|
constructor(provider, method, hint) {
|
|
63
|
-
super(
|
|
64
|
-
|
|
65
|
-
|
|
71
|
+
super(`${method} requires ${provider} \u2014 ${hint}`, {
|
|
72
|
+
operation: method,
|
|
73
|
+
reason: `${provider} was not configured`,
|
|
74
|
+
suggestion: hint
|
|
75
|
+
});
|
|
66
76
|
this.name = "ProviderNotConfiguredError";
|
|
67
77
|
this.provider = provider;
|
|
68
78
|
}
|
|
69
79
|
};
|
|
80
|
+
function wrapOperationError(operation, error) {
|
|
81
|
+
if (error instanceof WolbargError) {
|
|
82
|
+
return error;
|
|
83
|
+
}
|
|
84
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
85
|
+
const lower = raw.toLowerCase();
|
|
86
|
+
if (lower.includes("database is locked") || lower.includes("sqlite_busy")) {
|
|
87
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
88
|
+
cause: error instanceof Error ? error : void 0,
|
|
89
|
+
operation,
|
|
90
|
+
reason: "SQLite database locked",
|
|
91
|
+
suggestion: "Increase timeout or reduce concurrent writes."
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
if (lower.includes("no such file") || lower.includes("enoent")) {
|
|
95
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
96
|
+
cause: error instanceof Error ? error : void 0,
|
|
97
|
+
operation,
|
|
98
|
+
reason: "Database file not found",
|
|
99
|
+
suggestion: "Check the database path and ensure the directory exists."
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
if (lower.includes("readonly") || lower.includes("read-only")) {
|
|
103
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
104
|
+
cause: error instanceof Error ? error : void 0,
|
|
105
|
+
operation,
|
|
106
|
+
reason: "Database opened as read-only",
|
|
107
|
+
suggestion: "Open the database with write permissions or choose another path."
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
return new DatabaseError(formatOperationMessage(operation, raw), {
|
|
111
|
+
cause: error instanceof Error ? error : void 0,
|
|
112
|
+
operation,
|
|
113
|
+
reason: raw,
|
|
114
|
+
suggestion: "Inspect the underlying cause and retry the operation."
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
function formatOperationMessage(operation, reason) {
|
|
118
|
+
const reasonText = typeof reason === "string" ? reason : reason.reason ?? reason.message;
|
|
119
|
+
return `Failed to execute ${operation}()
|
|
120
|
+
Reason:
|
|
121
|
+
${reasonText}`;
|
|
122
|
+
}
|
|
70
123
|
|
|
71
124
|
// src/compression/index.ts
|
|
72
125
|
var SYSTEM_PROMPT = `You are a memory compression engine for multi-agent systems.
|
|
@@ -324,9 +377,9 @@ var AsyncMutex = class {
|
|
|
324
377
|
}
|
|
325
378
|
}
|
|
326
379
|
};
|
|
327
|
-
function joinUrl(baseUrl,
|
|
380
|
+
function joinUrl(baseUrl, path7) {
|
|
328
381
|
const base = baseUrl.replace(/\/+$/, "");
|
|
329
|
-
const suffix =
|
|
382
|
+
const suffix = path7.startsWith("/") ? path7 : `/${path7}`;
|
|
330
383
|
return `${base}${suffix}`;
|
|
331
384
|
}
|
|
332
385
|
|
|
@@ -623,7 +676,7 @@ function extOf(filename) {
|
|
|
623
676
|
if (!filename) {
|
|
624
677
|
return "";
|
|
625
678
|
}
|
|
626
|
-
return
|
|
679
|
+
return path5.extname(filename).toLowerCase();
|
|
627
680
|
}
|
|
628
681
|
var TextParser = class {
|
|
629
682
|
name = "text";
|
|
@@ -768,13 +821,31 @@ async function loadIngestSource(source) {
|
|
|
768
821
|
const buffer = await fs.readFile(source.path);
|
|
769
822
|
return {
|
|
770
823
|
buffer,
|
|
771
|
-
filename:
|
|
824
|
+
filename: path5.basename(source.path),
|
|
772
825
|
mimeType: source.mimeType
|
|
773
826
|
};
|
|
774
827
|
}
|
|
775
828
|
throw new ConfigurationError("ingest source must include path, buffer, or text");
|
|
776
829
|
}
|
|
777
830
|
|
|
831
|
+
// src/core/options.ts
|
|
832
|
+
function isEmbeddingProvider(value) {
|
|
833
|
+
return typeof value.embed === "function";
|
|
834
|
+
}
|
|
835
|
+
function isLlmProvider(value) {
|
|
836
|
+
return typeof value.complete === "function";
|
|
837
|
+
}
|
|
838
|
+
function isStorageProvider(value) {
|
|
839
|
+
return typeof value.open === "function";
|
|
840
|
+
}
|
|
841
|
+
function isTelemetryProvider(value) {
|
|
842
|
+
return typeof value.emit === "function";
|
|
843
|
+
}
|
|
844
|
+
function resolveDatabaseUrl(config) {
|
|
845
|
+
const url = "url" in config && config.url || "connectionString" in config && config.connectionString || "";
|
|
846
|
+
return typeof url === "string" ? url : "";
|
|
847
|
+
}
|
|
848
|
+
|
|
778
849
|
// src/filters/match.ts
|
|
779
850
|
function getField(metadata, field) {
|
|
780
851
|
const parts = field.split(".");
|
|
@@ -1256,12 +1327,16 @@ var SqliteStorageProvider = class {
|
|
|
1256
1327
|
constructor(options) {
|
|
1257
1328
|
this.connectionString = options.connectionString;
|
|
1258
1329
|
}
|
|
1330
|
+
/** Absolute or relative path / `:memory:` used by this provider. */
|
|
1331
|
+
get path() {
|
|
1332
|
+
return this.connectionString;
|
|
1333
|
+
}
|
|
1259
1334
|
async open() {
|
|
1260
1335
|
try {
|
|
1261
1336
|
const dbPath = this.resolvePath(this.connectionString);
|
|
1262
1337
|
this.resolvedPath = dbPath;
|
|
1263
1338
|
if (dbPath !== ":memory:") {
|
|
1264
|
-
|
|
1339
|
+
fs5.mkdirSync(path5.dirname(dbPath), { recursive: true });
|
|
1265
1340
|
}
|
|
1266
1341
|
const db = new DatabaseSync(dbPath, { allowExtension: true });
|
|
1267
1342
|
this.db = db;
|
|
@@ -1722,11 +1797,11 @@ var SqliteStorageProvider = class {
|
|
|
1722
1797
|
}
|
|
1723
1798
|
const dbPath = this.resolvedPath ?? this.resolvePath(this.connectionString);
|
|
1724
1799
|
try {
|
|
1725
|
-
let total =
|
|
1800
|
+
let total = fs5.statSync(dbPath).size;
|
|
1726
1801
|
for (const suffix of ["-wal", "-shm"]) {
|
|
1727
1802
|
const side = `${dbPath}${suffix}`;
|
|
1728
|
-
if (
|
|
1729
|
-
total +=
|
|
1803
|
+
if (fs5.existsSync(side)) {
|
|
1804
|
+
total += fs5.statSync(side).size;
|
|
1730
1805
|
}
|
|
1731
1806
|
}
|
|
1732
1807
|
return total;
|
|
@@ -2057,7 +2132,7 @@ var SqliteStorageProvider = class {
|
|
|
2057
2132
|
if (connectionString === ":memory:") {
|
|
2058
2133
|
return ":memory:";
|
|
2059
2134
|
}
|
|
2060
|
-
return
|
|
2135
|
+
return path5.isAbsolute(connectionString) ? connectionString : path5.resolve(process.cwd(), connectionString);
|
|
2061
2136
|
}
|
|
2062
2137
|
requireDb() {
|
|
2063
2138
|
if (!this.db) {
|
|
@@ -3121,14 +3196,20 @@ var PostgresStorageProvider = class {
|
|
|
3121
3196
|
|
|
3122
3197
|
// src/storage/index.ts
|
|
3123
3198
|
function createStorageProvider(config) {
|
|
3199
|
+
const connectionString = resolveDatabaseUrl(config);
|
|
3200
|
+
if (!connectionString) {
|
|
3201
|
+
throw new ConfigurationError(
|
|
3202
|
+
"database.url or database.connectionString is required"
|
|
3203
|
+
);
|
|
3204
|
+
}
|
|
3124
3205
|
if (config.provider === "sqlite") {
|
|
3125
3206
|
return new SqliteStorageProvider({
|
|
3126
|
-
connectionString
|
|
3207
|
+
connectionString
|
|
3127
3208
|
});
|
|
3128
3209
|
}
|
|
3129
3210
|
if (config.provider === "postgres") {
|
|
3130
3211
|
return new PostgresStorageProvider({
|
|
3131
|
-
connectionString
|
|
3212
|
+
connectionString,
|
|
3132
3213
|
maxPoolSize: config.maxPoolSize
|
|
3133
3214
|
});
|
|
3134
3215
|
}
|
|
@@ -3139,6 +3220,129 @@ function createStorageProvider(config) {
|
|
|
3139
3220
|
function createDatabaseProvider(config) {
|
|
3140
3221
|
return createStorageProvider(config);
|
|
3141
3222
|
}
|
|
3223
|
+
var SDK_VERSION = "0.3.0";
|
|
3224
|
+
var SqliteMemoryTransferProvider = class {
|
|
3225
|
+
async exportTo(exportPath, sourcePath, organization) {
|
|
3226
|
+
const resolvedSource = resolvePath(sourcePath);
|
|
3227
|
+
if (resolvedSource === ":memory:") {
|
|
3228
|
+
throw new ConfigurationError(
|
|
3229
|
+
"Cannot export an in-memory database. Use a file-backed SQLite database."
|
|
3230
|
+
);
|
|
3231
|
+
}
|
|
3232
|
+
if (!fs5.existsSync(resolvedSource)) {
|
|
3233
|
+
throw new DatabaseError(
|
|
3234
|
+
`Failed to execute export()
|
|
3235
|
+
Reason:
|
|
3236
|
+
Source database not found at ${resolvedSource}
|
|
3237
|
+
Suggestion:
|
|
3238
|
+
Initialize Wolbarg and verify the database path.`
|
|
3239
|
+
);
|
|
3240
|
+
}
|
|
3241
|
+
const resolvedExport = resolvePath(exportPath);
|
|
3242
|
+
fs5.mkdirSync(path5.dirname(resolvedExport), { recursive: true });
|
|
3243
|
+
const dbExportPath = resolvedExport.endsWith(".db") ? resolvedExport : `${resolvedExport}.db`;
|
|
3244
|
+
const manifestPath = `${dbExportPath}.manifest.json`;
|
|
3245
|
+
await copySqlite(resolvedSource, dbExportPath);
|
|
3246
|
+
const manifest = {
|
|
3247
|
+
format: "wolbarg-export-v1",
|
|
3248
|
+
exportedAt: nowIso(),
|
|
3249
|
+
sdkVersion: SDK_VERSION,
|
|
3250
|
+
provider: "sqlite",
|
|
3251
|
+
sourcePath: resolvedSource,
|
|
3252
|
+
organization
|
|
3253
|
+
};
|
|
3254
|
+
fs5.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), "utf8");
|
|
3255
|
+
return {
|
|
3256
|
+
path: dbExportPath,
|
|
3257
|
+
manifest,
|
|
3258
|
+
sizeBytes: fs5.statSync(dbExportPath).size
|
|
3259
|
+
};
|
|
3260
|
+
}
|
|
3261
|
+
async importFrom(exportPath, targetPath) {
|
|
3262
|
+
const resolvedExport = resolvePath(exportPath);
|
|
3263
|
+
const dbExportPath = resolvedExport.endsWith(".db") ? resolvedExport : fs5.existsSync(`${resolvedExport}.db`) ? `${resolvedExport}.db` : resolvedExport;
|
|
3264
|
+
if (!fs5.existsSync(dbExportPath)) {
|
|
3265
|
+
throw new ValidationError(
|
|
3266
|
+
`Failed to execute import()
|
|
3267
|
+
Reason:
|
|
3268
|
+
Export file not found at ${dbExportPath}
|
|
3269
|
+
Suggestion:
|
|
3270
|
+
Pass the path returned by export().`
|
|
3271
|
+
);
|
|
3272
|
+
}
|
|
3273
|
+
const manifestPath = `${dbExportPath}.manifest.json`;
|
|
3274
|
+
let manifest;
|
|
3275
|
+
if (fs5.existsSync(manifestPath)) {
|
|
3276
|
+
manifest = JSON.parse(fs5.readFileSync(manifestPath, "utf8"));
|
|
3277
|
+
if (manifest.format !== "wolbarg-export-v1") {
|
|
3278
|
+
throw new ValidationError(
|
|
3279
|
+
`Unsupported export format: ${String(manifest.format)}`
|
|
3280
|
+
);
|
|
3281
|
+
}
|
|
3282
|
+
} else {
|
|
3283
|
+
manifest = {
|
|
3284
|
+
format: "wolbarg-export-v1",
|
|
3285
|
+
exportedAt: nowIso(),
|
|
3286
|
+
sdkVersion: SDK_VERSION,
|
|
3287
|
+
provider: "sqlite",
|
|
3288
|
+
sourcePath: dbExportPath
|
|
3289
|
+
};
|
|
3290
|
+
}
|
|
3291
|
+
const resolvedTarget = resolvePath(targetPath);
|
|
3292
|
+
if (resolvedTarget === ":memory:") {
|
|
3293
|
+
throw new ConfigurationError("Cannot import into an in-memory database.");
|
|
3294
|
+
}
|
|
3295
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
3296
|
+
const side = `${resolvedTarget}${suffix}`;
|
|
3297
|
+
if (fs5.existsSync(side)) {
|
|
3298
|
+
fs5.rmSync(side, { force: true });
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
await copySqlite(dbExportPath, resolvedTarget);
|
|
3302
|
+
return { path: resolvedTarget, manifest };
|
|
3303
|
+
}
|
|
3304
|
+
};
|
|
3305
|
+
async function copySqlite(sourcePath, destPath) {
|
|
3306
|
+
fs5.mkdirSync(path5.dirname(destPath), { recursive: true });
|
|
3307
|
+
let source = null;
|
|
3308
|
+
let dest = null;
|
|
3309
|
+
try {
|
|
3310
|
+
source = new DatabaseSync(sourcePath);
|
|
3311
|
+
try {
|
|
3312
|
+
source.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
3313
|
+
} catch {
|
|
3314
|
+
}
|
|
3315
|
+
dest = new DatabaseSync(destPath);
|
|
3316
|
+
const backupFn = source.backup;
|
|
3317
|
+
if (typeof backupFn === "function") {
|
|
3318
|
+
await backupFn.call(source, dest);
|
|
3319
|
+
} else {
|
|
3320
|
+
source.close();
|
|
3321
|
+
source = null;
|
|
3322
|
+
dest.close();
|
|
3323
|
+
dest = null;
|
|
3324
|
+
fs5.copyFileSync(sourcePath, destPath);
|
|
3325
|
+
}
|
|
3326
|
+
} catch (error) {
|
|
3327
|
+
throw new DatabaseError(
|
|
3328
|
+
`SQLite transfer failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3329
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
3330
|
+
);
|
|
3331
|
+
} finally {
|
|
3332
|
+
try {
|
|
3333
|
+
source?.close();
|
|
3334
|
+
} catch {
|
|
3335
|
+
}
|
|
3336
|
+
try {
|
|
3337
|
+
dest?.close();
|
|
3338
|
+
} catch {
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
function resolvePath(p) {
|
|
3343
|
+
if (p === ":memory:") return ":memory:";
|
|
3344
|
+
return path5.isAbsolute(p) ? p : path5.resolve(process.cwd(), p);
|
|
3345
|
+
}
|
|
3142
3346
|
|
|
3143
3347
|
// src/memory/index.ts
|
|
3144
3348
|
function toMemoryRecord(row) {
|
|
@@ -3264,17 +3468,6 @@ function adaptiveFetchK(topK, overFetchFactor, hasFilters) {
|
|
|
3264
3468
|
return Math.min(Math.max(Math.ceil(topK * factor), topK), 1e3);
|
|
3265
3469
|
}
|
|
3266
3470
|
|
|
3267
|
-
// src/core/options.ts
|
|
3268
|
-
function isEmbeddingProvider(value) {
|
|
3269
|
-
return typeof value.embed === "function";
|
|
3270
|
-
}
|
|
3271
|
-
function isLlmProvider(value) {
|
|
3272
|
-
return typeof value.complete === "function";
|
|
3273
|
-
}
|
|
3274
|
-
function isStorageProvider(value) {
|
|
3275
|
-
return typeof value.open === "function";
|
|
3276
|
-
}
|
|
3277
|
-
|
|
3278
3471
|
// src/core/validate.ts
|
|
3279
3472
|
function assertNonEmpty(value, fieldName) {
|
|
3280
3473
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
@@ -3327,6 +3520,64 @@ function validateLlmConfig(config) {
|
|
|
3327
3520
|
...config.timeoutMs !== void 0 ? { timeoutMs: config.timeoutMs } : {}
|
|
3328
3521
|
};
|
|
3329
3522
|
}
|
|
3523
|
+
function normalizeDatabaseConfig(config) {
|
|
3524
|
+
const provider = config.provider;
|
|
3525
|
+
if (provider !== "sqlite" && provider !== "postgres") {
|
|
3526
|
+
throw new ConfigurationError(
|
|
3527
|
+
`Unsupported database provider "${String(config.provider)}". Supported: "sqlite", "postgres".`
|
|
3528
|
+
);
|
|
3529
|
+
}
|
|
3530
|
+
const connectionString = resolveDatabaseUrl(config).trim();
|
|
3531
|
+
assertNonEmpty(connectionString, "database.url / database.connectionString");
|
|
3532
|
+
if (provider === "postgres") {
|
|
3533
|
+
return {
|
|
3534
|
+
provider: "postgres",
|
|
3535
|
+
connectionString,
|
|
3536
|
+
url: connectionString,
|
|
3537
|
+
..."maxPoolSize" in config && config.maxPoolSize !== void 0 ? { maxPoolSize: config.maxPoolSize } : {}
|
|
3538
|
+
};
|
|
3539
|
+
}
|
|
3540
|
+
return {
|
|
3541
|
+
provider: "sqlite",
|
|
3542
|
+
connectionString,
|
|
3543
|
+
url: connectionString
|
|
3544
|
+
};
|
|
3545
|
+
}
|
|
3546
|
+
function validateTelemetryConfig(config) {
|
|
3547
|
+
if (!config.database || typeof config.database !== "object") {
|
|
3548
|
+
throw new ConfigurationError("telemetry.database is required when telemetry is enabled");
|
|
3549
|
+
}
|
|
3550
|
+
if (config.database.provider !== "sqlite") {
|
|
3551
|
+
throw new ConfigurationError(
|
|
3552
|
+
`Unsupported telemetry provider "${config.database.provider}". Only "sqlite" is implemented in v0.3.0; PostgreSQL will be added later without changing application code.`,
|
|
3553
|
+
{
|
|
3554
|
+
reason: `provider=${config.database.provider}`,
|
|
3555
|
+
suggestion: 'Use telemetry: { database: { provider: "sqlite", url: "./telemetry.db" } }'
|
|
3556
|
+
}
|
|
3557
|
+
);
|
|
3558
|
+
}
|
|
3559
|
+
const url = config.database.url?.trim() || config.database.connectionString?.trim() || "";
|
|
3560
|
+
assertNonEmpty(url, "telemetry.database.url");
|
|
3561
|
+
const level = config.level ?? "info";
|
|
3562
|
+
const allowed = /* @__PURE__ */ new Set(["off", "error", "warn", "info", "debug", "trace"]);
|
|
3563
|
+
if (!allowed.has(level)) {
|
|
3564
|
+
throw new ConfigurationError(`Invalid telemetry.level "${level}"`);
|
|
3565
|
+
}
|
|
3566
|
+
return {
|
|
3567
|
+
enabled: config.enabled ?? true,
|
|
3568
|
+
database: {
|
|
3569
|
+
provider: "sqlite",
|
|
3570
|
+
url,
|
|
3571
|
+
connectionString: url
|
|
3572
|
+
},
|
|
3573
|
+
level,
|
|
3574
|
+
captureQueries: config.captureQueries ?? true,
|
|
3575
|
+
captureLatency: config.captureLatency ?? true,
|
|
3576
|
+
captureErrors: config.captureErrors ?? true,
|
|
3577
|
+
captureSimilarity: config.captureSimilarity ?? true,
|
|
3578
|
+
captureEmbeddings: config.captureEmbeddings ?? false
|
|
3579
|
+
};
|
|
3580
|
+
}
|
|
3330
3581
|
function validateInitOptions(options) {
|
|
3331
3582
|
if (options === null || typeof options !== "object") {
|
|
3332
3583
|
throw new ConfigurationError("init options must be an object");
|
|
@@ -3335,16 +3586,7 @@ function validateInitOptions(options) {
|
|
|
3335
3586
|
if (!options.database || typeof options.database !== "object") {
|
|
3336
3587
|
throw new ConfigurationError("database configuration is required");
|
|
3337
3588
|
}
|
|
3338
|
-
const
|
|
3339
|
-
if (provider !== "sqlite" && provider !== "postgres") {
|
|
3340
|
-
throw new ConfigurationError(
|
|
3341
|
-
`Unsupported database provider "${String(options.database.provider)}". Supported: "sqlite", "postgres".`
|
|
3342
|
-
);
|
|
3343
|
-
}
|
|
3344
|
-
assertNonEmpty(
|
|
3345
|
-
options.database.connectionString,
|
|
3346
|
-
"database.connectionString"
|
|
3347
|
-
);
|
|
3589
|
+
const database = normalizeDatabaseConfig(options.database);
|
|
3348
3590
|
if (!options.embedding || typeof options.embedding !== "object") {
|
|
3349
3591
|
throw new ConfigurationError("embedding configuration is required");
|
|
3350
3592
|
}
|
|
@@ -3352,31 +3594,32 @@ function validateInitOptions(options) {
|
|
|
3352
3594
|
const llm = options.llm ? validateLlmConfig(options.llm) : void 0;
|
|
3353
3595
|
return {
|
|
3354
3596
|
organization: options.organization.trim(),
|
|
3355
|
-
database
|
|
3356
|
-
provider: "postgres",
|
|
3357
|
-
connectionString: options.database.connectionString.trim(),
|
|
3358
|
-
..."maxPoolSize" in options.database && options.database.maxPoolSize !== void 0 ? { maxPoolSize: options.database.maxPoolSize } : {}
|
|
3359
|
-
} : {
|
|
3360
|
-
provider: "sqlite",
|
|
3361
|
-
connectionString: options.database.connectionString.trim()
|
|
3362
|
-
},
|
|
3597
|
+
database,
|
|
3363
3598
|
embedding,
|
|
3364
3599
|
...llm ? { llm } : {}
|
|
3365
3600
|
};
|
|
3366
3601
|
}
|
|
3602
|
+
function resolveStorageInput(options) {
|
|
3603
|
+
if (options.storage && options.database) {
|
|
3604
|
+
throw new ConfigurationError(
|
|
3605
|
+
"Pass either storage or database, not both"
|
|
3606
|
+
);
|
|
3607
|
+
}
|
|
3608
|
+
const input = options.storage ?? options.database;
|
|
3609
|
+
if (!input) {
|
|
3610
|
+
throw new ConfigurationError("storage or database is required");
|
|
3611
|
+
}
|
|
3612
|
+
return input;
|
|
3613
|
+
}
|
|
3367
3614
|
function validateWolbargOptions(options) {
|
|
3368
3615
|
if (options === null || typeof options !== "object") {
|
|
3369
3616
|
throw new ConfigurationError("Wolbarg options must be an object");
|
|
3370
3617
|
}
|
|
3371
3618
|
assertNonEmpty(options.organization, "organization");
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
if (typeof options.storage !== "object" || !("provider" in options.storage) || !("connectionString" in options.storage)) {
|
|
3377
|
-
throw new ConfigurationError("storage must be a provider instance or config");
|
|
3378
|
-
}
|
|
3379
|
-
assertNonEmpty(options.storage.connectionString, "storage.connectionString");
|
|
3619
|
+
const storageInput = resolveStorageInput(options);
|
|
3620
|
+
let storage = storageInput;
|
|
3621
|
+
if (!isStorageProvider(storageInput)) {
|
|
3622
|
+
storage = normalizeDatabaseConfig(storageInput);
|
|
3380
3623
|
}
|
|
3381
3624
|
if (!options.embedding) {
|
|
3382
3625
|
throw new ConfigurationError("embedding is required");
|
|
@@ -3387,82 +3630,1010 @@ function validateWolbargOptions(options) {
|
|
|
3387
3630
|
if (options.llm !== void 0 && !isLlmProvider(options.llm)) {
|
|
3388
3631
|
validateLlmConfig(options.llm);
|
|
3389
3632
|
}
|
|
3633
|
+
let telemetry = options.telemetry;
|
|
3634
|
+
if (telemetry && !isTelemetryProvider(telemetry)) {
|
|
3635
|
+
telemetry = validateTelemetryConfig(telemetry);
|
|
3636
|
+
}
|
|
3390
3637
|
return {
|
|
3391
3638
|
...options,
|
|
3392
|
-
organization: options.organization.trim()
|
|
3639
|
+
organization: options.organization.trim(),
|
|
3640
|
+
storage,
|
|
3641
|
+
database: void 0,
|
|
3642
|
+
...telemetry ? { telemetry } : {}
|
|
3393
3643
|
};
|
|
3394
3644
|
}
|
|
3395
3645
|
|
|
3396
|
-
// src/
|
|
3397
|
-
var
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3646
|
+
// src/telemetry/logger.ts
|
|
3647
|
+
var LEVEL_ORDER = {
|
|
3648
|
+
off: 100,
|
|
3649
|
+
error: 50,
|
|
3650
|
+
warn: 40,
|
|
3651
|
+
info: 30,
|
|
3652
|
+
debug: 20,
|
|
3653
|
+
trace: 10
|
|
3654
|
+
};
|
|
3655
|
+
var WolbargLogger = class {
|
|
3656
|
+
constructor(level = "info") {
|
|
3657
|
+
this.level = level;
|
|
3658
|
+
}
|
|
3659
|
+
level;
|
|
3660
|
+
setLevel(level) {
|
|
3661
|
+
this.level = level;
|
|
3662
|
+
}
|
|
3663
|
+
getLevel() {
|
|
3664
|
+
return this.level;
|
|
3665
|
+
}
|
|
3666
|
+
error(message, extra) {
|
|
3667
|
+
this.write("error", message, extra);
|
|
3668
|
+
}
|
|
3669
|
+
warn(message, extra) {
|
|
3670
|
+
this.write("warn", message, extra);
|
|
3671
|
+
}
|
|
3672
|
+
info(message, extra) {
|
|
3673
|
+
this.write("info", message, extra);
|
|
3674
|
+
}
|
|
3675
|
+
debug(message, extra) {
|
|
3676
|
+
this.write("debug", message, extra);
|
|
3677
|
+
}
|
|
3678
|
+
trace(message, extra) {
|
|
3679
|
+
this.write("trace", message, extra);
|
|
3680
|
+
}
|
|
3681
|
+
write(level, message, extra) {
|
|
3682
|
+
if (LEVEL_ORDER[level] < LEVEL_ORDER[this.level]) {
|
|
3415
3683
|
return;
|
|
3416
3684
|
}
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
if (
|
|
3422
|
-
|
|
3423
|
-
|
|
3685
|
+
if (this.level === "off") {
|
|
3686
|
+
return;
|
|
3687
|
+
}
|
|
3688
|
+
const line = `[wolbarg:${level}] ${message}`;
|
|
3689
|
+
if (level === "error") {
|
|
3690
|
+
console.error(line, extra ?? "");
|
|
3691
|
+
} else if (level === "warn") {
|
|
3692
|
+
console.warn(line, extra ?? "");
|
|
3424
3693
|
} else {
|
|
3425
|
-
|
|
3694
|
+
console.log(line, extra ?? "");
|
|
3426
3695
|
}
|
|
3427
|
-
this.reranker = validated.reranker ?? null;
|
|
3428
|
-
this.keywordSearch = validated.keywordSearch ?? null;
|
|
3429
|
-
this.ocr = validated.ocr ?? null;
|
|
3430
|
-
this.vision = validated.vision ?? null;
|
|
3431
|
-
this.chunking = validated.chunking ?? null;
|
|
3432
|
-
this.retrievalConfig = validated.retrieval ?? {};
|
|
3433
3696
|
}
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3697
|
+
};
|
|
3698
|
+
|
|
3699
|
+
// src/telemetry/trace.ts
|
|
3700
|
+
function createSessionId() {
|
|
3701
|
+
return createId();
|
|
3702
|
+
}
|
|
3703
|
+
function createTraceId() {
|
|
3704
|
+
return createId();
|
|
3705
|
+
}
|
|
3706
|
+
function createRootTrace(sessionId) {
|
|
3707
|
+
return {
|
|
3708
|
+
sessionId,
|
|
3709
|
+
traceId: createTraceId(),
|
|
3710
|
+
parentTraceId: null
|
|
3711
|
+
};
|
|
3712
|
+
}
|
|
3713
|
+
function createChildTrace(parent) {
|
|
3714
|
+
return {
|
|
3715
|
+
sessionId: parent.sessionId,
|
|
3716
|
+
traceId: createTraceId(),
|
|
3717
|
+
parentTraceId: parent.traceId
|
|
3718
|
+
};
|
|
3719
|
+
}
|
|
3720
|
+
|
|
3721
|
+
// src/telemetry/emitter.ts
|
|
3722
|
+
var NoopTelemetryProvider = class {
|
|
3723
|
+
name = "noop";
|
|
3724
|
+
async open() {
|
|
3725
|
+
}
|
|
3726
|
+
async close() {
|
|
3727
|
+
}
|
|
3728
|
+
emit(_event) {
|
|
3729
|
+
}
|
|
3730
|
+
async flush() {
|
|
3731
|
+
}
|
|
3732
|
+
};
|
|
3733
|
+
var TelemetryEmitter = class {
|
|
3734
|
+
sessionId;
|
|
3735
|
+
logger;
|
|
3736
|
+
provider;
|
|
3737
|
+
config;
|
|
3738
|
+
context;
|
|
3739
|
+
constructor(provider, config, context = {}) {
|
|
3740
|
+
this.sessionId = createSessionId();
|
|
3741
|
+
this.provider = provider ?? new NoopTelemetryProvider();
|
|
3742
|
+
this.config = {
|
|
3743
|
+
enabled: config?.enabled ?? Boolean(provider),
|
|
3744
|
+
level: config?.level ?? "info",
|
|
3745
|
+
captureQueries: config?.captureQueries ?? true,
|
|
3746
|
+
captureLatency: config?.captureLatency ?? true,
|
|
3747
|
+
captureErrors: config?.captureErrors ?? true,
|
|
3748
|
+
captureSimilarity: config?.captureSimilarity ?? true,
|
|
3749
|
+
captureEmbeddings: config?.captureEmbeddings ?? false
|
|
3750
|
+
};
|
|
3751
|
+
this.context = context;
|
|
3752
|
+
this.logger = new WolbargLogger(this.config.level);
|
|
3753
|
+
}
|
|
3754
|
+
get enabled() {
|
|
3755
|
+
return this.config.enabled && this.provider.name !== "noop";
|
|
3756
|
+
}
|
|
3757
|
+
setProvider(provider) {
|
|
3758
|
+
this.provider = provider;
|
|
3759
|
+
}
|
|
3760
|
+
async open() {
|
|
3761
|
+
if (!this.config.enabled) {
|
|
3762
|
+
return;
|
|
3441
3763
|
}
|
|
3442
|
-
|
|
3764
|
+
await this.provider.open();
|
|
3765
|
+
}
|
|
3766
|
+
async close() {
|
|
3767
|
+
await this.provider.flush();
|
|
3768
|
+
await this.provider.close();
|
|
3769
|
+
}
|
|
3770
|
+
async flush() {
|
|
3771
|
+
await this.provider.flush();
|
|
3772
|
+
}
|
|
3773
|
+
start(operation, parent) {
|
|
3774
|
+
const context = parent ? createChildTrace(parent) : createRootTrace(this.sessionId);
|
|
3775
|
+
const startedAt = performance.now();
|
|
3776
|
+
const latency = {};
|
|
3777
|
+
const spans = [];
|
|
3778
|
+
const self = this;
|
|
3779
|
+
const finish = (status, fields, error) => {
|
|
3780
|
+
const totalMs = performance.now() - startedAt;
|
|
3781
|
+
const errMessage = error instanceof Error ? error.message : error ? String(error) : fields?.error ?? null;
|
|
3782
|
+
const errStack = error instanceof Error ? error.stack ?? null : fields?.errorStack ?? null;
|
|
3783
|
+
if (status === "error") {
|
|
3784
|
+
self.logger.error(`${operation} failed: ${errMessage ?? "unknown"}`);
|
|
3785
|
+
} else {
|
|
3786
|
+
self.logger.debug(`${operation} completed in ${totalMs.toFixed(2)}ms`);
|
|
3787
|
+
}
|
|
3788
|
+
if (!self.config.enabled) {
|
|
3789
|
+
return;
|
|
3790
|
+
}
|
|
3791
|
+
const event = {
|
|
3792
|
+
id: createId(),
|
|
3793
|
+
timestamp: nowIso(),
|
|
3794
|
+
operation,
|
|
3795
|
+
status,
|
|
3796
|
+
sessionId: context.sessionId,
|
|
3797
|
+
traceId: context.traceId,
|
|
3798
|
+
parentTraceId: context.parentTraceId,
|
|
3799
|
+
organization: fields?.organization ?? self.context.organization ?? null,
|
|
3800
|
+
agentId: fields?.agentId ?? null,
|
|
3801
|
+
tags: fields?.tags ?? null,
|
|
3802
|
+
checkpointId: fields?.checkpointId ?? null,
|
|
3803
|
+
durationMs: totalMs,
|
|
3804
|
+
provider: fields?.provider ?? null,
|
|
3805
|
+
query: self.config.captureQueries ? fields?.query ?? null : null,
|
|
3806
|
+
filters: fields?.filters ?? null,
|
|
3807
|
+
returnedCount: fields?.returnedCount ?? null,
|
|
3808
|
+
memoryIds: fields?.memoryIds ?? null,
|
|
3809
|
+
similarityScores: self.config.captureSimilarity ? fields?.similarityScores ?? null : null,
|
|
3810
|
+
metadata: fields?.metadata ?? null,
|
|
3811
|
+
embeddingProvider: fields?.embeddingProvider ?? null,
|
|
3812
|
+
model: fields?.model ?? null,
|
|
3813
|
+
error: self.config.captureErrors ? errMessage : null,
|
|
3814
|
+
errorStack: self.config.captureErrors ? errStack : null,
|
|
3815
|
+
userMetadata: fields?.userMetadata ?? null,
|
|
3816
|
+
extra: fields?.extra ?? null,
|
|
3817
|
+
latency: self.config.captureLatency ? { ...latency, totalMs, ...fields?.latency ?? {} } : { totalMs },
|
|
3818
|
+
explain: fields?.explain ?? null,
|
|
3819
|
+
spans: self.config.captureLatency ? [...spans, ...fields?.spans ?? []] : null
|
|
3820
|
+
};
|
|
3821
|
+
self.provider.emit(event);
|
|
3822
|
+
};
|
|
3823
|
+
return {
|
|
3824
|
+
context,
|
|
3825
|
+
startedAt,
|
|
3826
|
+
latency,
|
|
3827
|
+
child(childOp) {
|
|
3828
|
+
return self.start(childOp, context);
|
|
3829
|
+
},
|
|
3830
|
+
mark(stage, ms) {
|
|
3831
|
+
latency[stage] = ms;
|
|
3832
|
+
const elapsed = performance.now() - startedAt;
|
|
3833
|
+
spans.push({
|
|
3834
|
+
name: stage,
|
|
3835
|
+
startMs: Math.max(0, elapsed - ms),
|
|
3836
|
+
durationMs: ms
|
|
3837
|
+
});
|
|
3838
|
+
},
|
|
3839
|
+
success(fields) {
|
|
3840
|
+
finish("ok", fields);
|
|
3841
|
+
},
|
|
3842
|
+
failure(error, fields) {
|
|
3843
|
+
finish("error", fields, error);
|
|
3844
|
+
}
|
|
3845
|
+
};
|
|
3846
|
+
}
|
|
3847
|
+
emitStartup(provider) {
|
|
3848
|
+
const trace = this.start("startup");
|
|
3849
|
+
trace.success({ provider });
|
|
3850
|
+
}
|
|
3851
|
+
emitShutdown(provider) {
|
|
3852
|
+
const trace = this.start("shutdown");
|
|
3853
|
+
trace.success({ provider });
|
|
3854
|
+
}
|
|
3855
|
+
};
|
|
3856
|
+
var SCHEMA_SQL = `
|
|
3857
|
+
CREATE TABLE IF NOT EXISTS telemetry_events (
|
|
3858
|
+
id TEXT PRIMARY KEY,
|
|
3859
|
+
timestamp TEXT NOT NULL,
|
|
3860
|
+
operation TEXT NOT NULL,
|
|
3861
|
+
provider TEXT,
|
|
3862
|
+
duration_ms REAL,
|
|
3863
|
+
status TEXT NOT NULL,
|
|
3864
|
+
query TEXT,
|
|
3865
|
+
filters_json TEXT,
|
|
3866
|
+
returned_count INTEGER,
|
|
3867
|
+
memory_ids_json TEXT,
|
|
3868
|
+
similarity_scores_json TEXT,
|
|
3869
|
+
metadata_json TEXT,
|
|
3870
|
+
embedding_provider TEXT,
|
|
3871
|
+
model TEXT,
|
|
3872
|
+
error TEXT,
|
|
3873
|
+
error_stack TEXT,
|
|
3874
|
+
session_id TEXT NOT NULL,
|
|
3875
|
+
trace_id TEXT NOT NULL,
|
|
3876
|
+
parent_trace_id TEXT,
|
|
3877
|
+
user_metadata_json TEXT,
|
|
3878
|
+
extra_json TEXT,
|
|
3879
|
+
latency_json TEXT,
|
|
3880
|
+
organization TEXT,
|
|
3881
|
+
agent_id TEXT,
|
|
3882
|
+
tags_json TEXT,
|
|
3883
|
+
checkpoint_id TEXT,
|
|
3884
|
+
explain_json TEXT,
|
|
3885
|
+
spans_json TEXT
|
|
3886
|
+
);
|
|
3887
|
+
|
|
3888
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_timestamp ON telemetry_events(timestamp);
|
|
3889
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_operation ON telemetry_events(operation);
|
|
3890
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_trace ON telemetry_events(trace_id);
|
|
3891
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_session ON telemetry_events(session_id);
|
|
3892
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_status ON telemetry_events(status);
|
|
3893
|
+
`;
|
|
3894
|
+
var V2_COLUMNS = {
|
|
3895
|
+
organization: "TEXT",
|
|
3896
|
+
agent_id: "TEXT",
|
|
3897
|
+
tags_json: "TEXT",
|
|
3898
|
+
checkpoint_id: "TEXT",
|
|
3899
|
+
explain_json: "TEXT",
|
|
3900
|
+
spans_json: "TEXT"
|
|
3901
|
+
};
|
|
3902
|
+
var V2_INDEXES_SQL = `
|
|
3903
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_agent ON telemetry_events(agent_id);
|
|
3904
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_org ON telemetry_events(organization);
|
|
3905
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_checkpoint ON telemetry_events(checkpoint_id);
|
|
3906
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_operation_time ON telemetry_events(operation, timestamp);
|
|
3907
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_status_time ON telemetry_events(status, timestamp);
|
|
3908
|
+
`;
|
|
3909
|
+
var SqliteEventDatabase = class {
|
|
3910
|
+
name = "sqlite";
|
|
3911
|
+
url;
|
|
3912
|
+
readonly;
|
|
3913
|
+
db = null;
|
|
3914
|
+
insertStmt = null;
|
|
3915
|
+
columns = /* @__PURE__ */ new Set();
|
|
3916
|
+
constructor(options) {
|
|
3917
|
+
this.url = options.url;
|
|
3918
|
+
this.readonly = options.readonly ?? false;
|
|
3919
|
+
}
|
|
3920
|
+
async open() {
|
|
3443
3921
|
try {
|
|
3444
|
-
|
|
3922
|
+
const dbPath = this.resolvePath(this.url);
|
|
3923
|
+
if (dbPath !== ":memory:" && !this.readonly) {
|
|
3924
|
+
fs5.mkdirSync(path5.dirname(dbPath), { recursive: true });
|
|
3925
|
+
}
|
|
3926
|
+
const db = new DatabaseSync(dbPath, {
|
|
3927
|
+
allowExtension: false,
|
|
3928
|
+
readOnly: this.readonly
|
|
3929
|
+
});
|
|
3930
|
+
this.db = db;
|
|
3931
|
+
if (!this.readonly) {
|
|
3932
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
3933
|
+
db.exec(`
|
|
3934
|
+
PRAGMA synchronous = NORMAL;
|
|
3935
|
+
PRAGMA busy_timeout = 5000;
|
|
3936
|
+
PRAGMA temp_store = MEMORY;
|
|
3937
|
+
`);
|
|
3938
|
+
db.exec(SCHEMA_SQL);
|
|
3939
|
+
this.migrateToV2(db);
|
|
3940
|
+
}
|
|
3941
|
+
this.columns = readColumns(db);
|
|
3942
|
+
if (!this.readonly) {
|
|
3943
|
+
this.insertStmt = db.prepare(`
|
|
3944
|
+
INSERT INTO telemetry_events (
|
|
3945
|
+
id, timestamp, operation, provider, duration_ms, status,
|
|
3946
|
+
query, filters_json, returned_count, memory_ids_json,
|
|
3947
|
+
similarity_scores_json, metadata_json, embedding_provider, model,
|
|
3948
|
+
error, error_stack, session_id, trace_id, parent_trace_id,
|
|
3949
|
+
user_metadata_json, extra_json, latency_json, organization,
|
|
3950
|
+
agent_id, tags_json, checkpoint_id, explain_json, spans_json
|
|
3951
|
+
) VALUES (
|
|
3952
|
+
?, ?, ?, ?, ?, ?,
|
|
3953
|
+
?, ?, ?, ?,
|
|
3954
|
+
?, ?, ?, ?,
|
|
3955
|
+
?, ?, ?, ?, ?,
|
|
3956
|
+
?, ?, ?, ?,
|
|
3957
|
+
?, ?, ?, ?, ?
|
|
3958
|
+
)
|
|
3959
|
+
`);
|
|
3960
|
+
}
|
|
3445
3961
|
} catch (error) {
|
|
3446
|
-
|
|
3447
|
-
|
|
3962
|
+
try {
|
|
3963
|
+
this.db?.close();
|
|
3964
|
+
} catch {
|
|
3448
3965
|
}
|
|
3449
|
-
|
|
3450
|
-
|
|
3966
|
+
this.db = null;
|
|
3967
|
+
this.columns.clear();
|
|
3968
|
+
throw new InitializationError(
|
|
3969
|
+
`Failed to open telemetry EventDatabase: ${describe(error)}`,
|
|
3451
3970
|
{ cause: error instanceof Error ? error : void 0 }
|
|
3452
3971
|
);
|
|
3453
3972
|
}
|
|
3454
|
-
this.organization = validated.organization;
|
|
3455
|
-
this.storage = createStorageProvider(validated.database);
|
|
3456
|
-
this.embedding = createEmbeddingProvider(validated.embedding);
|
|
3457
|
-
if (validated.llm) {
|
|
3458
|
-
this.llm = createLlmProvider(validated.llm);
|
|
3459
|
-
this.compression = createCompressionProvider(this.llm);
|
|
3460
|
-
}
|
|
3461
|
-
await this.ready();
|
|
3462
3973
|
}
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3974
|
+
async close() {
|
|
3975
|
+
if (!this.db) return;
|
|
3976
|
+
try {
|
|
3977
|
+
this.db.close();
|
|
3978
|
+
} finally {
|
|
3979
|
+
this.db = null;
|
|
3980
|
+
this.insertStmt = null;
|
|
3981
|
+
this.columns.clear();
|
|
3982
|
+
}
|
|
3983
|
+
}
|
|
3984
|
+
async insertEvent(input) {
|
|
3985
|
+
const event = normalizeEvent(input);
|
|
3986
|
+
const stmt = this.insertStmt;
|
|
3987
|
+
if (!stmt) {
|
|
3988
|
+
throw new DatabaseError("Telemetry EventDatabase is not open for writes");
|
|
3989
|
+
}
|
|
3990
|
+
try {
|
|
3991
|
+
stmt.run(
|
|
3992
|
+
event.id,
|
|
3993
|
+
event.timestamp,
|
|
3994
|
+
event.operation,
|
|
3995
|
+
event.provider,
|
|
3996
|
+
event.durationMs,
|
|
3997
|
+
event.status,
|
|
3998
|
+
event.query,
|
|
3999
|
+
jsonOrNull(event.filters),
|
|
4000
|
+
event.returnedCount,
|
|
4001
|
+
jsonOrNull(event.memoryIds),
|
|
4002
|
+
jsonOrNull(event.similarityScores),
|
|
4003
|
+
jsonOrNull(event.metadata),
|
|
4004
|
+
event.embeddingProvider,
|
|
4005
|
+
event.model,
|
|
4006
|
+
event.error,
|
|
4007
|
+
event.errorStack,
|
|
4008
|
+
event.sessionId,
|
|
4009
|
+
event.traceId,
|
|
4010
|
+
event.parentTraceId,
|
|
4011
|
+
jsonOrNull(event.userMetadata),
|
|
4012
|
+
jsonOrNull(event.extra),
|
|
4013
|
+
jsonOrNull(event.latency),
|
|
4014
|
+
event.organization,
|
|
4015
|
+
event.agentId,
|
|
4016
|
+
jsonOrNull(event.tags),
|
|
4017
|
+
event.checkpointId,
|
|
4018
|
+
jsonOrNull(event.explain),
|
|
4019
|
+
jsonOrNull(event.spans)
|
|
4020
|
+
);
|
|
4021
|
+
return event;
|
|
4022
|
+
} catch (error) {
|
|
4023
|
+
throw new DatabaseError(
|
|
4024
|
+
`Failed to write telemetry event: ${describe(error)}`,
|
|
4025
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4026
|
+
);
|
|
4027
|
+
}
|
|
4028
|
+
}
|
|
4029
|
+
async insertEvents(inputs) {
|
|
4030
|
+
const db = this.requireDb();
|
|
4031
|
+
const out = [];
|
|
4032
|
+
db.exec("BEGIN");
|
|
4033
|
+
try {
|
|
4034
|
+
for (const input of inputs) {
|
|
4035
|
+
out.push(await this.insertEvent(input));
|
|
4036
|
+
}
|
|
4037
|
+
db.exec("COMMIT");
|
|
4038
|
+
return out;
|
|
4039
|
+
} catch (error) {
|
|
4040
|
+
try {
|
|
4041
|
+
db.exec("ROLLBACK");
|
|
4042
|
+
} catch {
|
|
4043
|
+
}
|
|
4044
|
+
throw error;
|
|
4045
|
+
}
|
|
4046
|
+
}
|
|
4047
|
+
async query(options) {
|
|
4048
|
+
const db = this.requireDb();
|
|
4049
|
+
const clauses = [];
|
|
4050
|
+
const params = [];
|
|
4051
|
+
if (options.operation) {
|
|
4052
|
+
const ops = Array.isArray(options.operation) ? options.operation : [options.operation];
|
|
4053
|
+
clauses.push(`operation IN (${ops.map(() => "?").join(",")})`);
|
|
4054
|
+
params.push(...ops);
|
|
4055
|
+
}
|
|
4056
|
+
if (options.status) {
|
|
4057
|
+
clauses.push(`status = ?`);
|
|
4058
|
+
params.push(options.status);
|
|
4059
|
+
}
|
|
4060
|
+
if (options.traceId) {
|
|
4061
|
+
clauses.push(`(trace_id = ? OR parent_trace_id = ?)`);
|
|
4062
|
+
params.push(options.traceId, options.traceId);
|
|
4063
|
+
}
|
|
4064
|
+
if (options.sessionId) {
|
|
4065
|
+
clauses.push(`session_id = ?`);
|
|
4066
|
+
params.push(options.sessionId);
|
|
4067
|
+
}
|
|
4068
|
+
this.addOptionalColumnFilter(
|
|
4069
|
+
clauses,
|
|
4070
|
+
params,
|
|
4071
|
+
"organization",
|
|
4072
|
+
options.organization
|
|
4073
|
+
);
|
|
4074
|
+
this.addOptionalColumnFilter(clauses, params, "agent_id", options.agentId);
|
|
4075
|
+
this.addOptionalColumnFilter(
|
|
4076
|
+
clauses,
|
|
4077
|
+
params,
|
|
4078
|
+
"checkpoint_id",
|
|
4079
|
+
options.checkpointId
|
|
4080
|
+
);
|
|
4081
|
+
if (options.tag) {
|
|
4082
|
+
if (this.columns.has("tags_json")) {
|
|
4083
|
+
clauses.push(
|
|
4084
|
+
`EXISTS (SELECT 1 FROM json_each(tags_json) WHERE json_each.value = ?)`
|
|
4085
|
+
);
|
|
4086
|
+
params.push(options.tag);
|
|
4087
|
+
} else {
|
|
4088
|
+
clauses.push("0 = 1");
|
|
4089
|
+
}
|
|
4090
|
+
}
|
|
4091
|
+
if (options.memoryId) {
|
|
4092
|
+
clauses.push(`memory_ids_json LIKE ?`);
|
|
4093
|
+
params.push(`%${options.memoryId}%`);
|
|
4094
|
+
}
|
|
4095
|
+
if (options.queryText) {
|
|
4096
|
+
clauses.push(`query LIKE ?`);
|
|
4097
|
+
params.push(`%${options.queryText}%`);
|
|
4098
|
+
}
|
|
4099
|
+
if (options.since) {
|
|
4100
|
+
clauses.push(`timestamp >= ?`);
|
|
4101
|
+
params.push(options.since);
|
|
4102
|
+
}
|
|
4103
|
+
if (options.until) {
|
|
4104
|
+
clauses.push(`timestamp <= ?`);
|
|
4105
|
+
params.push(options.until);
|
|
4106
|
+
}
|
|
4107
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
4108
|
+
const sortCol = options.sortBy === "duration_ms" ? "duration_ms" : "timestamp";
|
|
4109
|
+
const sortDir = options.sortDir === "asc" ? "ASC" : "DESC";
|
|
4110
|
+
const limit = options.limit ?? 50;
|
|
4111
|
+
const offset = options.offset ?? 0;
|
|
4112
|
+
const totalRow = db.prepare(`SELECT COUNT(*) AS c FROM telemetry_events ${where}`).get(...params);
|
|
4113
|
+
const rows = db.prepare(
|
|
4114
|
+
`SELECT * FROM telemetry_events ${where}
|
|
4115
|
+
ORDER BY ${sortCol} ${sortDir}
|
|
4116
|
+
LIMIT ? OFFSET ?`
|
|
4117
|
+
).all(...params, limit, offset);
|
|
4118
|
+
return {
|
|
4119
|
+
events: rows.map(rowToEvent),
|
|
4120
|
+
total: Number(totalRow.c),
|
|
4121
|
+
limit,
|
|
4122
|
+
offset
|
|
4123
|
+
};
|
|
4124
|
+
}
|
|
4125
|
+
async getEvent(id) {
|
|
4126
|
+
const db = this.requireDb();
|
|
4127
|
+
const row = db.prepare(`SELECT * FROM telemetry_events WHERE id = ?`).get(id);
|
|
4128
|
+
return row ? rowToEvent(row) : null;
|
|
4129
|
+
}
|
|
4130
|
+
async countEvents(filter) {
|
|
4131
|
+
const db = this.requireDb();
|
|
4132
|
+
const clauses = [];
|
|
4133
|
+
const params = [];
|
|
4134
|
+
if (filter?.since) {
|
|
4135
|
+
clauses.push(`timestamp >= ?`);
|
|
4136
|
+
params.push(filter.since);
|
|
4137
|
+
}
|
|
4138
|
+
if (filter?.operation) {
|
|
4139
|
+
clauses.push(`operation = ?`);
|
|
4140
|
+
params.push(filter.operation);
|
|
4141
|
+
}
|
|
4142
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
4143
|
+
const row = db.prepare(`SELECT COUNT(*) AS c FROM telemetry_events ${where}`).get(...params);
|
|
4144
|
+
return Number(row.c);
|
|
4145
|
+
}
|
|
4146
|
+
requireDb() {
|
|
4147
|
+
if (!this.db) {
|
|
4148
|
+
throw new DatabaseError("Telemetry EventDatabase is not open");
|
|
4149
|
+
}
|
|
4150
|
+
return this.db;
|
|
4151
|
+
}
|
|
4152
|
+
migrateToV2(db) {
|
|
4153
|
+
const columns = readColumns(db);
|
|
4154
|
+
for (const [name, type] of Object.entries(V2_COLUMNS)) {
|
|
4155
|
+
if (!columns.has(name)) {
|
|
4156
|
+
db.exec(`ALTER TABLE telemetry_events ADD COLUMN ${name} ${type};`);
|
|
4157
|
+
}
|
|
4158
|
+
}
|
|
4159
|
+
db.exec(`
|
|
4160
|
+
CREATE TABLE IF NOT EXISTS telemetry_meta (
|
|
4161
|
+
key TEXT PRIMARY KEY,
|
|
4162
|
+
value TEXT NOT NULL
|
|
4163
|
+
);
|
|
4164
|
+
INSERT INTO telemetry_meta(key, value) VALUES ('schema_version', '2')
|
|
4165
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value;
|
|
4166
|
+
${V2_INDEXES_SQL}
|
|
4167
|
+
`);
|
|
4168
|
+
}
|
|
4169
|
+
addOptionalColumnFilter(clauses, params, column, value) {
|
|
4170
|
+
if (!value) return;
|
|
4171
|
+
if (!this.columns.has(column)) {
|
|
4172
|
+
clauses.push("0 = 1");
|
|
4173
|
+
return;
|
|
4174
|
+
}
|
|
4175
|
+
clauses.push(`${column} = ?`);
|
|
4176
|
+
params.push(value);
|
|
4177
|
+
}
|
|
4178
|
+
resolvePath(url) {
|
|
4179
|
+
if (url === ":memory:") return ":memory:";
|
|
4180
|
+
return path5.isAbsolute(url) ? url : path5.resolve(process.cwd(), url);
|
|
4181
|
+
}
|
|
4182
|
+
};
|
|
4183
|
+
function normalizeEvent(input) {
|
|
4184
|
+
return {
|
|
4185
|
+
id: input.id ?? createId(),
|
|
4186
|
+
timestamp: input.timestamp ?? nowIso(),
|
|
4187
|
+
operation: input.operation,
|
|
4188
|
+
provider: input.provider ?? null,
|
|
4189
|
+
durationMs: input.durationMs ?? null,
|
|
4190
|
+
status: input.status,
|
|
4191
|
+
query: input.query ?? null,
|
|
4192
|
+
filters: input.filters ?? null,
|
|
4193
|
+
returnedCount: input.returnedCount ?? null,
|
|
4194
|
+
memoryIds: input.memoryIds ?? null,
|
|
4195
|
+
similarityScores: input.similarityScores ?? null,
|
|
4196
|
+
metadata: input.metadata ?? null,
|
|
4197
|
+
embeddingProvider: input.embeddingProvider ?? null,
|
|
4198
|
+
model: input.model ?? null,
|
|
4199
|
+
error: input.error ?? null,
|
|
4200
|
+
errorStack: input.errorStack ?? null,
|
|
4201
|
+
sessionId: input.sessionId,
|
|
4202
|
+
traceId: input.traceId,
|
|
4203
|
+
parentTraceId: input.parentTraceId ?? null,
|
|
4204
|
+
organization: input.organization ?? null,
|
|
4205
|
+
agentId: input.agentId ?? null,
|
|
4206
|
+
tags: input.tags ?? null,
|
|
4207
|
+
checkpointId: input.checkpointId ?? null,
|
|
4208
|
+
userMetadata: input.userMetadata ?? null,
|
|
4209
|
+
extra: input.extra ?? null,
|
|
4210
|
+
latency: input.latency ?? null,
|
|
4211
|
+
explain: input.explain ?? null,
|
|
4212
|
+
spans: input.spans ?? null
|
|
4213
|
+
};
|
|
4214
|
+
}
|
|
4215
|
+
function rowToEvent(row) {
|
|
4216
|
+
return {
|
|
4217
|
+
id: row.id,
|
|
4218
|
+
timestamp: row.timestamp,
|
|
4219
|
+
operation: row.operation,
|
|
4220
|
+
provider: row.provider,
|
|
4221
|
+
durationMs: row.duration_ms,
|
|
4222
|
+
status: row.status,
|
|
4223
|
+
query: row.query,
|
|
4224
|
+
filters: parseJson(row.filters_json),
|
|
4225
|
+
returnedCount: row.returned_count,
|
|
4226
|
+
memoryIds: parseJson(row.memory_ids_json),
|
|
4227
|
+
similarityScores: parseJson(row.similarity_scores_json),
|
|
4228
|
+
metadata: parseJson(row.metadata_json),
|
|
4229
|
+
embeddingProvider: row.embedding_provider,
|
|
4230
|
+
model: row.model,
|
|
4231
|
+
error: row.error,
|
|
4232
|
+
errorStack: row.error_stack,
|
|
4233
|
+
sessionId: row.session_id,
|
|
4234
|
+
traceId: row.trace_id,
|
|
4235
|
+
parentTraceId: row.parent_trace_id,
|
|
4236
|
+
organization: row.organization ?? null,
|
|
4237
|
+
agentId: row.agent_id ?? null,
|
|
4238
|
+
tags: parseJson(row.tags_json ?? null),
|
|
4239
|
+
checkpointId: row.checkpoint_id ?? null,
|
|
4240
|
+
userMetadata: parseJson(row.user_metadata_json),
|
|
4241
|
+
extra: parseJson(row.extra_json),
|
|
4242
|
+
latency: parseJson(row.latency_json),
|
|
4243
|
+
explain: parseJson(
|
|
4244
|
+
row.explain_json ?? null
|
|
4245
|
+
),
|
|
4246
|
+
spans: parseJson(row.spans_json ?? null)
|
|
4247
|
+
};
|
|
4248
|
+
}
|
|
4249
|
+
function readColumns(db) {
|
|
4250
|
+
const rows = db.prepare("PRAGMA table_info(telemetry_events)").all();
|
|
4251
|
+
return new Set(rows.map((row) => row.name));
|
|
4252
|
+
}
|
|
4253
|
+
function jsonOrNull(value) {
|
|
4254
|
+
if (value === void 0 || value === null) return null;
|
|
4255
|
+
return JSON.stringify(value);
|
|
4256
|
+
}
|
|
4257
|
+
function parseJson(value) {
|
|
4258
|
+
if (!value) return null;
|
|
4259
|
+
try {
|
|
4260
|
+
return JSON.parse(value);
|
|
4261
|
+
} catch {
|
|
4262
|
+
return null;
|
|
4263
|
+
}
|
|
4264
|
+
}
|
|
4265
|
+
function describe(error) {
|
|
4266
|
+
return error instanceof Error ? error.message : String(error);
|
|
4267
|
+
}
|
|
4268
|
+
|
|
4269
|
+
// src/providers/sqlite/sqliteTelemetryProvider.ts
|
|
4270
|
+
var SqliteTelemetryProvider = class {
|
|
4271
|
+
name = "sqlite";
|
|
4272
|
+
db;
|
|
4273
|
+
queue = [];
|
|
4274
|
+
flushing = null;
|
|
4275
|
+
closed = false;
|
|
4276
|
+
openPromise = null;
|
|
4277
|
+
constructor(options) {
|
|
4278
|
+
this.db = new SqliteEventDatabase({ url: options.url });
|
|
4279
|
+
}
|
|
4280
|
+
async open() {
|
|
4281
|
+
if (!this.openPromise) {
|
|
4282
|
+
this.openPromise = this.db.open();
|
|
4283
|
+
}
|
|
4284
|
+
await this.openPromise;
|
|
4285
|
+
this.closed = false;
|
|
4286
|
+
}
|
|
4287
|
+
async close() {
|
|
4288
|
+
this.closed = true;
|
|
4289
|
+
await this.flush();
|
|
4290
|
+
await this.db.close();
|
|
4291
|
+
this.openPromise = null;
|
|
4292
|
+
}
|
|
4293
|
+
emit(event) {
|
|
4294
|
+
if (this.closed) {
|
|
4295
|
+
return;
|
|
4296
|
+
}
|
|
4297
|
+
this.queue.push(event);
|
|
4298
|
+
void this.scheduleFlush();
|
|
4299
|
+
}
|
|
4300
|
+
async flush() {
|
|
4301
|
+
while (this.queue.length > 0 || this.flushing) {
|
|
4302
|
+
await this.scheduleFlush();
|
|
4303
|
+
if (this.flushing) {
|
|
4304
|
+
await this.flushing;
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
4307
|
+
}
|
|
4308
|
+
async query(options) {
|
|
4309
|
+
await this.open();
|
|
4310
|
+
return this.db.query(options);
|
|
4311
|
+
}
|
|
4312
|
+
async getEvent(id) {
|
|
4313
|
+
await this.open();
|
|
4314
|
+
return this.db.getEvent(id);
|
|
4315
|
+
}
|
|
4316
|
+
scheduleFlush() {
|
|
4317
|
+
if (this.flushing) {
|
|
4318
|
+
return this.flushing;
|
|
4319
|
+
}
|
|
4320
|
+
this.flushing = this.runFlush().finally(() => {
|
|
4321
|
+
this.flushing = null;
|
|
4322
|
+
});
|
|
4323
|
+
return this.flushing;
|
|
4324
|
+
}
|
|
4325
|
+
async runFlush() {
|
|
4326
|
+
if (this.queue.length === 0) {
|
|
4327
|
+
return;
|
|
4328
|
+
}
|
|
4329
|
+
await this.open();
|
|
4330
|
+
const batch = this.queue.splice(0, this.queue.length);
|
|
4331
|
+
try {
|
|
4332
|
+
if (typeof this.db.insertEvents === "function") {
|
|
4333
|
+
await this.db.insertEvents(batch);
|
|
4334
|
+
} else {
|
|
4335
|
+
for (const event of batch) {
|
|
4336
|
+
await this.db.insertEvent(event);
|
|
4337
|
+
}
|
|
4338
|
+
}
|
|
4339
|
+
} catch {
|
|
4340
|
+
}
|
|
4341
|
+
}
|
|
4342
|
+
};
|
|
4343
|
+
var SDK_VERSION2 = "0.3.0";
|
|
4344
|
+
var SqliteCheckpointProvider = class {
|
|
4345
|
+
name = "sqlite";
|
|
4346
|
+
directory;
|
|
4347
|
+
ready = false;
|
|
4348
|
+
constructor(options) {
|
|
4349
|
+
this.directory = options?.directory ?? path5.resolve(process.cwd(), ".wolbarg", "checkpoints");
|
|
4350
|
+
}
|
|
4351
|
+
async open() {
|
|
4352
|
+
fs5.mkdirSync(this.directory, { recursive: true });
|
|
4353
|
+
this.ready = true;
|
|
4354
|
+
}
|
|
4355
|
+
async close() {
|
|
4356
|
+
this.ready = false;
|
|
4357
|
+
}
|
|
4358
|
+
async checkpoint(name, sourcePath, options) {
|
|
4359
|
+
this.requireReady();
|
|
4360
|
+
assertCheckpointName(name);
|
|
4361
|
+
const metaPath = this.metaPath(name);
|
|
4362
|
+
if (fs5.existsSync(metaPath)) {
|
|
4363
|
+
throw new ValidationError(
|
|
4364
|
+
`Checkpoint "${name}" already exists. Choose a different name \u2014 checkpoints are never overwritten.`
|
|
4365
|
+
);
|
|
4366
|
+
}
|
|
4367
|
+
const resolvedSource = resolveDbPath(sourcePath);
|
|
4368
|
+
if (resolvedSource === ":memory:") {
|
|
4369
|
+
throw new ConfigurationError(
|
|
4370
|
+
"Cannot checkpoint an in-memory database. Use a file-backed SQLite database."
|
|
4371
|
+
);
|
|
4372
|
+
}
|
|
4373
|
+
if (!fs5.existsSync(resolvedSource)) {
|
|
4374
|
+
throw new DatabaseError(
|
|
4375
|
+
`Failed to execute checkpoint()
|
|
4376
|
+
Reason:
|
|
4377
|
+
Memory database not found at ${resolvedSource}
|
|
4378
|
+
Suggestion:
|
|
4379
|
+
Ensure Wolbarg has been initialized and the database path is correct.`
|
|
4380
|
+
);
|
|
4381
|
+
}
|
|
4382
|
+
const snapshotPath = this.snapshotPath(name);
|
|
4383
|
+
await safeSqliteBackup(resolvedSource, snapshotPath);
|
|
4384
|
+
const stats = fs5.statSync(snapshotPath);
|
|
4385
|
+
const meta2 = {
|
|
4386
|
+
name,
|
|
4387
|
+
description: options?.description ?? null,
|
|
4388
|
+
createdAt: nowIso(),
|
|
4389
|
+
sdkVersion: SDK_VERSION2,
|
|
4390
|
+
provider: this.name,
|
|
4391
|
+
snapshotPath,
|
|
4392
|
+
sourcePath: resolvedSource,
|
|
4393
|
+
sizeBytes: stats.size
|
|
4394
|
+
};
|
|
4395
|
+
fs5.writeFileSync(metaPath, JSON.stringify(meta2, null, 2), "utf8");
|
|
4396
|
+
return meta2;
|
|
4397
|
+
}
|
|
4398
|
+
async rollback(name, targetPath) {
|
|
4399
|
+
this.requireReady();
|
|
4400
|
+
const meta2 = await this.getCheckpoint(name);
|
|
4401
|
+
if (!meta2) {
|
|
4402
|
+
throw new ValidationError(`Checkpoint "${name}" was not found`);
|
|
4403
|
+
}
|
|
4404
|
+
const resolvedTarget = resolveDbPath(targetPath);
|
|
4405
|
+
if (resolvedTarget === ":memory:") {
|
|
4406
|
+
throw new ConfigurationError(
|
|
4407
|
+
"Cannot rollback into an in-memory database."
|
|
4408
|
+
);
|
|
4409
|
+
}
|
|
4410
|
+
fs5.mkdirSync(path5.dirname(resolvedTarget), { recursive: true });
|
|
4411
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
4412
|
+
const side = `${resolvedTarget}${suffix}`;
|
|
4413
|
+
if (fs5.existsSync(side)) {
|
|
4414
|
+
fs5.rmSync(side, { force: true });
|
|
4415
|
+
}
|
|
4416
|
+
}
|
|
4417
|
+
await safeSqliteBackup(meta2.snapshotPath, resolvedTarget);
|
|
4418
|
+
return meta2;
|
|
4419
|
+
}
|
|
4420
|
+
async deleteCheckpoint(name) {
|
|
4421
|
+
this.requireReady();
|
|
4422
|
+
const metaPath = this.metaPath(name);
|
|
4423
|
+
const snapshotPath = this.snapshotPath(name);
|
|
4424
|
+
let removed = false;
|
|
4425
|
+
if (fs5.existsSync(metaPath)) {
|
|
4426
|
+
fs5.rmSync(metaPath, { force: true });
|
|
4427
|
+
removed = true;
|
|
4428
|
+
}
|
|
4429
|
+
if (fs5.existsSync(snapshotPath)) {
|
|
4430
|
+
fs5.rmSync(snapshotPath, { force: true });
|
|
4431
|
+
removed = true;
|
|
4432
|
+
}
|
|
4433
|
+
return removed;
|
|
4434
|
+
}
|
|
4435
|
+
async listCheckpoints() {
|
|
4436
|
+
this.requireReady();
|
|
4437
|
+
if (!fs5.existsSync(this.directory)) {
|
|
4438
|
+
return [];
|
|
4439
|
+
}
|
|
4440
|
+
const files = fs5.readdirSync(this.directory).filter((f) => f.endsWith(".json"));
|
|
4441
|
+
const out = [];
|
|
4442
|
+
for (const file of files) {
|
|
4443
|
+
try {
|
|
4444
|
+
const raw = fs5.readFileSync(path5.join(this.directory, file), "utf8");
|
|
4445
|
+
out.push(JSON.parse(raw));
|
|
4446
|
+
} catch {
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
4449
|
+
return out.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
4450
|
+
}
|
|
4451
|
+
async getCheckpoint(name) {
|
|
4452
|
+
this.requireReady();
|
|
4453
|
+
const metaPath = this.metaPath(name);
|
|
4454
|
+
if (!fs5.existsSync(metaPath)) {
|
|
4455
|
+
return null;
|
|
4456
|
+
}
|
|
4457
|
+
try {
|
|
4458
|
+
return JSON.parse(fs5.readFileSync(metaPath, "utf8"));
|
|
4459
|
+
} catch {
|
|
4460
|
+
return null;
|
|
4461
|
+
}
|
|
4462
|
+
}
|
|
4463
|
+
metaPath(name) {
|
|
4464
|
+
return path5.join(this.directory, `${sanitize(name)}.json`);
|
|
4465
|
+
}
|
|
4466
|
+
snapshotPath(name) {
|
|
4467
|
+
return path5.join(this.directory, `${sanitize(name)}.db`);
|
|
4468
|
+
}
|
|
4469
|
+
requireReady() {
|
|
4470
|
+
if (!this.ready) {
|
|
4471
|
+
throw new DatabaseError(
|
|
4472
|
+
"Checkpoint provider is not open. Call ready() / open first."
|
|
4473
|
+
);
|
|
4474
|
+
}
|
|
4475
|
+
}
|
|
4476
|
+
};
|
|
4477
|
+
function sanitize(name) {
|
|
4478
|
+
return name.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
4479
|
+
}
|
|
4480
|
+
function assertCheckpointName(name) {
|
|
4481
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
4482
|
+
throw new ValidationError("checkpoint name must be a non-empty string");
|
|
4483
|
+
}
|
|
4484
|
+
if (name.length > 128) {
|
|
4485
|
+
throw new ValidationError("checkpoint name must be <= 128 characters");
|
|
4486
|
+
}
|
|
4487
|
+
}
|
|
4488
|
+
function resolveDbPath(connectionString) {
|
|
4489
|
+
if (connectionString === ":memory:") return ":memory:";
|
|
4490
|
+
return path5.isAbsolute(connectionString) ? connectionString : path5.resolve(process.cwd(), connectionString);
|
|
4491
|
+
}
|
|
4492
|
+
async function safeSqliteBackup(sourcePath, destPath) {
|
|
4493
|
+
fs5.mkdirSync(path5.dirname(destPath), { recursive: true });
|
|
4494
|
+
if (fs5.existsSync(destPath)) {
|
|
4495
|
+
fs5.rmSync(destPath, { force: true });
|
|
4496
|
+
}
|
|
4497
|
+
let source = null;
|
|
4498
|
+
let dest = null;
|
|
4499
|
+
try {
|
|
4500
|
+
source = new DatabaseSync(sourcePath);
|
|
4501
|
+
try {
|
|
4502
|
+
source.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
4503
|
+
} catch {
|
|
4504
|
+
}
|
|
4505
|
+
dest = new DatabaseSync(destPath);
|
|
4506
|
+
const backupFn = source.backup;
|
|
4507
|
+
if (typeof backupFn === "function") {
|
|
4508
|
+
await backupFn.call(source, dest);
|
|
4509
|
+
} else {
|
|
4510
|
+
source.close();
|
|
4511
|
+
source = null;
|
|
4512
|
+
dest.close();
|
|
4513
|
+
dest = null;
|
|
4514
|
+
fs5.copyFileSync(sourcePath, destPath);
|
|
4515
|
+
}
|
|
4516
|
+
} catch (error) {
|
|
4517
|
+
throw new DatabaseError(
|
|
4518
|
+
`Failed to execute checkpoint()
|
|
4519
|
+
Reason:
|
|
4520
|
+
${error instanceof Error ? error.message : String(error)}
|
|
4521
|
+
Suggestion:
|
|
4522
|
+
Ensure no exclusive lock is held and the destination directory is writable.`,
|
|
4523
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4524
|
+
);
|
|
4525
|
+
} finally {
|
|
4526
|
+
try {
|
|
4527
|
+
source?.close();
|
|
4528
|
+
} catch {
|
|
4529
|
+
}
|
|
4530
|
+
try {
|
|
4531
|
+
dest?.close();
|
|
4532
|
+
} catch {
|
|
4533
|
+
}
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
4536
|
+
|
|
4537
|
+
// src/core/wolbarg.ts
|
|
4538
|
+
var Wolbarg = class {
|
|
4539
|
+
initialized = false;
|
|
4540
|
+
booting = null;
|
|
4541
|
+
organization = null;
|
|
4542
|
+
storage = null;
|
|
4543
|
+
embedding = null;
|
|
4544
|
+
llm = null;
|
|
4545
|
+
compression = null;
|
|
4546
|
+
reranker = null;
|
|
4547
|
+
keywordSearch = null;
|
|
4548
|
+
ocr = null;
|
|
4549
|
+
vision = null;
|
|
4550
|
+
chunking = null;
|
|
4551
|
+
retrievalConfig = {};
|
|
4552
|
+
embeddingDimensions = null;
|
|
4553
|
+
writeMutex = new AsyncMutex();
|
|
4554
|
+
telemetry;
|
|
4555
|
+
checkpointProvider = null;
|
|
4556
|
+
memoryDbPath = null;
|
|
4557
|
+
transfer = new SqliteMemoryTransferProvider();
|
|
4558
|
+
constructor(options) {
|
|
4559
|
+
this.telemetry = new TelemetryEmitter(null, { enabled: false, level: "off" });
|
|
4560
|
+
if (!options) {
|
|
4561
|
+
return;
|
|
4562
|
+
}
|
|
4563
|
+
const validated = validateWolbargOptions(options);
|
|
4564
|
+
this.organization = validated.organization;
|
|
4565
|
+
const storageInput = validated.storage;
|
|
4566
|
+
this.storage = isStorageProvider(storageInput) ? storageInput : createStorageProvider(storageInput);
|
|
4567
|
+
if (!isStorageProvider(storageInput)) {
|
|
4568
|
+
this.memoryDbPath = resolveDatabaseUrl(storageInput);
|
|
4569
|
+
} else if (storageInput instanceof SqliteStorageProvider) {
|
|
4570
|
+
this.memoryDbPath = storageInput.path;
|
|
4571
|
+
}
|
|
4572
|
+
this.embedding = isEmbeddingProvider(validated.embedding) ? validated.embedding : createEmbeddingProvider(validated.embedding);
|
|
4573
|
+
if (validated.llm) {
|
|
4574
|
+
this.llm = isLlmProvider(validated.llm) ? validated.llm : createLlmProvider(validated.llm);
|
|
4575
|
+
this.compression = validated.compression ?? createCompressionProvider(this.llm);
|
|
4576
|
+
} else {
|
|
4577
|
+
this.compression = validated.compression ?? null;
|
|
4578
|
+
}
|
|
4579
|
+
this.reranker = validated.reranker ?? null;
|
|
4580
|
+
this.keywordSearch = validated.keywordSearch ?? null;
|
|
4581
|
+
this.ocr = validated.ocr ?? null;
|
|
4582
|
+
this.vision = validated.vision ?? null;
|
|
4583
|
+
this.chunking = validated.chunking ?? null;
|
|
4584
|
+
this.retrievalConfig = validated.retrieval ?? {};
|
|
4585
|
+
if (validated.telemetry) {
|
|
4586
|
+
if (isTelemetryProvider(validated.telemetry)) {
|
|
4587
|
+
this.telemetry = new TelemetryEmitter(validated.telemetry, {
|
|
4588
|
+
enabled: true
|
|
4589
|
+
}, { organization: validated.organization });
|
|
4590
|
+
} else {
|
|
4591
|
+
const url = validated.telemetry.database.url ?? validated.telemetry.database.connectionString ?? "";
|
|
4592
|
+
const provider = new SqliteTelemetryProvider({ url });
|
|
4593
|
+
this.telemetry = new TelemetryEmitter(provider, validated.telemetry, {
|
|
4594
|
+
organization: validated.organization
|
|
4595
|
+
});
|
|
4596
|
+
}
|
|
4597
|
+
}
|
|
4598
|
+
this.checkpointProvider = validated.checkpoint ?? new SqliteCheckpointProvider({
|
|
4599
|
+
directory: validated.checkpointDirectory
|
|
4600
|
+
});
|
|
4601
|
+
}
|
|
4602
|
+
/**
|
|
4603
|
+
* Backwards-compatible initialization (v0.1 API).
|
|
4604
|
+
*/
|
|
4605
|
+
async init(options) {
|
|
4606
|
+
if (this.initialized || this.storage) {
|
|
4607
|
+
throw new InitializationError("Wolbarg is already initialized");
|
|
4608
|
+
}
|
|
4609
|
+
let validated;
|
|
4610
|
+
try {
|
|
4611
|
+
validated = validateInitOptions(options);
|
|
4612
|
+
} catch (error) {
|
|
4613
|
+
if (error instanceof ConfigurationError || error instanceof ValidationError) {
|
|
4614
|
+
throw error;
|
|
4615
|
+
}
|
|
4616
|
+
throw new ConfigurationError(
|
|
4617
|
+
`Invalid configuration: ${error instanceof Error ? error.message : String(error)}`,
|
|
4618
|
+
{ cause: error instanceof Error ? error : void 0 }
|
|
4619
|
+
);
|
|
4620
|
+
}
|
|
4621
|
+
this.organization = validated.organization;
|
|
4622
|
+
this.storage = createStorageProvider(validated.database);
|
|
4623
|
+
this.memoryDbPath = resolveDatabaseUrl(validated.database);
|
|
4624
|
+
this.embedding = createEmbeddingProvider(validated.embedding);
|
|
4625
|
+
if (validated.llm) {
|
|
4626
|
+
this.llm = createLlmProvider(validated.llm);
|
|
4627
|
+
this.compression = createCompressionProvider(this.llm);
|
|
4628
|
+
}
|
|
4629
|
+
if (!this.checkpointProvider) {
|
|
4630
|
+
this.checkpointProvider = new SqliteCheckpointProvider();
|
|
4631
|
+
}
|
|
4632
|
+
await this.ready();
|
|
4633
|
+
}
|
|
4634
|
+
/** Ensure storage (and optional telemetry) are open. */
|
|
4635
|
+
async ready() {
|
|
4636
|
+
if (this.initialized) {
|
|
3466
4637
|
return;
|
|
3467
4638
|
}
|
|
3468
4639
|
if (this.booting) {
|
|
@@ -3484,12 +4655,16 @@ var Wolbarg = class {
|
|
|
3484
4655
|
}
|
|
3485
4656
|
try {
|
|
3486
4657
|
await this.storage.open();
|
|
4658
|
+
await this.telemetry.open();
|
|
4659
|
+
await this.checkpointProvider?.open();
|
|
3487
4660
|
const probe = await this.embedding.validate();
|
|
3488
4661
|
await this.storage.ensureVectorSchema(probe.dimensions);
|
|
3489
4662
|
this.embeddingDimensions = probe.dimensions;
|
|
3490
4663
|
this.initialized = true;
|
|
4664
|
+
this.telemetry.emitStartup(this.storage.name);
|
|
3491
4665
|
} catch (error) {
|
|
3492
4666
|
await this.storage.close().catch(() => void 0);
|
|
4667
|
+
await this.telemetry.close().catch(() => void 0);
|
|
3493
4668
|
this.initialized = false;
|
|
3494
4669
|
if (error instanceof ConfigurationError || error instanceof InitializationError || error instanceof ValidationError) {
|
|
3495
4670
|
throw error;
|
|
@@ -3502,484 +4677,946 @@ var Wolbarg = class {
|
|
|
3502
4677
|
}
|
|
3503
4678
|
/** Store a semantic memory for an agent. */
|
|
3504
4679
|
async remember(options) {
|
|
3505
|
-
const
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
const
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
4680
|
+
const trace = this.telemetry.start("remember");
|
|
4681
|
+
try {
|
|
4682
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4683
|
+
assertNonEmptyString(options.agent, "agent");
|
|
4684
|
+
if (!options.content || typeof options.content.text !== "string") {
|
|
4685
|
+
throw new ValidationError("content.text must be a string");
|
|
4686
|
+
}
|
|
4687
|
+
assertNonEmptyString(options.content.text, "content.text");
|
|
4688
|
+
const metadata = options.metadata ?? {};
|
|
4689
|
+
if (metadata === null || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
4690
|
+
throw new ValidationError("metadata must be a plain object when provided");
|
|
4691
|
+
}
|
|
4692
|
+
const tEmbed = performance.now();
|
|
4693
|
+
const vector = await embedding.embed(options.content.text);
|
|
4694
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
4695
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
4696
|
+
const id = createId();
|
|
4697
|
+
const timestamp = nowIso();
|
|
4698
|
+
const record = await this.withWriteLock(async () => {
|
|
4699
|
+
const tStore = performance.now();
|
|
4700
|
+
const row = await storage.insertMemory({
|
|
4701
|
+
id,
|
|
4702
|
+
organization,
|
|
4703
|
+
agent: options.agent.trim(),
|
|
4704
|
+
contentText: options.content.text,
|
|
4705
|
+
metadata,
|
|
4706
|
+
embedding: vector,
|
|
4707
|
+
createdAt: timestamp,
|
|
4708
|
+
updatedAt: timestamp
|
|
4709
|
+
});
|
|
4710
|
+
trace.mark("databaseWriteMs", performance.now() - tStore);
|
|
4711
|
+
return toMemoryRecord(row);
|
|
4712
|
+
});
|
|
4713
|
+
trace.success({
|
|
4714
|
+
provider: storage.name,
|
|
4715
|
+
memoryIds: [record.id],
|
|
4716
|
+
returnedCount: 1,
|
|
4717
|
+
embeddingProvider: embedding.model,
|
|
4718
|
+
model: embedding.model,
|
|
3529
4719
|
metadata,
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
updatedAt: timestamp
|
|
4720
|
+
agentId: options.agent.trim(),
|
|
4721
|
+
tags: telemetryTags(metadata)
|
|
3533
4722
|
});
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
);
|
|
3540
|
-
}
|
|
3541
|
-
return toMemoryRecord(row);
|
|
3542
|
-
});
|
|
4723
|
+
return record;
|
|
4724
|
+
} catch (error) {
|
|
4725
|
+
trace.failure(error, { agentId: options.agent });
|
|
4726
|
+
throw wrapOperationError("remember", error);
|
|
4727
|
+
}
|
|
3543
4728
|
}
|
|
3544
|
-
/**
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
assertNonEmptyString(options.query, "query");
|
|
3551
|
-
const topK = options.topK ?? 5;
|
|
3552
|
-
const threshold = options.threshold ?? 0;
|
|
3553
|
-
assertFiniteNumber(topK, "topK", { min: 1, max: 1e3 });
|
|
3554
|
-
assertFiniteNumber(threshold, "threshold", { min: 0, max: 1 });
|
|
3555
|
-
const query = options.query.trim();
|
|
3556
|
-
const vector = await embedding.embed(query);
|
|
3557
|
-
this.assertEmbeddingDimensions(vector.length);
|
|
3558
|
-
const hasFilters = Boolean(
|
|
3559
|
-
options.filter?.agent || options.filter?.metadata
|
|
3560
|
-
);
|
|
3561
|
-
const overFetch = this.retrievalConfig.overFetchFactor ?? (hasFilters ? 4 : 2);
|
|
3562
|
-
const fetchK = adaptiveFetchK(topK, overFetch, hasFilters);
|
|
3563
|
-
const semanticScores = /* @__PURE__ */ new Map();
|
|
3564
|
-
const byId = /* @__PURE__ */ new Map();
|
|
3565
|
-
const joined = storage.searchVectorsWithMemories;
|
|
3566
|
-
if (typeof joined === "function") {
|
|
3567
|
-
const joinedHits = await joined.call(
|
|
3568
|
-
storage,
|
|
3569
|
-
vector,
|
|
3570
|
-
fetchK,
|
|
3571
|
-
organization,
|
|
3572
|
-
{
|
|
3573
|
-
agent: options.filter?.agent,
|
|
3574
|
-
includeArchived: options.filter?.includeArchived
|
|
3575
|
-
}
|
|
3576
|
-
);
|
|
3577
|
-
for (const { row, distance } of joinedHits) {
|
|
3578
|
-
if (options.filter?.metadata && !matchesMetadata(
|
|
3579
|
-
deserializeMetadata(row.metadata_json),
|
|
3580
|
-
options.filter.metadata
|
|
3581
|
-
)) {
|
|
3582
|
-
continue;
|
|
3583
|
-
}
|
|
3584
|
-
const similarity = distanceToSimilarity(distance);
|
|
3585
|
-
if (similarity < threshold) {
|
|
3586
|
-
continue;
|
|
3587
|
-
}
|
|
3588
|
-
const result = toRecallResult(row, similarity);
|
|
3589
|
-
semanticScores.set(result.id, similarity);
|
|
3590
|
-
byId.set(result.id, result);
|
|
4729
|
+
/** Batch remember — one parent event + child traces per item. */
|
|
4730
|
+
async rememberBatch(items) {
|
|
4731
|
+
const parent = this.telemetry.start("rememberBatch");
|
|
4732
|
+
try {
|
|
4733
|
+
if (!Array.isArray(items) || items.length === 0) {
|
|
4734
|
+
throw new ValidationError("rememberBatch requires a non-empty array");
|
|
3591
4735
|
}
|
|
3592
|
-
|
|
3593
|
-
const
|
|
3594
|
-
const
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
} else {
|
|
3599
|
-
rowMap = /* @__PURE__ */ new Map();
|
|
3600
|
-
const looked = await Promise.all(
|
|
3601
|
-
hits.map(async (hit) => ({
|
|
3602
|
-
hit,
|
|
3603
|
-
row: await storage.getMemoryByRowid(hit.memoryRowid, organization)
|
|
3604
|
-
}))
|
|
3605
|
-
);
|
|
3606
|
-
for (const { hit, row } of looked) {
|
|
3607
|
-
if (row) rowMap.set(hit.memoryRowid, row);
|
|
4736
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4737
|
+
const tEmbed = performance.now();
|
|
4738
|
+
const texts = items.map((item, i) => {
|
|
4739
|
+
assertNonEmptyString(item.agent, `items[${i}].agent`);
|
|
4740
|
+
if (!item.content || typeof item.content.text !== "string") {
|
|
4741
|
+
throw new ValidationError(`items[${i}].content.text must be a string`);
|
|
3608
4742
|
}
|
|
4743
|
+
assertNonEmptyString(item.content.text, `items[${i}].content.text`);
|
|
4744
|
+
return item.content.text;
|
|
4745
|
+
});
|
|
4746
|
+
const vectors = await embedMany(embedding, texts);
|
|
4747
|
+
parent.mark("embeddingMs", performance.now() - tEmbed);
|
|
4748
|
+
for (const vector of vectors) {
|
|
4749
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
3609
4750
|
}
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
byId.set(result.id, result);
|
|
4751
|
+
const timestamp = nowIso();
|
|
4752
|
+
const inputs = items.map((item, i) => ({
|
|
4753
|
+
id: createId(),
|
|
4754
|
+
organization,
|
|
4755
|
+
agent: item.agent.trim(),
|
|
4756
|
+
contentText: item.content.text,
|
|
4757
|
+
metadata: item.metadata ?? {},
|
|
4758
|
+
embedding: vectors[i],
|
|
4759
|
+
createdAt: timestamp,
|
|
4760
|
+
updatedAt: timestamp
|
|
4761
|
+
}));
|
|
4762
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
4763
|
+
const child = parent.child("remember");
|
|
4764
|
+
child.success({
|
|
4765
|
+
provider: storage.name,
|
|
4766
|
+
memoryIds: [inputs[i].id],
|
|
4767
|
+
returnedCount: 1,
|
|
4768
|
+
metadata: inputs[i].metadata,
|
|
4769
|
+
agentId: inputs[i].agent,
|
|
4770
|
+
tags: telemetryTags(inputs[i].metadata)
|
|
4771
|
+
});
|
|
3632
4772
|
}
|
|
4773
|
+
const rows = await this.withWriteLock(async () => {
|
|
4774
|
+
const tStore = performance.now();
|
|
4775
|
+
const result = await storage.insertMemoriesBatch(inputs);
|
|
4776
|
+
parent.mark("databaseWriteMs", performance.now() - tStore);
|
|
4777
|
+
return result;
|
|
4778
|
+
});
|
|
4779
|
+
const records = rows.map(toMemoryRecord);
|
|
4780
|
+
parent.success({
|
|
4781
|
+
provider: storage.name,
|
|
4782
|
+
memoryIds: records.map((r) => r.id),
|
|
4783
|
+
returnedCount: records.length,
|
|
4784
|
+
embeddingProvider: embedding.model,
|
|
4785
|
+
model: embedding.model,
|
|
4786
|
+
agentId: commonAgent(items.map((item) => item.agent.trim()))
|
|
4787
|
+
});
|
|
4788
|
+
return records;
|
|
4789
|
+
} catch (error) {
|
|
4790
|
+
parent.failure(error);
|
|
4791
|
+
throw wrapOperationError("rememberBatch", error);
|
|
3633
4792
|
}
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
4793
|
+
}
|
|
4794
|
+
async recall(options) {
|
|
4795
|
+
const trace = this.telemetry.start("recall");
|
|
4796
|
+
const explain = options.explain === true;
|
|
4797
|
+
try {
|
|
4798
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
4799
|
+
assertNonEmptyString(options.query, "query");
|
|
4800
|
+
const topK = options.topK ?? 5;
|
|
4801
|
+
const threshold = options.threshold ?? 0;
|
|
4802
|
+
assertFiniteNumber(topK, "topK", { min: 1, max: 1e3 });
|
|
4803
|
+
assertFiniteNumber(threshold, "threshold", { min: 0, max: 1 });
|
|
4804
|
+
const query = options.query.trim();
|
|
4805
|
+
const tEmbed = performance.now();
|
|
4806
|
+
const vector = await embedding.embed(query);
|
|
4807
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
4808
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
4809
|
+
const hasFilters = Boolean(
|
|
4810
|
+
options.filter?.agent || options.filter?.metadata
|
|
4811
|
+
);
|
|
4812
|
+
const overFetch = this.retrievalConfig.overFetchFactor ?? (hasFilters ? 4 : 2);
|
|
4813
|
+
const fetchK = adaptiveFetchK(topK, overFetch, hasFilters);
|
|
4814
|
+
const semanticScores = /* @__PURE__ */ new Map();
|
|
4815
|
+
const distances = /* @__PURE__ */ new Map();
|
|
4816
|
+
const byId = /* @__PURE__ */ new Map();
|
|
4817
|
+
const metadataMatched = /* @__PURE__ */ new Map();
|
|
4818
|
+
const tSearch = performance.now();
|
|
4819
|
+
const joined = storage.searchVectorsWithMemories;
|
|
4820
|
+
if (typeof joined === "function") {
|
|
4821
|
+
const joinedHits = await joined.call(
|
|
4822
|
+
storage,
|
|
4823
|
+
vector,
|
|
4824
|
+
fetchK,
|
|
3640
4825
|
organization,
|
|
3641
|
-
fetchK
|
|
3642
|
-
);
|
|
3643
|
-
keywordScores = new Map(
|
|
3644
|
-
ftsHits.map((h) => [h.memoryId, h.score])
|
|
3645
|
-
);
|
|
3646
|
-
const missingIds = [];
|
|
3647
|
-
for (const id of keywordScores.keys()) {
|
|
3648
|
-
if (!byId.has(id)) missingIds.push(id);
|
|
3649
|
-
}
|
|
3650
|
-
if (missingIds.length > 0) {
|
|
3651
|
-
const fetched = await Promise.all(
|
|
3652
|
-
missingIds.map((id) => storage.getMemoryById(id, organization))
|
|
3653
|
-
);
|
|
3654
|
-
for (const row of fetched) {
|
|
3655
|
-
if (!row) continue;
|
|
3656
|
-
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
3657
|
-
continue;
|
|
3658
|
-
}
|
|
3659
|
-
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
3660
|
-
continue;
|
|
3661
|
-
}
|
|
3662
|
-
if (options.filter?.metadata && !matchesMetadata(
|
|
3663
|
-
deserializeMetadata(row.metadata_json),
|
|
3664
|
-
options.filter.metadata
|
|
3665
|
-
)) {
|
|
3666
|
-
continue;
|
|
3667
|
-
}
|
|
3668
|
-
byId.set(row.id, toRecallResult(row, 0));
|
|
3669
|
-
}
|
|
3670
|
-
}
|
|
3671
|
-
} else if (this.keywordSearch) {
|
|
3672
|
-
const corpus = await storage.listMemories(
|
|
3673
4826
|
{
|
|
3674
|
-
organization,
|
|
3675
4827
|
agent: options.filter?.agent,
|
|
3676
|
-
includeArchived: options.filter?.includeArchived
|
|
3677
|
-
|
|
3678
|
-
},
|
|
3679
|
-
Math.min(fetchK * 2, 2e3)
|
|
3680
|
-
);
|
|
3681
|
-
const keywordHits = await this.keywordSearch.search(
|
|
3682
|
-
query,
|
|
3683
|
-
corpus.map((row) => ({ id: row.id, text: row.content_text })),
|
|
3684
|
-
fetchK
|
|
3685
|
-
);
|
|
3686
|
-
keywordScores = new Map(
|
|
3687
|
-
keywordHits.map((h) => [h.memoryId, h.score])
|
|
4828
|
+
includeArchived: options.filter?.includeArchived
|
|
4829
|
+
}
|
|
3688
4830
|
);
|
|
3689
|
-
|
|
3690
|
-
|
|
4831
|
+
const tFilter = performance.now();
|
|
4832
|
+
for (const { row, distance } of joinedHits) {
|
|
4833
|
+
let metaOk = true;
|
|
4834
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
4835
|
+
deserializeMetadata(row.metadata_json),
|
|
4836
|
+
options.filter.metadata
|
|
4837
|
+
)) {
|
|
4838
|
+
metaOk = false;
|
|
3691
4839
|
continue;
|
|
3692
4840
|
}
|
|
3693
|
-
|
|
4841
|
+
const similarity = distanceToSimilarity(distance);
|
|
4842
|
+
if (similarity < threshold) {
|
|
3694
4843
|
continue;
|
|
3695
4844
|
}
|
|
4845
|
+
const result = toRecallResult(row, similarity);
|
|
4846
|
+
semanticScores.set(result.id, similarity);
|
|
4847
|
+
distances.set(result.id, distance);
|
|
4848
|
+
metadataMatched.set(result.id, metaOk);
|
|
4849
|
+
byId.set(result.id, result);
|
|
4850
|
+
}
|
|
4851
|
+
trace.mark("metadataFilteringMs", performance.now() - tFilter);
|
|
4852
|
+
} else {
|
|
4853
|
+
const hits = await storage.searchVectors(vector, fetchK);
|
|
4854
|
+
const rowids = hits.map((h) => h.memoryRowid);
|
|
4855
|
+
let rowMap;
|
|
4856
|
+
if (typeof storage.getMemoriesByRowids === "function") {
|
|
4857
|
+
rowMap = await storage.getMemoriesByRowids(rowids, organization);
|
|
4858
|
+
} else {
|
|
4859
|
+
rowMap = /* @__PURE__ */ new Map();
|
|
4860
|
+
const looked = await Promise.all(
|
|
4861
|
+
hits.map(async (hit) => ({
|
|
4862
|
+
hit,
|
|
4863
|
+
row: await storage.getMemoryByRowid(hit.memoryRowid, organization)
|
|
4864
|
+
}))
|
|
4865
|
+
);
|
|
4866
|
+
for (const { hit, row } of looked) {
|
|
4867
|
+
if (row) rowMap.set(hit.memoryRowid, row);
|
|
4868
|
+
}
|
|
4869
|
+
}
|
|
4870
|
+
const tFilter = performance.now();
|
|
4871
|
+
for (const hit of hits) {
|
|
4872
|
+
const row = rowMap.get(hit.memoryRowid);
|
|
4873
|
+
if (!row) continue;
|
|
4874
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) continue;
|
|
4875
|
+
if (!options.filter?.includeArchived && row.archived === 1) continue;
|
|
3696
4876
|
const metadata = deserializeMetadata(row.metadata_json);
|
|
3697
4877
|
if (options.filter?.metadata && !matchesMetadata(metadata, options.filter.metadata)) {
|
|
3698
4878
|
continue;
|
|
3699
4879
|
}
|
|
3700
|
-
|
|
3701
|
-
|
|
4880
|
+
const similarity = distanceToSimilarity(hit.distance);
|
|
4881
|
+
if (similarity < threshold) continue;
|
|
4882
|
+
const result = toRecallResult(row, similarity);
|
|
4883
|
+
semanticScores.set(result.id, similarity);
|
|
4884
|
+
distances.set(result.id, hit.distance);
|
|
4885
|
+
metadataMatched.set(result.id, true);
|
|
4886
|
+
byId.set(result.id, result);
|
|
4887
|
+
}
|
|
4888
|
+
trace.mark("metadataFilteringMs", performance.now() - tFilter);
|
|
4889
|
+
}
|
|
4890
|
+
const searchTime = performance.now() - tSearch;
|
|
4891
|
+
trace.mark("vectorSearchMs", searchTime);
|
|
4892
|
+
trace.mark("databaseReadMs", searchTime);
|
|
4893
|
+
const hybridWeights = resolveHybridWeights(options.hybrid) ?? (options.hybrid === void 0 ? resolveHybridWeights(this.retrievalConfig.hybrid) : null);
|
|
4894
|
+
let keywordSignal = "disabled";
|
|
4895
|
+
if (hybridWeights) {
|
|
4896
|
+
let keywordScores = /* @__PURE__ */ new Map();
|
|
4897
|
+
if (typeof storage.searchKeyword === "function") {
|
|
4898
|
+
keywordSignal = "enabled";
|
|
4899
|
+
const ftsHits = await storage.searchKeyword(
|
|
4900
|
+
query,
|
|
4901
|
+
organization,
|
|
4902
|
+
fetchK
|
|
4903
|
+
);
|
|
4904
|
+
keywordScores = new Map(ftsHits.map((h) => [h.memoryId, h.score]));
|
|
4905
|
+
const missingIds = [];
|
|
4906
|
+
for (const id of keywordScores.keys()) {
|
|
4907
|
+
if (!byId.has(id)) missingIds.push(id);
|
|
4908
|
+
}
|
|
4909
|
+
if (missingIds.length > 0) {
|
|
4910
|
+
const fetched = await Promise.all(
|
|
4911
|
+
missingIds.map((id) => storage.getMemoryById(id, organization))
|
|
4912
|
+
);
|
|
4913
|
+
for (const row of fetched) {
|
|
4914
|
+
if (!row) continue;
|
|
4915
|
+
if (options.filter?.agent && row.agent !== options.filter.agent) {
|
|
4916
|
+
continue;
|
|
4917
|
+
}
|
|
4918
|
+
if (!options.filter?.includeArchived && row.archived === 1) {
|
|
4919
|
+
continue;
|
|
4920
|
+
}
|
|
4921
|
+
if (options.filter?.metadata && !matchesMetadata(
|
|
4922
|
+
deserializeMetadata(row.metadata_json),
|
|
4923
|
+
options.filter.metadata
|
|
4924
|
+
)) {
|
|
4925
|
+
continue;
|
|
4926
|
+
}
|
|
4927
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
4928
|
+
metadataMatched.set(row.id, true);
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
} else if (this.keywordSearch) {
|
|
4932
|
+
keywordSignal = "enabled";
|
|
4933
|
+
const corpus = await storage.listMemories(
|
|
4934
|
+
{
|
|
4935
|
+
organization,
|
|
4936
|
+
agent: options.filter?.agent,
|
|
4937
|
+
includeArchived: options.filter?.includeArchived,
|
|
4938
|
+
metadata: options.filter?.metadata
|
|
4939
|
+
},
|
|
4940
|
+
Math.min(fetchK * 2, 2e3)
|
|
4941
|
+
);
|
|
4942
|
+
const keywordHits = await this.keywordSearch.search(
|
|
4943
|
+
query,
|
|
4944
|
+
corpus.map((row) => ({ id: row.id, text: row.content_text })),
|
|
4945
|
+
fetchK
|
|
4946
|
+
);
|
|
4947
|
+
keywordScores = new Map(
|
|
4948
|
+
keywordHits.map((h) => [h.memoryId, h.score])
|
|
4949
|
+
);
|
|
4950
|
+
for (const row of corpus) {
|
|
4951
|
+
if (byId.has(row.id) || !keywordScores.has(row.id)) continue;
|
|
4952
|
+
byId.set(row.id, toRecallResult(row, 0));
|
|
4953
|
+
metadataMatched.set(row.id, true);
|
|
3702
4954
|
}
|
|
3703
|
-
byId.set(row.id, toRecallResult(row, 0));
|
|
3704
4955
|
}
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
4956
|
+
if (keywordScores.size > 0) {
|
|
4957
|
+
const fused = fuseScores(semanticScores, keywordScores, hybridWeights);
|
|
4958
|
+
for (const [id, score] of fused) {
|
|
4959
|
+
const existing = byId.get(id);
|
|
4960
|
+
if (existing) {
|
|
4961
|
+
byId.set(id, { ...existing, similarity: score });
|
|
4962
|
+
}
|
|
3712
4963
|
}
|
|
3713
4964
|
}
|
|
3714
4965
|
}
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
);
|
|
3719
|
-
const mmrLambda = resolveMmr(options.mmr) ?? resolveMmr(
|
|
3720
|
-
this.retrievalConfig.mmr ? { lambda: this.retrievalConfig.mmr.lambda } : void 0
|
|
3721
|
-
);
|
|
3722
|
-
if (mmrLambda !== null) {
|
|
3723
|
-
results = applyMmr(results, Math.max(topK * 2, topK), mmrLambda);
|
|
3724
|
-
}
|
|
3725
|
-
if (options.rerank && this.reranker) {
|
|
3726
|
-
const reranked = await this.reranker.rerank(
|
|
3727
|
-
query,
|
|
3728
|
-
results.map((r) => ({ id: r.id, text: r.content.text })),
|
|
3729
|
-
topK
|
|
4966
|
+
const tRank = performance.now();
|
|
4967
|
+
let results = [...byId.values()].sort(
|
|
4968
|
+
(a, b) => b.similarity - a.similarity
|
|
3730
4969
|
);
|
|
3731
|
-
const
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
4970
|
+
const mmrLambda = resolveMmr(options.mmr) ?? resolveMmr(
|
|
4971
|
+
this.retrievalConfig.mmr ? { lambda: this.retrievalConfig.mmr.lambda } : void 0
|
|
4972
|
+
);
|
|
4973
|
+
let rankingReason = "cosine similarity";
|
|
4974
|
+
if (mmrLambda !== null) {
|
|
4975
|
+
results = applyMmr(results, Math.max(topK * 2, topK), mmrLambda);
|
|
4976
|
+
rankingReason = `MMR(lambda=${mmrLambda})`;
|
|
4977
|
+
}
|
|
4978
|
+
if (options.rerank && this.reranker) {
|
|
4979
|
+
const reranked = await this.reranker.rerank(
|
|
4980
|
+
query,
|
|
4981
|
+
results.map((r) => ({ id: r.id, text: r.content.text })),
|
|
4982
|
+
topK
|
|
4983
|
+
);
|
|
4984
|
+
const order = new Map(
|
|
4985
|
+
reranked.map((r, i) => [r.id, { score: r.score, i }])
|
|
4986
|
+
);
|
|
4987
|
+
results = results.filter((r) => order.has(r.id)).sort((a, b) => order.get(a.id).i - order.get(b.id).i).map((r) => ({
|
|
4988
|
+
...r,
|
|
4989
|
+
similarity: order.get(r.id)?.score ?? r.similarity
|
|
4990
|
+
}));
|
|
4991
|
+
rankingReason = `reranker:${this.reranker.name ?? "custom"}`;
|
|
4992
|
+
}
|
|
4993
|
+
const rankingTime = performance.now() - tRank;
|
|
4994
|
+
trace.mark("rankingMs", rankingTime);
|
|
4995
|
+
results = results.slice(0, topK);
|
|
4996
|
+
const tSer = performance.now();
|
|
4997
|
+
const serialized = results.map((r) => ({ ...r }));
|
|
4998
|
+
trace.mark("serializationMs", performance.now() - tSer);
|
|
4999
|
+
const telemetryFields = {
|
|
5000
|
+
provider: storage.name,
|
|
5001
|
+
query,
|
|
5002
|
+
filters: options.filter ?? null,
|
|
5003
|
+
returnedCount: serialized.length,
|
|
5004
|
+
memoryIds: serialized.map((r) => r.id),
|
|
5005
|
+
similarityScores: serialized.map((r) => r.similarity),
|
|
5006
|
+
embeddingProvider: embedding.model,
|
|
5007
|
+
model: embedding.model,
|
|
5008
|
+
agentId: options.filter?.agent ?? commonAgent(serialized.map((result) => result.agent)),
|
|
5009
|
+
tags: commonTags(serialized.map((result) => result.metadata))
|
|
5010
|
+
};
|
|
5011
|
+
if (!explain) {
|
|
5012
|
+
trace.success(telemetryFields);
|
|
5013
|
+
return serialized;
|
|
5014
|
+
}
|
|
5015
|
+
const explanations = serialized.map((memory) => {
|
|
5016
|
+
const matchedFields = ["content"];
|
|
5017
|
+
if (options.filter?.agent) matchedFields.push("agent");
|
|
5018
|
+
if (options.filter?.metadata) matchedFields.push("metadata");
|
|
5019
|
+
return {
|
|
5020
|
+
memory,
|
|
5021
|
+
score: memory.similarity,
|
|
5022
|
+
distance: distances.get(memory.id) ?? 1 - memory.similarity,
|
|
5023
|
+
rankingReason,
|
|
5024
|
+
matchedFields,
|
|
5025
|
+
metadataMatch: metadataMatched.get(memory.id) ?? true,
|
|
5026
|
+
providerUsed: storage.name,
|
|
5027
|
+
searchTime,
|
|
5028
|
+
rankingTime
|
|
5029
|
+
};
|
|
5030
|
+
});
|
|
5031
|
+
const totalTime = performance.now() - trace.startedAt;
|
|
5032
|
+
const persistedExplain = {
|
|
5033
|
+
enabled: true,
|
|
5034
|
+
providerUsed: storage.name,
|
|
5035
|
+
rankingStrategy: rankingReason,
|
|
5036
|
+
signals: {
|
|
5037
|
+
semantic: "enabled",
|
|
5038
|
+
keyword: keywordSignal,
|
|
5039
|
+
reranker: options.rerank === true && this.reranker ? "enabled" : "disabled",
|
|
5040
|
+
mmr: mmrLambda === null ? "disabled" : "enabled",
|
|
5041
|
+
recency: "disabled"
|
|
5042
|
+
},
|
|
5043
|
+
results: explanations.map((hit) => ({
|
|
5044
|
+
memoryId: hit.memory.id,
|
|
5045
|
+
score: hit.score,
|
|
5046
|
+
distance: hit.distance,
|
|
5047
|
+
rankingReason: hit.rankingReason,
|
|
5048
|
+
matchedFields: hit.matchedFields,
|
|
5049
|
+
metadataMatch: hit.metadataMatch
|
|
5050
|
+
})),
|
|
5051
|
+
searchTimeMs: searchTime,
|
|
5052
|
+
rankingTimeMs: rankingTime,
|
|
5053
|
+
totalTimeMs: totalTime
|
|
5054
|
+
};
|
|
5055
|
+
trace.success({ ...telemetryFields, explain: persistedExplain });
|
|
5056
|
+
return {
|
|
5057
|
+
results: explanations,
|
|
5058
|
+
providerUsed: storage.name,
|
|
5059
|
+
searchTime,
|
|
5060
|
+
rankingTime,
|
|
5061
|
+
totalTime,
|
|
5062
|
+
traceId: trace.context.traceId
|
|
5063
|
+
};
|
|
5064
|
+
} catch (error) {
|
|
5065
|
+
trace.failure(error, {
|
|
5066
|
+
query: options.query,
|
|
5067
|
+
agentId: options.filter?.agent ?? null
|
|
5068
|
+
});
|
|
5069
|
+
throw wrapOperationError("recall", error);
|
|
5070
|
+
}
|
|
5071
|
+
}
|
|
5072
|
+
/** Batch recall — parent event + child traces. */
|
|
5073
|
+
async recallBatch(queries) {
|
|
5074
|
+
const parent = this.telemetry.start("recallBatch");
|
|
5075
|
+
try {
|
|
5076
|
+
if (!Array.isArray(queries) || queries.length === 0) {
|
|
5077
|
+
throw new ValidationError("recallBatch requires a non-empty array");
|
|
5078
|
+
}
|
|
5079
|
+
const out = [];
|
|
5080
|
+
for (const q of queries) {
|
|
5081
|
+
const child = parent.child("recall");
|
|
5082
|
+
try {
|
|
5083
|
+
const hits = await this.recall({ ...q, explain: false });
|
|
5084
|
+
child.success({
|
|
5085
|
+
query: q.query,
|
|
5086
|
+
returnedCount: hits.length,
|
|
5087
|
+
memoryIds: hits.map((h) => h.id),
|
|
5088
|
+
similarityScores: hits.map((h) => h.similarity)
|
|
5089
|
+
});
|
|
5090
|
+
out.push(hits);
|
|
5091
|
+
} catch (error) {
|
|
5092
|
+
child.failure(error, { query: q.query });
|
|
5093
|
+
throw error;
|
|
5094
|
+
}
|
|
5095
|
+
}
|
|
5096
|
+
parent.success({
|
|
5097
|
+
returnedCount: out.reduce((n, r) => n + r.length, 0),
|
|
5098
|
+
extra: { queryCount: queries.length }
|
|
5099
|
+
});
|
|
5100
|
+
return out;
|
|
5101
|
+
} catch (error) {
|
|
5102
|
+
parent.failure(error);
|
|
5103
|
+
throw wrapOperationError("recallBatch", error);
|
|
3736
5104
|
}
|
|
3737
|
-
return results.slice(0, topK);
|
|
3738
5105
|
}
|
|
3739
|
-
/**
|
|
3740
|
-
* Compress related memories for an agent into a single summarized memory.
|
|
3741
|
-
* Only available when `llm` (or `compression`) was configured at construction.
|
|
3742
|
-
*/
|
|
3743
5106
|
async compress(options) {
|
|
3744
5107
|
return this.runCompress(options);
|
|
3745
5108
|
}
|
|
3746
5109
|
/** @internal */
|
|
3747
5110
|
async runCompress(options) {
|
|
3748
|
-
const
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
5111
|
+
const trace = this.telemetry.start("compress");
|
|
5112
|
+
try {
|
|
5113
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5114
|
+
if (!this.compression) {
|
|
5115
|
+
throw new ProviderNotConfiguredError(
|
|
5116
|
+
"llm",
|
|
5117
|
+
"compress",
|
|
5118
|
+
"pass llm: openaiLlm(...) (or compression) in the constructor"
|
|
5119
|
+
);
|
|
5120
|
+
}
|
|
5121
|
+
assertNonEmptyString(options.agent, "agent");
|
|
5122
|
+
const limit = options.limit ?? 50;
|
|
5123
|
+
assertFiniteNumber(limit, "limit", { min: 2, max: 500 });
|
|
5124
|
+
const tRead = performance.now();
|
|
5125
|
+
const rows = await storage.listMemories(
|
|
5126
|
+
{
|
|
5127
|
+
organization,
|
|
5128
|
+
agent: options.agent.trim(),
|
|
5129
|
+
includeArchived: false
|
|
5130
|
+
},
|
|
5131
|
+
limit
|
|
3754
5132
|
);
|
|
5133
|
+
trace.mark("databaseReadMs", performance.now() - tRead);
|
|
5134
|
+
if (rows.length < 2) {
|
|
5135
|
+
throw new ValidationError(
|
|
5136
|
+
"compress requires at least 2 active memories for the given agent"
|
|
5137
|
+
);
|
|
5138
|
+
}
|
|
5139
|
+
const records = rows.map(toMemoryRecord);
|
|
5140
|
+
const summaryText = await this.compression.compress(records);
|
|
5141
|
+
const tEmbed = performance.now();
|
|
5142
|
+
const vector = await embedding.embed(summaryText);
|
|
5143
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
5144
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
5145
|
+
const summaryId = createId();
|
|
5146
|
+
const timestamp = nowIso();
|
|
5147
|
+
const result = await this.withWriteLock(async () => {
|
|
5148
|
+
const tWrite = performance.now();
|
|
5149
|
+
const summaryRow = await storage.insertMemory({
|
|
5150
|
+
id: summaryId,
|
|
5151
|
+
organization,
|
|
5152
|
+
agent: options.agent.trim(),
|
|
5153
|
+
contentText: summaryText,
|
|
5154
|
+
metadata: {
|
|
5155
|
+
compressed: true,
|
|
5156
|
+
sourceCount: records.length,
|
|
5157
|
+
sourceIds: records.map((r) => r.id)
|
|
5158
|
+
},
|
|
5159
|
+
embedding: vector,
|
|
5160
|
+
createdAt: timestamp,
|
|
5161
|
+
updatedAt: timestamp
|
|
5162
|
+
});
|
|
5163
|
+
const archivedIds = await storage.archiveMemories(
|
|
5164
|
+
records.map((r) => r.id),
|
|
5165
|
+
organization,
|
|
5166
|
+
summaryId,
|
|
5167
|
+
timestamp
|
|
5168
|
+
);
|
|
5169
|
+
trace.mark("databaseWriteMs", performance.now() - tWrite);
|
|
5170
|
+
return {
|
|
5171
|
+
summary: toMemoryRecord(summaryRow),
|
|
5172
|
+
archivedIds
|
|
5173
|
+
};
|
|
5174
|
+
});
|
|
5175
|
+
trace.success({
|
|
5176
|
+
provider: storage.name,
|
|
5177
|
+
memoryIds: [result.summary.id, ...result.archivedIds],
|
|
5178
|
+
returnedCount: 1,
|
|
5179
|
+
agentId: options.agent.trim()
|
|
5180
|
+
});
|
|
5181
|
+
return result;
|
|
5182
|
+
} catch (error) {
|
|
5183
|
+
trace.failure(error, { agentId: options.agent });
|
|
5184
|
+
throw wrapOperationError("compress", error);
|
|
3755
5185
|
}
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
{
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
3765
|
-
|
|
3766
|
-
|
|
3767
|
-
|
|
3768
|
-
|
|
3769
|
-
|
|
5186
|
+
}
|
|
5187
|
+
async ingest(options) {
|
|
5188
|
+
const trace = this.telemetry.start("ingest");
|
|
5189
|
+
try {
|
|
5190
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5191
|
+
assertNonEmptyString(options.agent, "agent");
|
|
5192
|
+
const loaded = await loadIngestSource(options.source);
|
|
5193
|
+
let usedOcr = false;
|
|
5194
|
+
let usedVision = false;
|
|
5195
|
+
let text = loaded.rawText ?? "";
|
|
5196
|
+
if (!loaded.rawText) {
|
|
5197
|
+
const parser = resolveParser(loaded.filename, loaded.mimeType);
|
|
5198
|
+
const parsed = await parser.parse({
|
|
5199
|
+
buffer: loaded.buffer,
|
|
5200
|
+
filename: loaded.filename,
|
|
5201
|
+
mimeType: loaded.mimeType
|
|
5202
|
+
});
|
|
5203
|
+
text = parsed.text;
|
|
5204
|
+
if (parsed.isImage && parsed.imageBuffer) {
|
|
5205
|
+
const parts = [];
|
|
5206
|
+
if (this.ocr) {
|
|
5207
|
+
const ocrResult = await this.ocr.recognize(
|
|
5208
|
+
parsed.imageBuffer,
|
|
5209
|
+
parsed.mimeType
|
|
5210
|
+
);
|
|
5211
|
+
if (ocrResult.text) {
|
|
5212
|
+
parts.push(ocrResult.text);
|
|
5213
|
+
usedOcr = true;
|
|
5214
|
+
}
|
|
5215
|
+
}
|
|
5216
|
+
if (this.vision) {
|
|
5217
|
+
const visionResult = await this.vision.analyze(
|
|
5218
|
+
parsed.imageBuffer,
|
|
5219
|
+
parsed.mimeType
|
|
5220
|
+
);
|
|
5221
|
+
if (visionResult.caption) {
|
|
5222
|
+
parts.push(`Caption: ${visionResult.caption}`);
|
|
5223
|
+
}
|
|
5224
|
+
if (visionResult.description) {
|
|
5225
|
+
parts.push(visionResult.description);
|
|
5226
|
+
}
|
|
5227
|
+
if (visionResult.entities.length > 0) {
|
|
5228
|
+
parts.push(`Entities: ${visionResult.entities.join(", ")}`);
|
|
5229
|
+
}
|
|
5230
|
+
if (parts.length > 0) {
|
|
5231
|
+
usedVision = true;
|
|
5232
|
+
}
|
|
5233
|
+
}
|
|
5234
|
+
text = parts.join("\n\n").trim();
|
|
5235
|
+
}
|
|
5236
|
+
}
|
|
5237
|
+
if (!text) {
|
|
5238
|
+
throw new ValidationError(
|
|
5239
|
+
"ingest could not extract text from the document (configure ocr/vision for images, or provide text)"
|
|
5240
|
+
);
|
|
5241
|
+
}
|
|
5242
|
+
const strategy = this.chunking ?? (options.chunking?.strategy ? createChunkingStrategy(options.chunking.strategy) : inferChunkingStrategy(text));
|
|
5243
|
+
const chunks = strategy.chunk(text, {
|
|
5244
|
+
chunkSize: options.chunking?.chunkSize,
|
|
5245
|
+
overlap: options.chunking?.overlap
|
|
5246
|
+
});
|
|
5247
|
+
if (chunks.length === 0) {
|
|
5248
|
+
throw new ValidationError("ingest produced zero chunks");
|
|
5249
|
+
}
|
|
5250
|
+
const tEmbed = performance.now();
|
|
5251
|
+
const vectors = await embedMany(
|
|
5252
|
+
embedding,
|
|
5253
|
+
chunks.map((c) => c.text)
|
|
3770
5254
|
);
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
const summaryRow = await storage.insertMemory({
|
|
3780
|
-
id: summaryId,
|
|
5255
|
+
trace.mark("embeddingMs", performance.now() - tEmbed);
|
|
5256
|
+
for (const vector of vectors) {
|
|
5257
|
+
this.assertEmbeddingDimensions(vector.length);
|
|
5258
|
+
}
|
|
5259
|
+
const baseMeta = options.metadata ?? {};
|
|
5260
|
+
const timestamp = nowIso();
|
|
5261
|
+
const inputs = chunks.map((chunk, i) => ({
|
|
5262
|
+
id: createId(),
|
|
3781
5263
|
organization,
|
|
3782
5264
|
agent: options.agent.trim(),
|
|
3783
|
-
contentText:
|
|
5265
|
+
contentText: chunk.text,
|
|
3784
5266
|
metadata: {
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
5267
|
+
...baseMeta,
|
|
5268
|
+
ingest: true,
|
|
5269
|
+
chunkIndex: chunk.index,
|
|
5270
|
+
chunkCount: chunks.length,
|
|
5271
|
+
sourceFilename: loaded.filename ?? null
|
|
3788
5272
|
},
|
|
3789
|
-
embedding:
|
|
5273
|
+
embedding: vectors[i],
|
|
3790
5274
|
createdAt: timestamp,
|
|
3791
5275
|
updatedAt: timestamp
|
|
5276
|
+
}));
|
|
5277
|
+
const rows = await this.withWriteLock(async () => {
|
|
5278
|
+
const tWrite = performance.now();
|
|
5279
|
+
const result2 = await storage.insertMemoriesBatch(inputs);
|
|
5280
|
+
trace.mark("databaseWriteMs", performance.now() - tWrite);
|
|
5281
|
+
return result2;
|
|
3792
5282
|
});
|
|
3793
|
-
const
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
return {
|
|
3800
|
-
summary: toMemoryRecord(summaryRow),
|
|
3801
|
-
archivedIds
|
|
5283
|
+
const result = {
|
|
5284
|
+
memories: rows.map(toMemoryRecord),
|
|
5285
|
+
extractedChars: text.length,
|
|
5286
|
+
chunkCount: chunks.length,
|
|
5287
|
+
usedOcr,
|
|
5288
|
+
usedVision
|
|
3802
5289
|
};
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
const loaded = await loadIngestSource(options.source);
|
|
3810
|
-
let usedOcr = false;
|
|
3811
|
-
let usedVision = false;
|
|
3812
|
-
let text = loaded.rawText ?? "";
|
|
3813
|
-
if (!loaded.rawText) {
|
|
3814
|
-
const parser = resolveParser(loaded.filename, loaded.mimeType);
|
|
3815
|
-
const parsed = await parser.parse({
|
|
3816
|
-
buffer: loaded.buffer,
|
|
3817
|
-
filename: loaded.filename,
|
|
3818
|
-
mimeType: loaded.mimeType
|
|
5290
|
+
trace.success({
|
|
5291
|
+
provider: storage.name,
|
|
5292
|
+
memoryIds: result.memories.map((m) => m.id),
|
|
5293
|
+
returnedCount: result.memories.length,
|
|
5294
|
+
agentId: options.agent.trim(),
|
|
5295
|
+
tags: telemetryTags(baseMeta)
|
|
3819
5296
|
});
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
5297
|
+
return result;
|
|
5298
|
+
} catch (error) {
|
|
5299
|
+
trace.failure(error, { agentId: options.agent });
|
|
5300
|
+
throw wrapOperationError("ingest", error);
|
|
5301
|
+
}
|
|
5302
|
+
}
|
|
5303
|
+
async forget(options) {
|
|
5304
|
+
const trace = this.telemetry.start("forget");
|
|
5305
|
+
try {
|
|
5306
|
+
const { storage, organization } = await this.requireReady();
|
|
5307
|
+
const deleted = await this.withWriteLock(async () => {
|
|
5308
|
+
if ("id" in options && options.id !== void 0) {
|
|
5309
|
+
assertNonEmptyString(options.id, "id");
|
|
5310
|
+
const ok = await storage.deleteMemoryById(
|
|
5311
|
+
options.id.trim(),
|
|
5312
|
+
organization
|
|
3827
5313
|
);
|
|
3828
|
-
|
|
3829
|
-
parts.push(ocrResult.text);
|
|
3830
|
-
usedOcr = true;
|
|
3831
|
-
}
|
|
5314
|
+
return ok ? 1 : 0;
|
|
3832
5315
|
}
|
|
3833
|
-
if (
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
}
|
|
3841
|
-
if (visionResult.description) {
|
|
3842
|
-
parts.push(visionResult.description);
|
|
3843
|
-
}
|
|
3844
|
-
if (visionResult.entities.length > 0) {
|
|
3845
|
-
parts.push(`Entities: ${visionResult.entities.join(", ")}`);
|
|
3846
|
-
}
|
|
3847
|
-
if (parts.length > 0) {
|
|
3848
|
-
usedVision = true;
|
|
3849
|
-
}
|
|
5316
|
+
if ("filter" in options && options.filter?.agent) {
|
|
5317
|
+
assertNonEmptyString(options.filter.agent, "filter.agent");
|
|
5318
|
+
return storage.deleteMemoriesByFilter({
|
|
5319
|
+
organization,
|
|
5320
|
+
agent: options.filter.agent.trim(),
|
|
5321
|
+
includeArchived: true
|
|
5322
|
+
});
|
|
3850
5323
|
}
|
|
3851
|
-
|
|
5324
|
+
throw new ValidationError(
|
|
5325
|
+
"forget requires either { id } or { filter: { agent } }"
|
|
5326
|
+
);
|
|
5327
|
+
});
|
|
5328
|
+
trace.success({
|
|
5329
|
+
provider: storage.name,
|
|
5330
|
+
returnedCount: deleted,
|
|
5331
|
+
memoryIds: "id" in options && options.id ? [options.id.trim()] : null,
|
|
5332
|
+
filters: "filter" in options ? options.filter : null,
|
|
5333
|
+
agentId: "filter" in options ? options.filter?.agent?.trim() ?? null : null
|
|
5334
|
+
});
|
|
5335
|
+
return deleted;
|
|
5336
|
+
} catch (error) {
|
|
5337
|
+
trace.failure(error, {
|
|
5338
|
+
agentId: "filter" in options ? options.filter?.agent ?? null : null
|
|
5339
|
+
});
|
|
5340
|
+
throw wrapOperationError("forget", error);
|
|
5341
|
+
}
|
|
5342
|
+
}
|
|
5343
|
+
async history(options) {
|
|
5344
|
+
const trace = this.telemetry.start("history");
|
|
5345
|
+
try {
|
|
5346
|
+
const { storage, organization } = await this.requireReady();
|
|
5347
|
+
assertNonEmptyString(options.id, "id");
|
|
5348
|
+
const row = await storage.getMemoryById(options.id.trim(), organization);
|
|
5349
|
+
if (!row) {
|
|
5350
|
+
throw new MemoryNotFoundError(`Memory not found: ${options.id}`);
|
|
3852
5351
|
}
|
|
5352
|
+
const events = await storage.getHistory(row.id);
|
|
5353
|
+
const result = {
|
|
5354
|
+
memory: toMemoryRecord(row),
|
|
5355
|
+
events: events.map(toHistoryEvent)
|
|
5356
|
+
};
|
|
5357
|
+
trace.success({
|
|
5358
|
+
provider: storage.name,
|
|
5359
|
+
memoryIds: [row.id],
|
|
5360
|
+
returnedCount: events.length,
|
|
5361
|
+
agentId: row.agent,
|
|
5362
|
+
tags: telemetryTags(deserializeMetadata(row.metadata_json))
|
|
5363
|
+
});
|
|
5364
|
+
return result;
|
|
5365
|
+
} catch (error) {
|
|
5366
|
+
trace.failure(error);
|
|
5367
|
+
throw wrapOperationError("history", error);
|
|
3853
5368
|
}
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
5369
|
+
}
|
|
5370
|
+
async stats() {
|
|
5371
|
+
const trace = this.telemetry.start("stats");
|
|
5372
|
+
try {
|
|
5373
|
+
const { storage, embedding, organization } = await this.requireReady();
|
|
5374
|
+
const counts = await storage.getStats(organization);
|
|
5375
|
+
const databaseSizeBytes = await storage.getDatabaseSizeBytes();
|
|
5376
|
+
const result = {
|
|
5377
|
+
totalMemories: counts.totalMemories,
|
|
5378
|
+
activeMemories: counts.activeMemories,
|
|
5379
|
+
archivedMemories: counts.archivedMemories,
|
|
5380
|
+
totalAgents: counts.totalAgents,
|
|
5381
|
+
databaseSizeBytes,
|
|
5382
|
+
embeddingModel: embedding.model,
|
|
5383
|
+
llmModel: this.llm?.model ?? null,
|
|
5384
|
+
organization,
|
|
5385
|
+
embeddingDimensions: this.embeddingDimensions ?? 0
|
|
5386
|
+
};
|
|
5387
|
+
trace.success({
|
|
5388
|
+
provider: storage.name,
|
|
5389
|
+
returnedCount: result.totalMemories
|
|
5390
|
+
});
|
|
5391
|
+
return result;
|
|
5392
|
+
} catch (error) {
|
|
5393
|
+
trace.failure(error);
|
|
5394
|
+
throw wrapOperationError("stats", error);
|
|
3858
5395
|
}
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
5396
|
+
}
|
|
5397
|
+
async clear(options) {
|
|
5398
|
+
const trace = this.telemetry.start("clear");
|
|
5399
|
+
try {
|
|
5400
|
+
const { storage, organization } = await this.requireReady();
|
|
5401
|
+
if (!options || options.confirm !== true) {
|
|
5402
|
+
throw new ValidationError(
|
|
5403
|
+
"clear requires { confirm: true } to permanently delete all memories"
|
|
5404
|
+
);
|
|
5405
|
+
}
|
|
5406
|
+
const deleted = await this.withWriteLock(async () => {
|
|
5407
|
+
return storage.clearOrganization(organization);
|
|
5408
|
+
});
|
|
5409
|
+
trace.success({ provider: storage.name, returnedCount: deleted });
|
|
5410
|
+
return deleted;
|
|
5411
|
+
} catch (error) {
|
|
5412
|
+
trace.failure(error);
|
|
5413
|
+
throw wrapOperationError("clear", error);
|
|
3866
5414
|
}
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
);
|
|
3871
|
-
|
|
3872
|
-
this.
|
|
5415
|
+
}
|
|
5416
|
+
/** Create an immutable named checkpoint of the memory database. */
|
|
5417
|
+
async checkpoint(name, options) {
|
|
5418
|
+
const trace = this.telemetry.start("checkpoint");
|
|
5419
|
+
try {
|
|
5420
|
+
await this.requireReady();
|
|
5421
|
+
const provider = this.requireCheckpointProvider();
|
|
5422
|
+
const source = this.requireMemoryDbPath();
|
|
5423
|
+
const tCheckpoint = performance.now();
|
|
5424
|
+
const meta2 = await provider.checkpoint(name, source, options);
|
|
5425
|
+
trace.mark("databaseWriteMs", performance.now() - tCheckpoint);
|
|
5426
|
+
trace.success({
|
|
5427
|
+
provider: provider.name,
|
|
5428
|
+
checkpointId: meta2.name,
|
|
5429
|
+
extra: { checkpoint: meta2.name }
|
|
5430
|
+
});
|
|
5431
|
+
return meta2;
|
|
5432
|
+
} catch (error) {
|
|
5433
|
+
trace.failure(error, { checkpointId: name });
|
|
5434
|
+
throw wrapOperationError("checkpoint", error);
|
|
3873
5435
|
}
|
|
3874
|
-
const baseMeta = options.metadata ?? {};
|
|
3875
|
-
const timestamp = nowIso();
|
|
3876
|
-
const inputs = chunks.map((chunk, i) => ({
|
|
3877
|
-
id: createId(),
|
|
3878
|
-
organization,
|
|
3879
|
-
agent: options.agent.trim(),
|
|
3880
|
-
contentText: chunk.text,
|
|
3881
|
-
metadata: {
|
|
3882
|
-
...baseMeta,
|
|
3883
|
-
ingest: true,
|
|
3884
|
-
chunkIndex: chunk.index,
|
|
3885
|
-
chunkCount: chunks.length,
|
|
3886
|
-
sourceFilename: loaded.filename ?? null
|
|
3887
|
-
},
|
|
3888
|
-
embedding: vectors[i],
|
|
3889
|
-
createdAt: timestamp,
|
|
3890
|
-
updatedAt: timestamp
|
|
3891
|
-
}));
|
|
3892
|
-
const rows = await this.withWriteLock(async () => {
|
|
3893
|
-
return storage.insertMemoriesBatch(inputs);
|
|
3894
|
-
});
|
|
3895
|
-
return {
|
|
3896
|
-
memories: rows.map(toMemoryRecord),
|
|
3897
|
-
extractedChars: text.length,
|
|
3898
|
-
chunkCount: chunks.length,
|
|
3899
|
-
usedOcr,
|
|
3900
|
-
usedVision
|
|
3901
|
-
};
|
|
3902
5436
|
}
|
|
3903
|
-
/**
|
|
3904
|
-
async
|
|
3905
|
-
const
|
|
3906
|
-
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
3913
|
-
|
|
5437
|
+
/** Restore the memory database from a named checkpoint. */
|
|
5438
|
+
async rollback(name) {
|
|
5439
|
+
const trace = this.telemetry.start("rollback");
|
|
5440
|
+
let storageClosed = false;
|
|
5441
|
+
try {
|
|
5442
|
+
await this.requireReady();
|
|
5443
|
+
const provider = this.requireCheckpointProvider();
|
|
5444
|
+
const target = this.requireMemoryDbPath();
|
|
5445
|
+
const existing = await provider.getCheckpoint(name);
|
|
5446
|
+
if (!existing) {
|
|
5447
|
+
throw new ValidationError(`Checkpoint "${name}" was not found`);
|
|
3914
5448
|
}
|
|
3915
|
-
if (
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
organization,
|
|
3919
|
-
agent: options.filter.agent.trim(),
|
|
3920
|
-
includeArchived: true
|
|
3921
|
-
});
|
|
5449
|
+
if (this.storage) {
|
|
5450
|
+
await this.storage.close();
|
|
5451
|
+
storageClosed = true;
|
|
3922
5452
|
}
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
);
|
|
3926
|
-
|
|
5453
|
+
const tRollback = performance.now();
|
|
5454
|
+
const meta2 = await provider.rollback(name, target);
|
|
5455
|
+
trace.mark("databaseWriteMs", performance.now() - tRollback);
|
|
5456
|
+
await this.storage?.open();
|
|
5457
|
+
storageClosed = false;
|
|
5458
|
+
if (this.embeddingDimensions !== null) {
|
|
5459
|
+
await this.storage?.ensureVectorSchema(this.embeddingDimensions);
|
|
5460
|
+
}
|
|
5461
|
+
trace.success({
|
|
5462
|
+
provider: provider.name,
|
|
5463
|
+
checkpointId: meta2.name,
|
|
5464
|
+
extra: { checkpoint: meta2.name }
|
|
5465
|
+
});
|
|
5466
|
+
return meta2;
|
|
5467
|
+
} catch (error) {
|
|
5468
|
+
if (storageClosed && this.storage) {
|
|
5469
|
+
await this.storage.open().catch(() => void 0);
|
|
5470
|
+
if (this.embeddingDimensions !== null) {
|
|
5471
|
+
await this.storage.ensureVectorSchema(this.embeddingDimensions).catch(() => void 0);
|
|
5472
|
+
}
|
|
5473
|
+
}
|
|
5474
|
+
trace.failure(error, { checkpointId: name });
|
|
5475
|
+
throw wrapOperationError("rollback", error);
|
|
5476
|
+
}
|
|
3927
5477
|
}
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
|
|
3934
|
-
|
|
5478
|
+
async deleteCheckpoint(name) {
|
|
5479
|
+
const trace = this.telemetry.start("deleteCheckpoint");
|
|
5480
|
+
try {
|
|
5481
|
+
await this.requireReady();
|
|
5482
|
+
const provider = this.requireCheckpointProvider();
|
|
5483
|
+
const tDelete = performance.now();
|
|
5484
|
+
const removed = await provider.deleteCheckpoint(name);
|
|
5485
|
+
trace.mark("databaseWriteMs", performance.now() - tDelete);
|
|
5486
|
+
trace.success({
|
|
5487
|
+
provider: provider.name,
|
|
5488
|
+
checkpointId: name,
|
|
5489
|
+
returnedCount: removed ? 1 : 0
|
|
5490
|
+
});
|
|
5491
|
+
return removed;
|
|
5492
|
+
} catch (error) {
|
|
5493
|
+
trace.failure(error, { checkpointId: name });
|
|
5494
|
+
throw wrapOperationError("deleteCheckpoint", error);
|
|
3935
5495
|
}
|
|
3936
|
-
const events = await storage.getHistory(row.id);
|
|
3937
|
-
return {
|
|
3938
|
-
memory: toMemoryRecord(row),
|
|
3939
|
-
events: events.map(toHistoryEvent)
|
|
3940
|
-
};
|
|
3941
5496
|
}
|
|
3942
|
-
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
5497
|
+
async listCheckpoints() {
|
|
5498
|
+
const trace = this.telemetry.start("listCheckpoints");
|
|
5499
|
+
try {
|
|
5500
|
+
await this.requireReady();
|
|
5501
|
+
const provider = this.requireCheckpointProvider();
|
|
5502
|
+
const tList = performance.now();
|
|
5503
|
+
const list = await provider.listCheckpoints();
|
|
5504
|
+
trace.mark("databaseReadMs", performance.now() - tList);
|
|
5505
|
+
trace.success({
|
|
5506
|
+
provider: provider.name,
|
|
5507
|
+
returnedCount: list.length
|
|
5508
|
+
});
|
|
5509
|
+
return list;
|
|
5510
|
+
} catch (error) {
|
|
5511
|
+
trace.failure(error);
|
|
5512
|
+
throw wrapOperationError("listCheckpoints", error);
|
|
5513
|
+
}
|
|
3958
5514
|
}
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
|
|
5515
|
+
async getCheckpoint(name) {
|
|
5516
|
+
const trace = this.telemetry.start("getCheckpoint");
|
|
5517
|
+
try {
|
|
5518
|
+
await this.requireReady();
|
|
5519
|
+
const provider = this.requireCheckpointProvider();
|
|
5520
|
+
const tGet = performance.now();
|
|
5521
|
+
const meta2 = await provider.getCheckpoint(name);
|
|
5522
|
+
trace.mark("databaseReadMs", performance.now() - tGet);
|
|
5523
|
+
trace.success({
|
|
5524
|
+
provider: provider.name,
|
|
5525
|
+
checkpointId: name,
|
|
5526
|
+
returnedCount: meta2 ? 1 : 0
|
|
5527
|
+
});
|
|
5528
|
+
return meta2;
|
|
5529
|
+
} catch (error) {
|
|
5530
|
+
trace.failure(error, { checkpointId: name });
|
|
5531
|
+
throw wrapOperationError("getCheckpoint", error);
|
|
5532
|
+
}
|
|
5533
|
+
}
|
|
5534
|
+
/** Export the memory database to a portable SQLite + manifest bundle. */
|
|
5535
|
+
async export(exportPath) {
|
|
5536
|
+
const trace = this.telemetry.start("export");
|
|
5537
|
+
try {
|
|
5538
|
+
await this.requireReady();
|
|
5539
|
+
const source = this.requireMemoryDbPath();
|
|
5540
|
+
if (this.storage?.name === "sqlite") {
|
|
5541
|
+
}
|
|
5542
|
+
const result = await this.transfer.exportTo(
|
|
5543
|
+
exportPath,
|
|
5544
|
+
source,
|
|
5545
|
+
this.organization ?? void 0
|
|
5546
|
+
);
|
|
5547
|
+
trace.success({
|
|
5548
|
+
provider: "sqlite",
|
|
5549
|
+
extra: { path: result.path, sizeBytes: result.sizeBytes }
|
|
5550
|
+
});
|
|
5551
|
+
return {
|
|
5552
|
+
path: result.path,
|
|
5553
|
+
sizeBytes: result.sizeBytes,
|
|
5554
|
+
exportedAt: result.manifest.exportedAt
|
|
5555
|
+
};
|
|
5556
|
+
} catch (error) {
|
|
5557
|
+
trace.failure(error);
|
|
5558
|
+
throw wrapOperationError("export", error);
|
|
5559
|
+
}
|
|
5560
|
+
}
|
|
5561
|
+
/** Import a previously exported memory database, replacing the current file. */
|
|
5562
|
+
async import(exportPath) {
|
|
5563
|
+
const trace = this.telemetry.start("import");
|
|
5564
|
+
let storageClosed = false;
|
|
5565
|
+
try {
|
|
5566
|
+
await this.requireReady();
|
|
5567
|
+
const target = this.requireMemoryDbPath();
|
|
5568
|
+
if (this.storage) {
|
|
5569
|
+
await this.storage.close();
|
|
5570
|
+
storageClosed = true;
|
|
5571
|
+
}
|
|
5572
|
+
const result = await this.transfer.importFrom(
|
|
5573
|
+
exportPath,
|
|
5574
|
+
target
|
|
3965
5575
|
);
|
|
5576
|
+
await this.storage?.open();
|
|
5577
|
+
storageClosed = false;
|
|
5578
|
+
if (this.embeddingDimensions !== null) {
|
|
5579
|
+
await this.storage?.ensureVectorSchema(this.embeddingDimensions);
|
|
5580
|
+
}
|
|
5581
|
+
trace.success({
|
|
5582
|
+
provider: "sqlite",
|
|
5583
|
+
extra: { path: result.path }
|
|
5584
|
+
});
|
|
5585
|
+
return {
|
|
5586
|
+
path: result.path,
|
|
5587
|
+
importedAt: nowIso()
|
|
5588
|
+
};
|
|
5589
|
+
} catch (error) {
|
|
5590
|
+
if (storageClosed && this.storage) {
|
|
5591
|
+
await this.storage.open().catch(() => void 0);
|
|
5592
|
+
if (this.embeddingDimensions !== null) {
|
|
5593
|
+
await this.storage.ensureVectorSchema(this.embeddingDimensions).catch(() => void 0);
|
|
5594
|
+
}
|
|
5595
|
+
}
|
|
5596
|
+
trace.failure(error);
|
|
5597
|
+
throw wrapOperationError("import", error);
|
|
3966
5598
|
}
|
|
3967
|
-
return this.withWriteLock(async () => {
|
|
3968
|
-
return storage.clearOrganization(organization);
|
|
3969
|
-
});
|
|
3970
5599
|
}
|
|
3971
|
-
/**
|
|
3972
|
-
|
|
3973
|
-
|
|
3974
|
-
|
|
5600
|
+
/** Flush pending telemetry (useful in tests). */
|
|
5601
|
+
async flushTelemetry() {
|
|
5602
|
+
await this.telemetry.flush();
|
|
5603
|
+
}
|
|
5604
|
+
/** Session id for this SDK instance (telemetry traces). */
|
|
5605
|
+
get sessionId() {
|
|
5606
|
+
return this.telemetry.sessionId;
|
|
5607
|
+
}
|
|
3975
5608
|
withWriteLock(fn) {
|
|
3976
5609
|
if (this.storage?.name === "sqlite") {
|
|
3977
5610
|
return this.writeMutex.runExclusive(fn);
|
|
3978
5611
|
}
|
|
3979
5612
|
return fn();
|
|
3980
5613
|
}
|
|
3981
|
-
/** Close the database connection and release resources. */
|
|
3982
5614
|
async close() {
|
|
5615
|
+
if (this.storage) {
|
|
5616
|
+
this.telemetry.emitShutdown(this.storage.name);
|
|
5617
|
+
}
|
|
5618
|
+
await this.telemetry.close().catch(() => void 0);
|
|
5619
|
+
await this.checkpointProvider?.close().catch(() => void 0);
|
|
3983
5620
|
if (this.storage) {
|
|
3984
5621
|
await this.storage.close();
|
|
3985
5622
|
}
|
|
@@ -3991,7 +5628,6 @@ var Wolbarg = class {
|
|
|
3991
5628
|
this.embeddingDimensions = null;
|
|
3992
5629
|
this.initialized = false;
|
|
3993
5630
|
}
|
|
3994
|
-
/** Whether providers are ready for API calls. */
|
|
3995
5631
|
get isInitialized() {
|
|
3996
5632
|
return this.initialized;
|
|
3997
5633
|
}
|
|
@@ -4015,7 +5651,56 @@ var Wolbarg = class {
|
|
|
4015
5651
|
);
|
|
4016
5652
|
}
|
|
4017
5653
|
}
|
|
5654
|
+
requireCheckpointProvider() {
|
|
5655
|
+
if (!this.checkpointProvider) {
|
|
5656
|
+
throw new ProviderNotConfiguredError(
|
|
5657
|
+
"checkpoint",
|
|
5658
|
+
"checkpoint",
|
|
5659
|
+
"construct Wolbarg with a file-backed database"
|
|
5660
|
+
);
|
|
5661
|
+
}
|
|
5662
|
+
return this.checkpointProvider;
|
|
5663
|
+
}
|
|
5664
|
+
requireMemoryDbPath() {
|
|
5665
|
+
if (!this.memoryDbPath || this.memoryDbPath === ":memory:") {
|
|
5666
|
+
throw new ConfigurationError(
|
|
5667
|
+
"This operation requires a file-backed SQLite memory database (not :memory:).",
|
|
5668
|
+
{
|
|
5669
|
+
suggestion: 'Use database: { provider: "sqlite", url: "./memory.db" }'
|
|
5670
|
+
}
|
|
5671
|
+
);
|
|
5672
|
+
}
|
|
5673
|
+
return path5.isAbsolute(this.memoryDbPath) ? this.memoryDbPath : path5.resolve(process.cwd(), this.memoryDbPath);
|
|
5674
|
+
}
|
|
4018
5675
|
};
|
|
5676
|
+
function wolbarg(options) {
|
|
5677
|
+
return new Wolbarg(options);
|
|
5678
|
+
}
|
|
5679
|
+
function telemetryTags(metadata) {
|
|
5680
|
+
const value = metadata.tags;
|
|
5681
|
+
if (typeof value === "string") {
|
|
5682
|
+
const tag = value.trim();
|
|
5683
|
+
return tag ? [tag] : null;
|
|
5684
|
+
}
|
|
5685
|
+
if (Array.isArray(value)) {
|
|
5686
|
+
const tags = value.filter(
|
|
5687
|
+
(tag) => typeof tag === "string" && tag.trim().length > 0
|
|
5688
|
+
);
|
|
5689
|
+
return tags.length > 0 ? tags.map((tag) => tag.trim()) : null;
|
|
5690
|
+
}
|
|
5691
|
+
return null;
|
|
5692
|
+
}
|
|
5693
|
+
function commonAgent(agents) {
|
|
5694
|
+
const first = agents[0];
|
|
5695
|
+
return first && agents.every((agent) => agent === first) ? first : null;
|
|
5696
|
+
}
|
|
5697
|
+
function commonTags(metadata) {
|
|
5698
|
+
const tags = /* @__PURE__ */ new Set();
|
|
5699
|
+
for (const item of metadata) {
|
|
5700
|
+
for (const tag of telemetryTags(item) ?? []) tags.add(tag);
|
|
5701
|
+
}
|
|
5702
|
+
return tags.size > 0 ? [...tags] : null;
|
|
5703
|
+
}
|
|
4019
5704
|
|
|
4020
5705
|
// src/filters/types.ts
|
|
4021
5706
|
var meta = {
|
|
@@ -4057,7 +5742,11 @@ function sqlite(connectionString) {
|
|
|
4057
5742
|
return new SqliteStorageProvider({ connectionString });
|
|
4058
5743
|
}
|
|
4059
5744
|
function sqliteConfig(connectionString) {
|
|
4060
|
-
return {
|
|
5745
|
+
return {
|
|
5746
|
+
provider: "sqlite",
|
|
5747
|
+
connectionString,
|
|
5748
|
+
url: connectionString
|
|
5749
|
+
};
|
|
4061
5750
|
}
|
|
4062
5751
|
function postgres(options) {
|
|
4063
5752
|
const opts = typeof options === "string" ? { connectionString: options } : options;
|
|
@@ -4067,9 +5756,31 @@ function postgresConfig(connectionString, options) {
|
|
|
4067
5756
|
return {
|
|
4068
5757
|
provider: "postgres",
|
|
4069
5758
|
connectionString,
|
|
5759
|
+
url: connectionString,
|
|
4070
5760
|
...options
|
|
4071
5761
|
};
|
|
4072
5762
|
}
|
|
5763
|
+
function sqliteTelemetry(url) {
|
|
5764
|
+
return new SqliteTelemetryProvider({ url });
|
|
5765
|
+
}
|
|
5766
|
+
function sqliteCheckpoint(directory) {
|
|
5767
|
+
return new SqliteCheckpointProvider({ directory });
|
|
5768
|
+
}
|
|
5769
|
+
function createTelemetryProvider(config) {
|
|
5770
|
+
if (config.database.provider !== "sqlite") {
|
|
5771
|
+
throw new ConfigurationError(
|
|
5772
|
+
`Unsupported telemetry provider "${config.database.provider}". Only "sqlite" is implemented in v0.3.0.`
|
|
5773
|
+
);
|
|
5774
|
+
}
|
|
5775
|
+
const url = config.database.url ?? config.database.connectionString ?? "";
|
|
5776
|
+
if (!url) {
|
|
5777
|
+
throw new ConfigurationError("telemetry.database.url is required");
|
|
5778
|
+
}
|
|
5779
|
+
return new SqliteTelemetryProvider({ url });
|
|
5780
|
+
}
|
|
5781
|
+
function wolbarg2(options) {
|
|
5782
|
+
return new Wolbarg(options);
|
|
5783
|
+
}
|
|
4073
5784
|
|
|
4074
5785
|
// src/keyword/index.ts
|
|
4075
5786
|
var Bm25KeywordSearchProvider = class {
|
|
@@ -4465,6 +6176,56 @@ function openaiVision(options) {
|
|
|
4465
6176
|
});
|
|
4466
6177
|
}
|
|
4467
6178
|
|
|
4468
|
-
|
|
6179
|
+
// src/benchmark/index.ts
|
|
6180
|
+
async function runBenchmark(name, iterations, fn) {
|
|
6181
|
+
const samples = [];
|
|
6182
|
+
for (let i = 0; i < iterations; i += 1) {
|
|
6183
|
+
const t0 = performance.now();
|
|
6184
|
+
try {
|
|
6185
|
+
await fn();
|
|
6186
|
+
samples.push({
|
|
6187
|
+
name,
|
|
6188
|
+
durationMs: performance.now() - t0,
|
|
6189
|
+
ok: true
|
|
6190
|
+
});
|
|
6191
|
+
} catch (error) {
|
|
6192
|
+
samples.push({
|
|
6193
|
+
name,
|
|
6194
|
+
durationMs: performance.now() - t0,
|
|
6195
|
+
ok: false,
|
|
6196
|
+
error: error instanceof Error ? error.message : String(error)
|
|
6197
|
+
});
|
|
6198
|
+
}
|
|
6199
|
+
}
|
|
6200
|
+
return samples;
|
|
6201
|
+
}
|
|
6202
|
+
function summarizeBenchmark(samples, startedAt, finishedAt) {
|
|
6203
|
+
const durations = samples.filter((s) => s.ok).map((s) => s.durationMs).sort((a, b) => a - b);
|
|
6204
|
+
const avg = durations.length === 0 ? 0 : durations.reduce((a, b) => a + b, 0) / durations.length;
|
|
6205
|
+
return {
|
|
6206
|
+
startedAt,
|
|
6207
|
+
finishedAt,
|
|
6208
|
+
samples,
|
|
6209
|
+
summary: {
|
|
6210
|
+
count: samples.length,
|
|
6211
|
+
ok: samples.filter((s) => s.ok).length,
|
|
6212
|
+
failed: samples.filter((s) => !s.ok).length,
|
|
6213
|
+
avgMs: avg,
|
|
6214
|
+
p50Ms: percentile(durations, 0.5),
|
|
6215
|
+
p95Ms: percentile(durations, 0.95),
|
|
6216
|
+
p99Ms: percentile(durations, 0.99)
|
|
6217
|
+
}
|
|
6218
|
+
};
|
|
6219
|
+
}
|
|
6220
|
+
function percentile(sorted, p) {
|
|
6221
|
+
if (sorted.length === 0) return 0;
|
|
6222
|
+
const idx = Math.min(
|
|
6223
|
+
sorted.length - 1,
|
|
6224
|
+
Math.max(0, Math.ceil(p * sorted.length) - 1)
|
|
6225
|
+
);
|
|
6226
|
+
return sorted[idx];
|
|
6227
|
+
}
|
|
6228
|
+
|
|
6229
|
+
export { CompressionError, ConfigurationError, DatabaseError, EmbeddingError, FixedChunkingStrategy, HeadingChunkingStrategy, InitializationError, LlmCompressionProvider, MarkdownChunkingStrategy, MemoryNotFoundError, NoopTelemetryProvider, ParagraphChunkingStrategy, PostgresStorageProvider, ProviderNotConfiguredError, SentenceChunkingStrategy, SqliteCheckpointProvider, SqliteDatabaseProvider, SqliteEventDatabase, SqliteStorageProvider, SqliteTelemetryProvider, TelemetryEmitter, ValidationError, Wolbarg, WolbargError, WolbargLogger, bgeReranker, bm25, cohereReranker, createChunkingStrategy, createCompressionProvider, createDatabaseProvider, createEmbeddingProvider, createLlmProvider, createStorageProvider, createTelemetryProvider, wolbarg2 as createWolbarg, crossEncoder, geminiEmbedding, geminiVision, jinaReranker, lmStudioEmbedding, meta, ollamaEmbedding, ollamaLlm, openRouterEmbedding, openRouterLlm, openaiCompatibleEmbedding, openaiCompatibleLlm, openaiEmbedding, openaiLlm, openaiReranker, openaiVision, postgres, postgresConfig, runBenchmark, sqlite, sqliteCheckpoint, sqliteConfig, sqliteTelemetry, summarizeBenchmark, tesseract, togetherEmbedding, vllmEmbedding, wolbarg, wrapOperationError };
|
|
4469
6230
|
//# sourceMappingURL=index.js.map
|
|
4470
6231
|
//# sourceMappingURL=index.js.map
|