teleton 0.2.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -96
- package/dist/{chunk-LCMHAUNK.js → chunk-67QC5FBN.js} +0 -2
- package/dist/{chunk-DR6WM6B5.js → chunk-7UPH62J2.js} +252 -93
- package/dist/{chunk-DAMFGHXV.js → chunk-A64NPEFL.js} +1 -1
- package/dist/chunk-EHEV7FJ7.js +157 -0
- package/dist/{chunk-2X4PCE7V.js → chunk-OA5L7GM6.js} +7 -1
- package/dist/chunk-QBGUCUOW.js +16 -0
- package/dist/{chunk-KS7B2CVM.js → chunk-QU4ZOR35.js} +6647 -6852
- package/dist/cli/index.js +13 -14
- package/dist/endpoint-FT2B2RZ2.js +7 -0
- package/dist/format-transactions-FD74HI5N.js +9 -0
- package/dist/index.js +7 -5
- package/dist/{memory-PDEJIPK6.js → memory-SYTQ5P7P.js} +6 -6
- package/dist/{migrate-6BQZVOBN.js → migrate-ITXMRRSZ.js} +4 -4
- package/dist/{scraper-2O6Z3ZHT.js → scraper-SH7GS7TO.js} +6 -6
- package/dist/{task-dependency-resolver-WNYOTWWM.js → task-dependency-resolver-GY6PEBIS.js} +4 -4
- package/dist/{task-executor-4SVB72GR.js → task-executor-4QKTZZ3P.js} +1 -1
- package/package.json +2 -1
- package/src/templates/IDENTITY.md +1 -1
- package/src/templates/MEMORY.md +1 -0
- package/src/templates/SECURITY.md +5 -0
- package/src/templates/SOUL.md +3 -2
- package/dist/timeouts-ZAK6NELA.js +0 -63
|
@@ -1,27 +1,30 @@
|
|
|
1
1
|
import {
|
|
2
2
|
VOYAGE_API_URL,
|
|
3
3
|
fetchWithTimeout
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-A64NPEFL.js";
|
|
5
5
|
import {
|
|
6
|
+
EMBEDDING_CACHE_EVICTION_INTERVAL,
|
|
7
|
+
EMBEDDING_CACHE_MAX_ENTRIES,
|
|
8
|
+
EMBEDDING_CACHE_TTL_DAYS,
|
|
6
9
|
KNOWLEDGE_CHUNK_OVERLAP,
|
|
7
10
|
KNOWLEDGE_CHUNK_SIZE,
|
|
8
11
|
SQLITE_CACHE_SIZE_KB,
|
|
9
12
|
SQLITE_MMAP_SIZE,
|
|
10
13
|
VOYAGE_BATCH_SIZE
|
|
11
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-OA5L7GM6.js";
|
|
12
15
|
import {
|
|
13
16
|
TELETON_ROOT
|
|
14
17
|
} from "./chunk-EYWNOHMJ.js";
|
|
15
18
|
|
|
16
19
|
// src/memory/database.ts
|
|
17
20
|
import Database2 from "better-sqlite3";
|
|
18
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync2 } from "fs";
|
|
21
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2, chmodSync as chmodSync2 } from "fs";
|
|
19
22
|
import { dirname as dirname2 } from "path";
|
|
20
23
|
import * as sqliteVec from "sqlite-vec";
|
|
21
24
|
|
|
22
25
|
// src/utils/module-db.ts
|
|
23
26
|
import Database from "better-sqlite3";
|
|
24
|
-
import { existsSync, mkdirSync } from "fs";
|
|
27
|
+
import { existsSync, mkdirSync, chmodSync } from "fs";
|
|
25
28
|
import { dirname, join } from "path";
|
|
26
29
|
var JOURNAL_SCHEMA = `
|
|
27
30
|
CREATE TABLE IF NOT EXISTS journal (
|
|
@@ -72,6 +75,10 @@ function openModuleDb(path) {
|
|
|
72
75
|
mkdirSync(dir, { recursive: true });
|
|
73
76
|
}
|
|
74
77
|
const db = new Database(path);
|
|
78
|
+
try {
|
|
79
|
+
chmodSync(path, 384);
|
|
80
|
+
} catch {
|
|
81
|
+
}
|
|
75
82
|
db.pragma("journal_mode = WAL");
|
|
76
83
|
return db;
|
|
77
84
|
}
|
|
@@ -92,6 +99,11 @@ function createDbWrapper(getDb, moduleName) {
|
|
|
92
99
|
var MAIN_DB_PATH = join(TELETON_ROOT, "memory.db");
|
|
93
100
|
function migrateFromMainDb(moduleDb, tables) {
|
|
94
101
|
let totalMigrated = 0;
|
|
102
|
+
for (const table of tables) {
|
|
103
|
+
if (!/^[a-z_]+$/.test(table)) {
|
|
104
|
+
throw new Error(`Invalid table name for migration: "${table}"`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
95
107
|
for (const table of tables) {
|
|
96
108
|
try {
|
|
97
109
|
const row = moduleDb.prepare(`SELECT COUNT(*) as c FROM ${table}`).get();
|
|
@@ -362,16 +374,16 @@ function ensureSchema(db) {
|
|
|
362
374
|
-- ============================================
|
|
363
375
|
|
|
364
376
|
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
365
|
-
hash TEXT
|
|
366
|
-
embedding TEXT NOT NULL,
|
|
377
|
+
hash TEXT NOT NULL,
|
|
367
378
|
model TEXT NOT NULL,
|
|
368
379
|
provider TEXT NOT NULL,
|
|
369
|
-
|
|
380
|
+
embedding BLOB NOT NULL,
|
|
381
|
+
dims INTEGER NOT NULL,
|
|
370
382
|
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
371
|
-
accessed_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
383
|
+
accessed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
384
|
+
PRIMARY KEY (hash, model, provider)
|
|
372
385
|
);
|
|
373
386
|
|
|
374
|
-
CREATE INDEX IF NOT EXISTS idx_embedding_cache_model ON embedding_cache(provider, model);
|
|
375
387
|
CREATE INDEX IF NOT EXISTS idx_embedding_cache_accessed ON embedding_cache(accessed_at);
|
|
376
388
|
|
|
377
389
|
-- =====================================================
|
|
@@ -416,7 +428,7 @@ function setSchemaVersion(db, version) {
|
|
|
416
428
|
`
|
|
417
429
|
).run(version);
|
|
418
430
|
}
|
|
419
|
-
var CURRENT_SCHEMA_VERSION = "1.
|
|
431
|
+
var CURRENT_SCHEMA_VERSION = "1.9.0";
|
|
420
432
|
function runMigrations(db) {
|
|
421
433
|
const currentVersion = getSchemaVersion(db);
|
|
422
434
|
if (!currentVersion || versionLessThan(currentVersion, "1.1.0")) {
|
|
@@ -500,6 +512,29 @@ function runMigrations(db) {
|
|
|
500
512
|
throw error;
|
|
501
513
|
}
|
|
502
514
|
}
|
|
515
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.9.0")) {
|
|
516
|
+
console.log("\u{1F504} Running migration 1.9.0: Upgrade embedding_cache to BLOB storage");
|
|
517
|
+
try {
|
|
518
|
+
db.exec(`DROP TABLE IF EXISTS embedding_cache`);
|
|
519
|
+
db.exec(`
|
|
520
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
521
|
+
hash TEXT NOT NULL,
|
|
522
|
+
model TEXT NOT NULL,
|
|
523
|
+
provider TEXT NOT NULL,
|
|
524
|
+
embedding BLOB NOT NULL,
|
|
525
|
+
dims INTEGER NOT NULL,
|
|
526
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
527
|
+
accessed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
528
|
+
PRIMARY KEY (hash, model, provider)
|
|
529
|
+
);
|
|
530
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_cache_accessed ON embedding_cache(accessed_at);
|
|
531
|
+
`);
|
|
532
|
+
console.log("\u2705 Migration 1.9.0 complete: embedding_cache upgraded to BLOB storage");
|
|
533
|
+
} catch (error) {
|
|
534
|
+
console.error("\u274C Migration 1.9.0 failed:", error);
|
|
535
|
+
throw error;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
503
538
|
setSchemaVersion(db, CURRENT_SCHEMA_VERSION);
|
|
504
539
|
}
|
|
505
540
|
|
|
@@ -517,6 +552,10 @@ var MemoryDatabase = class {
|
|
|
517
552
|
this.db = new Database2(config.path, {
|
|
518
553
|
verbose: process.env.DEBUG_SQL ? console.log : void 0
|
|
519
554
|
});
|
|
555
|
+
try {
|
|
556
|
+
chmodSync2(config.path, 384);
|
|
557
|
+
} catch {
|
|
558
|
+
}
|
|
520
559
|
this.db.pragma("journal_mode = WAL");
|
|
521
560
|
this.db.pragma("synchronous = NORMAL");
|
|
522
561
|
this.db.pragma(`cache_size = -${SQLITE_CACHE_SIZE_KB}`);
|
|
@@ -541,13 +580,12 @@ var MemoryDatabase = class {
|
|
|
541
580
|
if (this.config.enableVectorSearch) {
|
|
542
581
|
this.loadVectorExtension();
|
|
543
582
|
}
|
|
583
|
+
this.db.exec("ANALYZE");
|
|
544
584
|
}
|
|
545
585
|
loadVectorExtension() {
|
|
546
586
|
try {
|
|
547
587
|
sqliteVec.load(this.db);
|
|
548
|
-
|
|
549
|
-
const { vec_version } = this.db.prepare("SELECT vec_version() as vec_version").get();
|
|
550
|
-
console.log(` Version: ${vec_version}`);
|
|
588
|
+
this.db.prepare("SELECT vec_version() as vec_version").get();
|
|
551
589
|
const dims = this.config.vectorDimensions ?? 512;
|
|
552
590
|
ensureVectorTables(this.db, dims);
|
|
553
591
|
this.vectorReady = true;
|
|
@@ -691,6 +729,9 @@ function closeDatabase() {
|
|
|
691
729
|
}
|
|
692
730
|
}
|
|
693
731
|
|
|
732
|
+
// src/memory/embeddings/index.ts
|
|
733
|
+
import { createHash } from "crypto";
|
|
734
|
+
|
|
694
735
|
// src/memory/embeddings/provider.ts
|
|
695
736
|
var NoopEmbeddingProvider = class {
|
|
696
737
|
id = "noop";
|
|
@@ -762,32 +803,159 @@ var AnthropicEmbeddingProvider = class {
|
|
|
762
803
|
};
|
|
763
804
|
|
|
764
805
|
// src/memory/embeddings/local.ts
|
|
806
|
+
import { pipeline } from "@huggingface/transformers";
|
|
807
|
+
var extractorPromise = null;
|
|
808
|
+
function getExtractor(model) {
|
|
809
|
+
if (!extractorPromise) {
|
|
810
|
+
console.log(`\u{1F4E6} Loading local embedding model: ${model} \u2026`);
|
|
811
|
+
extractorPromise = pipeline("feature-extraction", model, {
|
|
812
|
+
dtype: "fp32"
|
|
813
|
+
}).then((ext) => {
|
|
814
|
+
console.log(`\u2705 Local embedding model ready`);
|
|
815
|
+
return ext;
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
return extractorPromise;
|
|
819
|
+
}
|
|
765
820
|
var LocalEmbeddingProvider = class {
|
|
766
821
|
id = "local";
|
|
767
822
|
model;
|
|
768
823
|
dimensions;
|
|
769
|
-
hasWarned = false;
|
|
770
824
|
constructor(config) {
|
|
771
|
-
this.model = config.model
|
|
825
|
+
this.model = config.model || "Xenova/all-MiniLM-L6-v2";
|
|
772
826
|
this.dimensions = 384;
|
|
773
827
|
}
|
|
774
828
|
async embedQuery(text) {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
829
|
+
const extractor = await getExtractor(this.model);
|
|
830
|
+
const output = await extractor(text, { pooling: "mean", normalize: true });
|
|
831
|
+
return Array.from(output.data);
|
|
832
|
+
}
|
|
833
|
+
async embedBatch(texts) {
|
|
834
|
+
if (texts.length === 0) return [];
|
|
835
|
+
const extractor = await getExtractor(this.model);
|
|
836
|
+
const output = await extractor(texts, { pooling: "mean", normalize: true });
|
|
837
|
+
const data = output.data;
|
|
838
|
+
const dims = this.dimensions;
|
|
839
|
+
const results = [];
|
|
840
|
+
for (let i = 0; i < texts.length; i++) {
|
|
841
|
+
results.push(Array.from(data.slice(i * dims, (i + 1) * dims)));
|
|
780
842
|
}
|
|
781
|
-
return
|
|
843
|
+
return results;
|
|
844
|
+
}
|
|
845
|
+
};
|
|
846
|
+
|
|
847
|
+
// src/memory/embeddings/cached.ts
|
|
848
|
+
var CachedEmbeddingProvider = class {
|
|
849
|
+
constructor(inner, db) {
|
|
850
|
+
this.inner = inner;
|
|
851
|
+
this.db = db;
|
|
852
|
+
this.id = inner.id;
|
|
853
|
+
this.model = inner.model;
|
|
854
|
+
this.dimensions = inner.dimensions;
|
|
855
|
+
}
|
|
856
|
+
id;
|
|
857
|
+
model;
|
|
858
|
+
dimensions;
|
|
859
|
+
hits = 0;
|
|
860
|
+
misses = 0;
|
|
861
|
+
ops = 0;
|
|
862
|
+
cacheGet(hash) {
|
|
863
|
+
return this.db.prepare(
|
|
864
|
+
`SELECT embedding FROM embedding_cache WHERE hash = ? AND model = ? AND provider = ?`
|
|
865
|
+
).get(hash, this.model, this.id);
|
|
866
|
+
}
|
|
867
|
+
cachePut(hash, blob) {
|
|
868
|
+
this.db.prepare(
|
|
869
|
+
`INSERT OR REPLACE INTO embedding_cache (hash, embedding, model, provider, dims, created_at, accessed_at)
|
|
870
|
+
VALUES (?, ?, ?, ?, ?, unixepoch(), unixepoch())`
|
|
871
|
+
).run(hash, blob, this.model, this.id, this.dimensions);
|
|
872
|
+
}
|
|
873
|
+
cacheTouch(hash) {
|
|
874
|
+
this.db.prepare(
|
|
875
|
+
`UPDATE embedding_cache SET accessed_at = unixepoch() WHERE hash = ? AND model = ? AND provider = ?`
|
|
876
|
+
).run(hash, this.model, this.id);
|
|
877
|
+
}
|
|
878
|
+
async embedQuery(text) {
|
|
879
|
+
const hash = hashText(text);
|
|
880
|
+
const row = this.cacheGet(hash);
|
|
881
|
+
if (row) {
|
|
882
|
+
this.hits++;
|
|
883
|
+
this.cacheTouch(hash);
|
|
884
|
+
this.tick();
|
|
885
|
+
return deserializeEmbedding(row.embedding);
|
|
886
|
+
}
|
|
887
|
+
this.misses++;
|
|
888
|
+
const embedding = await this.inner.embedQuery(text);
|
|
889
|
+
this.cachePut(hash, serializeEmbedding(embedding));
|
|
890
|
+
this.tick();
|
|
891
|
+
return embedding;
|
|
782
892
|
}
|
|
783
893
|
async embedBatch(texts) {
|
|
784
|
-
if (
|
|
785
|
-
|
|
786
|
-
|
|
894
|
+
if (texts.length === 0) return [];
|
|
895
|
+
const hashes = texts.map(hashText);
|
|
896
|
+
const results = new Array(texts.length).fill(null);
|
|
897
|
+
const missIndices = [];
|
|
898
|
+
const missTexts = [];
|
|
899
|
+
for (let i = 0; i < texts.length; i++) {
|
|
900
|
+
const row = this.cacheGet(hashes[i]);
|
|
901
|
+
if (row) {
|
|
902
|
+
this.hits++;
|
|
903
|
+
this.cacheTouch(hashes[i]);
|
|
904
|
+
results[i] = deserializeEmbedding(row.embedding);
|
|
905
|
+
} else {
|
|
906
|
+
this.misses++;
|
|
907
|
+
missIndices.push(i);
|
|
908
|
+
missTexts.push(texts[i]);
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
if (missTexts.length > 0) {
|
|
912
|
+
const newEmbeddings = await this.inner.embedBatch(missTexts);
|
|
913
|
+
for (let j = 0; j < missIndices.length; j++) {
|
|
914
|
+
const idx = missIndices[j];
|
|
915
|
+
const embedding = newEmbeddings[j] ?? [];
|
|
916
|
+
results[idx] = embedding;
|
|
917
|
+
if (embedding.length > 0) {
|
|
918
|
+
this.cachePut(hashes[idx], serializeEmbedding(embedding));
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
this.ops += texts.length;
|
|
923
|
+
this.maybeEvict();
|
|
924
|
+
this.maybeLogStats();
|
|
925
|
+
return results;
|
|
926
|
+
}
|
|
927
|
+
tick() {
|
|
928
|
+
this.ops++;
|
|
929
|
+
this.maybeEvict();
|
|
930
|
+
this.maybeLogStats();
|
|
931
|
+
}
|
|
932
|
+
maybeLogStats() {
|
|
933
|
+
const total = this.hits + this.misses;
|
|
934
|
+
if (total > 0 && total % 100 === 0) {
|
|
935
|
+
const rate = (this.hits / total * 100).toFixed(0);
|
|
936
|
+
console.log(
|
|
937
|
+
`\u{1F4CA} Embedding cache: ${this.hits} hits, ${this.misses} misses (${rate}% hit rate)`
|
|
787
938
|
);
|
|
788
|
-
this.hasWarned = true;
|
|
789
939
|
}
|
|
790
|
-
|
|
940
|
+
}
|
|
941
|
+
maybeEvict() {
|
|
942
|
+
if (this.ops % EMBEDDING_CACHE_EVICTION_INTERVAL !== 0) return;
|
|
943
|
+
try {
|
|
944
|
+
const cutoff = Math.floor(Date.now() / 1e3) - EMBEDDING_CACHE_TTL_DAYS * 86400;
|
|
945
|
+
this.db.prepare(`DELETE FROM embedding_cache WHERE accessed_at < ?`).run(cutoff);
|
|
946
|
+
const count = this.db.prepare(`SELECT COUNT(*) as cnt FROM embedding_cache`).get().cnt;
|
|
947
|
+
if (count > EMBEDDING_CACHE_MAX_ENTRIES) {
|
|
948
|
+
const toDelete = Math.ceil(count * 0.1);
|
|
949
|
+
this.db.prepare(
|
|
950
|
+
`DELETE FROM embedding_cache WHERE (hash, model, provider) IN (
|
|
951
|
+
SELECT hash, model, provider FROM embedding_cache ORDER BY accessed_at ASC LIMIT ?
|
|
952
|
+
)`
|
|
953
|
+
).run(toDelete);
|
|
954
|
+
console.log(`\u{1F9F9} Embedding cache eviction: removed ${toDelete} entries (${count} total)`);
|
|
955
|
+
}
|
|
956
|
+
} catch (err) {
|
|
957
|
+
console.warn("\u26A0\uFE0F Embedding cache eviction error:", err);
|
|
958
|
+
}
|
|
791
959
|
}
|
|
792
960
|
};
|
|
793
961
|
|
|
@@ -813,27 +981,22 @@ function createEmbeddingProvider(config) {
|
|
|
813
981
|
}
|
|
814
982
|
}
|
|
815
983
|
function hashText(text) {
|
|
816
|
-
|
|
817
|
-
for (let i = 0; i < text.length; i++) {
|
|
818
|
-
const char = text.charCodeAt(i);
|
|
819
|
-
hash = (hash << 5) - hash + char;
|
|
820
|
-
hash = hash & hash;
|
|
821
|
-
}
|
|
822
|
-
return Math.abs(hash).toString(36);
|
|
984
|
+
return createHash("sha256").update(text).digest("hex");
|
|
823
985
|
}
|
|
824
986
|
function serializeEmbedding(embedding) {
|
|
825
|
-
return
|
|
987
|
+
return Buffer.from(new Float32Array(embedding).buffer);
|
|
826
988
|
}
|
|
827
989
|
function deserializeEmbedding(data) {
|
|
828
990
|
try {
|
|
991
|
+
if (Buffer.isBuffer(data)) {
|
|
992
|
+
const floats = new Float32Array(data.buffer, data.byteOffset, data.byteLength / 4);
|
|
993
|
+
return Array.from(floats);
|
|
994
|
+
}
|
|
829
995
|
return JSON.parse(data);
|
|
830
996
|
} catch {
|
|
831
997
|
return [];
|
|
832
998
|
}
|
|
833
999
|
}
|
|
834
|
-
function embeddingToBlob(embedding) {
|
|
835
|
-
return Buffer.from(new Float32Array(embedding).buffer);
|
|
836
|
-
}
|
|
837
1000
|
|
|
838
1001
|
// src/memory/agent/knowledge.ts
|
|
839
1002
|
import { readFileSync, existsSync as existsSync3, readdirSync, statSync } from "fs";
|
|
@@ -876,6 +1039,13 @@ var KnowledgeIndexer = class {
|
|
|
876
1039
|
if (existing?.hash === fileHash) {
|
|
877
1040
|
return false;
|
|
878
1041
|
}
|
|
1042
|
+
if (this.vectorEnabled) {
|
|
1043
|
+
this.db.prepare(
|
|
1044
|
+
`DELETE FROM knowledge_vec WHERE id IN (
|
|
1045
|
+
SELECT id FROM knowledge WHERE path = ? AND source = 'memory'
|
|
1046
|
+
)`
|
|
1047
|
+
).run(relPath);
|
|
1048
|
+
}
|
|
879
1049
|
this.db.prepare(`DELETE FROM knowledge WHERE path = ? AND source = 'memory'`).run(relPath);
|
|
880
1050
|
const chunks = this.chunkMarkdown(content, relPath);
|
|
881
1051
|
const texts = chunks.map((c) => c.text);
|
|
@@ -898,7 +1068,7 @@ var KnowledgeIndexer = class {
|
|
|
898
1068
|
chunk.hash
|
|
899
1069
|
);
|
|
900
1070
|
if (insertVec && embedding.length > 0) {
|
|
901
|
-
insertVec.run(chunk.id,
|
|
1071
|
+
insertVec.run(chunk.id, serializeEmbedding(embedding));
|
|
902
1072
|
}
|
|
903
1073
|
}
|
|
904
1074
|
return true;
|
|
@@ -1095,7 +1265,7 @@ var SessionStore = class {
|
|
|
1095
1265
|
const knowledgeId = `session:${sessionId}`;
|
|
1096
1266
|
const text = `Session from ${session.startedAt.toISOString()}:
|
|
1097
1267
|
${session.summary}`;
|
|
1098
|
-
const hash =
|
|
1268
|
+
const hash = hashText(text);
|
|
1099
1269
|
let embedding = null;
|
|
1100
1270
|
if (this.vectorEnabled) {
|
|
1101
1271
|
embedding = await this.embedder.embedQuery(text);
|
|
@@ -1111,15 +1281,10 @@ ${session.summary}`;
|
|
|
1111
1281
|
`
|
|
1112
1282
|
).run(knowledgeId, sessionId, text, hash);
|
|
1113
1283
|
if (embedding && this.vectorEnabled) {
|
|
1114
|
-
const embeddingBuffer =
|
|
1284
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1115
1285
|
const rowid = this.db.prepare(`SELECT rowid FROM knowledge WHERE id = ?`).get(knowledgeId);
|
|
1116
|
-
this.db.prepare(
|
|
1117
|
-
|
|
1118
|
-
INSERT INTO knowledge_vec (rowid, embedding)
|
|
1119
|
-
VALUES (?, ?)
|
|
1120
|
-
ON CONFLICT(rowid) DO UPDATE SET embedding = excluded.embedding
|
|
1121
|
-
`
|
|
1122
|
-
).run(rowid.rowid, embeddingBuffer);
|
|
1286
|
+
this.db.prepare(`DELETE FROM knowledge_vec WHERE rowid = ?`).run(rowid.rowid);
|
|
1287
|
+
this.db.prepare(`INSERT INTO knowledge_vec (rowid, embedding) VALUES (?, ?)`).run(rowid.rowid, embeddingBuffer);
|
|
1123
1288
|
}
|
|
1124
1289
|
console.log(`Indexed session ${sessionId} to knowledge base`);
|
|
1125
1290
|
} catch (error) {
|
|
@@ -1130,21 +1295,12 @@ ${session.summary}`;
|
|
|
1130
1295
|
* Delete a session
|
|
1131
1296
|
*/
|
|
1132
1297
|
deleteSession(sessionId) {
|
|
1133
|
-
|
|
1134
|
-
this.
|
|
1135
|
-
|
|
1136
|
-
hashText(text) {
|
|
1137
|
-
let hash = 0;
|
|
1138
|
-
for (let i = 0; i < text.length; i++) {
|
|
1139
|
-
const char = text.charCodeAt(i);
|
|
1140
|
-
hash = (hash << 5) - hash + char;
|
|
1141
|
-
hash = hash & hash;
|
|
1298
|
+
const knowledgeId = `session:${sessionId}`;
|
|
1299
|
+
if (this.vectorEnabled) {
|
|
1300
|
+
this.db.prepare(`DELETE FROM knowledge_vec WHERE id = ?`).run(knowledgeId);
|
|
1142
1301
|
}
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
serializeEmbedding(embedding) {
|
|
1146
|
-
const float32 = new Float32Array(embedding);
|
|
1147
|
-
return Buffer.from(float32.buffer);
|
|
1302
|
+
this.db.prepare(`DELETE FROM sessions WHERE id = ?`).run(sessionId);
|
|
1303
|
+
this.db.prepare(`DELETE FROM knowledge WHERE id = ?`).run(knowledgeId);
|
|
1148
1304
|
}
|
|
1149
1305
|
};
|
|
1150
1306
|
|
|
@@ -1182,7 +1338,8 @@ var MessageStore = class {
|
|
|
1182
1338
|
if (message.senderId) {
|
|
1183
1339
|
this.ensureUser(message.senderId);
|
|
1184
1340
|
}
|
|
1185
|
-
const embedding = message.text ? await this.embedder.embedQuery(message.text) : [];
|
|
1341
|
+
const embedding = this.vectorEnabled && message.text ? await this.embedder.embedQuery(message.text) : [];
|
|
1342
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1186
1343
|
this.db.prepare(
|
|
1187
1344
|
`
|
|
1188
1345
|
INSERT OR REPLACE INTO tg_messages (
|
|
@@ -1195,7 +1352,7 @@ var MessageStore = class {
|
|
|
1195
1352
|
message.chatId,
|
|
1196
1353
|
message.senderId,
|
|
1197
1354
|
message.text,
|
|
1198
|
-
|
|
1355
|
+
embeddingBuffer,
|
|
1199
1356
|
message.replyToId,
|
|
1200
1357
|
message.isFromAgent ? 1 : 0,
|
|
1201
1358
|
message.hasMedia ? 1 : 0,
|
|
@@ -1203,7 +1360,8 @@ var MessageStore = class {
|
|
|
1203
1360
|
message.timestamp
|
|
1204
1361
|
);
|
|
1205
1362
|
if (this.vectorEnabled && embedding.length > 0 && message.text) {
|
|
1206
|
-
this.db.prepare(`
|
|
1363
|
+
this.db.prepare(`DELETE FROM tg_messages_vec WHERE id = ?`).run(message.id);
|
|
1364
|
+
this.db.prepare(`INSERT INTO tg_messages_vec (id, embedding) VALUES (?, ?)`).run(message.id, embeddingBuffer);
|
|
1207
1365
|
}
|
|
1208
1366
|
this.db.prepare(`UPDATE tg_chats SET last_message_at = ?, last_message_id = ? WHERE id = ?`).run(message.timestamp, message.id, message.chatId);
|
|
1209
1367
|
}
|
|
@@ -1571,8 +1729,8 @@ var HybridSearch = class {
|
|
|
1571
1729
|
const limit = options.limit ?? 10;
|
|
1572
1730
|
const vectorWeight = options.vectorWeight ?? 0.7;
|
|
1573
1731
|
const keywordWeight = options.keywordWeight ?? 0.3;
|
|
1574
|
-
const vectorResults = this.vectorEnabled ? this.vectorSearchKnowledge(queryEmbedding, limit *
|
|
1575
|
-
const keywordResults = this.keywordSearchKnowledge(query, limit *
|
|
1732
|
+
const vectorResults = this.vectorEnabled ? this.vectorSearchKnowledge(queryEmbedding, Math.ceil(limit * 1.5)) : [];
|
|
1733
|
+
const keywordResults = this.keywordSearchKnowledge(query, Math.ceil(limit * 1.5));
|
|
1576
1734
|
return this.mergeResults(vectorResults, keywordResults, vectorWeight, keywordWeight, limit);
|
|
1577
1735
|
}
|
|
1578
1736
|
/**
|
|
@@ -1582,22 +1740,27 @@ var HybridSearch = class {
|
|
|
1582
1740
|
const limit = options.limit ?? 10;
|
|
1583
1741
|
const vectorWeight = options.vectorWeight ?? 0.7;
|
|
1584
1742
|
const keywordWeight = options.keywordWeight ?? 0.3;
|
|
1585
|
-
const vectorResults = this.vectorEnabled ? this.vectorSearchMessages(queryEmbedding, limit *
|
|
1586
|
-
const keywordResults = this.keywordSearchMessages(
|
|
1743
|
+
const vectorResults = this.vectorEnabled ? this.vectorSearchMessages(queryEmbedding, Math.ceil(limit * 1.5), options.chatId) : [];
|
|
1744
|
+
const keywordResults = this.keywordSearchMessages(
|
|
1745
|
+
query,
|
|
1746
|
+
Math.ceil(limit * 1.5),
|
|
1747
|
+
options.chatId
|
|
1748
|
+
);
|
|
1587
1749
|
return this.mergeResults(vectorResults, keywordResults, vectorWeight, keywordWeight, limit);
|
|
1588
1750
|
}
|
|
1589
1751
|
vectorSearchKnowledge(embedding, limit) {
|
|
1590
1752
|
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1591
1753
|
try {
|
|
1592
|
-
const embeddingBuffer =
|
|
1754
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1593
1755
|
const rows = this.db.prepare(
|
|
1594
1756
|
`
|
|
1595
1757
|
SELECT kv.id, k.text, k.source, kv.distance
|
|
1596
|
-
FROM
|
|
1758
|
+
FROM (
|
|
1759
|
+
SELECT id, distance
|
|
1760
|
+
FROM knowledge_vec
|
|
1761
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1762
|
+
) kv
|
|
1597
1763
|
JOIN knowledge k ON k.id = kv.id
|
|
1598
|
-
WHERE kv.embedding MATCH ?
|
|
1599
|
-
ORDER BY kv.distance
|
|
1600
|
-
LIMIT ?
|
|
1601
1764
|
`
|
|
1602
1765
|
).all(embeddingBuffer, limit);
|
|
1603
1766
|
return rows.map((row) => ({
|
|
@@ -1605,7 +1768,6 @@ var HybridSearch = class {
|
|
|
1605
1768
|
text: row.text,
|
|
1606
1769
|
source: row.source,
|
|
1607
1770
|
score: 1 - row.distance,
|
|
1608
|
-
// Convert distance to similarity
|
|
1609
1771
|
vectorScore: 1 - row.distance
|
|
1610
1772
|
}));
|
|
1611
1773
|
} catch (error) {
|
|
@@ -1639,23 +1801,26 @@ var HybridSearch = class {
|
|
|
1639
1801
|
vectorSearchMessages(embedding, limit, chatId) {
|
|
1640
1802
|
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1641
1803
|
try {
|
|
1642
|
-
const embeddingBuffer =
|
|
1804
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1643
1805
|
const sql = chatId ? `
|
|
1644
1806
|
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1645
|
-
FROM
|
|
1807
|
+
FROM (
|
|
1808
|
+
SELECT id, distance
|
|
1809
|
+
FROM tg_messages_vec
|
|
1810
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1811
|
+
) mv
|
|
1646
1812
|
JOIN tg_messages m ON m.id = mv.id
|
|
1647
|
-
WHERE
|
|
1648
|
-
ORDER BY mv.distance
|
|
1649
|
-
LIMIT ?
|
|
1813
|
+
WHERE m.chat_id = ?
|
|
1650
1814
|
` : `
|
|
1651
1815
|
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1652
|
-
FROM
|
|
1816
|
+
FROM (
|
|
1817
|
+
SELECT id, distance
|
|
1818
|
+
FROM tg_messages_vec
|
|
1819
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1820
|
+
) mv
|
|
1653
1821
|
JOIN tg_messages m ON m.id = mv.id
|
|
1654
|
-
WHERE mv.embedding MATCH ?
|
|
1655
|
-
ORDER BY mv.distance
|
|
1656
|
-
LIMIT ?
|
|
1657
1822
|
`;
|
|
1658
|
-
const rows = chatId ? this.db.prepare(sql).all(embeddingBuffer,
|
|
1823
|
+
const rows = chatId ? this.db.prepare(sql).all(embeddingBuffer, limit, chatId) : this.db.prepare(sql).all(embeddingBuffer, limit);
|
|
1659
1824
|
return rows.map((row) => ({
|
|
1660
1825
|
id: row.id,
|
|
1661
1826
|
text: row.text ?? "",
|
|
@@ -1717,13 +1882,6 @@ var HybridSearch = class {
|
|
|
1717
1882
|
bm25ToScore(rank) {
|
|
1718
1883
|
return 1 / (1 + Math.abs(rank));
|
|
1719
1884
|
}
|
|
1720
|
-
/**
|
|
1721
|
-
* Serialize embedding (number[]) to Buffer for sqlite-vec
|
|
1722
|
-
*/
|
|
1723
|
-
serializeEmbedding(embedding) {
|
|
1724
|
-
const float32 = new Float32Array(embedding);
|
|
1725
|
-
return Buffer.from(float32.buffer);
|
|
1726
|
-
}
|
|
1727
1885
|
};
|
|
1728
1886
|
|
|
1729
1887
|
// src/memory/search/context.ts
|
|
@@ -1808,9 +1966,10 @@ var ContextBuilder = class {
|
|
|
1808
1966
|
// src/memory/index.ts
|
|
1809
1967
|
function initializeMemory(config) {
|
|
1810
1968
|
const db = getDatabase(config.database);
|
|
1811
|
-
const
|
|
1969
|
+
const rawEmbedder = createEmbeddingProvider(config.embeddings);
|
|
1812
1970
|
const vectorEnabled = db.isVectorSearchReady();
|
|
1813
1971
|
const database = db.getDb();
|
|
1972
|
+
const embedder = rawEmbedder.id === "noop" ? rawEmbedder : new CachedEmbeddingProvider(rawEmbedder, database);
|
|
1814
1973
|
return {
|
|
1815
1974
|
db: database,
|
|
1816
1975
|
embedder,
|
|
@@ -1838,11 +1997,11 @@ export {
|
|
|
1838
1997
|
NoopEmbeddingProvider,
|
|
1839
1998
|
AnthropicEmbeddingProvider,
|
|
1840
1999
|
LocalEmbeddingProvider,
|
|
2000
|
+
CachedEmbeddingProvider,
|
|
1841
2001
|
createEmbeddingProvider,
|
|
1842
2002
|
hashText,
|
|
1843
2003
|
serializeEmbedding,
|
|
1844
2004
|
deserializeEmbedding,
|
|
1845
|
-
embeddingToBlob,
|
|
1846
2005
|
KnowledgeIndexer,
|
|
1847
2006
|
SessionStore,
|
|
1848
2007
|
MessageStore,
|