teleton 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{chunk-WMIN6AGX.js → chunk-DAMFGHXV.js} +1 -1
- package/dist/{chunk-74MTM3AB.js → chunk-ILDG4OPK.js} +879 -365
- package/dist/{chunk-LJXYESJJ.js → chunk-LCMHAUNK.js} +7 -1
- package/dist/{chunk-TBIMVWQZ.js → chunk-QQW6KE7Q.js} +221 -75
- package/dist/{chunk-QMN6ZOA5.js → chunk-UYF4TT44.js} +11 -1
- package/dist/cli/index.js +9 -13
- package/dist/index.js +6 -6
- package/dist/{memory-OEOLPEXP.js → memory-ZXDAJBL6.js} +6 -6
- package/dist/{migrate-7XOO67O5.js → migrate-7OG67FXP.js} +4 -4
- package/dist/{scraper-DSAYK6QJ.js → scraper-KXRBQMVQ.js} +4 -4
- package/dist/{task-dependency-resolver-CWG6DTU4.js → task-dependency-resolver-S45DFI5C.js} +4 -2
- package/dist/{task-executor-MNI4VIZL.js → task-executor-AUTT3VAL.js} +1 -1
- package/dist/timeouts-ZAK6NELA.js +63 -0
- package/package.json +5 -1
|
@@ -15,6 +15,7 @@ var SCRAPER_PRE_SCROLL_MS = 4e3;
|
|
|
15
15
|
var SCRAPER_COLLECTION_SCROLL_MS = 200;
|
|
16
16
|
var SCRAPER_SCROLL_INCREMENT_PX = 250;
|
|
17
17
|
var SCRAPER_SCROLL_PADDING_PX = 500;
|
|
18
|
+
var SCRAPER_WINDOW_SCROLL_PX = 2e3;
|
|
18
19
|
var SCRAPER_MAX_SCROLL_ITERATIONS = 15;
|
|
19
20
|
var SCRAPER_COLLECTION_NAV_MS = 6e4;
|
|
20
21
|
var RETRY_DEFAULT_MAX_ATTEMPTS = 3;
|
|
@@ -25,6 +26,8 @@ var RETRY_BLOCKCHAIN_BASE_DELAY_MS = 2e3;
|
|
|
25
26
|
var RETRY_BLOCKCHAIN_MAX_DELAY_MS = 15e3;
|
|
26
27
|
var RETRY_BLOCKCHAIN_TIMEOUT_MS = 3e4;
|
|
27
28
|
var GRAMJS_RETRY_DELAY_MS = 1e3;
|
|
29
|
+
var TOOL_EXECUTION_TIMEOUT_MS = 9e4;
|
|
30
|
+
var SHUTDOWN_TIMEOUT_MS = 1e4;
|
|
28
31
|
|
|
29
32
|
export {
|
|
30
33
|
TTS_TIMEOUT_MS,
|
|
@@ -43,6 +46,7 @@ export {
|
|
|
43
46
|
SCRAPER_COLLECTION_SCROLL_MS,
|
|
44
47
|
SCRAPER_SCROLL_INCREMENT_PX,
|
|
45
48
|
SCRAPER_SCROLL_PADDING_PX,
|
|
49
|
+
SCRAPER_WINDOW_SCROLL_PX,
|
|
46
50
|
SCRAPER_MAX_SCROLL_ITERATIONS,
|
|
47
51
|
SCRAPER_COLLECTION_NAV_MS,
|
|
48
52
|
RETRY_DEFAULT_MAX_ATTEMPTS,
|
|
@@ -52,5 +56,7 @@ export {
|
|
|
52
56
|
RETRY_BLOCKCHAIN_BASE_DELAY_MS,
|
|
53
57
|
RETRY_BLOCKCHAIN_MAX_DELAY_MS,
|
|
54
58
|
RETRY_BLOCKCHAIN_TIMEOUT_MS,
|
|
55
|
-
GRAMJS_RETRY_DELAY_MS
|
|
59
|
+
GRAMJS_RETRY_DELAY_MS,
|
|
60
|
+
TOOL_EXECUTION_TIMEOUT_MS,
|
|
61
|
+
SHUTDOWN_TIMEOUT_MS
|
|
56
62
|
};
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import {
|
|
2
2
|
VOYAGE_API_URL,
|
|
3
3
|
fetchWithTimeout
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-DAMFGHXV.js";
|
|
5
5
|
import {
|
|
6
|
+
EMBEDDING_CACHE_EVICTION_INTERVAL,
|
|
7
|
+
EMBEDDING_CACHE_MAX_ENTRIES,
|
|
8
|
+
EMBEDDING_CACHE_TTL_DAYS,
|
|
6
9
|
KNOWLEDGE_CHUNK_OVERLAP,
|
|
7
10
|
KNOWLEDGE_CHUNK_SIZE,
|
|
8
11
|
SQLITE_CACHE_SIZE_KB,
|
|
9
12
|
SQLITE_MMAP_SIZE,
|
|
10
13
|
VOYAGE_BATCH_SIZE
|
|
11
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-UYF4TT44.js";
|
|
12
15
|
import {
|
|
13
16
|
TELETON_ROOT
|
|
14
17
|
} from "./chunk-EYWNOHMJ.js";
|
|
@@ -362,16 +365,16 @@ function ensureSchema(db) {
|
|
|
362
365
|
-- ============================================
|
|
363
366
|
|
|
364
367
|
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
365
|
-
hash TEXT
|
|
366
|
-
embedding TEXT NOT NULL,
|
|
368
|
+
hash TEXT NOT NULL,
|
|
367
369
|
model TEXT NOT NULL,
|
|
368
370
|
provider TEXT NOT NULL,
|
|
369
|
-
|
|
371
|
+
embedding BLOB NOT NULL,
|
|
372
|
+
dims INTEGER NOT NULL,
|
|
370
373
|
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
371
|
-
accessed_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
374
|
+
accessed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
375
|
+
PRIMARY KEY (hash, model, provider)
|
|
372
376
|
);
|
|
373
377
|
|
|
374
|
-
CREATE INDEX IF NOT EXISTS idx_embedding_cache_model ON embedding_cache(provider, model);
|
|
375
378
|
CREATE INDEX IF NOT EXISTS idx_embedding_cache_accessed ON embedding_cache(accessed_at);
|
|
376
379
|
|
|
377
380
|
-- =====================================================
|
|
@@ -416,7 +419,7 @@ function setSchemaVersion(db, version) {
|
|
|
416
419
|
`
|
|
417
420
|
).run(version);
|
|
418
421
|
}
|
|
419
|
-
var CURRENT_SCHEMA_VERSION = "1.
|
|
422
|
+
var CURRENT_SCHEMA_VERSION = "1.9.0";
|
|
420
423
|
function runMigrations(db) {
|
|
421
424
|
const currentVersion = getSchemaVersion(db);
|
|
422
425
|
if (!currentVersion || versionLessThan(currentVersion, "1.1.0")) {
|
|
@@ -500,6 +503,29 @@ function runMigrations(db) {
|
|
|
500
503
|
throw error;
|
|
501
504
|
}
|
|
502
505
|
}
|
|
506
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.9.0")) {
|
|
507
|
+
console.log("\u{1F504} Running migration 1.9.0: Upgrade embedding_cache to BLOB storage");
|
|
508
|
+
try {
|
|
509
|
+
db.exec(`DROP TABLE IF EXISTS embedding_cache`);
|
|
510
|
+
db.exec(`
|
|
511
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
512
|
+
hash TEXT NOT NULL,
|
|
513
|
+
model TEXT NOT NULL,
|
|
514
|
+
provider TEXT NOT NULL,
|
|
515
|
+
embedding BLOB NOT NULL,
|
|
516
|
+
dims INTEGER NOT NULL,
|
|
517
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
518
|
+
accessed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
519
|
+
PRIMARY KEY (hash, model, provider)
|
|
520
|
+
);
|
|
521
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_cache_accessed ON embedding_cache(accessed_at);
|
|
522
|
+
`);
|
|
523
|
+
console.log("\u2705 Migration 1.9.0 complete: embedding_cache upgraded to BLOB storage");
|
|
524
|
+
} catch (error) {
|
|
525
|
+
console.error("\u274C Migration 1.9.0 failed:", error);
|
|
526
|
+
throw error;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
503
529
|
setSchemaVersion(db, CURRENT_SCHEMA_VERSION);
|
|
504
530
|
}
|
|
505
531
|
|
|
@@ -691,6 +717,9 @@ function closeDatabase() {
|
|
|
691
717
|
}
|
|
692
718
|
}
|
|
693
719
|
|
|
720
|
+
// src/memory/embeddings/index.ts
|
|
721
|
+
import { createHash } from "crypto";
|
|
722
|
+
|
|
694
723
|
// src/memory/embeddings/provider.ts
|
|
695
724
|
var NoopEmbeddingProvider = class {
|
|
696
725
|
id = "noop";
|
|
@@ -762,32 +791,159 @@ var AnthropicEmbeddingProvider = class {
|
|
|
762
791
|
};
|
|
763
792
|
|
|
764
793
|
// src/memory/embeddings/local.ts
|
|
794
|
+
import { pipeline } from "@huggingface/transformers";
|
|
795
|
+
var extractorPromise = null;
|
|
796
|
+
function getExtractor(model) {
|
|
797
|
+
if (!extractorPromise) {
|
|
798
|
+
console.log(`\u{1F4E6} Loading local embedding model: ${model} \u2026`);
|
|
799
|
+
extractorPromise = pipeline("feature-extraction", model, {
|
|
800
|
+
dtype: "fp32"
|
|
801
|
+
}).then((ext) => {
|
|
802
|
+
console.log(`\u2705 Local embedding model ready`);
|
|
803
|
+
return ext;
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
return extractorPromise;
|
|
807
|
+
}
|
|
765
808
|
var LocalEmbeddingProvider = class {
|
|
766
809
|
id = "local";
|
|
767
810
|
model;
|
|
768
811
|
dimensions;
|
|
769
|
-
hasWarned = false;
|
|
770
812
|
constructor(config) {
|
|
771
|
-
this.model = config.model
|
|
813
|
+
this.model = config.model || "Xenova/all-MiniLM-L6-v2";
|
|
772
814
|
this.dimensions = 384;
|
|
773
815
|
}
|
|
774
816
|
async embedQuery(text) {
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
817
|
+
const extractor = await getExtractor(this.model);
|
|
818
|
+
const output = await extractor(text, { pooling: "mean", normalize: true });
|
|
819
|
+
return Array.from(output.data);
|
|
820
|
+
}
|
|
821
|
+
async embedBatch(texts) {
|
|
822
|
+
if (texts.length === 0) return [];
|
|
823
|
+
const extractor = await getExtractor(this.model);
|
|
824
|
+
const output = await extractor(texts, { pooling: "mean", normalize: true });
|
|
825
|
+
const data = output.data;
|
|
826
|
+
const dims = this.dimensions;
|
|
827
|
+
const results = [];
|
|
828
|
+
for (let i = 0; i < texts.length; i++) {
|
|
829
|
+
results.push(Array.from(data.slice(i * dims, (i + 1) * dims)));
|
|
780
830
|
}
|
|
781
|
-
return
|
|
831
|
+
return results;
|
|
832
|
+
}
|
|
833
|
+
};
|
|
834
|
+
|
|
835
|
+
// src/memory/embeddings/cached.ts
|
|
836
|
+
var CachedEmbeddingProvider = class {
|
|
837
|
+
constructor(inner, db) {
|
|
838
|
+
this.inner = inner;
|
|
839
|
+
this.db = db;
|
|
840
|
+
this.id = inner.id;
|
|
841
|
+
this.model = inner.model;
|
|
842
|
+
this.dimensions = inner.dimensions;
|
|
843
|
+
}
|
|
844
|
+
id;
|
|
845
|
+
model;
|
|
846
|
+
dimensions;
|
|
847
|
+
hits = 0;
|
|
848
|
+
misses = 0;
|
|
849
|
+
ops = 0;
|
|
850
|
+
cacheGet(hash) {
|
|
851
|
+
return this.db.prepare(
|
|
852
|
+
`SELECT embedding FROM embedding_cache WHERE hash = ? AND model = ? AND provider = ?`
|
|
853
|
+
).get(hash, this.model, this.id);
|
|
854
|
+
}
|
|
855
|
+
cachePut(hash, blob) {
|
|
856
|
+
this.db.prepare(
|
|
857
|
+
`INSERT OR REPLACE INTO embedding_cache (hash, embedding, model, provider, dims, created_at, accessed_at)
|
|
858
|
+
VALUES (?, ?, ?, ?, ?, unixepoch(), unixepoch())`
|
|
859
|
+
).run(hash, blob, this.model, this.id, this.dimensions);
|
|
860
|
+
}
|
|
861
|
+
cacheTouch(hash) {
|
|
862
|
+
this.db.prepare(
|
|
863
|
+
`UPDATE embedding_cache SET accessed_at = unixepoch() WHERE hash = ? AND model = ? AND provider = ?`
|
|
864
|
+
).run(hash, this.model, this.id);
|
|
865
|
+
}
|
|
866
|
+
async embedQuery(text) {
|
|
867
|
+
const hash = hashText(text);
|
|
868
|
+
const row = this.cacheGet(hash);
|
|
869
|
+
if (row) {
|
|
870
|
+
this.hits++;
|
|
871
|
+
this.cacheTouch(hash);
|
|
872
|
+
this.tick();
|
|
873
|
+
return deserializeEmbedding(row.embedding);
|
|
874
|
+
}
|
|
875
|
+
this.misses++;
|
|
876
|
+
const embedding = await this.inner.embedQuery(text);
|
|
877
|
+
this.cachePut(hash, serializeEmbedding(embedding));
|
|
878
|
+
this.tick();
|
|
879
|
+
return embedding;
|
|
782
880
|
}
|
|
783
881
|
async embedBatch(texts) {
|
|
784
|
-
if (
|
|
785
|
-
|
|
786
|
-
|
|
882
|
+
if (texts.length === 0) return [];
|
|
883
|
+
const hashes = texts.map(hashText);
|
|
884
|
+
const results = new Array(texts.length).fill(null);
|
|
885
|
+
const missIndices = [];
|
|
886
|
+
const missTexts = [];
|
|
887
|
+
for (let i = 0; i < texts.length; i++) {
|
|
888
|
+
const row = this.cacheGet(hashes[i]);
|
|
889
|
+
if (row) {
|
|
890
|
+
this.hits++;
|
|
891
|
+
this.cacheTouch(hashes[i]);
|
|
892
|
+
results[i] = deserializeEmbedding(row.embedding);
|
|
893
|
+
} else {
|
|
894
|
+
this.misses++;
|
|
895
|
+
missIndices.push(i);
|
|
896
|
+
missTexts.push(texts[i]);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
if (missTexts.length > 0) {
|
|
900
|
+
const newEmbeddings = await this.inner.embedBatch(missTexts);
|
|
901
|
+
for (let j = 0; j < missIndices.length; j++) {
|
|
902
|
+
const idx = missIndices[j];
|
|
903
|
+
const embedding = newEmbeddings[j] ?? [];
|
|
904
|
+
results[idx] = embedding;
|
|
905
|
+
if (embedding.length > 0) {
|
|
906
|
+
this.cachePut(hashes[idx], serializeEmbedding(embedding));
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
this.ops += texts.length;
|
|
911
|
+
this.maybeEvict();
|
|
912
|
+
this.maybeLogStats();
|
|
913
|
+
return results;
|
|
914
|
+
}
|
|
915
|
+
tick() {
|
|
916
|
+
this.ops++;
|
|
917
|
+
this.maybeEvict();
|
|
918
|
+
this.maybeLogStats();
|
|
919
|
+
}
|
|
920
|
+
maybeLogStats() {
|
|
921
|
+
const total = this.hits + this.misses;
|
|
922
|
+
if (total > 0 && total % 100 === 0) {
|
|
923
|
+
const rate = (this.hits / total * 100).toFixed(0);
|
|
924
|
+
console.log(
|
|
925
|
+
`\u{1F4CA} Embedding cache: ${this.hits} hits, ${this.misses} misses (${rate}% hit rate)`
|
|
787
926
|
);
|
|
788
|
-
this.hasWarned = true;
|
|
789
927
|
}
|
|
790
|
-
|
|
928
|
+
}
|
|
929
|
+
maybeEvict() {
|
|
930
|
+
if (this.ops % EMBEDDING_CACHE_EVICTION_INTERVAL !== 0) return;
|
|
931
|
+
try {
|
|
932
|
+
const cutoff = Math.floor(Date.now() / 1e3) - EMBEDDING_CACHE_TTL_DAYS * 86400;
|
|
933
|
+
this.db.prepare(`DELETE FROM embedding_cache WHERE accessed_at < ?`).run(cutoff);
|
|
934
|
+
const count = this.db.prepare(`SELECT COUNT(*) as cnt FROM embedding_cache`).get().cnt;
|
|
935
|
+
if (count > EMBEDDING_CACHE_MAX_ENTRIES) {
|
|
936
|
+
const toDelete = Math.ceil(count * 0.1);
|
|
937
|
+
this.db.prepare(
|
|
938
|
+
`DELETE FROM embedding_cache WHERE (hash, model, provider) IN (
|
|
939
|
+
SELECT hash, model, provider FROM embedding_cache ORDER BY accessed_at ASC LIMIT ?
|
|
940
|
+
)`
|
|
941
|
+
).run(toDelete);
|
|
942
|
+
console.log(`\u{1F9F9} Embedding cache eviction: removed ${toDelete} entries (${count} total)`);
|
|
943
|
+
}
|
|
944
|
+
} catch (err) {
|
|
945
|
+
console.warn("\u26A0\uFE0F Embedding cache eviction error:", err);
|
|
946
|
+
}
|
|
791
947
|
}
|
|
792
948
|
};
|
|
793
949
|
|
|
@@ -813,27 +969,22 @@ function createEmbeddingProvider(config) {
|
|
|
813
969
|
}
|
|
814
970
|
}
|
|
815
971
|
function hashText(text) {
|
|
816
|
-
|
|
817
|
-
for (let i = 0; i < text.length; i++) {
|
|
818
|
-
const char = text.charCodeAt(i);
|
|
819
|
-
hash = (hash << 5) - hash + char;
|
|
820
|
-
hash = hash & hash;
|
|
821
|
-
}
|
|
822
|
-
return Math.abs(hash).toString(36);
|
|
972
|
+
return createHash("sha256").update(text).digest("hex");
|
|
823
973
|
}
|
|
824
974
|
function serializeEmbedding(embedding) {
|
|
825
|
-
return
|
|
975
|
+
return Buffer.from(new Float32Array(embedding).buffer);
|
|
826
976
|
}
|
|
827
977
|
function deserializeEmbedding(data) {
|
|
828
978
|
try {
|
|
979
|
+
if (Buffer.isBuffer(data)) {
|
|
980
|
+
const floats = new Float32Array(data.buffer, data.byteOffset, data.byteLength / 4);
|
|
981
|
+
return Array.from(floats);
|
|
982
|
+
}
|
|
829
983
|
return JSON.parse(data);
|
|
830
984
|
} catch {
|
|
831
985
|
return [];
|
|
832
986
|
}
|
|
833
987
|
}
|
|
834
|
-
function embeddingToBlob(embedding) {
|
|
835
|
-
return Buffer.from(new Float32Array(embedding).buffer);
|
|
836
|
-
}
|
|
837
988
|
|
|
838
989
|
// src/memory/agent/knowledge.ts
|
|
839
990
|
import { readFileSync, existsSync as existsSync3, readdirSync, statSync } from "fs";
|
|
@@ -876,6 +1027,13 @@ var KnowledgeIndexer = class {
|
|
|
876
1027
|
if (existing?.hash === fileHash) {
|
|
877
1028
|
return false;
|
|
878
1029
|
}
|
|
1030
|
+
if (this.vectorEnabled) {
|
|
1031
|
+
this.db.prepare(
|
|
1032
|
+
`DELETE FROM knowledge_vec WHERE id IN (
|
|
1033
|
+
SELECT id FROM knowledge WHERE path = ? AND source = 'memory'
|
|
1034
|
+
)`
|
|
1035
|
+
).run(relPath);
|
|
1036
|
+
}
|
|
879
1037
|
this.db.prepare(`DELETE FROM knowledge WHERE path = ? AND source = 'memory'`).run(relPath);
|
|
880
1038
|
const chunks = this.chunkMarkdown(content, relPath);
|
|
881
1039
|
const texts = chunks.map((c) => c.text);
|
|
@@ -898,7 +1056,7 @@ var KnowledgeIndexer = class {
|
|
|
898
1056
|
chunk.hash
|
|
899
1057
|
);
|
|
900
1058
|
if (insertVec && embedding.length > 0) {
|
|
901
|
-
insertVec.run(chunk.id,
|
|
1059
|
+
insertVec.run(chunk.id, serializeEmbedding(embedding));
|
|
902
1060
|
}
|
|
903
1061
|
}
|
|
904
1062
|
return true;
|
|
@@ -1095,7 +1253,7 @@ var SessionStore = class {
|
|
|
1095
1253
|
const knowledgeId = `session:${sessionId}`;
|
|
1096
1254
|
const text = `Session from ${session.startedAt.toISOString()}:
|
|
1097
1255
|
${session.summary}`;
|
|
1098
|
-
const hash =
|
|
1256
|
+
const hash = hashText(text);
|
|
1099
1257
|
let embedding = null;
|
|
1100
1258
|
if (this.vectorEnabled) {
|
|
1101
1259
|
embedding = await this.embedder.embedQuery(text);
|
|
@@ -1111,7 +1269,7 @@ ${session.summary}`;
|
|
|
1111
1269
|
`
|
|
1112
1270
|
).run(knowledgeId, sessionId, text, hash);
|
|
1113
1271
|
if (embedding && this.vectorEnabled) {
|
|
1114
|
-
const embeddingBuffer =
|
|
1272
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1115
1273
|
const rowid = this.db.prepare(`SELECT rowid FROM knowledge WHERE id = ?`).get(knowledgeId);
|
|
1116
1274
|
this.db.prepare(
|
|
1117
1275
|
`
|
|
@@ -1130,21 +1288,12 @@ ${session.summary}`;
|
|
|
1130
1288
|
* Delete a session
|
|
1131
1289
|
*/
|
|
1132
1290
|
deleteSession(sessionId) {
|
|
1133
|
-
|
|
1134
|
-
this.
|
|
1135
|
-
|
|
1136
|
-
hashText(text) {
|
|
1137
|
-
let hash = 0;
|
|
1138
|
-
for (let i = 0; i < text.length; i++) {
|
|
1139
|
-
const char = text.charCodeAt(i);
|
|
1140
|
-
hash = (hash << 5) - hash + char;
|
|
1141
|
-
hash = hash & hash;
|
|
1291
|
+
const knowledgeId = `session:${sessionId}`;
|
|
1292
|
+
if (this.vectorEnabled) {
|
|
1293
|
+
this.db.prepare(`DELETE FROM knowledge_vec WHERE id = ?`).run(knowledgeId);
|
|
1142
1294
|
}
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
serializeEmbedding(embedding) {
|
|
1146
|
-
const float32 = new Float32Array(embedding);
|
|
1147
|
-
return Buffer.from(float32.buffer);
|
|
1295
|
+
this.db.prepare(`DELETE FROM sessions WHERE id = ?`).run(sessionId);
|
|
1296
|
+
this.db.prepare(`DELETE FROM knowledge WHERE id = ?`).run(knowledgeId);
|
|
1148
1297
|
}
|
|
1149
1298
|
};
|
|
1150
1299
|
|
|
@@ -1203,7 +1352,7 @@ var MessageStore = class {
|
|
|
1203
1352
|
message.timestamp
|
|
1204
1353
|
);
|
|
1205
1354
|
if (this.vectorEnabled && embedding.length > 0 && message.text) {
|
|
1206
|
-
this.db.prepare(`INSERT OR REPLACE INTO tg_messages_vec (id, embedding) VALUES (?, ?)`).run(message.id,
|
|
1355
|
+
this.db.prepare(`INSERT OR REPLACE INTO tg_messages_vec (id, embedding) VALUES (?, ?)`).run(message.id, serializeEmbedding(embedding));
|
|
1207
1356
|
}
|
|
1208
1357
|
this.db.prepare(`UPDATE tg_chats SET last_message_at = ?, last_message_id = ? WHERE id = ?`).run(message.timestamp, message.id, message.chatId);
|
|
1209
1358
|
}
|
|
@@ -1589,15 +1738,16 @@ var HybridSearch = class {
|
|
|
1589
1738
|
vectorSearchKnowledge(embedding, limit) {
|
|
1590
1739
|
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1591
1740
|
try {
|
|
1592
|
-
const embeddingBuffer =
|
|
1741
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1593
1742
|
const rows = this.db.prepare(
|
|
1594
1743
|
`
|
|
1595
1744
|
SELECT kv.id, k.text, k.source, kv.distance
|
|
1596
|
-
FROM
|
|
1745
|
+
FROM (
|
|
1746
|
+
SELECT id, distance
|
|
1747
|
+
FROM knowledge_vec
|
|
1748
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1749
|
+
) kv
|
|
1597
1750
|
JOIN knowledge k ON k.id = kv.id
|
|
1598
|
-
WHERE kv.embedding MATCH ?
|
|
1599
|
-
ORDER BY kv.distance
|
|
1600
|
-
LIMIT ?
|
|
1601
1751
|
`
|
|
1602
1752
|
).all(embeddingBuffer, limit);
|
|
1603
1753
|
return rows.map((row) => ({
|
|
@@ -1605,7 +1755,6 @@ var HybridSearch = class {
|
|
|
1605
1755
|
text: row.text,
|
|
1606
1756
|
source: row.source,
|
|
1607
1757
|
score: 1 - row.distance,
|
|
1608
|
-
// Convert distance to similarity
|
|
1609
1758
|
vectorScore: 1 - row.distance
|
|
1610
1759
|
}));
|
|
1611
1760
|
} catch (error) {
|
|
@@ -1639,23 +1788,26 @@ var HybridSearch = class {
|
|
|
1639
1788
|
vectorSearchMessages(embedding, limit, chatId) {
|
|
1640
1789
|
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1641
1790
|
try {
|
|
1642
|
-
const embeddingBuffer =
|
|
1791
|
+
const embeddingBuffer = serializeEmbedding(embedding);
|
|
1643
1792
|
const sql = chatId ? `
|
|
1644
1793
|
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1645
|
-
FROM
|
|
1794
|
+
FROM (
|
|
1795
|
+
SELECT id, distance
|
|
1796
|
+
FROM tg_messages_vec
|
|
1797
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1798
|
+
) mv
|
|
1646
1799
|
JOIN tg_messages m ON m.id = mv.id
|
|
1647
|
-
WHERE
|
|
1648
|
-
ORDER BY mv.distance
|
|
1649
|
-
LIMIT ?
|
|
1800
|
+
WHERE m.chat_id = ?
|
|
1650
1801
|
` : `
|
|
1651
1802
|
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1652
|
-
FROM
|
|
1803
|
+
FROM (
|
|
1804
|
+
SELECT id, distance
|
|
1805
|
+
FROM tg_messages_vec
|
|
1806
|
+
WHERE embedding MATCH ? AND k = ?
|
|
1807
|
+
) mv
|
|
1653
1808
|
JOIN tg_messages m ON m.id = mv.id
|
|
1654
|
-
WHERE mv.embedding MATCH ?
|
|
1655
|
-
ORDER BY mv.distance
|
|
1656
|
-
LIMIT ?
|
|
1657
1809
|
`;
|
|
1658
|
-
const rows = chatId ? this.db.prepare(sql).all(embeddingBuffer,
|
|
1810
|
+
const rows = chatId ? this.db.prepare(sql).all(embeddingBuffer, limit, chatId) : this.db.prepare(sql).all(embeddingBuffer, limit);
|
|
1659
1811
|
return rows.map((row) => ({
|
|
1660
1812
|
id: row.id,
|
|
1661
1813
|
text: row.text ?? "",
|
|
@@ -1717,13 +1869,6 @@ var HybridSearch = class {
|
|
|
1717
1869
|
bm25ToScore(rank) {
|
|
1718
1870
|
return 1 / (1 + Math.abs(rank));
|
|
1719
1871
|
}
|
|
1720
|
-
/**
|
|
1721
|
-
* Serialize embedding (number[]) to Buffer for sqlite-vec
|
|
1722
|
-
*/
|
|
1723
|
-
serializeEmbedding(embedding) {
|
|
1724
|
-
const float32 = new Float32Array(embedding);
|
|
1725
|
-
return Buffer.from(float32.buffer);
|
|
1726
|
-
}
|
|
1727
1872
|
};
|
|
1728
1873
|
|
|
1729
1874
|
// src/memory/search/context.ts
|
|
@@ -1808,9 +1953,10 @@ var ContextBuilder = class {
|
|
|
1808
1953
|
// src/memory/index.ts
|
|
1809
1954
|
function initializeMemory(config) {
|
|
1810
1955
|
const db = getDatabase(config.database);
|
|
1811
|
-
const
|
|
1956
|
+
const rawEmbedder = createEmbeddingProvider(config.embeddings);
|
|
1812
1957
|
const vectorEnabled = db.isVectorSearchReady();
|
|
1813
1958
|
const database = db.getDb();
|
|
1959
|
+
const embedder = rawEmbedder.id === "noop" ? rawEmbedder : new CachedEmbeddingProvider(rawEmbedder, database);
|
|
1814
1960
|
return {
|
|
1815
1961
|
db: database,
|
|
1816
1962
|
embedder,
|
|
@@ -1838,11 +1984,11 @@ export {
|
|
|
1838
1984
|
NoopEmbeddingProvider,
|
|
1839
1985
|
AnthropicEmbeddingProvider,
|
|
1840
1986
|
LocalEmbeddingProvider,
|
|
1987
|
+
CachedEmbeddingProvider,
|
|
1841
1988
|
createEmbeddingProvider,
|
|
1842
1989
|
hashText,
|
|
1843
1990
|
serializeEmbedding,
|
|
1844
1991
|
deserializeEmbedding,
|
|
1845
|
-
embeddingToBlob,
|
|
1846
1992
|
KnowledgeIndexer,
|
|
1847
1993
|
SessionStore,
|
|
1848
1994
|
MessageStore,
|
|
@@ -29,6 +29,11 @@ var KNOWLEDGE_CHUNK_OVERLAP = 50;
|
|
|
29
29
|
var PAYMENT_TOLERANCE_RATIO = 0.99;
|
|
30
30
|
var TELEGRAM_CONNECTION_RETRIES = 5;
|
|
31
31
|
var TELEGRAM_FLOOD_SLEEP_THRESHOLD = 60;
|
|
32
|
+
var MAX_DEPENDENTS_PER_TASK = 10;
|
|
33
|
+
var EMBEDDING_CACHE_MAX_ENTRIES = 5e4;
|
|
34
|
+
var EMBEDDING_CACHE_TTL_DAYS = 60;
|
|
35
|
+
var EMBEDDING_CACHE_EVICTION_INTERVAL = 1e3;
|
|
36
|
+
var MAX_WRITE_SIZE = 50 * 1024 * 1024;
|
|
32
37
|
|
|
33
38
|
export {
|
|
34
39
|
MAX_TOOL_RESULT_SIZE,
|
|
@@ -60,5 +65,10 @@ export {
|
|
|
60
65
|
KNOWLEDGE_CHUNK_OVERLAP,
|
|
61
66
|
PAYMENT_TOLERANCE_RATIO,
|
|
62
67
|
TELEGRAM_CONNECTION_RETRIES,
|
|
63
|
-
TELEGRAM_FLOOD_SLEEP_THRESHOLD
|
|
68
|
+
TELEGRAM_FLOOD_SLEEP_THRESHOLD,
|
|
69
|
+
MAX_DEPENDENTS_PER_TASK,
|
|
70
|
+
EMBEDDING_CACHE_MAX_ENTRIES,
|
|
71
|
+
EMBEDDING_CACHE_TTL_DAYS,
|
|
72
|
+
EMBEDDING_CACHE_EVICTION_INTERVAL,
|
|
73
|
+
MAX_WRITE_SIZE
|
|
64
74
|
};
|
package/dist/cli/index.js
CHANGED
|
@@ -17,25 +17,25 @@ import {
|
|
|
17
17
|
saveWallet,
|
|
18
18
|
validateApiKeyFormat,
|
|
19
19
|
walletExists
|
|
20
|
-
} from "../chunk-
|
|
20
|
+
} from "../chunk-ILDG4OPK.js";
|
|
21
|
+
import "../chunk-U7FQYCBQ.js";
|
|
21
22
|
import "../chunk-DUW5VBAZ.js";
|
|
22
23
|
import "../chunk-OQGNS2FV.js";
|
|
23
|
-
import "../chunk-
|
|
24
|
+
import "../chunk-QQW6KE7Q.js";
|
|
24
25
|
import {
|
|
25
26
|
fetchWithTimeout
|
|
26
|
-
} from "../chunk-
|
|
27
|
+
} from "../chunk-DAMFGHXV.js";
|
|
27
28
|
import {
|
|
28
29
|
TELEGRAM_MAX_MESSAGE_LENGTH
|
|
29
|
-
} from "../chunk-
|
|
30
|
-
import {
|
|
31
|
-
ONBOARDING_PROMPT_TIMEOUT_MS
|
|
32
|
-
} from "../chunk-LJXYESJJ.js";
|
|
30
|
+
} from "../chunk-UYF4TT44.js";
|
|
33
31
|
import {
|
|
34
32
|
TELETON_ROOT
|
|
35
33
|
} from "../chunk-EYWNOHMJ.js";
|
|
34
|
+
import {
|
|
35
|
+
ONBOARDING_PROMPT_TIMEOUT_MS
|
|
36
|
+
} from "../chunk-LCMHAUNK.js";
|
|
36
37
|
import "../chunk-E2NXSWOS.js";
|
|
37
38
|
import "../chunk-B2PRMXOH.js";
|
|
38
|
-
import "../chunk-U7FQYCBQ.js";
|
|
39
39
|
import "../chunk-QGM4M3NI.js";
|
|
40
40
|
|
|
41
41
|
// src/cli/index.ts
|
|
@@ -523,11 +523,7 @@ Get it at: ${providerMeta.consoleUrl}`,
|
|
|
523
523
|
sellMinFloorPercent = parseInt(sellInput, 10);
|
|
524
524
|
}
|
|
525
525
|
}
|
|
526
|
-
|
|
527
|
-
message: "Set up a Telegram bot for deal confirmations? (inline buttons)",
|
|
528
|
-
initialValue: true
|
|
529
|
-
}) : false;
|
|
530
|
-
if (setupBot) {
|
|
526
|
+
if (dealsEnabled) {
|
|
531
527
|
prompter.note(
|
|
532
528
|
"Create a bot with @BotFather on Telegram:\n1. Send /newbot and follow the instructions\n2. Copy the bot token\n3. Enable inline mode: /setinline on the bot",
|
|
533
529
|
"Deals Bot"
|
package/dist/index.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import {
|
|
2
2
|
TonnetApp,
|
|
3
3
|
main
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-ILDG4OPK.js";
|
|
5
|
+
import "./chunk-U7FQYCBQ.js";
|
|
5
6
|
import "./chunk-DUW5VBAZ.js";
|
|
6
7
|
import "./chunk-OQGNS2FV.js";
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
10
|
-
import "./chunk-LJXYESJJ.js";
|
|
8
|
+
import "./chunk-QQW6KE7Q.js";
|
|
9
|
+
import "./chunk-DAMFGHXV.js";
|
|
10
|
+
import "./chunk-UYF4TT44.js";
|
|
11
11
|
import "./chunk-EYWNOHMJ.js";
|
|
12
|
+
import "./chunk-LCMHAUNK.js";
|
|
12
13
|
import "./chunk-E2NXSWOS.js";
|
|
13
14
|
import "./chunk-B2PRMXOH.js";
|
|
14
|
-
import "./chunk-U7FQYCBQ.js";
|
|
15
15
|
import "./chunk-QGM4M3NI.js";
|
|
16
16
|
export {
|
|
17
17
|
TonnetApp,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AnthropicEmbeddingProvider,
|
|
3
3
|
CURRENT_SCHEMA_VERSION,
|
|
4
|
+
CachedEmbeddingProvider,
|
|
4
5
|
ChatStore,
|
|
5
6
|
ContextBuilder,
|
|
6
7
|
HybridSearch,
|
|
@@ -14,7 +15,6 @@ import {
|
|
|
14
15
|
closeDatabase,
|
|
15
16
|
createEmbeddingProvider,
|
|
16
17
|
deserializeEmbedding,
|
|
17
|
-
embeddingToBlob,
|
|
18
18
|
ensureSchema,
|
|
19
19
|
ensureVectorTables,
|
|
20
20
|
getDatabase,
|
|
@@ -24,11 +24,11 @@ import {
|
|
|
24
24
|
runMigrations,
|
|
25
25
|
serializeEmbedding,
|
|
26
26
|
setSchemaVersion
|
|
27
|
-
} from "./chunk-
|
|
28
|
-
import "./chunk-
|
|
29
|
-
import "./chunk-
|
|
30
|
-
import "./chunk-LJXYESJJ.js";
|
|
27
|
+
} from "./chunk-QQW6KE7Q.js";
|
|
28
|
+
import "./chunk-DAMFGHXV.js";
|
|
29
|
+
import "./chunk-UYF4TT44.js";
|
|
31
30
|
import "./chunk-EYWNOHMJ.js";
|
|
31
|
+
import "./chunk-LCMHAUNK.js";
|
|
32
32
|
import {
|
|
33
33
|
TaskStore,
|
|
34
34
|
getTaskStore
|
|
@@ -37,6 +37,7 @@ import "./chunk-QGM4M3NI.js";
|
|
|
37
37
|
export {
|
|
38
38
|
AnthropicEmbeddingProvider,
|
|
39
39
|
CURRENT_SCHEMA_VERSION,
|
|
40
|
+
CachedEmbeddingProvider,
|
|
40
41
|
ChatStore,
|
|
41
42
|
ContextBuilder,
|
|
42
43
|
HybridSearch,
|
|
@@ -51,7 +52,6 @@ export {
|
|
|
51
52
|
closeDatabase,
|
|
52
53
|
createEmbeddingProvider,
|
|
53
54
|
deserializeEmbedding,
|
|
54
|
-
embeddingToBlob,
|
|
55
55
|
ensureSchema,
|
|
56
56
|
ensureVectorTables,
|
|
57
57
|
getDatabase,
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getDatabase
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-LJXYESJJ.js";
|
|
3
|
+
} from "./chunk-QQW6KE7Q.js";
|
|
4
|
+
import "./chunk-DAMFGHXV.js";
|
|
5
|
+
import "./chunk-UYF4TT44.js";
|
|
7
6
|
import {
|
|
8
7
|
TELETON_ROOT
|
|
9
8
|
} from "./chunk-EYWNOHMJ.js";
|
|
9
|
+
import "./chunk-LCMHAUNK.js";
|
|
10
10
|
import "./chunk-E2NXSWOS.js";
|
|
11
11
|
import "./chunk-QGM4M3NI.js";
|
|
12
12
|
|
|
@@ -8,10 +8,11 @@ import {
|
|
|
8
8
|
} from "./chunk-DUW5VBAZ.js";
|
|
9
9
|
import {
|
|
10
10
|
MARKETAPP_BASE_URL
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-DAMFGHXV.js";
|
|
12
12
|
import {
|
|
13
13
|
SCRAPER_PARALLEL_WORKERS
|
|
14
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-UYF4TT44.js";
|
|
15
|
+
import "./chunk-EYWNOHMJ.js";
|
|
15
16
|
import {
|
|
16
17
|
BROWSER_NAVIGATION_TIMEOUT_MS,
|
|
17
18
|
SCRAPER_COLLECTION_NAV_MS,
|
|
@@ -26,8 +27,7 @@ import {
|
|
|
26
27
|
SCRAPER_SCROLL_INCREMENT_PX,
|
|
27
28
|
SCRAPER_SCROLL_PADDING_PX,
|
|
28
29
|
SCRAPER_SCROLL_STEP_MS
|
|
29
|
-
} from "./chunk-
|
|
30
|
-
import "./chunk-EYWNOHMJ.js";
|
|
30
|
+
} from "./chunk-LCMHAUNK.js";
|
|
31
31
|
import "./chunk-QGM4M3NI.js";
|
|
32
32
|
|
|
33
33
|
// src/market/scraper.ts
|