teleton 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +395 -0
- package/bin/teleton.js +2 -0
- package/dist/chunk-7NJ46ZIX.js +74 -0
- package/dist/chunk-UR2LQEKR.js +319 -0
- package/dist/chunk-WDUHRPGA.js +1930 -0
- package/dist/chunk-WXVHT6CI.js +18938 -0
- package/dist/chunk-XBGUNXF2.js +176 -0
- package/dist/cli/index.js +1055 -0
- package/dist/get-my-gifts-YKUHPRGS.js +8 -0
- package/dist/index.js +12 -0
- package/dist/memory-O5NYYWF3.js +60 -0
- package/dist/migrate-25RH22HJ.js +59 -0
- package/dist/paths-STCOKEXS.js +14 -0
- package/dist/scraper-DW5Z2AP5.js +377 -0
- package/dist/task-dependency-resolver-5I62EU67.js +133 -0
- package/dist/task-executor-ZMXWLMI7.js +144 -0
- package/dist/tasks-NUFMZNV5.js +8 -0
- package/package.json +85 -0
- package/src/templates/BOOTSTRAP.md +48 -0
- package/src/templates/IDENTITY.md +33 -0
- package/src/templates/MEMORY.md +34 -0
- package/src/templates/SOUL.md +43 -0
- package/src/templates/USER.md +36 -0
|
@@ -0,0 +1,1930 @@
|
|
|
1
|
+
// src/memory/database.ts
|
|
2
|
+
import Database from "better-sqlite3";
|
|
3
|
+
import { existsSync, mkdirSync } from "fs";
|
|
4
|
+
import { dirname } from "path";
|
|
5
|
+
import * as sqliteVec from "sqlite-vec";
|
|
6
|
+
|
|
7
|
+
// src/memory/schema.ts
|
|
8
|
+
function compareSemver(a, b) {
|
|
9
|
+
const parseVersion = (v) => {
|
|
10
|
+
const parts = v.split("-")[0].split(".").map(Number);
|
|
11
|
+
return {
|
|
12
|
+
major: parts[0] || 0,
|
|
13
|
+
minor: parts[1] || 0,
|
|
14
|
+
patch: parts[2] || 0
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
const va = parseVersion(a);
|
|
18
|
+
const vb = parseVersion(b);
|
|
19
|
+
if (va.major !== vb.major) return va.major < vb.major ? -1 : 1;
|
|
20
|
+
if (va.minor !== vb.minor) return va.minor < vb.minor ? -1 : 1;
|
|
21
|
+
if (va.patch !== vb.patch) return va.patch < vb.patch ? -1 : 1;
|
|
22
|
+
return 0;
|
|
23
|
+
}
|
|
24
|
+
function versionLessThan(a, b) {
|
|
25
|
+
return compareSemver(a, b) < 0;
|
|
26
|
+
}
|
|
27
|
+
function ensureSchema(db) {
|
|
28
|
+
db.exec(`
|
|
29
|
+
-- ============================================
|
|
30
|
+
-- METADATA
|
|
31
|
+
-- ============================================
|
|
32
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
33
|
+
key TEXT PRIMARY KEY,
|
|
34
|
+
value TEXT NOT NULL,
|
|
35
|
+
updated_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
-- ============================================
|
|
39
|
+
-- AGENT MEMORY (Knowledge Base)
|
|
40
|
+
-- ============================================
|
|
41
|
+
|
|
42
|
+
-- Knowledge chunks from MEMORY.md, memory/*.md, learned facts
|
|
43
|
+
CREATE TABLE IF NOT EXISTS knowledge (
|
|
44
|
+
id TEXT PRIMARY KEY,
|
|
45
|
+
source TEXT NOT NULL CHECK(source IN ('memory', 'session', 'learned')),
|
|
46
|
+
path TEXT,
|
|
47
|
+
text TEXT NOT NULL,
|
|
48
|
+
embedding TEXT,
|
|
49
|
+
start_line INTEGER,
|
|
50
|
+
end_line INTEGER,
|
|
51
|
+
hash TEXT NOT NULL,
|
|
52
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
53
|
+
updated_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_source ON knowledge(source);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_hash ON knowledge(hash);
|
|
58
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_updated ON knowledge(updated_at DESC);
|
|
59
|
+
|
|
60
|
+
-- Full-text search for knowledge
|
|
61
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts USING fts5(
|
|
62
|
+
text,
|
|
63
|
+
id UNINDEXED,
|
|
64
|
+
path UNINDEXED,
|
|
65
|
+
source UNINDEXED,
|
|
66
|
+
content='knowledge',
|
|
67
|
+
content_rowid='rowid'
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
-- FTS triggers
|
|
71
|
+
CREATE TRIGGER IF NOT EXISTS knowledge_fts_insert AFTER INSERT ON knowledge BEGIN
|
|
72
|
+
INSERT INTO knowledge_fts(rowid, text, id, path, source)
|
|
73
|
+
VALUES (new.rowid, new.text, new.id, new.path, new.source);
|
|
74
|
+
END;
|
|
75
|
+
|
|
76
|
+
CREATE TRIGGER IF NOT EXISTS knowledge_fts_delete AFTER DELETE ON knowledge BEGIN
|
|
77
|
+
DELETE FROM knowledge_fts WHERE rowid = old.rowid;
|
|
78
|
+
END;
|
|
79
|
+
|
|
80
|
+
CREATE TRIGGER IF NOT EXISTS knowledge_fts_update AFTER UPDATE ON knowledge BEGIN
|
|
81
|
+
DELETE FROM knowledge_fts WHERE rowid = old.rowid;
|
|
82
|
+
INSERT INTO knowledge_fts(rowid, text, id, path, source)
|
|
83
|
+
VALUES (new.rowid, new.text, new.id, new.path, new.source);
|
|
84
|
+
END;
|
|
85
|
+
|
|
86
|
+
-- Sessions/Conversations
|
|
87
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
88
|
+
id TEXT PRIMARY KEY, -- session_id (UUID)
|
|
89
|
+
chat_id TEXT UNIQUE NOT NULL, -- telegram:chat_id
|
|
90
|
+
started_at INTEGER NOT NULL, -- createdAt (Unix timestamp ms)
|
|
91
|
+
updated_at INTEGER NOT NULL, -- updatedAt (Unix timestamp ms)
|
|
92
|
+
ended_at INTEGER, -- Optional end time
|
|
93
|
+
summary TEXT, -- Session summary
|
|
94
|
+
message_count INTEGER DEFAULT 0, -- Number of messages
|
|
95
|
+
tokens_used INTEGER DEFAULT 0, -- Deprecated (use context_tokens)
|
|
96
|
+
last_message_id INTEGER, -- Last Telegram message ID
|
|
97
|
+
last_channel TEXT, -- Last channel (telegram/discord/etc)
|
|
98
|
+
last_to TEXT, -- Last recipient
|
|
99
|
+
context_tokens INTEGER, -- Current context size
|
|
100
|
+
model TEXT, -- Model used (claude-opus-4-5-20251101)
|
|
101
|
+
provider TEXT, -- Provider (anthropic)
|
|
102
|
+
last_reset_date TEXT -- YYYY-MM-DD of last daily reset
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_chat ON sessions(chat_id);
|
|
106
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_started ON sessions(started_at DESC);
|
|
107
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at DESC);
|
|
108
|
+
|
|
109
|
+
-- Tasks
|
|
110
|
+
CREATE TABLE IF NOT EXISTS tasks (
|
|
111
|
+
id TEXT PRIMARY KEY,
|
|
112
|
+
description TEXT NOT NULL,
|
|
113
|
+
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'in_progress', 'done', 'failed', 'cancelled')),
|
|
114
|
+
priority INTEGER DEFAULT 0,
|
|
115
|
+
created_by TEXT,
|
|
116
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
117
|
+
started_at INTEGER,
|
|
118
|
+
completed_at INTEGER,
|
|
119
|
+
result TEXT,
|
|
120
|
+
error TEXT,
|
|
121
|
+
scheduled_for INTEGER,
|
|
122
|
+
payload TEXT,
|
|
123
|
+
reason TEXT,
|
|
124
|
+
scheduled_message_id INTEGER
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
|
|
128
|
+
CREATE INDEX IF NOT EXISTS idx_tasks_priority ON tasks(priority DESC, created_at ASC);
|
|
129
|
+
CREATE INDEX IF NOT EXISTS idx_tasks_scheduled ON tasks(scheduled_for) WHERE scheduled_for IS NOT NULL;
|
|
130
|
+
|
|
131
|
+
-- Task Dependencies (for chained tasks)
|
|
132
|
+
CREATE TABLE IF NOT EXISTS task_dependencies (
|
|
133
|
+
task_id TEXT NOT NULL,
|
|
134
|
+
depends_on_task_id TEXT NOT NULL,
|
|
135
|
+
PRIMARY KEY (task_id, depends_on_task_id),
|
|
136
|
+
FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE,
|
|
137
|
+
FOREIGN KEY (depends_on_task_id) REFERENCES tasks(id) ON DELETE CASCADE
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
CREATE INDEX IF NOT EXISTS idx_task_deps_task ON task_dependencies(task_id);
|
|
141
|
+
CREATE INDEX IF NOT EXISTS idx_task_deps_parent ON task_dependencies(depends_on_task_id);
|
|
142
|
+
|
|
143
|
+
-- ============================================
|
|
144
|
+
-- TELEGRAM FEED
|
|
145
|
+
-- ============================================
|
|
146
|
+
|
|
147
|
+
-- Chats (groups, channels, DMs)
|
|
148
|
+
CREATE TABLE IF NOT EXISTS tg_chats (
|
|
149
|
+
id TEXT PRIMARY KEY,
|
|
150
|
+
type TEXT NOT NULL CHECK(type IN ('dm', 'group', 'channel')),
|
|
151
|
+
title TEXT,
|
|
152
|
+
username TEXT,
|
|
153
|
+
member_count INTEGER,
|
|
154
|
+
is_monitored INTEGER DEFAULT 1,
|
|
155
|
+
is_archived INTEGER DEFAULT 0,
|
|
156
|
+
last_message_id TEXT,
|
|
157
|
+
last_message_at INTEGER,
|
|
158
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
159
|
+
updated_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
CREATE INDEX IF NOT EXISTS idx_tg_chats_type ON tg_chats(type);
|
|
163
|
+
CREATE INDEX IF NOT EXISTS idx_tg_chats_monitored ON tg_chats(is_monitored, last_message_at DESC);
|
|
164
|
+
CREATE INDEX IF NOT EXISTS idx_tg_chats_username ON tg_chats(username) WHERE username IS NOT NULL;
|
|
165
|
+
|
|
166
|
+
-- Users
|
|
167
|
+
CREATE TABLE IF NOT EXISTS tg_users (
|
|
168
|
+
id TEXT PRIMARY KEY,
|
|
169
|
+
username TEXT,
|
|
170
|
+
first_name TEXT,
|
|
171
|
+
last_name TEXT,
|
|
172
|
+
is_bot INTEGER DEFAULT 0,
|
|
173
|
+
is_admin INTEGER DEFAULT 0,
|
|
174
|
+
is_allowed INTEGER DEFAULT 0,
|
|
175
|
+
first_seen_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
176
|
+
last_seen_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
177
|
+
message_count INTEGER DEFAULT 0
|
|
178
|
+
);
|
|
179
|
+
|
|
180
|
+
CREATE INDEX IF NOT EXISTS idx_tg_users_username ON tg_users(username) WHERE username IS NOT NULL;
|
|
181
|
+
CREATE INDEX IF NOT EXISTS idx_tg_users_admin ON tg_users(is_admin) WHERE is_admin = 1;
|
|
182
|
+
CREATE INDEX IF NOT EXISTS idx_tg_users_last_seen ON tg_users(last_seen_at DESC);
|
|
183
|
+
|
|
184
|
+
-- Messages
|
|
185
|
+
CREATE TABLE IF NOT EXISTS tg_messages (
|
|
186
|
+
id TEXT PRIMARY KEY,
|
|
187
|
+
chat_id TEXT NOT NULL,
|
|
188
|
+
sender_id TEXT,
|
|
189
|
+
text TEXT,
|
|
190
|
+
embedding TEXT,
|
|
191
|
+
reply_to_id TEXT,
|
|
192
|
+
forward_from_id TEXT,
|
|
193
|
+
is_from_agent INTEGER DEFAULT 0,
|
|
194
|
+
is_edited INTEGER DEFAULT 0,
|
|
195
|
+
has_media INTEGER DEFAULT 0,
|
|
196
|
+
media_type TEXT,
|
|
197
|
+
timestamp INTEGER NOT NULL,
|
|
198
|
+
indexed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
199
|
+
FOREIGN KEY (chat_id) REFERENCES tg_chats(id) ON DELETE CASCADE,
|
|
200
|
+
FOREIGN KEY (sender_id) REFERENCES tg_users(id) ON DELETE SET NULL
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
CREATE INDEX IF NOT EXISTS idx_tg_messages_chat ON tg_messages(chat_id, timestamp DESC);
|
|
204
|
+
CREATE INDEX IF NOT EXISTS idx_tg_messages_sender ON tg_messages(sender_id, timestamp DESC);
|
|
205
|
+
CREATE INDEX IF NOT EXISTS idx_tg_messages_timestamp ON tg_messages(timestamp DESC);
|
|
206
|
+
CREATE INDEX IF NOT EXISTS idx_tg_messages_reply ON tg_messages(reply_to_id) WHERE reply_to_id IS NOT NULL;
|
|
207
|
+
CREATE INDEX IF NOT EXISTS idx_tg_messages_from_agent ON tg_messages(is_from_agent, timestamp DESC) WHERE is_from_agent = 1;
|
|
208
|
+
|
|
209
|
+
-- Full-text search for messages
|
|
210
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS tg_messages_fts USING fts5(
|
|
211
|
+
text,
|
|
212
|
+
id UNINDEXED,
|
|
213
|
+
chat_id UNINDEXED,
|
|
214
|
+
sender_id UNINDEXED,
|
|
215
|
+
timestamp UNINDEXED,
|
|
216
|
+
content='tg_messages',
|
|
217
|
+
content_rowid='rowid'
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
-- FTS triggers for messages
|
|
221
|
+
CREATE TRIGGER IF NOT EXISTS tg_messages_fts_insert AFTER INSERT ON tg_messages BEGIN
|
|
222
|
+
INSERT INTO tg_messages_fts(rowid, text, id, chat_id, sender_id, timestamp)
|
|
223
|
+
VALUES (new.rowid, new.text, new.id, new.chat_id, new.sender_id, new.timestamp);
|
|
224
|
+
END;
|
|
225
|
+
|
|
226
|
+
CREATE TRIGGER IF NOT EXISTS tg_messages_fts_delete AFTER DELETE ON tg_messages BEGIN
|
|
227
|
+
DELETE FROM tg_messages_fts WHERE rowid = old.rowid;
|
|
228
|
+
END;
|
|
229
|
+
|
|
230
|
+
CREATE TRIGGER IF NOT EXISTS tg_messages_fts_update AFTER UPDATE ON tg_messages BEGIN
|
|
231
|
+
DELETE FROM tg_messages_fts WHERE rowid = old.rowid;
|
|
232
|
+
INSERT INTO tg_messages_fts(rowid, text, id, chat_id, sender_id, timestamp)
|
|
233
|
+
VALUES (new.rowid, new.text, new.id, new.chat_id, new.sender_id, new.timestamp);
|
|
234
|
+
END;
|
|
235
|
+
|
|
236
|
+
-- ============================================
|
|
237
|
+
-- EMBEDDING CACHE
|
|
238
|
+
-- ============================================
|
|
239
|
+
|
|
240
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
241
|
+
hash TEXT PRIMARY KEY,
|
|
242
|
+
embedding TEXT NOT NULL,
|
|
243
|
+
model TEXT NOT NULL,
|
|
244
|
+
provider TEXT NOT NULL,
|
|
245
|
+
dims INTEGER,
|
|
246
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
247
|
+
accessed_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
248
|
+
);
|
|
249
|
+
|
|
250
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_cache_model ON embedding_cache(provider, model);
|
|
251
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_cache_accessed ON embedding_cache(accessed_at);
|
|
252
|
+
|
|
253
|
+
-- =====================================================
|
|
254
|
+
-- JOURNAL (Trading & Business Operations)
|
|
255
|
+
-- =====================================================
|
|
256
|
+
|
|
257
|
+
CREATE TABLE IF NOT EXISTS journal (
|
|
258
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
259
|
+
timestamp INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
260
|
+
type TEXT NOT NULL CHECK(type IN ('trade', 'gift', 'middleman', 'kol')),
|
|
261
|
+
action TEXT NOT NULL,
|
|
262
|
+
asset_from TEXT,
|
|
263
|
+
asset_to TEXT,
|
|
264
|
+
amount_from REAL,
|
|
265
|
+
amount_to REAL,
|
|
266
|
+
price_ton REAL,
|
|
267
|
+
counterparty TEXT,
|
|
268
|
+
platform TEXT,
|
|
269
|
+
reasoning TEXT,
|
|
270
|
+
outcome TEXT CHECK(outcome IN ('pending', 'profit', 'loss', 'neutral', 'cancelled')),
|
|
271
|
+
pnl_ton REAL,
|
|
272
|
+
pnl_pct REAL,
|
|
273
|
+
tx_hash TEXT,
|
|
274
|
+
tool_used TEXT,
|
|
275
|
+
chat_id TEXT,
|
|
276
|
+
user_id INTEGER,
|
|
277
|
+
closed_at INTEGER,
|
|
278
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
CREATE INDEX IF NOT EXISTS idx_journal_type ON journal(type);
|
|
282
|
+
CREATE INDEX IF NOT EXISTS idx_journal_timestamp ON journal(timestamp DESC);
|
|
283
|
+
CREATE INDEX IF NOT EXISTS idx_journal_asset_from ON journal(asset_from);
|
|
284
|
+
CREATE INDEX IF NOT EXISTS idx_journal_outcome ON journal(outcome);
|
|
285
|
+
CREATE INDEX IF NOT EXISTS idx_journal_type_timestamp ON journal(type, timestamp DESC);
|
|
286
|
+
`);
|
|
287
|
+
}
|
|
288
|
+
function ensureVectorTables(db, dimensions) {
|
|
289
|
+
const existingDims = db.prepare(
|
|
290
|
+
`
|
|
291
|
+
SELECT sql FROM sqlite_master
|
|
292
|
+
WHERE type='table' AND name='knowledge_vec'
|
|
293
|
+
`
|
|
294
|
+
).get();
|
|
295
|
+
if (existingDims?.sql && !existingDims.sql.includes(`[${dimensions}]`)) {
|
|
296
|
+
db.exec(`DROP TABLE IF EXISTS knowledge_vec`);
|
|
297
|
+
db.exec(`DROP TABLE IF EXISTS tg_messages_vec`);
|
|
298
|
+
}
|
|
299
|
+
db.exec(`
|
|
300
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
|
|
301
|
+
id TEXT PRIMARY KEY,
|
|
302
|
+
embedding FLOAT[${dimensions}] distance_metric=cosine
|
|
303
|
+
);
|
|
304
|
+
|
|
305
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS tg_messages_vec USING vec0(
|
|
306
|
+
id TEXT PRIMARY KEY,
|
|
307
|
+
embedding FLOAT[${dimensions}] distance_metric=cosine
|
|
308
|
+
);
|
|
309
|
+
`);
|
|
310
|
+
}
|
|
311
|
+
function getSchemaVersion(db) {
|
|
312
|
+
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get();
|
|
313
|
+
return row?.value ?? null;
|
|
314
|
+
}
|
|
315
|
+
function setSchemaVersion(db, version) {
|
|
316
|
+
db.prepare(
|
|
317
|
+
`
|
|
318
|
+
INSERT INTO meta (key, value, updated_at)
|
|
319
|
+
VALUES ('schema_version', ?, unixepoch())
|
|
320
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
|
|
321
|
+
`
|
|
322
|
+
).run(version);
|
|
323
|
+
}
|
|
324
|
+
var CURRENT_SCHEMA_VERSION = "1.6.0";
|
|
325
|
+
function runMigrations(db) {
|
|
326
|
+
const currentVersion = getSchemaVersion(db);
|
|
327
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.1.0")) {
|
|
328
|
+
console.log("\u{1F4E6} Running migration: Adding scheduled task columns...");
|
|
329
|
+
try {
|
|
330
|
+
const tableExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='tasks'").get();
|
|
331
|
+
if (!tableExists) {
|
|
332
|
+
console.log(" Tasks table doesn't exist yet, skipping column migration");
|
|
333
|
+
setSchemaVersion(db, CURRENT_SCHEMA_VERSION);
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const tableInfo = db.prepare("PRAGMA table_info(tasks)").all();
|
|
337
|
+
const existingColumns = tableInfo.map((col) => col.name);
|
|
338
|
+
if (!existingColumns.includes("scheduled_for")) {
|
|
339
|
+
db.exec(`ALTER TABLE tasks ADD COLUMN scheduled_for INTEGER`);
|
|
340
|
+
}
|
|
341
|
+
if (!existingColumns.includes("payload")) {
|
|
342
|
+
db.exec(`ALTER TABLE tasks ADD COLUMN payload TEXT`);
|
|
343
|
+
}
|
|
344
|
+
if (!existingColumns.includes("reason")) {
|
|
345
|
+
db.exec(`ALTER TABLE tasks ADD COLUMN reason TEXT`);
|
|
346
|
+
}
|
|
347
|
+
if (!existingColumns.includes("scheduled_message_id")) {
|
|
348
|
+
db.exec(`ALTER TABLE tasks ADD COLUMN scheduled_message_id INTEGER`);
|
|
349
|
+
}
|
|
350
|
+
db.exec(
|
|
351
|
+
`CREATE INDEX IF NOT EXISTS idx_tasks_scheduled ON tasks(scheduled_for) WHERE scheduled_for IS NOT NULL`
|
|
352
|
+
);
|
|
353
|
+
db.exec(`
|
|
354
|
+
CREATE TABLE IF NOT EXISTS task_dependencies (
|
|
355
|
+
task_id TEXT NOT NULL,
|
|
356
|
+
depends_on_task_id TEXT NOT NULL,
|
|
357
|
+
PRIMARY KEY (task_id, depends_on_task_id),
|
|
358
|
+
FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE,
|
|
359
|
+
FOREIGN KEY (depends_on_task_id) REFERENCES tasks(id) ON DELETE CASCADE
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
CREATE INDEX IF NOT EXISTS idx_task_deps_task ON task_dependencies(task_id);
|
|
363
|
+
CREATE INDEX IF NOT EXISTS idx_task_deps_parent ON task_dependencies(depends_on_task_id);
|
|
364
|
+
`);
|
|
365
|
+
console.log("\u2705 Migration 1.1.0 complete: Scheduled tasks support added");
|
|
366
|
+
} catch (error) {
|
|
367
|
+
console.error("\u274C Migration 1.1.0 failed:", error);
|
|
368
|
+
throw error;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.2.0")) {
|
|
372
|
+
try {
|
|
373
|
+
console.log("\u{1F504} Running migration 1.2.0: Extend sessions table for SQLite backend");
|
|
374
|
+
const addColumnIfNotExists = (table, column, type) => {
|
|
375
|
+
try {
|
|
376
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
|
|
377
|
+
} catch (e) {
|
|
378
|
+
if (!e.message.includes("duplicate column name")) {
|
|
379
|
+
throw e;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
addColumnIfNotExists(
|
|
384
|
+
"sessions",
|
|
385
|
+
"updated_at",
|
|
386
|
+
"INTEGER NOT NULL DEFAULT (unixepoch() * 1000)"
|
|
387
|
+
);
|
|
388
|
+
addColumnIfNotExists("sessions", "last_message_id", "INTEGER");
|
|
389
|
+
addColumnIfNotExists("sessions", "last_channel", "TEXT");
|
|
390
|
+
addColumnIfNotExists("sessions", "last_to", "TEXT");
|
|
391
|
+
addColumnIfNotExists("sessions", "context_tokens", "INTEGER");
|
|
392
|
+
addColumnIfNotExists("sessions", "model", "TEXT");
|
|
393
|
+
addColumnIfNotExists("sessions", "provider", "TEXT");
|
|
394
|
+
addColumnIfNotExists("sessions", "last_reset_date", "TEXT");
|
|
395
|
+
const sessions = db.prepare("SELECT started_at FROM sessions LIMIT 1").all();
|
|
396
|
+
if (sessions.length > 0 && sessions[0].started_at < 1e12) {
|
|
397
|
+
db.exec(
|
|
398
|
+
"UPDATE sessions SET started_at = started_at * 1000 WHERE started_at < 1000000000000"
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at DESC)");
|
|
402
|
+
console.log("\u2705 Migration 1.2.0 complete: Sessions table extended");
|
|
403
|
+
} catch (error) {
|
|
404
|
+
console.error("\u274C Migration 1.2.0 failed:", error);
|
|
405
|
+
throw error;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.5.0")) {
|
|
409
|
+
try {
|
|
410
|
+
console.log("\u{1F504} Running migration 1.5.0: Add deals system for secure trading");
|
|
411
|
+
db.exec(`
|
|
412
|
+
CREATE TABLE IF NOT EXISTS deals (
|
|
413
|
+
id TEXT PRIMARY KEY,
|
|
414
|
+
status TEXT NOT NULL CHECK(status IN (
|
|
415
|
+
'proposed', 'accepted', 'payment_claimed', 'verified', 'completed',
|
|
416
|
+
'declined', 'expired', 'cancelled', 'failed'
|
|
417
|
+
)),
|
|
418
|
+
|
|
419
|
+
-- Parties
|
|
420
|
+
user_telegram_id INTEGER NOT NULL,
|
|
421
|
+
user_username TEXT,
|
|
422
|
+
chat_id TEXT NOT NULL,
|
|
423
|
+
proposal_message_id INTEGER,
|
|
424
|
+
|
|
425
|
+
-- What USER gives
|
|
426
|
+
user_gives_type TEXT NOT NULL CHECK(user_gives_type IN ('ton', 'gift')),
|
|
427
|
+
user_gives_ton_amount REAL,
|
|
428
|
+
user_gives_gift_id TEXT,
|
|
429
|
+
user_gives_gift_slug TEXT,
|
|
430
|
+
user_gives_value_ton REAL NOT NULL,
|
|
431
|
+
|
|
432
|
+
-- What AGENT gives
|
|
433
|
+
agent_gives_type TEXT NOT NULL CHECK(agent_gives_type IN ('ton', 'gift')),
|
|
434
|
+
agent_gives_ton_amount REAL,
|
|
435
|
+
agent_gives_gift_id TEXT,
|
|
436
|
+
agent_gives_gift_slug TEXT,
|
|
437
|
+
agent_gives_value_ton REAL NOT NULL,
|
|
438
|
+
|
|
439
|
+
-- Payment/Gift verification
|
|
440
|
+
user_payment_verified_at INTEGER,
|
|
441
|
+
user_payment_tx_hash TEXT,
|
|
442
|
+
user_payment_gift_msgid TEXT,
|
|
443
|
+
user_payment_wallet TEXT,
|
|
444
|
+
|
|
445
|
+
-- Agent send tracking
|
|
446
|
+
agent_sent_at INTEGER,
|
|
447
|
+
agent_sent_tx_hash TEXT,
|
|
448
|
+
agent_sent_gift_msgid TEXT,
|
|
449
|
+
|
|
450
|
+
-- Business logic
|
|
451
|
+
strategy_check TEXT,
|
|
452
|
+
profit_ton REAL,
|
|
453
|
+
|
|
454
|
+
-- Timestamps
|
|
455
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
456
|
+
expires_at INTEGER NOT NULL,
|
|
457
|
+
completed_at INTEGER,
|
|
458
|
+
|
|
459
|
+
notes TEXT
|
|
460
|
+
);
|
|
461
|
+
|
|
462
|
+
CREATE INDEX IF NOT EXISTS idx_deals_status ON deals(status);
|
|
463
|
+
CREATE INDEX IF NOT EXISTS idx_deals_user ON deals(user_telegram_id);
|
|
464
|
+
CREATE INDEX IF NOT EXISTS idx_deals_chat ON deals(chat_id);
|
|
465
|
+
CREATE INDEX IF NOT EXISTS idx_deals_expires ON deals(expires_at)
|
|
466
|
+
WHERE status IN ('proposed', 'accepted');
|
|
467
|
+
`);
|
|
468
|
+
console.log("\u2705 Migration 1.5.0 complete: Deals system added");
|
|
469
|
+
} catch (error) {
|
|
470
|
+
console.error("\u274C Migration 1.5.0 failed:", error);
|
|
471
|
+
throw error;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
if (!currentVersion || versionLessThan(currentVersion, "1.6.0")) {
|
|
475
|
+
try {
|
|
476
|
+
console.log("\u{1F504} Running migration 1.6.0: Add bot inline tracking + payment_claimed status");
|
|
477
|
+
const dealsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='deals'").get();
|
|
478
|
+
if (dealsExists) {
|
|
479
|
+
const tableSql = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='deals'").get()?.sql || "";
|
|
480
|
+
if (!tableSql.includes("payment_claimed")) {
|
|
481
|
+
db.exec(`
|
|
482
|
+
ALTER TABLE deals RENAME TO deals_old;
|
|
483
|
+
|
|
484
|
+
CREATE TABLE deals (
|
|
485
|
+
id TEXT PRIMARY KEY,
|
|
486
|
+
status TEXT NOT NULL CHECK(status IN (
|
|
487
|
+
'proposed', 'accepted', 'payment_claimed', 'verified', 'completed',
|
|
488
|
+
'declined', 'expired', 'cancelled', 'failed'
|
|
489
|
+
)),
|
|
490
|
+
user_telegram_id INTEGER NOT NULL,
|
|
491
|
+
user_username TEXT,
|
|
492
|
+
chat_id TEXT NOT NULL,
|
|
493
|
+
proposal_message_id INTEGER,
|
|
494
|
+
user_gives_type TEXT NOT NULL CHECK(user_gives_type IN ('ton', 'gift')),
|
|
495
|
+
user_gives_ton_amount REAL,
|
|
496
|
+
user_gives_gift_id TEXT,
|
|
497
|
+
user_gives_gift_slug TEXT,
|
|
498
|
+
user_gives_value_ton REAL NOT NULL,
|
|
499
|
+
agent_gives_type TEXT NOT NULL CHECK(agent_gives_type IN ('ton', 'gift')),
|
|
500
|
+
agent_gives_ton_amount REAL,
|
|
501
|
+
agent_gives_gift_id TEXT,
|
|
502
|
+
agent_gives_gift_slug TEXT,
|
|
503
|
+
agent_gives_value_ton REAL NOT NULL,
|
|
504
|
+
user_payment_verified_at INTEGER,
|
|
505
|
+
user_payment_tx_hash TEXT,
|
|
506
|
+
user_payment_gift_msgid TEXT,
|
|
507
|
+
user_payment_wallet TEXT,
|
|
508
|
+
agent_sent_at INTEGER,
|
|
509
|
+
agent_sent_tx_hash TEXT,
|
|
510
|
+
agent_sent_gift_msgid TEXT,
|
|
511
|
+
strategy_check TEXT,
|
|
512
|
+
profit_ton REAL,
|
|
513
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
514
|
+
expires_at INTEGER NOT NULL,
|
|
515
|
+
completed_at INTEGER,
|
|
516
|
+
notes TEXT,
|
|
517
|
+
inline_message_id TEXT,
|
|
518
|
+
payment_claimed_at INTEGER
|
|
519
|
+
);
|
|
520
|
+
|
|
521
|
+
INSERT INTO deals (
|
|
522
|
+
id, status, user_telegram_id, user_username, chat_id, proposal_message_id,
|
|
523
|
+
user_gives_type, user_gives_ton_amount, user_gives_gift_id, user_gives_gift_slug, user_gives_value_ton,
|
|
524
|
+
agent_gives_type, agent_gives_ton_amount, agent_gives_gift_id, agent_gives_gift_slug, agent_gives_value_ton,
|
|
525
|
+
user_payment_verified_at, user_payment_tx_hash, user_payment_gift_msgid, user_payment_wallet,
|
|
526
|
+
agent_sent_at, agent_sent_tx_hash, agent_sent_gift_msgid,
|
|
527
|
+
strategy_check, profit_ton, created_at, expires_at, completed_at, notes
|
|
528
|
+
)
|
|
529
|
+
SELECT
|
|
530
|
+
id, status, user_telegram_id, user_username, chat_id, proposal_message_id,
|
|
531
|
+
user_gives_type, user_gives_ton_amount, user_gives_gift_id, user_gives_gift_slug, user_gives_value_ton,
|
|
532
|
+
agent_gives_type, agent_gives_ton_amount, agent_gives_gift_id, agent_gives_gift_slug, agent_gives_value_ton,
|
|
533
|
+
user_payment_verified_at, user_payment_tx_hash, user_payment_gift_msgid, user_payment_wallet,
|
|
534
|
+
agent_sent_at, agent_sent_tx_hash, agent_sent_gift_msgid,
|
|
535
|
+
strategy_check, profit_ton, created_at, expires_at, completed_at, notes
|
|
536
|
+
FROM deals_old;
|
|
537
|
+
|
|
538
|
+
DROP TABLE deals_old;
|
|
539
|
+
|
|
540
|
+
CREATE INDEX IF NOT EXISTS idx_deals_status ON deals(status);
|
|
541
|
+
CREATE INDEX IF NOT EXISTS idx_deals_user ON deals(user_telegram_id);
|
|
542
|
+
CREATE INDEX IF NOT EXISTS idx_deals_chat ON deals(chat_id);
|
|
543
|
+
CREATE INDEX IF NOT EXISTS idx_deals_inline_msg ON deals(inline_message_id)
|
|
544
|
+
WHERE inline_message_id IS NOT NULL;
|
|
545
|
+
CREATE INDEX IF NOT EXISTS idx_deals_payment_claimed ON deals(payment_claimed_at)
|
|
546
|
+
WHERE payment_claimed_at IS NOT NULL;
|
|
547
|
+
`);
|
|
548
|
+
} else {
|
|
549
|
+
const columns = db.prepare(`PRAGMA table_info(deals)`).all();
|
|
550
|
+
const columnNames = columns.map((c) => c.name);
|
|
551
|
+
if (!columnNames.includes("inline_message_id")) {
|
|
552
|
+
db.exec(`ALTER TABLE deals ADD COLUMN inline_message_id TEXT`);
|
|
553
|
+
}
|
|
554
|
+
if (!columnNames.includes("payment_claimed_at")) {
|
|
555
|
+
db.exec(`ALTER TABLE deals ADD COLUMN payment_claimed_at INTEGER`);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
db.exec(`
|
|
560
|
+
CREATE TABLE IF NOT EXISTS user_trade_stats (
|
|
561
|
+
telegram_id INTEGER PRIMARY KEY,
|
|
562
|
+
username TEXT,
|
|
563
|
+
first_trade_at INTEGER DEFAULT (unixepoch()),
|
|
564
|
+
total_deals INTEGER DEFAULT 0,
|
|
565
|
+
completed_deals INTEGER DEFAULT 0,
|
|
566
|
+
declined_deals INTEGER DEFAULT 0,
|
|
567
|
+
total_ton_sent REAL DEFAULT 0,
|
|
568
|
+
total_ton_received REAL DEFAULT 0,
|
|
569
|
+
total_gifts_sent INTEGER DEFAULT 0,
|
|
570
|
+
total_gifts_received INTEGER DEFAULT 0,
|
|
571
|
+
last_deal_at INTEGER
|
|
572
|
+
);
|
|
573
|
+
`);
|
|
574
|
+
console.log("\u2705 Migration 1.6.0 complete: Bot inline tracking + payment_claimed added");
|
|
575
|
+
} catch (error) {
|
|
576
|
+
console.error("\u274C Migration 1.6.0 failed:", error);
|
|
577
|
+
throw error;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
setSchemaVersion(db, CURRENT_SCHEMA_VERSION);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// src/memory/database.ts
|
|
584
|
+
var MemoryDatabase = class {
|
|
585
|
+
db;
|
|
586
|
+
config;
|
|
587
|
+
vectorReady = false;
|
|
588
|
+
constructor(config) {
|
|
589
|
+
this.config = config;
|
|
590
|
+
const dir = dirname(config.path);
|
|
591
|
+
if (!existsSync(dir)) {
|
|
592
|
+
mkdirSync(dir, { recursive: true });
|
|
593
|
+
}
|
|
594
|
+
this.db = new Database(config.path, {
|
|
595
|
+
verbose: process.env.DEBUG_SQL ? console.log : void 0
|
|
596
|
+
});
|
|
597
|
+
this.db.pragma("journal_mode = WAL");
|
|
598
|
+
this.db.pragma("synchronous = NORMAL");
|
|
599
|
+
this.db.pragma("cache_size = -64000");
|
|
600
|
+
this.db.pragma("temp_store = MEMORY");
|
|
601
|
+
this.db.pragma("mmap_size = 30000000000");
|
|
602
|
+
this.db.pragma("foreign_keys = ON");
|
|
603
|
+
this.initialize();
|
|
604
|
+
}
|
|
605
|
+
initialize() {
|
|
606
|
+
let currentVersion = null;
|
|
607
|
+
try {
|
|
608
|
+
currentVersion = getSchemaVersion(this.db);
|
|
609
|
+
} catch {
|
|
610
|
+
currentVersion = null;
|
|
611
|
+
}
|
|
612
|
+
if (!currentVersion) {
|
|
613
|
+
ensureSchema(this.db);
|
|
614
|
+
runMigrations(this.db);
|
|
615
|
+
} else if (currentVersion !== CURRENT_SCHEMA_VERSION) {
|
|
616
|
+
this.migrate(currentVersion, CURRENT_SCHEMA_VERSION);
|
|
617
|
+
}
|
|
618
|
+
if (this.config.enableVectorSearch) {
|
|
619
|
+
this.loadVectorExtension();
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
loadVectorExtension() {
|
|
623
|
+
try {
|
|
624
|
+
sqliteVec.load(this.db);
|
|
625
|
+
console.log("\u2705 sqlite-vec loaded successfully");
|
|
626
|
+
const { vec_version } = this.db.prepare("SELECT vec_version() as vec_version").get();
|
|
627
|
+
console.log(` Version: ${vec_version}`);
|
|
628
|
+
const dims = this.config.vectorDimensions ?? 512;
|
|
629
|
+
ensureVectorTables(this.db, dims);
|
|
630
|
+
this.vectorReady = true;
|
|
631
|
+
} catch (error) {
|
|
632
|
+
console.warn(
|
|
633
|
+
`\u26A0\uFE0F sqlite-vec not available, vector search disabled: ${error.message}`
|
|
634
|
+
);
|
|
635
|
+
console.warn(" Falling back to keyword-only search");
|
|
636
|
+
this.config.enableVectorSearch = false;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
migrate(from, to) {
|
|
640
|
+
console.log(`Migrating database from ${from} to ${to}...`);
|
|
641
|
+
runMigrations(this.db);
|
|
642
|
+
ensureSchema(this.db);
|
|
643
|
+
console.log("Migration complete");
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* Get the underlying better-sqlite3 database
|
|
647
|
+
*/
|
|
648
|
+
getDb() {
|
|
649
|
+
return this.db;
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Check if vector search is available
|
|
653
|
+
*/
|
|
654
|
+
isVectorSearchReady() {
|
|
655
|
+
return this.vectorReady;
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* Get vector dimensions
|
|
659
|
+
*/
|
|
660
|
+
getVectorDimensions() {
|
|
661
|
+
return this.config.vectorDimensions;
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Execute a function in a transaction
|
|
665
|
+
*/
|
|
666
|
+
transaction(fn) {
|
|
667
|
+
return this.db.transaction(fn)();
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Execute an async function in a transaction
|
|
671
|
+
*/
|
|
672
|
+
async asyncTransaction(fn) {
|
|
673
|
+
const beginTrans = this.db.prepare("BEGIN");
|
|
674
|
+
const commitTrans = this.db.prepare("COMMIT");
|
|
675
|
+
const rollbackTrans = this.db.prepare("ROLLBACK");
|
|
676
|
+
beginTrans.run();
|
|
677
|
+
try {
|
|
678
|
+
const result = await fn();
|
|
679
|
+
commitTrans.run();
|
|
680
|
+
return result;
|
|
681
|
+
} catch (error) {
|
|
682
|
+
rollbackTrans.run();
|
|
683
|
+
throw error;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Get database stats
|
|
688
|
+
*/
|
|
689
|
+
getStats() {
|
|
690
|
+
const knowledge = this.db.prepare(`SELECT COUNT(*) as c FROM knowledge`).get();
|
|
691
|
+
const sessions = this.db.prepare(`SELECT COUNT(*) as c FROM sessions`).get();
|
|
692
|
+
const tasks = this.db.prepare(`SELECT COUNT(*) as c FROM tasks`).get();
|
|
693
|
+
const tgChats = this.db.prepare(`SELECT COUNT(*) as c FROM tg_chats`).get();
|
|
694
|
+
const tgUsers = this.db.prepare(`SELECT COUNT(*) as c FROM tg_users`).get();
|
|
695
|
+
const tgMessages = this.db.prepare(`SELECT COUNT(*) as c FROM tg_messages`).get();
|
|
696
|
+
const embeddingCache = this.db.prepare(`SELECT COUNT(*) as c FROM embedding_cache`).get();
|
|
697
|
+
return {
|
|
698
|
+
knowledge: knowledge.c,
|
|
699
|
+
sessions: sessions.c,
|
|
700
|
+
tasks: tasks.c,
|
|
701
|
+
tgChats: tgChats.c,
|
|
702
|
+
tgUsers: tgUsers.c,
|
|
703
|
+
tgMessages: tgMessages.c,
|
|
704
|
+
embeddingCache: embeddingCache.c,
|
|
705
|
+
vectorSearchEnabled: this.vectorReady
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
/**
|
|
709
|
+
* Vacuum the database to reclaim space
|
|
710
|
+
*/
|
|
711
|
+
vacuum() {
|
|
712
|
+
this.db.exec("VACUUM");
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Optimize the database (ANALYZE)
|
|
716
|
+
*/
|
|
717
|
+
optimize() {
|
|
718
|
+
this.db.exec("ANALYZE");
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* Rebuild FTS indexes from existing data
|
|
722
|
+
* Call this if FTS triggers didn't fire correctly
|
|
723
|
+
*/
|
|
724
|
+
rebuildFtsIndexes() {
|
|
725
|
+
this.db.exec(`DELETE FROM knowledge_fts`);
|
|
726
|
+
const knowledgeRows = this.db.prepare(`SELECT rowid, text, id, path, source FROM knowledge`).all();
|
|
727
|
+
const insertKnowledge = this.db.prepare(
|
|
728
|
+
`INSERT INTO knowledge_fts(rowid, text, id, path, source) VALUES (?, ?, ?, ?, ?)`
|
|
729
|
+
);
|
|
730
|
+
for (const row of knowledgeRows) {
|
|
731
|
+
insertKnowledge.run(row.rowid, row.text, row.id, row.path, row.source);
|
|
732
|
+
}
|
|
733
|
+
this.db.exec(`DELETE FROM tg_messages_fts`);
|
|
734
|
+
const messageRows = this.db.prepare(
|
|
735
|
+
`SELECT rowid, text, id, chat_id, sender_id, timestamp FROM tg_messages WHERE text IS NOT NULL`
|
|
736
|
+
).all();
|
|
737
|
+
const insertMessage = this.db.prepare(
|
|
738
|
+
`INSERT INTO tg_messages_fts(rowid, text, id, chat_id, sender_id, timestamp) VALUES (?, ?, ?, ?, ?, ?)`
|
|
739
|
+
);
|
|
740
|
+
for (const row of messageRows) {
|
|
741
|
+
insertMessage.run(row.rowid, row.text, row.id, row.chat_id, row.sender_id, row.timestamp);
|
|
742
|
+
}
|
|
743
|
+
return { knowledge: knowledgeRows.length, messages: messageRows.length };
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* Close the database connection
|
|
747
|
+
*/
|
|
748
|
+
close() {
|
|
749
|
+
if (this.db.open) {
|
|
750
|
+
this.db.close();
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
};
|
|
754
|
+
var instance = null;
|
|
755
|
+
function getDatabase(config) {
|
|
756
|
+
if (!instance && !config) {
|
|
757
|
+
throw new Error("Database not initialized. Provide config on first call.");
|
|
758
|
+
}
|
|
759
|
+
if (!instance && config) {
|
|
760
|
+
instance = new MemoryDatabase(config);
|
|
761
|
+
}
|
|
762
|
+
return instance;
|
|
763
|
+
}
|
|
764
|
+
function closeDatabase() {
|
|
765
|
+
if (instance) {
|
|
766
|
+
instance.close();
|
|
767
|
+
instance = null;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
// src/memory/embeddings/provider.ts
|
|
772
|
+
var NoopEmbeddingProvider = class {
|
|
773
|
+
id = "noop";
|
|
774
|
+
model = "none";
|
|
775
|
+
dimensions = 0;
|
|
776
|
+
async embedQuery(_text) {
|
|
777
|
+
return [];
|
|
778
|
+
}
|
|
779
|
+
async embedBatch(_texts) {
|
|
780
|
+
return [];
|
|
781
|
+
}
|
|
782
|
+
};
|
|
783
|
+
|
|
784
|
+
// src/memory/embeddings/anthropic.ts
|
|
785
|
+
var AnthropicEmbeddingProvider = class {
|
|
786
|
+
id = "anthropic";
|
|
787
|
+
model;
|
|
788
|
+
dimensions;
|
|
789
|
+
apiKey;
|
|
790
|
+
baseUrl = "https://api.voyageai.com/v1";
|
|
791
|
+
constructor(config) {
|
|
792
|
+
this.apiKey = config.apiKey;
|
|
793
|
+
this.model = config.model ?? "voyage-3-lite";
|
|
794
|
+
const dims = {
|
|
795
|
+
"voyage-3": 1024,
|
|
796
|
+
"voyage-3-lite": 512,
|
|
797
|
+
"voyage-code-3": 1024,
|
|
798
|
+
"voyage-finance-2": 1024,
|
|
799
|
+
"voyage-multilingual-2": 1024,
|
|
800
|
+
"voyage-law-2": 1024
|
|
801
|
+
};
|
|
802
|
+
this.dimensions = dims[this.model] ?? 512;
|
|
803
|
+
}
|
|
804
|
+
async embedQuery(text) {
|
|
805
|
+
const result = await this.embed([text]);
|
|
806
|
+
return result[0] ?? [];
|
|
807
|
+
}
|
|
808
|
+
async embedBatch(texts) {
|
|
809
|
+
if (texts.length === 0) return [];
|
|
810
|
+
const batchSize = 128;
|
|
811
|
+
const results = [];
|
|
812
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
813
|
+
const batch = texts.slice(i, i + batchSize);
|
|
814
|
+
const embeddings = await this.embed(batch);
|
|
815
|
+
results.push(...embeddings);
|
|
816
|
+
}
|
|
817
|
+
return results;
|
|
818
|
+
}
|
|
819
|
+
async embed(texts) {
|
|
820
|
+
const response = await fetch(`${this.baseUrl}/embeddings`, {
|
|
821
|
+
method: "POST",
|
|
822
|
+
headers: {
|
|
823
|
+
"Content-Type": "application/json",
|
|
824
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
825
|
+
},
|
|
826
|
+
body: JSON.stringify({
|
|
827
|
+
input: texts,
|
|
828
|
+
model: this.model,
|
|
829
|
+
input_type: "document"
|
|
830
|
+
})
|
|
831
|
+
});
|
|
832
|
+
if (!response.ok) {
|
|
833
|
+
const error = await response.text();
|
|
834
|
+
throw new Error(`Voyage API error: ${response.status} ${error}`);
|
|
835
|
+
}
|
|
836
|
+
const data = await response.json();
|
|
837
|
+
return data.data.map((item) => item.embedding);
|
|
838
|
+
}
|
|
839
|
+
};
|
|
840
|
+
|
|
841
|
+
// src/memory/embeddings/local.ts
|
|
842
|
+
var LocalEmbeddingProvider = class {
|
|
843
|
+
id = "local";
|
|
844
|
+
model;
|
|
845
|
+
dimensions;
|
|
846
|
+
hasWarned = false;
|
|
847
|
+
constructor(config) {
|
|
848
|
+
this.model = config.model ?? "all-MiniLM-L6-v2";
|
|
849
|
+
this.dimensions = 384;
|
|
850
|
+
}
|
|
851
|
+
async embedQuery(text) {
|
|
852
|
+
if (!this.hasWarned) {
|
|
853
|
+
console.warn(
|
|
854
|
+
"\u26A0\uFE0F Local embeddings not yet implemented. Returning zero vectors. This will not work for semantic search. Consider using 'anthropic' embedding provider."
|
|
855
|
+
);
|
|
856
|
+
this.hasWarned = true;
|
|
857
|
+
}
|
|
858
|
+
return new Array(this.dimensions).fill(0);
|
|
859
|
+
}
|
|
860
|
+
async embedBatch(texts) {
|
|
861
|
+
if (!this.hasWarned) {
|
|
862
|
+
console.warn(
|
|
863
|
+
"\u26A0\uFE0F Local embeddings not yet implemented. Returning zero vectors. This will not work for semantic search. Consider using 'anthropic' embedding provider."
|
|
864
|
+
);
|
|
865
|
+
this.hasWarned = true;
|
|
866
|
+
}
|
|
867
|
+
return texts.map(() => new Array(this.dimensions).fill(0));
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
|
|
871
|
+
// src/memory/embeddings/index.ts
|
|
872
|
+
function createEmbeddingProvider(config) {
|
|
873
|
+
switch (config.provider) {
|
|
874
|
+
case "anthropic":
|
|
875
|
+
if (!config.apiKey) {
|
|
876
|
+
throw new Error("API key required for Anthropic embedding provider");
|
|
877
|
+
}
|
|
878
|
+
return new AnthropicEmbeddingProvider({
|
|
879
|
+
apiKey: config.apiKey,
|
|
880
|
+
model: config.model
|
|
881
|
+
});
|
|
882
|
+
case "local":
|
|
883
|
+
return new LocalEmbeddingProvider({
|
|
884
|
+
model: config.model
|
|
885
|
+
});
|
|
886
|
+
case "none":
|
|
887
|
+
return new NoopEmbeddingProvider();
|
|
888
|
+
default:
|
|
889
|
+
throw new Error(`Unknown embedding provider: ${config.provider}`);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
function hashText(text) {
|
|
893
|
+
let hash = 0;
|
|
894
|
+
for (let i = 0; i < text.length; i++) {
|
|
895
|
+
const char = text.charCodeAt(i);
|
|
896
|
+
hash = (hash << 5) - hash + char;
|
|
897
|
+
hash = hash & hash;
|
|
898
|
+
}
|
|
899
|
+
return Math.abs(hash).toString(36);
|
|
900
|
+
}
|
|
901
|
+
function serializeEmbedding(embedding) {
|
|
902
|
+
return JSON.stringify(embedding);
|
|
903
|
+
}
|
|
904
|
+
function deserializeEmbedding(data) {
|
|
905
|
+
try {
|
|
906
|
+
return JSON.parse(data);
|
|
907
|
+
} catch {
|
|
908
|
+
return [];
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
function embeddingToBlob(embedding) {
|
|
912
|
+
return Buffer.from(new Float32Array(embedding).buffer);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// src/memory/agent/knowledge.ts
|
|
916
|
+
import { readFileSync, existsSync as existsSync2, readdirSync, statSync } from "fs";
|
|
917
|
+
import { join } from "path";
|
|
918
|
+
var KnowledgeIndexer = class {
|
|
919
|
+
constructor(db, workspaceDir, embedder, vectorEnabled) {
|
|
920
|
+
this.db = db;
|
|
921
|
+
this.workspaceDir = workspaceDir;
|
|
922
|
+
this.embedder = embedder;
|
|
923
|
+
this.vectorEnabled = vectorEnabled;
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Index all memory files
|
|
927
|
+
*/
|
|
928
|
+
async indexAll() {
|
|
929
|
+
const files = this.listMemoryFiles();
|
|
930
|
+
let indexed = 0;
|
|
931
|
+
let skipped = 0;
|
|
932
|
+
for (const file of files) {
|
|
933
|
+
const wasIndexed = await this.indexFile(file);
|
|
934
|
+
if (wasIndexed) {
|
|
935
|
+
indexed++;
|
|
936
|
+
} else {
|
|
937
|
+
skipped++;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
return { indexed, skipped };
|
|
941
|
+
}
|
|
942
|
+
/**
|
|
943
|
+
* Index a single file
|
|
944
|
+
*/
|
|
945
|
+
async indexFile(absPath) {
|
|
946
|
+
if (!existsSync2(absPath) || !absPath.endsWith(".md")) {
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
949
|
+
const content = readFileSync(absPath, "utf-8");
|
|
950
|
+
const relPath = absPath.replace(this.workspaceDir + "/", "");
|
|
951
|
+
const fileHash = hashText(content);
|
|
952
|
+
const existing = this.db.prepare(`SELECT hash FROM knowledge WHERE path = ? AND source = 'memory' LIMIT 1`).get(relPath);
|
|
953
|
+
if (existing?.hash === fileHash) {
|
|
954
|
+
return false;
|
|
955
|
+
}
|
|
956
|
+
this.db.prepare(`DELETE FROM knowledge WHERE path = ? AND source = 'memory'`).run(relPath);
|
|
957
|
+
const chunks = this.chunkMarkdown(content, relPath);
|
|
958
|
+
const texts = chunks.map((c) => c.text);
|
|
959
|
+
const embeddings = await this.embedder.embedBatch(texts);
|
|
960
|
+
const insert = this.db.prepare(`
|
|
961
|
+
INSERT INTO knowledge (id, source, path, text, embedding, start_line, end_line, hash)
|
|
962
|
+
VALUES (?, 'memory', ?, ?, ?, ?, ?, ?)
|
|
963
|
+
`);
|
|
964
|
+
const insertVec = this.vectorEnabled ? this.db.prepare(`INSERT INTO knowledge_vec (id, embedding) VALUES (?, ?)`) : null;
|
|
965
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
966
|
+
const chunk = chunks[i];
|
|
967
|
+
const embedding = embeddings[i] ?? [];
|
|
968
|
+
insert.run(
|
|
969
|
+
chunk.id,
|
|
970
|
+
chunk.path,
|
|
971
|
+
chunk.text,
|
|
972
|
+
serializeEmbedding(embedding),
|
|
973
|
+
chunk.startLine,
|
|
974
|
+
chunk.endLine,
|
|
975
|
+
chunk.hash
|
|
976
|
+
);
|
|
977
|
+
if (insertVec && embedding.length > 0) {
|
|
978
|
+
insertVec.run(chunk.id, embeddingToBlob(embedding));
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
return true;
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* List all memory files
|
|
985
|
+
*/
|
|
986
|
+
listMemoryFiles() {
|
|
987
|
+
const files = [];
|
|
988
|
+
const memoryMd = join(this.workspaceDir, "MEMORY.md");
|
|
989
|
+
if (existsSync2(memoryMd)) {
|
|
990
|
+
files.push(memoryMd);
|
|
991
|
+
}
|
|
992
|
+
const memoryDir = join(this.workspaceDir, "memory");
|
|
993
|
+
if (existsSync2(memoryDir)) {
|
|
994
|
+
const entries = readdirSync(memoryDir);
|
|
995
|
+
for (const entry of entries) {
|
|
996
|
+
const absPath = join(memoryDir, entry);
|
|
997
|
+
if (statSync(absPath).isFile() && entry.endsWith(".md")) {
|
|
998
|
+
files.push(absPath);
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
return files;
|
|
1003
|
+
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Chunk markdown content
|
|
1006
|
+
*/
|
|
1007
|
+
chunkMarkdown(content, path) {
|
|
1008
|
+
const lines = content.split("\n");
|
|
1009
|
+
const chunks = [];
|
|
1010
|
+
const chunkSize = 500;
|
|
1011
|
+
const overlap = 50;
|
|
1012
|
+
let currentChunk = "";
|
|
1013
|
+
let startLine = 1;
|
|
1014
|
+
let currentLine = 1;
|
|
1015
|
+
for (const line of lines) {
|
|
1016
|
+
currentChunk += line + "\n";
|
|
1017
|
+
if (currentChunk.length >= chunkSize) {
|
|
1018
|
+
const text2 = currentChunk.trim();
|
|
1019
|
+
if (text2.length > 0) {
|
|
1020
|
+
chunks.push({
|
|
1021
|
+
id: hashText(`${path}:${startLine}:${currentLine}`),
|
|
1022
|
+
source: "memory",
|
|
1023
|
+
path,
|
|
1024
|
+
text: text2,
|
|
1025
|
+
startLine,
|
|
1026
|
+
endLine: currentLine,
|
|
1027
|
+
hash: hashText(text2)
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
const overlapText = currentChunk.slice(-overlap);
|
|
1031
|
+
currentChunk = overlapText;
|
|
1032
|
+
startLine = currentLine + 1;
|
|
1033
|
+
}
|
|
1034
|
+
currentLine++;
|
|
1035
|
+
}
|
|
1036
|
+
const text = currentChunk.trim();
|
|
1037
|
+
if (text.length > 0) {
|
|
1038
|
+
chunks.push({
|
|
1039
|
+
id: hashText(`${path}:${startLine}:${currentLine}`),
|
|
1040
|
+
source: "memory",
|
|
1041
|
+
path,
|
|
1042
|
+
text,
|
|
1043
|
+
startLine,
|
|
1044
|
+
endLine: currentLine,
|
|
1045
|
+
hash: hashText(text)
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
return chunks;
|
|
1049
|
+
}
|
|
1050
|
+
};
|
|
1051
|
+
|
|
1052
|
+
// src/memory/agent/sessions.ts
|
|
1053
|
+
import { randomUUID } from "crypto";
|
|
1054
|
+
var SessionStore = class {
|
|
1055
|
+
constructor(db, embedder, vectorEnabled) {
|
|
1056
|
+
this.db = db;
|
|
1057
|
+
this.embedder = embedder;
|
|
1058
|
+
this.vectorEnabled = vectorEnabled;
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* Create a new session
|
|
1062
|
+
*/
|
|
1063
|
+
createSession(chatId) {
|
|
1064
|
+
const id = randomUUID();
|
|
1065
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
1066
|
+
this.db.prepare(
|
|
1067
|
+
`
|
|
1068
|
+
INSERT INTO sessions (id, chat_id, started_at, message_count, tokens_used)
|
|
1069
|
+
VALUES (?, ?, ?, 0, 0)
|
|
1070
|
+
`
|
|
1071
|
+
).run(id, chatId ?? null, now);
|
|
1072
|
+
return {
|
|
1073
|
+
id,
|
|
1074
|
+
chatId,
|
|
1075
|
+
startedAt: new Date(now * 1e3),
|
|
1076
|
+
messageCount: 0,
|
|
1077
|
+
tokensUsed: 0
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
/**
|
|
1081
|
+
* End a session with summary
|
|
1082
|
+
*/
|
|
1083
|
+
endSession(sessionId, summary, tokensUsed = 0) {
|
|
1084
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
1085
|
+
this.db.prepare(
|
|
1086
|
+
`
|
|
1087
|
+
UPDATE sessions
|
|
1088
|
+
SET ended_at = ?, summary = ?, tokens_used = ?
|
|
1089
|
+
WHERE id = ?
|
|
1090
|
+
`
|
|
1091
|
+
).run(now, summary, tokensUsed, sessionId);
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Update message count for a session
|
|
1095
|
+
*/
|
|
1096
|
+
incrementMessageCount(sessionId, count = 1) {
|
|
1097
|
+
this.db.prepare(
|
|
1098
|
+
`
|
|
1099
|
+
UPDATE sessions
|
|
1100
|
+
SET message_count = message_count + ?
|
|
1101
|
+
WHERE id = ?
|
|
1102
|
+
`
|
|
1103
|
+
).run(count, sessionId);
|
|
1104
|
+
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Get a session by ID
|
|
1107
|
+
*/
|
|
1108
|
+
getSession(id) {
|
|
1109
|
+
const row = this.db.prepare(`SELECT * FROM sessions WHERE id = ?`).get(id);
|
|
1110
|
+
if (!row) return void 0;
|
|
1111
|
+
return {
|
|
1112
|
+
id: row.id,
|
|
1113
|
+
chatId: row.chat_id,
|
|
1114
|
+
startedAt: new Date(row.started_at * 1e3),
|
|
1115
|
+
endedAt: row.ended_at ? new Date(row.ended_at * 1e3) : void 0,
|
|
1116
|
+
summary: row.summary,
|
|
1117
|
+
messageCount: row.message_count,
|
|
1118
|
+
tokensUsed: row.tokens_used
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
/**
|
|
1122
|
+
* Get active (not ended) sessions
|
|
1123
|
+
*/
|
|
1124
|
+
getActiveSessions() {
|
|
1125
|
+
const rows = this.db.prepare(
|
|
1126
|
+
`
|
|
1127
|
+
SELECT * FROM sessions
|
|
1128
|
+
WHERE ended_at IS NULL
|
|
1129
|
+
ORDER BY started_at DESC
|
|
1130
|
+
`
|
|
1131
|
+
).all();
|
|
1132
|
+
return rows.map((row) => ({
|
|
1133
|
+
id: row.id,
|
|
1134
|
+
chatId: row.chat_id,
|
|
1135
|
+
startedAt: new Date(row.started_at * 1e3),
|
|
1136
|
+
endedAt: void 0,
|
|
1137
|
+
summary: row.summary,
|
|
1138
|
+
messageCount: row.message_count,
|
|
1139
|
+
tokensUsed: row.tokens_used
|
|
1140
|
+
}));
|
|
1141
|
+
}
|
|
1142
|
+
/**
|
|
1143
|
+
* Get sessions for a specific chat
|
|
1144
|
+
*/
|
|
1145
|
+
getSessionsByChat(chatId, limit = 50) {
|
|
1146
|
+
const rows = this.db.prepare(
|
|
1147
|
+
`
|
|
1148
|
+
SELECT * FROM sessions
|
|
1149
|
+
WHERE chat_id = ?
|
|
1150
|
+
ORDER BY started_at DESC
|
|
1151
|
+
LIMIT ?
|
|
1152
|
+
`
|
|
1153
|
+
).all(chatId, limit);
|
|
1154
|
+
return rows.map((row) => ({
|
|
1155
|
+
id: row.id,
|
|
1156
|
+
chatId: row.chat_id,
|
|
1157
|
+
startedAt: new Date(row.started_at * 1e3),
|
|
1158
|
+
endedAt: row.ended_at ? new Date(row.ended_at * 1e3) : void 0,
|
|
1159
|
+
summary: row.summary,
|
|
1160
|
+
messageCount: row.message_count,
|
|
1161
|
+
tokensUsed: row.tokens_used
|
|
1162
|
+
}));
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Index a session for search (after ending)
|
|
1166
|
+
* This creates a knowledge entry from the session summary
|
|
1167
|
+
*/
|
|
1168
|
+
async indexSession(sessionId) {
|
|
1169
|
+
const session = this.getSession(sessionId);
|
|
1170
|
+
if (!session || !session.summary) return;
|
|
1171
|
+
try {
|
|
1172
|
+
const knowledgeId = `session:${sessionId}`;
|
|
1173
|
+
const text = `Session from ${session.startedAt.toISOString()}:
|
|
1174
|
+
${session.summary}`;
|
|
1175
|
+
const hash = this.hashText(text);
|
|
1176
|
+
let embedding = null;
|
|
1177
|
+
if (this.vectorEnabled) {
|
|
1178
|
+
embedding = await this.embedder.embedQuery(text);
|
|
1179
|
+
}
|
|
1180
|
+
this.db.prepare(
|
|
1181
|
+
`
|
|
1182
|
+
INSERT INTO knowledge (id, source, path, text, hash, created_at, updated_at)
|
|
1183
|
+
VALUES (?, 'session', ?, ?, ?, unixepoch(), unixepoch())
|
|
1184
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1185
|
+
text = excluded.text,
|
|
1186
|
+
hash = excluded.hash,
|
|
1187
|
+
updated_at = excluded.updated_at
|
|
1188
|
+
`
|
|
1189
|
+
).run(knowledgeId, sessionId, text, hash);
|
|
1190
|
+
if (embedding && this.vectorEnabled) {
|
|
1191
|
+
const embeddingBuffer = this.serializeEmbedding(embedding);
|
|
1192
|
+
const rowid = this.db.prepare(`SELECT rowid FROM knowledge WHERE id = ?`).get(knowledgeId);
|
|
1193
|
+
this.db.prepare(
|
|
1194
|
+
`
|
|
1195
|
+
INSERT INTO knowledge_vec (rowid, embedding)
|
|
1196
|
+
VALUES (?, ?)
|
|
1197
|
+
ON CONFLICT(rowid) DO UPDATE SET embedding = excluded.embedding
|
|
1198
|
+
`
|
|
1199
|
+
).run(rowid.rowid, embeddingBuffer);
|
|
1200
|
+
}
|
|
1201
|
+
console.log(`Indexed session ${sessionId} to knowledge base`);
|
|
1202
|
+
} catch (error) {
|
|
1203
|
+
console.error("Error indexing session:", error);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
/**
|
|
1207
|
+
* Delete a session
|
|
1208
|
+
*/
|
|
1209
|
+
deleteSession(sessionId) {
|
|
1210
|
+
this.db.prepare(`DELETE FROM sessions WHERE id = ?`).run(sessionId);
|
|
1211
|
+
this.db.prepare(`DELETE FROM knowledge WHERE id = ?`).run(`session:${sessionId}`);
|
|
1212
|
+
}
|
|
1213
|
+
hashText(text) {
|
|
1214
|
+
let hash = 0;
|
|
1215
|
+
for (let i = 0; i < text.length; i++) {
|
|
1216
|
+
const char = text.charCodeAt(i);
|
|
1217
|
+
hash = (hash << 5) - hash + char;
|
|
1218
|
+
hash = hash & hash;
|
|
1219
|
+
}
|
|
1220
|
+
return hash.toString(36);
|
|
1221
|
+
}
|
|
1222
|
+
serializeEmbedding(embedding) {
|
|
1223
|
+
const float32 = new Float32Array(embedding);
|
|
1224
|
+
return Buffer.from(float32.buffer);
|
|
1225
|
+
}
|
|
1226
|
+
};
|
|
1227
|
+
|
|
1228
|
+
// src/memory/feed/messages.ts
|
|
1229
|
+
var MessageStore = class {
|
|
1230
|
+
constructor(db, embedder, vectorEnabled) {
|
|
1231
|
+
this.db = db;
|
|
1232
|
+
this.embedder = embedder;
|
|
1233
|
+
this.vectorEnabled = vectorEnabled;
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1236
|
+
* Ensure chat exists in database
|
|
1237
|
+
*/
|
|
1238
|
+
ensureChat(chatId, isGroup = false) {
|
|
1239
|
+
const existing = this.db.prepare(`SELECT id FROM tg_chats WHERE id = ?`).get(chatId);
|
|
1240
|
+
if (!existing) {
|
|
1241
|
+
this.db.prepare(
|
|
1242
|
+
`INSERT INTO tg_chats (id, type, is_monitored) VALUES (?, ?, 1)`
|
|
1243
|
+
).run(chatId, isGroup ? "group" : "dm");
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Ensure user exists in database
|
|
1248
|
+
*/
|
|
1249
|
+
ensureUser(userId) {
|
|
1250
|
+
if (!userId) return;
|
|
1251
|
+
const existing = this.db.prepare(`SELECT id FROM tg_users WHERE id = ?`).get(userId);
|
|
1252
|
+
if (!existing) {
|
|
1253
|
+
this.db.prepare(
|
|
1254
|
+
`INSERT INTO tg_users (id) VALUES (?)`
|
|
1255
|
+
).run(userId);
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Store a message
|
|
1260
|
+
*/
|
|
1261
|
+
async storeMessage(message) {
|
|
1262
|
+
this.ensureChat(message.chatId);
|
|
1263
|
+
if (message.senderId) {
|
|
1264
|
+
this.ensureUser(message.senderId);
|
|
1265
|
+
}
|
|
1266
|
+
const embedding = message.text ? await this.embedder.embedQuery(message.text) : [];
|
|
1267
|
+
this.db.prepare(
|
|
1268
|
+
`
|
|
1269
|
+
INSERT OR REPLACE INTO tg_messages (
|
|
1270
|
+
id, chat_id, sender_id, text, embedding, reply_to_id,
|
|
1271
|
+
is_from_agent, has_media, media_type, timestamp
|
|
1272
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1273
|
+
`
|
|
1274
|
+
).run(
|
|
1275
|
+
message.id,
|
|
1276
|
+
message.chatId,
|
|
1277
|
+
message.senderId,
|
|
1278
|
+
message.text,
|
|
1279
|
+
serializeEmbedding(embedding),
|
|
1280
|
+
message.replyToId,
|
|
1281
|
+
message.isFromAgent ? 1 : 0,
|
|
1282
|
+
message.hasMedia ? 1 : 0,
|
|
1283
|
+
message.mediaType,
|
|
1284
|
+
message.timestamp
|
|
1285
|
+
);
|
|
1286
|
+
if (this.vectorEnabled && embedding.length > 0 && message.text) {
|
|
1287
|
+
this.db.prepare(`INSERT OR REPLACE INTO tg_messages_vec (id, embedding) VALUES (?, ?)`).run(message.id, embeddingToBlob(embedding));
|
|
1288
|
+
}
|
|
1289
|
+
this.db.prepare(`UPDATE tg_chats SET last_message_at = ?, last_message_id = ? WHERE id = ?`).run(message.timestamp, message.id, message.chatId);
|
|
1290
|
+
}
|
|
1291
|
+
/**
|
|
1292
|
+
* Get recent messages from a chat
|
|
1293
|
+
*/
|
|
1294
|
+
getRecentMessages(chatId, limit = 20) {
|
|
1295
|
+
const rows = this.db.prepare(
|
|
1296
|
+
`
|
|
1297
|
+
SELECT id, chat_id, sender_id, text, reply_to_id, is_from_agent, has_media, media_type, timestamp
|
|
1298
|
+
FROM tg_messages
|
|
1299
|
+
WHERE chat_id = ?
|
|
1300
|
+
ORDER BY timestamp DESC
|
|
1301
|
+
LIMIT ?
|
|
1302
|
+
`
|
|
1303
|
+
).all(chatId, limit);
|
|
1304
|
+
return rows.reverse().map((row) => ({
|
|
1305
|
+
id: row.id,
|
|
1306
|
+
chatId: row.chat_id,
|
|
1307
|
+
senderId: row.sender_id,
|
|
1308
|
+
text: row.text,
|
|
1309
|
+
replyToId: row.reply_to_id ?? void 0,
|
|
1310
|
+
isFromAgent: Boolean(row.is_from_agent),
|
|
1311
|
+
hasMedia: Boolean(row.has_media),
|
|
1312
|
+
mediaType: row.media_type ?? void 0,
|
|
1313
|
+
timestamp: row.timestamp
|
|
1314
|
+
}));
|
|
1315
|
+
}
|
|
1316
|
+
};
|
|
1317
|
+
|
|
1318
|
+
// src/memory/feed/chats.ts
|
|
1319
|
+
var ChatStore = class {
|
|
1320
|
+
constructor(db) {
|
|
1321
|
+
this.db = db;
|
|
1322
|
+
}
|
|
1323
|
+
/**
|
|
1324
|
+
* Create or update a chat
|
|
1325
|
+
*/
|
|
1326
|
+
upsertChat(chat) {
|
|
1327
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
1328
|
+
this.db.prepare(
|
|
1329
|
+
`
|
|
1330
|
+
INSERT INTO tg_chats (
|
|
1331
|
+
id, type, title, username, member_count, is_monitored, is_archived,
|
|
1332
|
+
last_message_id, last_message_at, created_at, updated_at
|
|
1333
|
+
)
|
|
1334
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1335
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1336
|
+
title = COALESCE(excluded.title, title),
|
|
1337
|
+
username = COALESCE(excluded.username, username),
|
|
1338
|
+
member_count = COALESCE(excluded.member_count, member_count),
|
|
1339
|
+
last_message_id = COALESCE(excluded.last_message_id, last_message_id),
|
|
1340
|
+
last_message_at = COALESCE(excluded.last_message_at, last_message_at),
|
|
1341
|
+
updated_at = excluded.updated_at
|
|
1342
|
+
`
|
|
1343
|
+
).run(
|
|
1344
|
+
chat.id,
|
|
1345
|
+
chat.type,
|
|
1346
|
+
chat.title ?? null,
|
|
1347
|
+
chat.username ?? null,
|
|
1348
|
+
chat.memberCount ?? null,
|
|
1349
|
+
chat.isMonitored ?? 1,
|
|
1350
|
+
chat.isArchived ?? 0,
|
|
1351
|
+
chat.lastMessageId ?? null,
|
|
1352
|
+
chat.lastMessageAt ? Math.floor(chat.lastMessageAt.getTime() / 1e3) : null,
|
|
1353
|
+
now,
|
|
1354
|
+
now
|
|
1355
|
+
);
|
|
1356
|
+
}
|
|
1357
|
+
/**
|
|
1358
|
+
* Get a chat by ID
|
|
1359
|
+
*/
|
|
1360
|
+
getChat(id) {
|
|
1361
|
+
const row = this.db.prepare(
|
|
1362
|
+
`
|
|
1363
|
+
SELECT * FROM tg_chats WHERE id = ?
|
|
1364
|
+
`
|
|
1365
|
+
).get(id);
|
|
1366
|
+
if (!row) return void 0;
|
|
1367
|
+
return {
|
|
1368
|
+
id: row.id,
|
|
1369
|
+
type: row.type,
|
|
1370
|
+
title: row.title,
|
|
1371
|
+
username: row.username,
|
|
1372
|
+
memberCount: row.member_count,
|
|
1373
|
+
isMonitored: Boolean(row.is_monitored),
|
|
1374
|
+
isArchived: Boolean(row.is_archived),
|
|
1375
|
+
lastMessageId: row.last_message_id,
|
|
1376
|
+
lastMessageAt: row.last_message_at ? new Date(row.last_message_at * 1e3) : void 0,
|
|
1377
|
+
createdAt: new Date(row.created_at * 1e3),
|
|
1378
|
+
updatedAt: new Date(row.updated_at * 1e3)
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* Get active (monitored, non-archived) chats
|
|
1383
|
+
*/
|
|
1384
|
+
getActiveChats(limit = 50) {
|
|
1385
|
+
const rows = this.db.prepare(
|
|
1386
|
+
`
|
|
1387
|
+
SELECT * FROM tg_chats
|
|
1388
|
+
WHERE is_monitored = 1 AND is_archived = 0
|
|
1389
|
+
ORDER BY last_message_at DESC NULLS LAST
|
|
1390
|
+
LIMIT ?
|
|
1391
|
+
`
|
|
1392
|
+
).all(limit);
|
|
1393
|
+
return rows.map((row) => ({
|
|
1394
|
+
id: row.id,
|
|
1395
|
+
type: row.type,
|
|
1396
|
+
title: row.title,
|
|
1397
|
+
username: row.username,
|
|
1398
|
+
memberCount: row.member_count,
|
|
1399
|
+
isMonitored: Boolean(row.is_monitored),
|
|
1400
|
+
isArchived: Boolean(row.is_archived),
|
|
1401
|
+
lastMessageId: row.last_message_id,
|
|
1402
|
+
lastMessageAt: row.last_message_at ? new Date(row.last_message_at * 1e3) : void 0,
|
|
1403
|
+
createdAt: new Date(row.created_at * 1e3),
|
|
1404
|
+
updatedAt: new Date(row.updated_at * 1e3)
|
|
1405
|
+
}));
|
|
1406
|
+
}
|
|
1407
|
+
/**
|
|
1408
|
+
* Update last message info
|
|
1409
|
+
*/
|
|
1410
|
+
updateLastMessage(chatId, messageId, timestamp) {
|
|
1411
|
+
this.db.prepare(
|
|
1412
|
+
`
|
|
1413
|
+
UPDATE tg_chats
|
|
1414
|
+
SET last_message_id = ?, last_message_at = ?, updated_at = unixepoch()
|
|
1415
|
+
WHERE id = ?
|
|
1416
|
+
`
|
|
1417
|
+
).run(messageId, Math.floor(timestamp.getTime() / 1e3), chatId);
|
|
1418
|
+
}
|
|
1419
|
+
/**
|
|
1420
|
+
* Archive a chat
|
|
1421
|
+
*/
|
|
1422
|
+
archiveChat(chatId) {
|
|
1423
|
+
this.db.prepare(
|
|
1424
|
+
`
|
|
1425
|
+
UPDATE tg_chats
|
|
1426
|
+
SET is_archived = 1, updated_at = unixepoch()
|
|
1427
|
+
WHERE id = ?
|
|
1428
|
+
`
|
|
1429
|
+
).run(chatId);
|
|
1430
|
+
}
|
|
1431
|
+
/**
|
|
1432
|
+
* Unarchive a chat
|
|
1433
|
+
*/
|
|
1434
|
+
unarchiveChat(chatId) {
|
|
1435
|
+
this.db.prepare(
|
|
1436
|
+
`
|
|
1437
|
+
UPDATE tg_chats
|
|
1438
|
+
SET is_archived = 0, updated_at = unixepoch()
|
|
1439
|
+
WHERE id = ?
|
|
1440
|
+
`
|
|
1441
|
+
).run(chatId);
|
|
1442
|
+
}
|
|
1443
|
+
/**
|
|
1444
|
+
* Set monitoring status
|
|
1445
|
+
*/
|
|
1446
|
+
setMonitored(chatId, monitored) {
|
|
1447
|
+
this.db.prepare(
|
|
1448
|
+
`
|
|
1449
|
+
UPDATE tg_chats
|
|
1450
|
+
SET is_monitored = ?, updated_at = unixepoch()
|
|
1451
|
+
WHERE id = ?
|
|
1452
|
+
`
|
|
1453
|
+
).run(monitored ? 1 : 0, chatId);
|
|
1454
|
+
}
|
|
1455
|
+
};
|
|
1456
|
+
|
|
1457
|
+
// src/memory/feed/users.ts
|
|
1458
|
+
var UserStore = class {
|
|
1459
|
+
constructor(db) {
|
|
1460
|
+
this.db = db;
|
|
1461
|
+
}
|
|
1462
|
+
/**
|
|
1463
|
+
* Create or update a user
|
|
1464
|
+
*/
|
|
1465
|
+
upsertUser(user) {
|
|
1466
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
1467
|
+
const existing = this.db.prepare(`SELECT id FROM tg_users WHERE id = ?`).get(user.id);
|
|
1468
|
+
if (existing) {
|
|
1469
|
+
this.db.prepare(
|
|
1470
|
+
`
|
|
1471
|
+
UPDATE tg_users
|
|
1472
|
+
SET
|
|
1473
|
+
username = COALESCE(?, username),
|
|
1474
|
+
first_name = COALESCE(?, first_name),
|
|
1475
|
+
last_name = COALESCE(?, last_name),
|
|
1476
|
+
last_seen_at = ?
|
|
1477
|
+
WHERE id = ?
|
|
1478
|
+
`
|
|
1479
|
+
).run(user.username ?? null, user.firstName ?? null, user.lastName ?? null, now, user.id);
|
|
1480
|
+
} else {
|
|
1481
|
+
this.db.prepare(
|
|
1482
|
+
`
|
|
1483
|
+
INSERT INTO tg_users (
|
|
1484
|
+
id, username, first_name, last_name, is_bot, is_admin, is_allowed,
|
|
1485
|
+
first_seen_at, last_seen_at, message_count
|
|
1486
|
+
)
|
|
1487
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1488
|
+
`
|
|
1489
|
+
).run(
|
|
1490
|
+
user.id,
|
|
1491
|
+
user.username ?? null,
|
|
1492
|
+
user.firstName ?? null,
|
|
1493
|
+
user.lastName ?? null,
|
|
1494
|
+
user.isBot ?? 0,
|
|
1495
|
+
user.isAdmin ?? 0,
|
|
1496
|
+
user.isAllowed ?? 0,
|
|
1497
|
+
now,
|
|
1498
|
+
now,
|
|
1499
|
+
0
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
/**
|
|
1504
|
+
* Get a user by ID
|
|
1505
|
+
*/
|
|
1506
|
+
getUser(id) {
|
|
1507
|
+
const row = this.db.prepare(`SELECT * FROM tg_users WHERE id = ?`).get(id);
|
|
1508
|
+
if (!row) return void 0;
|
|
1509
|
+
return {
|
|
1510
|
+
id: row.id,
|
|
1511
|
+
username: row.username,
|
|
1512
|
+
firstName: row.first_name,
|
|
1513
|
+
lastName: row.last_name,
|
|
1514
|
+
isBot: Boolean(row.is_bot),
|
|
1515
|
+
isAdmin: Boolean(row.is_admin),
|
|
1516
|
+
isAllowed: Boolean(row.is_allowed),
|
|
1517
|
+
firstSeenAt: new Date(row.first_seen_at * 1e3),
|
|
1518
|
+
lastSeenAt: new Date(row.last_seen_at * 1e3),
|
|
1519
|
+
messageCount: row.message_count
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Get a user by username
|
|
1524
|
+
*/
|
|
1525
|
+
getUserByUsername(username) {
|
|
1526
|
+
const row = this.db.prepare(`SELECT * FROM tg_users WHERE username = ?`).get(username.replace("@", ""));
|
|
1527
|
+
if (!row) return void 0;
|
|
1528
|
+
return {
|
|
1529
|
+
id: row.id,
|
|
1530
|
+
username: row.username,
|
|
1531
|
+
firstName: row.first_name,
|
|
1532
|
+
lastName: row.last_name,
|
|
1533
|
+
isBot: Boolean(row.is_bot),
|
|
1534
|
+
isAdmin: Boolean(row.is_admin),
|
|
1535
|
+
isAllowed: Boolean(row.is_allowed),
|
|
1536
|
+
firstSeenAt: new Date(row.first_seen_at * 1e3),
|
|
1537
|
+
lastSeenAt: new Date(row.last_seen_at * 1e3),
|
|
1538
|
+
messageCount: row.message_count
|
|
1539
|
+
};
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* Update last seen timestamp
|
|
1543
|
+
*/
|
|
1544
|
+
updateLastSeen(userId) {
|
|
1545
|
+
this.db.prepare(
|
|
1546
|
+
`
|
|
1547
|
+
UPDATE tg_users
|
|
1548
|
+
SET last_seen_at = unixepoch()
|
|
1549
|
+
WHERE id = ?
|
|
1550
|
+
`
|
|
1551
|
+
).run(userId);
|
|
1552
|
+
}
|
|
1553
|
+
/**
|
|
1554
|
+
* Increment message count
|
|
1555
|
+
*/
|
|
1556
|
+
incrementMessageCount(userId) {
|
|
1557
|
+
this.db.prepare(
|
|
1558
|
+
`
|
|
1559
|
+
UPDATE tg_users
|
|
1560
|
+
SET message_count = message_count + 1, last_seen_at = unixepoch()
|
|
1561
|
+
WHERE id = ?
|
|
1562
|
+
`
|
|
1563
|
+
).run(userId);
|
|
1564
|
+
}
|
|
1565
|
+
/**
|
|
1566
|
+
* Set admin status
|
|
1567
|
+
*/
|
|
1568
|
+
setAdmin(userId, isAdmin) {
|
|
1569
|
+
this.db.prepare(
|
|
1570
|
+
`
|
|
1571
|
+
UPDATE tg_users
|
|
1572
|
+
SET is_admin = ?
|
|
1573
|
+
WHERE id = ?
|
|
1574
|
+
`
|
|
1575
|
+
).run(isAdmin ? 1 : 0, userId);
|
|
1576
|
+
}
|
|
1577
|
+
/**
|
|
1578
|
+
* Set allowed status
|
|
1579
|
+
*/
|
|
1580
|
+
setAllowed(userId, isAllowed) {
|
|
1581
|
+
this.db.prepare(
|
|
1582
|
+
`
|
|
1583
|
+
UPDATE tg_users
|
|
1584
|
+
SET is_allowed = ?
|
|
1585
|
+
WHERE id = ?
|
|
1586
|
+
`
|
|
1587
|
+
).run(isAllowed ? 1 : 0, userId);
|
|
1588
|
+
}
|
|
1589
|
+
/**
|
|
1590
|
+
* Get all admins
|
|
1591
|
+
*/
|
|
1592
|
+
getAdmins() {
|
|
1593
|
+
const rows = this.db.prepare(
|
|
1594
|
+
`
|
|
1595
|
+
SELECT * FROM tg_users
|
|
1596
|
+
WHERE is_admin = 1
|
|
1597
|
+
ORDER BY last_seen_at DESC
|
|
1598
|
+
`
|
|
1599
|
+
).all();
|
|
1600
|
+
return rows.map((row) => ({
|
|
1601
|
+
id: row.id,
|
|
1602
|
+
username: row.username,
|
|
1603
|
+
firstName: row.first_name,
|
|
1604
|
+
lastName: row.last_name,
|
|
1605
|
+
isBot: Boolean(row.is_bot),
|
|
1606
|
+
isAdmin: Boolean(row.is_admin),
|
|
1607
|
+
isAllowed: Boolean(row.is_allowed),
|
|
1608
|
+
firstSeenAt: new Date(row.first_seen_at * 1e3),
|
|
1609
|
+
lastSeenAt: new Date(row.last_seen_at * 1e3),
|
|
1610
|
+
messageCount: row.message_count
|
|
1611
|
+
}));
|
|
1612
|
+
}
|
|
1613
|
+
/**
|
|
1614
|
+
* Get recently active users
|
|
1615
|
+
*/
|
|
1616
|
+
getRecentUsers(limit = 50) {
|
|
1617
|
+
const rows = this.db.prepare(
|
|
1618
|
+
`
|
|
1619
|
+
SELECT * FROM tg_users
|
|
1620
|
+
ORDER BY last_seen_at DESC
|
|
1621
|
+
LIMIT ?
|
|
1622
|
+
`
|
|
1623
|
+
).all(limit);
|
|
1624
|
+
return rows.map((row) => ({
|
|
1625
|
+
id: row.id,
|
|
1626
|
+
username: row.username,
|
|
1627
|
+
firstName: row.first_name,
|
|
1628
|
+
lastName: row.last_name,
|
|
1629
|
+
isBot: Boolean(row.is_bot),
|
|
1630
|
+
isAdmin: Boolean(row.is_admin),
|
|
1631
|
+
isAllowed: Boolean(row.is_allowed),
|
|
1632
|
+
firstSeenAt: new Date(row.first_seen_at * 1e3),
|
|
1633
|
+
lastSeenAt: new Date(row.last_seen_at * 1e3),
|
|
1634
|
+
messageCount: row.message_count
|
|
1635
|
+
}));
|
|
1636
|
+
}
|
|
1637
|
+
};
|
|
1638
|
+
|
|
1639
|
+
// src/memory/search/hybrid.ts
|
|
1640
|
+
function escapeFts5Query(query) {
|
|
1641
|
+
return query.replace(/["\*\-\+\(\)\:\^\~\?\.\@\#\$\%\&\!\[\]\{\}\|\\\/<>=,;'`]/g, " ").replace(/\s+/g, " ").trim();
|
|
1642
|
+
}
|
|
1643
|
+
var HybridSearch = class {
|
|
1644
|
+
constructor(db, vectorEnabled) {
|
|
1645
|
+
this.db = db;
|
|
1646
|
+
this.vectorEnabled = vectorEnabled;
|
|
1647
|
+
}
|
|
1648
|
+
/**
|
|
1649
|
+
* Search in knowledge base
|
|
1650
|
+
*/
|
|
1651
|
+
async searchKnowledge(query, queryEmbedding, options = {}) {
|
|
1652
|
+
const limit = options.limit ?? 10;
|
|
1653
|
+
const vectorWeight = options.vectorWeight ?? 0.7;
|
|
1654
|
+
const keywordWeight = options.keywordWeight ?? 0.3;
|
|
1655
|
+
const vectorResults = this.vectorEnabled ? this.vectorSearchKnowledge(queryEmbedding, limit * 2) : [];
|
|
1656
|
+
const keywordResults = this.keywordSearchKnowledge(query, limit * 2);
|
|
1657
|
+
return this.mergeResults(vectorResults, keywordResults, vectorWeight, keywordWeight, limit);
|
|
1658
|
+
}
|
|
1659
|
+
/**
|
|
1660
|
+
* Search in Telegram messages
|
|
1661
|
+
*/
|
|
1662
|
+
async searchMessages(query, queryEmbedding, options = {}) {
|
|
1663
|
+
const limit = options.limit ?? 10;
|
|
1664
|
+
const vectorWeight = options.vectorWeight ?? 0.7;
|
|
1665
|
+
const keywordWeight = options.keywordWeight ?? 0.3;
|
|
1666
|
+
const vectorResults = this.vectorEnabled ? this.vectorSearchMessages(queryEmbedding, limit * 2, options.chatId) : [];
|
|
1667
|
+
const keywordResults = this.keywordSearchMessages(query, limit * 2, options.chatId);
|
|
1668
|
+
return this.mergeResults(vectorResults, keywordResults, vectorWeight, keywordWeight, limit);
|
|
1669
|
+
}
|
|
1670
|
+
vectorSearchKnowledge(embedding, limit) {
|
|
1671
|
+
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1672
|
+
try {
|
|
1673
|
+
const embeddingBuffer = this.serializeEmbedding(embedding);
|
|
1674
|
+
const rows = this.db.prepare(
|
|
1675
|
+
`
|
|
1676
|
+
SELECT kv.id, k.text, k.source, kv.distance
|
|
1677
|
+
FROM knowledge_vec kv
|
|
1678
|
+
JOIN knowledge k ON k.id = kv.id
|
|
1679
|
+
WHERE kv.embedding MATCH ?
|
|
1680
|
+
ORDER BY kv.distance
|
|
1681
|
+
LIMIT ?
|
|
1682
|
+
`
|
|
1683
|
+
).all(embeddingBuffer, limit);
|
|
1684
|
+
return rows.map((row) => ({
|
|
1685
|
+
id: row.id,
|
|
1686
|
+
text: row.text,
|
|
1687
|
+
source: row.source,
|
|
1688
|
+
score: 1 - row.distance,
|
|
1689
|
+
// Convert distance to similarity
|
|
1690
|
+
vectorScore: 1 - row.distance
|
|
1691
|
+
}));
|
|
1692
|
+
} catch (error) {
|
|
1693
|
+
console.error("Vector search error (knowledge):", error);
|
|
1694
|
+
return [];
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
keywordSearchKnowledge(query, limit) {
|
|
1698
|
+
const safeQuery = escapeFts5Query(query);
|
|
1699
|
+
if (!safeQuery) return [];
|
|
1700
|
+
try {
|
|
1701
|
+
const rows = this.db.prepare(
|
|
1702
|
+
`
|
|
1703
|
+
SELECT k.id, k.text, k.source, rank as score
|
|
1704
|
+
FROM knowledge_fts kf
|
|
1705
|
+
JOIN knowledge k ON k.rowid = kf.rowid
|
|
1706
|
+
WHERE knowledge_fts MATCH ?
|
|
1707
|
+
ORDER BY rank
|
|
1708
|
+
LIMIT ?
|
|
1709
|
+
`
|
|
1710
|
+
).all(safeQuery, limit);
|
|
1711
|
+
return rows.map((row) => ({
|
|
1712
|
+
...row,
|
|
1713
|
+
keywordScore: this.bm25ToScore(row.score)
|
|
1714
|
+
}));
|
|
1715
|
+
} catch (error) {
|
|
1716
|
+
console.error("FTS5 search error (knowledge):", error);
|
|
1717
|
+
return [];
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
vectorSearchMessages(embedding, limit, chatId) {
|
|
1721
|
+
if (!this.vectorEnabled || embedding.length === 0) return [];
|
|
1722
|
+
try {
|
|
1723
|
+
const embeddingBuffer = this.serializeEmbedding(embedding);
|
|
1724
|
+
const sql = chatId ? `
|
|
1725
|
+
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1726
|
+
FROM tg_messages_vec mv
|
|
1727
|
+
JOIN tg_messages m ON m.id = mv.id
|
|
1728
|
+
WHERE mv.embedding MATCH ? AND m.chat_id = ?
|
|
1729
|
+
ORDER BY mv.distance
|
|
1730
|
+
LIMIT ?
|
|
1731
|
+
` : `
|
|
1732
|
+
SELECT mv.id, m.text, m.chat_id as source, mv.distance
|
|
1733
|
+
FROM tg_messages_vec mv
|
|
1734
|
+
JOIN tg_messages m ON m.id = mv.id
|
|
1735
|
+
WHERE mv.embedding MATCH ?
|
|
1736
|
+
ORDER BY mv.distance
|
|
1737
|
+
LIMIT ?
|
|
1738
|
+
`;
|
|
1739
|
+
const rows = chatId ? this.db.prepare(sql).all(embeddingBuffer, chatId, limit) : this.db.prepare(sql).all(embeddingBuffer, limit);
|
|
1740
|
+
return rows.map((row) => ({
|
|
1741
|
+
id: row.id,
|
|
1742
|
+
text: row.text ?? "",
|
|
1743
|
+
source: row.source,
|
|
1744
|
+
score: 1 - row.distance,
|
|
1745
|
+
vectorScore: 1 - row.distance
|
|
1746
|
+
}));
|
|
1747
|
+
} catch (error) {
|
|
1748
|
+
console.error("Vector search error (messages):", error);
|
|
1749
|
+
return [];
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
keywordSearchMessages(query, limit, chatId) {
|
|
1753
|
+
const safeQuery = escapeFts5Query(query);
|
|
1754
|
+
if (!safeQuery) return [];
|
|
1755
|
+
try {
|
|
1756
|
+
const sql = chatId ? `
|
|
1757
|
+
SELECT m.id, m.text, m.chat_id as source, rank as score
|
|
1758
|
+
FROM tg_messages_fts mf
|
|
1759
|
+
JOIN tg_messages m ON m.rowid = mf.rowid
|
|
1760
|
+
WHERE tg_messages_fts MATCH ? AND m.chat_id = ?
|
|
1761
|
+
ORDER BY rank
|
|
1762
|
+
LIMIT ?
|
|
1763
|
+
` : `
|
|
1764
|
+
SELECT m.id, m.text, m.chat_id as source, rank as score
|
|
1765
|
+
FROM tg_messages_fts mf
|
|
1766
|
+
JOIN tg_messages m ON m.rowid = mf.rowid
|
|
1767
|
+
WHERE tg_messages_fts MATCH ?
|
|
1768
|
+
ORDER BY rank
|
|
1769
|
+
LIMIT ?
|
|
1770
|
+
`;
|
|
1771
|
+
const rows = chatId ? this.db.prepare(sql).all(safeQuery, chatId, limit) : this.db.prepare(sql).all(safeQuery, limit);
|
|
1772
|
+
return rows.map((row) => ({
|
|
1773
|
+
...row,
|
|
1774
|
+
text: row.text ?? "",
|
|
1775
|
+
keywordScore: this.bm25ToScore(row.score)
|
|
1776
|
+
}));
|
|
1777
|
+
} catch (error) {
|
|
1778
|
+
console.error("FTS5 search error (messages):", error);
|
|
1779
|
+
return [];
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
mergeResults(vectorResults, keywordResults, vectorWeight, keywordWeight, limit) {
|
|
1783
|
+
const byId = /* @__PURE__ */ new Map();
|
|
1784
|
+
for (const r of vectorResults) {
|
|
1785
|
+
byId.set(r.id, { ...r, vectorScore: r.score });
|
|
1786
|
+
}
|
|
1787
|
+
for (const r of keywordResults) {
|
|
1788
|
+
const existing = byId.get(r.id);
|
|
1789
|
+
if (existing) {
|
|
1790
|
+
existing.keywordScore = r.keywordScore;
|
|
1791
|
+
existing.score = vectorWeight * (existing.vectorScore ?? 0) + keywordWeight * (r.keywordScore ?? 0);
|
|
1792
|
+
} else {
|
|
1793
|
+
byId.set(r.id, { ...r, score: keywordWeight * (r.keywordScore ?? 0) });
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
return Array.from(byId.values()).sort((a, b) => b.score - a.score).slice(0, limit);
|
|
1797
|
+
}
|
|
1798
|
+
bm25ToScore(rank) {
|
|
1799
|
+
return 1 / (1 + Math.abs(rank));
|
|
1800
|
+
}
|
|
1801
|
+
/**
|
|
1802
|
+
* Serialize embedding (number[]) to Buffer for sqlite-vec
|
|
1803
|
+
*/
|
|
1804
|
+
serializeEmbedding(embedding) {
|
|
1805
|
+
const float32 = new Float32Array(embedding);
|
|
1806
|
+
return Buffer.from(float32.buffer);
|
|
1807
|
+
}
|
|
1808
|
+
};
|
|
1809
|
+
|
|
1810
|
+
// src/memory/search/context.ts
|
|
1811
|
+
var ContextBuilder = class {
|
|
1812
|
+
constructor(db, embedder, vectorEnabled) {
|
|
1813
|
+
this.db = db;
|
|
1814
|
+
this.embedder = embedder;
|
|
1815
|
+
this.hybridSearch = new HybridSearch(db, vectorEnabled);
|
|
1816
|
+
this.messageStore = new MessageStore(db, embedder, vectorEnabled);
|
|
1817
|
+
}
|
|
1818
|
+
hybridSearch;
|
|
1819
|
+
messageStore;
|
|
1820
|
+
async buildContext(options) {
|
|
1821
|
+
const {
|
|
1822
|
+
query,
|
|
1823
|
+
chatId,
|
|
1824
|
+
includeAgentMemory = true,
|
|
1825
|
+
includeFeedHistory = true,
|
|
1826
|
+
searchAllChats = false,
|
|
1827
|
+
maxRecentMessages = 20,
|
|
1828
|
+
maxRelevantChunks = 5
|
|
1829
|
+
} = options;
|
|
1830
|
+
const queryEmbedding = await this.embedder.embedQuery(query);
|
|
1831
|
+
const recentTgMessages = this.messageStore.getRecentMessages(chatId, maxRecentMessages);
|
|
1832
|
+
const recentMessages = recentTgMessages.map((m) => ({
|
|
1833
|
+
role: m.isFromAgent ? "assistant" : "user",
|
|
1834
|
+
content: m.text ?? ""
|
|
1835
|
+
}));
|
|
1836
|
+
const relevantKnowledge = [];
|
|
1837
|
+
if (includeAgentMemory) {
|
|
1838
|
+
try {
|
|
1839
|
+
const knowledgeResults = await this.hybridSearch.searchKnowledge(query, queryEmbedding, {
|
|
1840
|
+
limit: maxRelevantChunks
|
|
1841
|
+
});
|
|
1842
|
+
relevantKnowledge.push(...knowledgeResults.map((r) => r.text));
|
|
1843
|
+
} catch (error) {
|
|
1844
|
+
console.warn("Knowledge search failed:", error);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
const relevantFeed = [];
|
|
1848
|
+
if (includeFeedHistory) {
|
|
1849
|
+
try {
|
|
1850
|
+
const feedResults = await this.hybridSearch.searchMessages(query, queryEmbedding, {
|
|
1851
|
+
chatId,
|
|
1852
|
+
limit: maxRelevantChunks
|
|
1853
|
+
});
|
|
1854
|
+
relevantFeed.push(...feedResults.map((r) => r.text));
|
|
1855
|
+
if (searchAllChats) {
|
|
1856
|
+
const globalResults = await this.hybridSearch.searchMessages(query, queryEmbedding, {
|
|
1857
|
+
// No chatId = search all chats
|
|
1858
|
+
limit: maxRelevantChunks
|
|
1859
|
+
});
|
|
1860
|
+
const existingTexts = new Set(relevantFeed);
|
|
1861
|
+
for (const r of globalResults) {
|
|
1862
|
+
if (!existingTexts.has(r.text)) {
|
|
1863
|
+
relevantFeed.push(`[From chat ${r.source}]: ${r.text}`);
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
} catch (error) {
|
|
1868
|
+
console.warn("Feed search failed:", error);
|
|
1869
|
+
}
|
|
1870
|
+
if (relevantFeed.length === 0 && recentTgMessages.length > 0) {
|
|
1871
|
+
const recentTexts = recentTgMessages.filter((m) => m.text && m.text.length > 0).slice(-maxRelevantChunks).map((m) => {
|
|
1872
|
+
const sender = m.isFromAgent ? "Agent" : "User";
|
|
1873
|
+
return `[${sender}]: ${m.text}`;
|
|
1874
|
+
});
|
|
1875
|
+
relevantFeed.push(...recentTexts);
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
const allText = recentMessages.map((m) => m.content).join(" ") + relevantKnowledge.join(" ") + relevantFeed.join(" ");
|
|
1879
|
+
const estimatedTokens = Math.ceil(allText.length / 4);
|
|
1880
|
+
return {
|
|
1881
|
+
recentMessages,
|
|
1882
|
+
relevantKnowledge,
|
|
1883
|
+
relevantFeed,
|
|
1884
|
+
estimatedTokens
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
};
|
|
1888
|
+
|
|
1889
|
+
// src/memory/index.ts
|
|
1890
|
+
function initializeMemory(config) {
|
|
1891
|
+
const db = getDatabase(config.database);
|
|
1892
|
+
const embedder = createEmbeddingProvider(config.embeddings);
|
|
1893
|
+
const vectorEnabled = db.isVectorSearchReady();
|
|
1894
|
+
const database = db.getDb();
|
|
1895
|
+
return {
|
|
1896
|
+
db: database,
|
|
1897
|
+
embedder,
|
|
1898
|
+
knowledge: new KnowledgeIndexer(database, config.workspaceDir, embedder, vectorEnabled),
|
|
1899
|
+
messages: new MessageStore(database, embedder, vectorEnabled),
|
|
1900
|
+
context: new ContextBuilder(database, embedder, vectorEnabled)
|
|
1901
|
+
};
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
export {
|
|
1905
|
+
ensureSchema,
|
|
1906
|
+
ensureVectorTables,
|
|
1907
|
+
getSchemaVersion,
|
|
1908
|
+
setSchemaVersion,
|
|
1909
|
+
CURRENT_SCHEMA_VERSION,
|
|
1910
|
+
runMigrations,
|
|
1911
|
+
MemoryDatabase,
|
|
1912
|
+
getDatabase,
|
|
1913
|
+
closeDatabase,
|
|
1914
|
+
NoopEmbeddingProvider,
|
|
1915
|
+
AnthropicEmbeddingProvider,
|
|
1916
|
+
LocalEmbeddingProvider,
|
|
1917
|
+
createEmbeddingProvider,
|
|
1918
|
+
hashText,
|
|
1919
|
+
serializeEmbedding,
|
|
1920
|
+
deserializeEmbedding,
|
|
1921
|
+
embeddingToBlob,
|
|
1922
|
+
KnowledgeIndexer,
|
|
1923
|
+
SessionStore,
|
|
1924
|
+
MessageStore,
|
|
1925
|
+
ChatStore,
|
|
1926
|
+
UserStore,
|
|
1927
|
+
HybridSearch,
|
|
1928
|
+
ContextBuilder,
|
|
1929
|
+
initializeMemory
|
|
1930
|
+
};
|