th-memory-mcp 2.2.8 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE_v2.md +1 -1
- package/README.md +54 -3
- package/README.th.md +51 -1
- package/SECURITY.md +9 -0
- package/design.md +1 -1
- package/dist/cli/commands.js +507 -0
- package/dist/cli.js +4 -0
- package/dist/core/consolidation-engine.js +119 -2
- package/dist/core/graph-engine.js +11 -0
- package/dist/db/index.js +4 -0
- package/dist/db/migrations.js +47 -12
- package/dist/index.js +1 -1
- package/dist/lib/config.js +3 -1
- package/dist/lib/highlight.js +139 -0
- package/dist/lib/iso.js +11 -0
- package/dist/lib/memory-format.js +53 -0
- package/dist/memory/deduplicator.js +24 -9
- package/dist/retrieval/fts.js +17 -26
- package/dist/tools/consolidate.js +47 -21
- package/dist/tools/context.js +6 -2
- package/dist/tools/export_memory.js +43 -0
- package/dist/tools/forget.js +31 -4
- package/dist/tools/import_memory.js +404 -88
- package/dist/tools/merge_memory.js +14 -0
- package/dist/tools/profile.js +16 -3
- package/dist/tools/recall.js +3 -1
- package/dist/tools/update_memory.js +29 -3
- package/package.json +5 -3
- package/dist/db.js +0 -111
|
@@ -4,8 +4,9 @@ import { join, dirname, resolve, sep } from "node:path";
|
|
|
4
4
|
import { db, DB_PATH, nowISO, syncSearchIndex, upsertEmbedding, ok, err, } from "../db/index.js";
|
|
5
5
|
import { createMemory } from "../db/repositories/memories.js";
|
|
6
6
|
import { deduplicate } from "../memory/deduplicator.js";
|
|
7
|
-
import { MEMORY_TYPES, SOURCE_TYPES } from "../memory/types.js";
|
|
7
|
+
import { MEMORY_TYPES, SOURCE_TYPES, LIFECYCLE_STATES, LINK_RELATIONS } from "../memory/types.js";
|
|
8
8
|
import { EXPORTS_DIRNAME } from "../lib/config.js";
|
|
9
|
+
import { isIsoDateString } from "../lib/iso.js";
|
|
9
10
|
import { embed } from "../lib/embed.js";
|
|
10
11
|
export const importMemoryInput = {
|
|
11
12
|
file: z
|
|
@@ -26,23 +27,30 @@ export const importMemoryInput = {
|
|
|
26
27
|
.optional()
|
|
27
28
|
.describe("Scope imported memories to a user (USER scope)"),
|
|
28
29
|
};
|
|
29
|
-
function isIsoDateString(s) {
|
|
30
|
-
if (typeof s !== "string")
|
|
31
|
-
return false;
|
|
32
|
-
const d = Date.parse(s);
|
|
33
|
-
if (Number.isNaN(d))
|
|
34
|
-
return false;
|
|
35
|
-
try {
|
|
36
|
-
const iso = new Date(s).toISOString();
|
|
37
|
-
return iso.length >= 10;
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
return false;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
30
|
function isValidSource(s) {
|
|
44
31
|
return SOURCE_TYPES.includes(s);
|
|
45
32
|
}
|
|
33
|
+
// Batch A-2: normalize the per-item trust marker. Accepts boolean true,
|
|
34
|
+
// "trusted"/"trust"/"yes"/"1" strings as trusted; everything else (including
|
|
35
|
+
// missing) defaults to untrusted (false). No schema change — the flag is
|
|
36
|
+
// carried inside memory metadata as { trusted: boolean } for team B delimiters.
|
|
37
|
+
function parseTrustedFlag(v) {
|
|
38
|
+
// Strict: only a boolean true marks a source trusted. String aliases
|
|
39
|
+
// ("yes"/"true"/"1") are NOT accepted — otherwise a caller could pass
|
|
40
|
+
// trusted:"yes" and bypass the untrusted-import label.
|
|
41
|
+
return v === true;
|
|
42
|
+
}
|
|
43
|
+
function withTrustedFlag(meta, trusted) {
|
|
44
|
+
const flag = parseTrustedFlag(trusted);
|
|
45
|
+
if (meta == null)
|
|
46
|
+
return { trusted: flag };
|
|
47
|
+
if (typeof meta === "object" && !Array.isArray(meta)) {
|
|
48
|
+
return { ...meta, trusted: flag };
|
|
49
|
+
}
|
|
50
|
+
// Non-object legacy metadata cannot carry a field — wrap it so the
|
|
51
|
+
// contract metadata.trusted stays readable without altering the DB schema.
|
|
52
|
+
return { value: meta, trusted: flag };
|
|
53
|
+
}
|
|
46
54
|
export function importMemoryHandler(args) {
|
|
47
55
|
try {
|
|
48
56
|
let raw;
|
|
@@ -86,19 +94,15 @@ export function importMemoryHandler(args) {
|
|
|
86
94
|
let exportPrefs = null;
|
|
87
95
|
let exportLessons = null;
|
|
88
96
|
let exportProfile = null;
|
|
97
|
+
let exportLinks = null;
|
|
98
|
+
// Batch A: additive backup fields (may be absent in legacy v2 files -> null = backward compat).
|
|
99
|
+
let exportUsers = null;
|
|
100
|
+
let exportEntities = null;
|
|
101
|
+
let exportRelations = null;
|
|
89
102
|
if (Array.isArray(parsed)) {
|
|
90
103
|
memoryItems = parsed;
|
|
91
104
|
}
|
|
92
|
-
else if (parsed &&
|
|
93
|
-
typeof parsed === "object" &&
|
|
94
|
-
Array.isArray(parsed.memories)) {
|
|
95
|
-
memoryItems = parsed.memories;
|
|
96
|
-
}
|
|
97
|
-
else if (parsed &&
|
|
98
|
-
typeof parsed === "object" &&
|
|
99
|
-
("preferences" in parsed ||
|
|
100
|
-
"lessons" in parsed ||
|
|
101
|
-
"profile" in parsed)) {
|
|
105
|
+
else if (parsed && typeof parsed === "object") {
|
|
102
106
|
const p = parsed;
|
|
103
107
|
if (Array.isArray(p.preferences))
|
|
104
108
|
exportPrefs = p.preferences;
|
|
@@ -108,15 +112,63 @@ export function importMemoryHandler(args) {
|
|
|
108
112
|
exportProfile = p.profile;
|
|
109
113
|
if (Array.isArray(p.memories))
|
|
110
114
|
memoryItems = p.memories;
|
|
115
|
+
if (Array.isArray(p.memoryLinks))
|
|
116
|
+
exportLinks = p.memoryLinks;
|
|
117
|
+
// Accept both new keys and legacy absence (missing => null => skip, still pass).
|
|
118
|
+
if (Array.isArray(p.users))
|
|
119
|
+
exportUsers = p.users;
|
|
120
|
+
if (Array.isArray(p.entities))
|
|
121
|
+
exportEntities = p.entities;
|
|
122
|
+
if (Array.isArray(p.relations))
|
|
123
|
+
exportRelations = p.relations;
|
|
111
124
|
}
|
|
112
125
|
else {
|
|
113
126
|
memoryItems = [];
|
|
114
127
|
}
|
|
128
|
+
// Batch A-2 phase 1: parse + validate everything first (no DB writes).
|
|
129
|
+
// Phase 2 (below) applies all validated rows inside ONE db.transaction,
|
|
130
|
+
// so a mid-way error rolls back everything. Users are validated here and
|
|
131
|
+
// restored first inside the transaction so createMemory()->ensureUser
|
|
132
|
+
// reuses the restored row. Idempotent via UNIQUE(external_id).
|
|
133
|
+
let usersImported = 0;
|
|
134
|
+
let usersInvalid = 0;
|
|
135
|
+
const validUsers = [];
|
|
136
|
+
if (exportUsers != null) {
|
|
137
|
+
for (const u of exportUsers) {
|
|
138
|
+
const external = typeof u.externalId === "string"
|
|
139
|
+
? u.externalId
|
|
140
|
+
: typeof u.external_id === "string"
|
|
141
|
+
? u.external_id
|
|
142
|
+
: null;
|
|
143
|
+
if (!u ||
|
|
144
|
+
!external ||
|
|
145
|
+
external.length === 0 ||
|
|
146
|
+
external.length > 200 ||
|
|
147
|
+
/[\x00-\x1f]/.test(external)) {
|
|
148
|
+
usersInvalid++;
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const created = (typeof u.createdAt === "string" && isIsoDateString(u.createdAt)
|
|
152
|
+
? u.createdAt
|
|
153
|
+
: typeof u.created_at === "string" && isIsoDateString(u.created_at)
|
|
154
|
+
? u.created_at
|
|
155
|
+
: null) ?? nowISO();
|
|
156
|
+
const name = typeof u.name === "string" ? u.name.slice(0, 500) : null;
|
|
157
|
+
usersImported++;
|
|
158
|
+
validUsers.push({ external, name, created });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
115
161
|
let wouldImport = 0;
|
|
116
162
|
let skipped = 0;
|
|
117
163
|
let invalid = 0;
|
|
118
164
|
const log = [];
|
|
119
|
-
|
|
165
|
+
const importedIds = new Map();
|
|
166
|
+
// Phase-1 validated memory payloads (applied in the single transaction).
|
|
167
|
+
const validMemories = [];
|
|
168
|
+
// Intra-batch exact-dupe guard: deduplicate() only sees committed rows,
|
|
169
|
+
// so track normalized type+content accepted in THIS batch as well.
|
|
170
|
+
const batchSeen = new Set();
|
|
171
|
+
for (const [idx, it] of memoryItems.entries()) {
|
|
120
172
|
if (!it ||
|
|
121
173
|
typeof it.content !== "string" ||
|
|
122
174
|
it.content.length === 0 ||
|
|
@@ -141,6 +193,14 @@ export function importMemoryHandler(args) {
|
|
|
141
193
|
invalid++;
|
|
142
194
|
continue;
|
|
143
195
|
}
|
|
196
|
+
if (it.salience != null && (typeof it.salience !== "number" || it.salience < 0 || it.salience > 1)) {
|
|
197
|
+
invalid++;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (it.status != null && (typeof it.status !== "string" || !LIFECYCLE_STATES.includes(it.status))) {
|
|
201
|
+
invalid++;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
144
204
|
if (it.projectId != null && (typeof it.projectId !== "string" || it.projectId.length === 0 || it.projectId.length > 200)) {
|
|
145
205
|
invalid++;
|
|
146
206
|
continue;
|
|
@@ -155,10 +215,19 @@ export function importMemoryHandler(args) {
|
|
|
155
215
|
}
|
|
156
216
|
if (it.validFrom != null && (typeof it.validFrom !== "string" || !isIsoDateString(it.validFrom))) {
|
|
157
217
|
invalid++;
|
|
218
|
+
log.push(`invalid row ${idx}: validFrom '${String(it.validFrom)}' rejected — must be a full ISO datetime with timezone offset (e.g. 2024-01-01T00:00:00.000Z), date-only '2024-01-01' is not accepted`);
|
|
158
219
|
continue;
|
|
159
220
|
}
|
|
160
221
|
if (it.validUntil != null && (typeof it.validUntil !== "string" || !isIsoDateString(it.validUntil))) {
|
|
161
222
|
invalid++;
|
|
223
|
+
log.push(`invalid row ${idx}: validUntil '${String(it.validUntil)}' rejected — must be a full ISO datetime with timezone offset (e.g. 2024-01-01T00:00:00.000Z), date-only '2024-01-01' is not accepted`);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
if (typeof it.validFrom === "string" &&
|
|
227
|
+
typeof it.validUntil === "string" &&
|
|
228
|
+
new Date(it.validFrom) > new Date(it.validUntil)) {
|
|
229
|
+
invalid++;
|
|
230
|
+
log.push(`invalid row ${idx}: validFrom (${it.validFrom}) must not be later than validUntil (${it.validUntil})`);
|
|
162
231
|
continue;
|
|
163
232
|
}
|
|
164
233
|
const dup = deduplicate(it.type, it.content);
|
|
@@ -167,43 +236,153 @@ export function importMemoryHandler(args) {
|
|
|
167
236
|
log.push(`skip duplicate -> existing ${dup.existingId}`);
|
|
168
237
|
continue;
|
|
169
238
|
}
|
|
239
|
+
const batchKey = `${it.type}::${it.content.toLowerCase().trim()}`;
|
|
240
|
+
if (batchSeen.has(batchKey)) {
|
|
241
|
+
skipped++;
|
|
242
|
+
log.push(`skip duplicate -> duplicate within import batch (row ${idx})`);
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
batchSeen.add(batchKey);
|
|
170
246
|
wouldImport++;
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
247
|
+
validMemories.push({ idx, it });
|
|
248
|
+
}
|
|
249
|
+
let linksImported = 0;
|
|
250
|
+
let linksInvalid = 0;
|
|
251
|
+
const validLinks = [];
|
|
252
|
+
if (exportLinks != null) {
|
|
253
|
+
// Phase-1 remap precheck (no writes): a link is resolvable in phase 2
|
|
254
|
+
// iff both old ids belong to memories validated in THIS batch
|
|
255
|
+
// (mirrors the original importedIds-only rule, but works for dry-run too —
|
|
256
|
+
// the old code always reported links invalid on dry-run because the map
|
|
257
|
+
// was still empty at validate time).
|
|
258
|
+
const batchMemIds = new Set();
|
|
259
|
+
for (const { it } of validMemories) {
|
|
260
|
+
if (typeof it.id === "number" && Number.isInteger(it.id))
|
|
261
|
+
batchMemIds.add(it.id);
|
|
262
|
+
}
|
|
263
|
+
for (const link of exportLinks) {
|
|
264
|
+
if (typeof link.sourceId !== "number" ||
|
|
265
|
+
typeof link.targetId !== "number" ||
|
|
266
|
+
typeof link.relation !== "string" ||
|
|
267
|
+
!LINK_RELATIONS.includes(link.relation) ||
|
|
268
|
+
(link.confidence != null && (typeof link.confidence !== "number" || link.confidence < 0 || link.confidence > 1))) {
|
|
269
|
+
linksInvalid++;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
if (!batchMemIds.has(link.sourceId) || !batchMemIds.has(link.targetId)) {
|
|
273
|
+
linksInvalid++;
|
|
274
|
+
log.push(`invalid link ${link.sourceId}->${link.targetId}: unresolvable memory id`);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
linksImported++;
|
|
278
|
+
validLinks.push({ sourceId: link.sourceId, targetId: link.targetId, relation: link.relation, confidence: link.confidence });
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// Batch A: restore M003 entities/relations idempotently.
|
|
282
|
+
// - entities: dedupe by UNIQUE(canonical_name); keep oldId->newId map for relations.
|
|
283
|
+
// - relations: `relation` is free-form (e.g. co_occurs) so do NOT check LINK_RELATIONS
|
|
284
|
+
// here (that check stays only for memory_links above). Keep ISO validation as-is;
|
|
285
|
+
// another team will unify validation later. Missing arrays (legacy v2) => skip, still pass.
|
|
286
|
+
let entitiesImported = 0;
|
|
287
|
+
let entitiesInvalid = 0;
|
|
288
|
+
const importedEntityIds = new Map();
|
|
289
|
+
const validEntities = [];
|
|
290
|
+
if (exportEntities != null) {
|
|
291
|
+
for (const e of exportEntities) {
|
|
292
|
+
const name = typeof e.name === "string" ? e.name : "";
|
|
293
|
+
const canonical = typeof e.canonicalName === "string"
|
|
294
|
+
? e.canonicalName
|
|
295
|
+
: typeof e.canonical_name === "string"
|
|
296
|
+
? e.canonical_name
|
|
297
|
+
: "";
|
|
298
|
+
if (!e || name.length === 0 || name.length > 500 || canonical.length === 0 || canonical.length > 500) {
|
|
299
|
+
entitiesInvalid++;
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
entitiesImported++;
|
|
303
|
+
validEntities.push({ e, name, canonical });
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
let relationsImported = 0;
|
|
307
|
+
let relationsInvalid = 0;
|
|
308
|
+
const validRelations = [];
|
|
309
|
+
if (exportRelations != null) {
|
|
310
|
+
// Phase-1 entity-existence precheck (read-only): an old entity id is
|
|
311
|
+
// resolvable in phase 2 iff it is shipped in THIS batch (validEntities)
|
|
312
|
+
// or still exists in the DB. oldMem stays orphan-safe (NULL fallback).
|
|
313
|
+
const batchEntIds = new Set();
|
|
314
|
+
for (const { e } of validEntities) {
|
|
315
|
+
if (typeof e.id === "number" && Number.isInteger(e.id))
|
|
316
|
+
batchEntIds.add(e.id);
|
|
317
|
+
}
|
|
318
|
+
const selEntExists = db.prepare("SELECT id FROM entities WHERE id = ?");
|
|
319
|
+
for (const r of exportRelations) {
|
|
320
|
+
const oldSource = typeof r.sourceEntityId === "number"
|
|
321
|
+
? r.sourceEntityId
|
|
322
|
+
: typeof r.source_entity_id === "number"
|
|
323
|
+
? r.source_entity_id
|
|
324
|
+
: null;
|
|
325
|
+
const oldTarget = typeof r.targetEntityId === "number"
|
|
326
|
+
? r.targetEntityId
|
|
327
|
+
: typeof r.target_entity_id === "number"
|
|
328
|
+
? r.target_entity_id
|
|
329
|
+
: null;
|
|
330
|
+
const predicate = typeof r.relation === "string" ? r.relation : "";
|
|
331
|
+
const conf = r.confidence == null ? 0.5 : r.confidence;
|
|
332
|
+
const vf = typeof r.validFrom === "string"
|
|
333
|
+
? r.validFrom
|
|
334
|
+
: typeof r.valid_from === "string"
|
|
335
|
+
? r.valid_from
|
|
336
|
+
: null;
|
|
337
|
+
const vu = typeof r.validUntil === "string"
|
|
338
|
+
? r.validUntil
|
|
339
|
+
: typeof r.valid_until === "string"
|
|
340
|
+
? r.valid_until
|
|
341
|
+
: null;
|
|
342
|
+
const oldMem = typeof r.sourceMemoryId === "number"
|
|
343
|
+
? r.sourceMemoryId
|
|
344
|
+
: typeof r.source_memory_id === "number"
|
|
345
|
+
? r.source_memory_id
|
|
346
|
+
: null;
|
|
347
|
+
if (!r ||
|
|
348
|
+
oldSource == null ||
|
|
349
|
+
oldTarget == null ||
|
|
350
|
+
predicate.length === 0 ||
|
|
351
|
+
predicate.length > 200 ||
|
|
352
|
+
typeof conf !== "number" ||
|
|
353
|
+
conf < 0 ||
|
|
354
|
+
conf > 1 ||
|
|
355
|
+
(vf != null && !isIsoDateString(vf)) ||
|
|
356
|
+
(vu != null && !isIsoDateString(vu)) ||
|
|
357
|
+
(vf != null && vu != null && new Date(vf) > new Date(vu))) {
|
|
358
|
+
relationsInvalid++;
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
const srcOk = batchEntIds.has(oldSource) || !!selEntExists.get(oldSource);
|
|
362
|
+
const tgtOk = batchEntIds.has(oldTarget) || !!selEntExists.get(oldTarget);
|
|
363
|
+
if (!srcOk || !tgtOk) {
|
|
364
|
+
relationsInvalid++;
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
relationsImported++;
|
|
368
|
+
validRelations.push({ r, oldSource, oldTarget, predicate, conf, vf, vu, oldMem });
|
|
190
369
|
}
|
|
191
370
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
371
|
+
// Phase-1 validation for prefs/lessons/profile (no writes yet).
|
|
372
|
+
const prefArr = Array.isArray(exportPrefs) ? exportPrefs : [];
|
|
373
|
+
const lessonArr = Array.isArray(exportLessons) ? exportLessons : [];
|
|
374
|
+
const profileArr = Array.isArray(exportProfile) ? exportProfile : [];
|
|
375
|
+
const hasAux = exportPrefs != null || exportLessons != null || exportProfile != null;
|
|
376
|
+
const validCat = new Set(["work_style", "coding_pref", "language", "domain", "other"]);
|
|
377
|
+
let prefOk = 0;
|
|
378
|
+
let prefInvalid = 0;
|
|
379
|
+
let lessonOk = 0;
|
|
380
|
+
let lessonInvalid = 0;
|
|
381
|
+
let profileOk = 0;
|
|
382
|
+
const validPrefs = [];
|
|
383
|
+
const validLessons = [];
|
|
384
|
+
const validProfiles = [];
|
|
385
|
+
if (hasAux) {
|
|
207
386
|
for (const p of prefArr) {
|
|
208
387
|
if (!p ||
|
|
209
388
|
typeof p.category !== "string" ||
|
|
@@ -219,21 +398,7 @@ export function importMemoryHandler(args) {
|
|
|
219
398
|
continue;
|
|
220
399
|
}
|
|
221
400
|
prefOk++;
|
|
222
|
-
|
|
223
|
-
const existing = selectPref.get(p.category, p.key);
|
|
224
|
-
const conf = typeof p.confidence === "number" ? p.confidence : 0.5;
|
|
225
|
-
const src = typeof p.source === "string" ? p.source : "imported";
|
|
226
|
-
const ts = typeof p.updated_at === "string" && isIsoDateString(p.updated_at) ? p.updated_at : nowISO();
|
|
227
|
-
if (existing)
|
|
228
|
-
updatePref.run(p.value, conf, src, ts, p.category, p.key);
|
|
229
|
-
else
|
|
230
|
-
insertPref.run(p.category, p.key, p.value, conf, src, ts);
|
|
231
|
-
const row = db.prepare("SELECT id FROM preferences WHERE category = ? AND key = ?").get(p.category, p.key);
|
|
232
|
-
if (row) {
|
|
233
|
-
syncSearchIndex("preferences", row.id, `${p.category}/${p.key}`, `${p.key}: ${p.value}`);
|
|
234
|
-
upsertEmbedding("preferences", row.id, embed(`${p.category} ${p.key} ${p.value}`));
|
|
235
|
-
}
|
|
236
|
-
}
|
|
401
|
+
validPrefs.push(p);
|
|
237
402
|
}
|
|
238
403
|
for (const l of lessonArr) {
|
|
239
404
|
if (!l ||
|
|
@@ -250,29 +415,180 @@ export function importMemoryHandler(args) {
|
|
|
250
415
|
continue;
|
|
251
416
|
}
|
|
252
417
|
lessonOk++;
|
|
253
|
-
|
|
254
|
-
const ts = typeof l.created_at === "string" && isIsoDateString(l.created_at) ? l.created_at : nowISO();
|
|
255
|
-
const res = insertLesson.run(l.situation, l.mistake, l.correction, ts);
|
|
256
|
-
const id = Number(res.lastInsertRowid);
|
|
257
|
-
syncSearchIndex("lessons", id, l.situation.slice(0, 80), `${l.situation} | mistake: ${l.mistake} -> correction: ${l.correction}`);
|
|
258
|
-
upsertEmbedding("lessons", id, embed(`${l.situation} ${l.mistake} ${l.correction}`));
|
|
259
|
-
}
|
|
418
|
+
validLessons.push(l);
|
|
260
419
|
}
|
|
261
420
|
for (const pr of profileArr) {
|
|
262
421
|
if (!pr || typeof pr.section !== "string" || pr.section.length === 0 || typeof pr.content !== "string" || pr.content.length === 0)
|
|
263
422
|
continue;
|
|
264
423
|
profileOk++;
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
424
|
+
validProfiles.push(pr);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
// Batch A-2 phase 2: single-transaction apply. All validated rows go in
|
|
428
|
+
// together; any mid-way throw rolls back EVERYTHING (atomic import).
|
|
429
|
+
if (args.apply === true) {
|
|
430
|
+
try {
|
|
431
|
+
const applyAll = db.transaction(() => {
|
|
432
|
+
const insertUser = db.prepare("INSERT OR IGNORE INTO users (external_id, name, created_at) VALUES (?, ?, ?)");
|
|
433
|
+
for (const u of validUsers)
|
|
434
|
+
insertUser.run(u.external, u.name, u.created);
|
|
435
|
+
for (const { it } of validMemories) {
|
|
436
|
+
const newId = createMemory({
|
|
437
|
+
type: it.type,
|
|
438
|
+
content: it.content,
|
|
439
|
+
summary: typeof it.summary === "string" ? it.summary : null,
|
|
440
|
+
source: it.source ?? "imported",
|
|
441
|
+
confidence: typeof it.confidence === "number" ? it.confidence : 0.7,
|
|
442
|
+
importance: typeof it.importance === "number" ? it.importance : 0.5,
|
|
443
|
+
salience: typeof it.salience === "number" ? it.salience : 0.5,
|
|
444
|
+
projectId: typeof it.projectId === "string" ? it.projectId : null,
|
|
445
|
+
sessionId: typeof it.sessionId === "string" ? it.sessionId : null,
|
|
446
|
+
userId: typeof it.userId === "string"
|
|
447
|
+
? it.userId
|
|
448
|
+
: typeof args.userId === "string"
|
|
449
|
+
? args.userId
|
|
450
|
+
: null,
|
|
451
|
+
validFrom: typeof it.validFrom === "string" ? it.validFrom : null,
|
|
452
|
+
validUntil: typeof it.validUntil === "string" ? it.validUntil : null,
|
|
453
|
+
metadata: withTrustedFlag(it.metadata, it.trusted),
|
|
454
|
+
});
|
|
455
|
+
if (typeof it.id === "number" && Number.isInteger(it.id))
|
|
456
|
+
importedIds.set(it.id, newId);
|
|
457
|
+
if (typeof it.status === "string" && it.status !== "active") {
|
|
458
|
+
db.prepare("UPDATE memories SET status = ? WHERE id = ?").run(it.status, newId);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
// Entities first (relations remap needs importedEntityIds).
|
|
462
|
+
const selEntity = db.prepare("SELECT id FROM entities WHERE canonical_name = ?");
|
|
463
|
+
const insEntity = db.prepare("INSERT INTO entities (name, canonical_name, type, metadata) VALUES (?, ?, ?, ?)");
|
|
464
|
+
const batchCanonical = new Map();
|
|
465
|
+
for (const { e, name, canonical } of validEntities) {
|
|
466
|
+
if (batchCanonical.has(canonical)) {
|
|
467
|
+
if (typeof e.id === "number" && Number.isInteger(e.id))
|
|
468
|
+
importedEntityIds.set(e.id, batchCanonical.get(canonical));
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
const existing = selEntity.get(canonical);
|
|
472
|
+
let newId;
|
|
473
|
+
if (existing) {
|
|
474
|
+
newId = existing.id;
|
|
475
|
+
}
|
|
476
|
+
else {
|
|
477
|
+
const type = typeof e.type === "string" ? e.type.slice(0, 200) : "concept";
|
|
478
|
+
const meta = e.metadata == null
|
|
479
|
+
? JSON.stringify({})
|
|
480
|
+
: typeof e.metadata === "string"
|
|
481
|
+
? e.metadata
|
|
482
|
+
: JSON.stringify(e.metadata);
|
|
483
|
+
const res = insEntity.run(name.slice(0, 500), canonical, type, meta);
|
|
484
|
+
newId = Number(res.lastInsertRowid);
|
|
485
|
+
}
|
|
486
|
+
batchCanonical.set(canonical, newId);
|
|
487
|
+
if (typeof e.id === "number" && Number.isInteger(e.id))
|
|
488
|
+
importedEntityIds.set(e.id, newId);
|
|
489
|
+
}
|
|
490
|
+
const insLink = db.prepare("INSERT OR IGNORE INTO memory_links (source_memory_id, relation, target_memory_id, confidence, created_at) VALUES (?, ?, ?, ?, ?)");
|
|
491
|
+
for (const link of validLinks) {
|
|
492
|
+
const sourceId = importedIds.get(link.sourceId);
|
|
493
|
+
const targetId = importedIds.get(link.targetId);
|
|
494
|
+
if (!sourceId || !targetId) {
|
|
495
|
+
// Old id has no remapped row (points outside this import and
|
|
496
|
+
// the row is gone): reclassify as invalid, keep counts honest.
|
|
497
|
+
linksImported--;
|
|
498
|
+
linksInvalid++;
|
|
499
|
+
log.push(`invalid link ${link.sourceId}->${link.targetId}: unresolvable memory id`);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
insLink.run(sourceId, link.relation, targetId, link.confidence ?? 0.5, nowISO());
|
|
503
|
+
}
|
|
504
|
+
const selEntityById = db.prepare("SELECT id FROM entities WHERE id = ?");
|
|
505
|
+
const selMemById = db.prepare("SELECT id FROM memories WHERE id = ?");
|
|
506
|
+
const selRelation = db.prepare("SELECT id FROM relations WHERE source_entity_id = ? AND relation = ? AND target_entity_id = ? AND COALESCE(source_memory_id, -1) = COALESCE(?, -1)");
|
|
507
|
+
const insRelation = db.prepare("INSERT INTO relations (source_entity_id, relation, target_entity_id, confidence, valid_from, valid_until, source_memory_id, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
|
|
508
|
+
for (const v of validRelations) {
|
|
509
|
+
let newSource = null;
|
|
510
|
+
let newTarget = null;
|
|
511
|
+
if (importedEntityIds.has(v.oldSource))
|
|
512
|
+
newSource = importedEntityIds.get(v.oldSource);
|
|
513
|
+
else if (selEntityById.get(v.oldSource))
|
|
514
|
+
newSource = v.oldSource;
|
|
515
|
+
if (importedEntityIds.has(v.oldTarget))
|
|
516
|
+
newTarget = importedEntityIds.get(v.oldTarget);
|
|
517
|
+
else if (selEntityById.get(v.oldTarget))
|
|
518
|
+
newTarget = v.oldTarget;
|
|
519
|
+
if (newSource == null || newTarget == null) {
|
|
520
|
+
relationsImported--;
|
|
521
|
+
relationsInvalid++;
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
524
|
+
let newMem = null;
|
|
525
|
+
if (v.oldMem == null)
|
|
526
|
+
newMem = null;
|
|
527
|
+
else if (importedIds.has(v.oldMem))
|
|
528
|
+
newMem = importedIds.get(v.oldMem);
|
|
529
|
+
else if (selMemById.get(v.oldMem))
|
|
530
|
+
newMem = v.oldMem;
|
|
531
|
+
else
|
|
532
|
+
newMem = null;
|
|
533
|
+
const dup = selRelation.get(newSource, v.predicate, newTarget, newMem);
|
|
534
|
+
if (dup)
|
|
535
|
+
continue;
|
|
536
|
+
const meta = v.r.metadata == null
|
|
537
|
+
? JSON.stringify({})
|
|
538
|
+
: typeof v.r.metadata === "string"
|
|
539
|
+
? v.r.metadata
|
|
540
|
+
: JSON.stringify(v.r.metadata);
|
|
541
|
+
insRelation.run(newSource, v.predicate, newTarget, v.conf, v.vf, v.vu, newMem, meta);
|
|
542
|
+
}
|
|
543
|
+
if (hasAux) {
|
|
544
|
+
const selectPref = db.prepare("SELECT id FROM preferences WHERE category = ? AND key = ?");
|
|
545
|
+
const insertPref = db.prepare("INSERT INTO preferences (category, key, value, confidence, source, updated_at) VALUES (?, ?, ?, ?, ?, ?)");
|
|
546
|
+
const updatePref = db.prepare("UPDATE preferences SET value = ?, confidence = ?, source = ?, updated_at = ? WHERE category = ? AND key = ?");
|
|
547
|
+
const insertLesson = db.prepare("INSERT INTO lessons (situation, mistake, correction, created_at) VALUES (?, ?, ?, ?)");
|
|
548
|
+
const upsertProfile = db.prepare("INSERT INTO profile (section, content, updated_at) VALUES (?, ?, ?) ON CONFLICT(section) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at");
|
|
549
|
+
for (const p of validPrefs) {
|
|
550
|
+
const existing = selectPref.get(p.category, p.key);
|
|
551
|
+
const conf = typeof p.confidence === "number" ? p.confidence : 0.5;
|
|
552
|
+
const src = typeof p.source === "string" ? p.source : "imported";
|
|
553
|
+
const ts = typeof p.updated_at === "string" && isIsoDateString(p.updated_at) ? p.updated_at : nowISO();
|
|
554
|
+
if (existing)
|
|
555
|
+
updatePref.run(p.value, conf, src, ts, p.category, p.key);
|
|
556
|
+
else
|
|
557
|
+
insertPref.run(p.category, p.key, p.value, conf, src, ts);
|
|
558
|
+
const row = db.prepare("SELECT id FROM preferences WHERE category = ? AND key = ?").get(p.category, p.key);
|
|
559
|
+
if (row) {
|
|
560
|
+
syncSearchIndex("preferences", row.id, `${p.category}/${p.key}`, `${p.key}: ${p.value}`);
|
|
561
|
+
upsertEmbedding("preferences", row.id, embed(`${p.category} ${p.key} ${p.value}`));
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
for (const l of validLessons) {
|
|
565
|
+
const ts = typeof l.created_at === "string" && isIsoDateString(l.created_at) ? l.created_at : nowISO();
|
|
566
|
+
const res = insertLesson.run(l.situation, l.mistake, l.correction, ts);
|
|
567
|
+
const id = Number(res.lastInsertRowid);
|
|
568
|
+
syncSearchIndex("lessons", id, (l.situation ?? "").slice(0, 80), `${l.situation} | mistake: ${l.mistake} -> correction: ${l.correction}`);
|
|
569
|
+
upsertEmbedding("lessons", id, embed(`${l.situation} ${l.mistake} ${l.correction}`));
|
|
570
|
+
}
|
|
571
|
+
for (const pr of validProfiles) {
|
|
572
|
+
const ts = typeof pr.updated_at === "string" && isIsoDateString(pr.updated_at) ? pr.updated_at : nowISO();
|
|
573
|
+
upsertProfile.run(pr.section, pr.content, ts);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
});
|
|
577
|
+
applyAll();
|
|
578
|
+
}
|
|
579
|
+
catch (e) {
|
|
580
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
581
|
+
return err(`import failed and rolled back: ${reason} (validated ${wouldImport} memories, ${linksImported} links, ${entitiesImported} entities, ${relationsImported} relations; nothing was written)`);
|
|
269
582
|
}
|
|
583
|
+
}
|
|
584
|
+
const graphSuffix = `users ${usersImported} ok ${usersInvalid} invalid; entities ${entitiesImported} ok ${entitiesInvalid} invalid; relations ${relationsImported} ok ${relationsInvalid} invalid`;
|
|
585
|
+
if (hasAux) {
|
|
270
586
|
const mode = args.apply === true ? "applied" : "dry-run";
|
|
271
|
-
const summary = `import ${mode}: ${wouldImport} memories to import, ${skipped} duplicate(s) skipped, ${invalid} invalid memories; preferences ${prefOk} ok ${prefInvalid} invalid; lessons ${lessonOk} ok ${lessonInvalid} invalid; profile ${profileOk} ok`;
|
|
587
|
+
const summary = `import ${mode}: ${wouldImport} memories to import, ${skipped} duplicate(s) skipped, ${invalid} invalid memories; ${linksImported} link(s) imported, ${linksInvalid} invalid link(s); preferences ${prefOk} ok ${prefInvalid} invalid; lessons ${lessonOk} ok ${lessonInvalid} invalid; profile ${profileOk} ok; ${graphSuffix}`;
|
|
272
588
|
return ok(summary);
|
|
273
589
|
}
|
|
274
590
|
const mode = args.apply === true ? "applied" : "dry-run";
|
|
275
|
-
const summary = `import ${mode}: ${wouldImport} to import, ${skipped} duplicate(s) skipped, ${invalid} invalid`;
|
|
591
|
+
const summary = `import ${mode}: ${wouldImport} to import, ${skipped} duplicate(s) skipped, ${invalid} invalid, ${linksImported} link(s) imported, ${linksInvalid} invalid link(s); ${graphSuffix}`;
|
|
276
592
|
return ok(log.length ? `${summary}\n${log.join("\n")}` : summary);
|
|
277
593
|
}
|
|
278
594
|
catch (e) {
|
|
@@ -39,6 +39,20 @@ export function mergeMemoryHandler(args) {
|
|
|
39
39
|
return err("cannot merge a memory into itself");
|
|
40
40
|
if (src.status === "deleted" || tgt.status === "deleted")
|
|
41
41
|
return err("cannot merge deleted memories");
|
|
42
|
+
// Batch B-3 (defense-in-depth, mirrors link_memory): refuse to merge
|
|
43
|
+
// across scope boundaries. Note: userId/sessionId/projectId are
|
|
44
|
+
// caller-supplied with no auth layer (single-user local process) — these
|
|
45
|
+
// checks are only as trustworthy as the caller (see README/SECURITY).
|
|
46
|
+
if ((src.scope === "USER" || tgt.scope === "USER") &&
|
|
47
|
+
src.user_id !== tgt.user_id)
|
|
48
|
+
return err("cannot merge memories across different users");
|
|
49
|
+
if ((src.scope === "SESSION" || tgt.scope === "SESSION") &&
|
|
50
|
+
src.session_id !== tgt.session_id)
|
|
51
|
+
return err("cannot merge memories across different sessions");
|
|
52
|
+
if (src.scope === "PROJECT" &&
|
|
53
|
+
tgt.scope === "PROJECT" &&
|
|
54
|
+
src.project_id !== tgt.project_id)
|
|
55
|
+
return err("cannot merge memories across different projects");
|
|
42
56
|
db.prepare("UPDATE memories SET metadata = ?, updated_at = ? WHERE id = ?").run(mergeMetadata(tgt.metadata, src.id), nowISO(), tgt.id);
|
|
43
57
|
supersede(src.id, tgt.id);
|
|
44
58
|
return ok(`merged memory ${src.id} into ${tgt.id} (source superseded, provenance recorded in metadata.merged_from)`);
|
package/dist/tools/profile.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { db, truncate, ok, err, } from "../db/index.js";
|
|
2
|
+
import { MEMORY_REF_CLOSE, MEMORY_REF_GUIDANCE, MEMORY_REF_OPEN, UNTRUSTED_NOTE, UNTRUSTED_TAG, isUntrustedMetadata, } from "../lib/memory-format.js";
|
|
2
3
|
export const PROFILE_BUDGET = 3000;
|
|
3
4
|
const PROFILE_SECTION_MAX = 400;
|
|
4
5
|
const PREF_CATEGORY_MAX = 30;
|
|
@@ -9,7 +10,7 @@ const LESSON_CORRECTION_MAX = 150;
|
|
|
9
10
|
const profileRows = db.prepare("SELECT section, content FROM profile");
|
|
10
11
|
const topPrefs = db.prepare("SELECT category, key, value, confidence FROM preferences ORDER BY confidence DESC, updated_at DESC LIMIT 15");
|
|
11
12
|
const recentLessons = db.prepare("SELECT situation, mistake, correction FROM lessons ORDER BY created_at DESC, id DESC LIMIT 5");
|
|
12
|
-
const topMemories = db.prepare("SELECT type, content, importance, confidence FROM memories WHERE status = 'active' ORDER BY importance * confidence DESC, updated_at DESC LIMIT 15");
|
|
13
|
+
const topMemories = db.prepare("SELECT type, content, importance, confidence, metadata FROM memories WHERE status = 'active' ORDER BY importance * confidence DESC, updated_at DESC LIMIT 15");
|
|
13
14
|
export function buildProfileText() {
|
|
14
15
|
const parts = [];
|
|
15
16
|
const prof = profileRows.all();
|
|
@@ -40,11 +41,23 @@ export function buildProfileText() {
|
|
|
40
41
|
if (mems.length > 0) {
|
|
41
42
|
let block = "[memories]";
|
|
42
43
|
for (const m of mems) {
|
|
43
|
-
|
|
44
|
+
// Batch B-2: label untrusted imports (metadata.trusted=false) inline.
|
|
45
|
+
const tag = isUntrustedMetadata(m.metadata)
|
|
46
|
+
? ` ${UNTRUSTED_TAG} (${UNTRUSTED_NOTE})`
|
|
47
|
+
: "";
|
|
48
|
+
block += `\n- (${m.type} c${m.confidence.toFixed(2)})${tag} ${truncate(m.content, 200)}`;
|
|
44
49
|
}
|
|
45
50
|
parts.push(block);
|
|
46
51
|
}
|
|
47
|
-
|
|
52
|
+
// Batch B-2: profile/memory text is reference data, not instructions —
|
|
53
|
+
// wrap with delimiters. Reserve wrapper overhead inside PROFILE_BUDGET so
|
|
54
|
+
// the wrapped output still fits the 3000-char contract.
|
|
55
|
+
const overhead = MEMORY_REF_GUIDANCE.length +
|
|
56
|
+
MEMORY_REF_OPEN.length +
|
|
57
|
+
MEMORY_REF_CLOSE.length +
|
|
58
|
+
8;
|
|
59
|
+
const inner = truncate(parts.join("\n\n"), Math.max(0, PROFILE_BUDGET - overhead));
|
|
60
|
+
return `${MEMORY_REF_GUIDANCE}\n${MEMORY_REF_OPEN}\n${inner}\n${MEMORY_REF_CLOSE}`;
|
|
48
61
|
}
|
|
49
62
|
export async function getProfileHandler() {
|
|
50
63
|
try {
|