rag-memory-epf-mcp 6.4.0 → 6.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/index.d.ts +51 -2
- package/dist/index.js +456 -149
- package/dist/src/chunkerC.d.ts +1 -0
- package/dist/src/chunkerC.js +69 -22
- package/dist/src/kg/relation-evidence.d.ts +156 -0
- package/dist/src/kg/relation-evidence.js +438 -0
- package/dist/src/observations/projection.d.ts +2 -0
- package/dist/src/observations/projection.js +49 -0
- package/dist/src/tools/knowledge-graph-tools.js +7 -3
- package/docs/UPDATING.md +10 -0
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -23,9 +23,10 @@ import { getAllMCPTools, validateToolArgs, getSystemInfo } from './src/tools/too
|
|
|
23
23
|
// Import migration system
|
|
24
24
|
import { MigrationManager } from './src/migrations/migration-manager.js';
|
|
25
25
|
import { backupBeforeMigration } from './src/backup/preflight.js';
|
|
26
|
-
import { rebuildProjection, deleteStaleKgChunks, deleteKgRelationshipChunks } from './src/observations/projection.js';
|
|
26
|
+
import { rebuildProjection, deleteStaleKgChunks, deleteKgRelationshipChunks, deleteKgChunkRows, refreshKgRelationshipConfidence } from './src/observations/projection.js';
|
|
27
27
|
import { addRevision, correctRevision, transitionStatus, linkSources, nextProjectionOrder } from './src/observations/lifecycle.js';
|
|
28
28
|
import { getObservationHistory } from './src/observations/history.js';
|
|
29
|
+
import { kgComparisonValue, captureKgEmbedInput, parseEvidenceMetadata, fingerprintFromMetadata, sha256Utf8, EVIDENCE_FORMAT, resolveEvidenceItem, EvidenceInputError, EvidenceCommitConflictError, resolveEvidenceStatus, collectEvidenceRowidsCitingObservations, collectEvidenceRowidsCitingDocument, collectEvidenceRowidsCitingOwnerEntity } from './src/kg/relation-evidence.js';
|
|
29
30
|
// Import chunk text algorithm (extracted for publish-time invariant testing)
|
|
30
31
|
import { chunkStructured as chunkStructuredText, effectiveSignature, isCurrentFormatSignature, LEGACY_SIGNATURE, DEFAULT_MAX_TOKENS } from './src/chunkerC.js';
|
|
31
32
|
import { migrations } from './src/migrations/migrations.js';
|
|
@@ -186,6 +187,13 @@ export function setSyncFaultPoint(point, fn) {
|
|
|
186
187
|
__syncFaultHook = point && fn ? (p) => { if (p === point)
|
|
187
188
|
fn(); } : null;
|
|
188
189
|
}
|
|
190
|
+
// Test-only fault hook for the KG writers (kg-relation-evidence D1/D2⑤). Same rule as
|
|
191
|
+
// setSyncFaultPoint: an argument, never an environment variable.
|
|
192
|
+
let __kgFaultHook = null;
|
|
193
|
+
export function setKgFaultPoint(point, fn) {
|
|
194
|
+
__kgFaultHook = point && fn ? (p) => { if (p === point)
|
|
195
|
+
fn(); } : null;
|
|
196
|
+
}
|
|
189
197
|
// Pure vector-reuse decision (spec §5.2 조건 1~5). Returns an OWNED Buffer copy:
|
|
190
198
|
// a Buffer read back from SQLite has no byteOffset-0 guarantee, and inserting
|
|
191
199
|
// `.buffer` of a subarray would write the wrong 4,096 bytes (advisor r5-9).
|
|
@@ -499,11 +507,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
499
507
|
mutateEntityAndInvalidate(entityId, mutate) {
|
|
500
508
|
let changed = false;
|
|
501
509
|
const tx = this.db.transaction(() => {
|
|
502
|
-
|
|
510
|
+
// D2③ (kg-relation-evidence): the fifth obligation — evidence chunks citing a changed observation
|
|
511
|
+
// go in the same transaction. Collected before the mutation (purge deletes the rows they name).
|
|
512
|
+
const marked = new Set();
|
|
513
|
+
const kg = {
|
|
514
|
+
collect: (ids) => collectEvidenceRowidsCitingObservations(this.db, ids),
|
|
515
|
+
mark: (rowids) => { for (const r of rowids)
|
|
516
|
+
marked.add(r); },
|
|
517
|
+
};
|
|
518
|
+
changed = mutate(kg) !== false;
|
|
503
519
|
if (!changed)
|
|
504
520
|
return;
|
|
505
521
|
rebuildProjection(this.db, entityId);
|
|
506
522
|
this.invalidateDerivedForEntity(entityId);
|
|
523
|
+
deleteKgChunkRows(this.db, [...marked]);
|
|
507
524
|
});
|
|
508
525
|
tx();
|
|
509
526
|
if (changed)
|
|
@@ -543,15 +560,70 @@ export class RAGKnowledgeGraphManager {
|
|
|
543
560
|
tx();
|
|
544
561
|
this.coordinator?.invalidateCoverage();
|
|
545
562
|
}
|
|
563
|
+
// D2⑤ (kg-relation-evidence): the one write-back for KG chunk vectors, shared by the dedicated
|
|
564
|
+
// embedKnowledgeGraphChunks loop and the backfill callback. `captured` was taken immediately before
|
|
565
|
+
// inference; every part must still hold at write time: same rowid -> same chunk_id, same text hash,
|
|
566
|
+
// same comparison value (evidence fingerprint, or for legacy rows source identity + body hash), and
|
|
567
|
+
// the profile that was current when inference started. null comparison values never match. A
|
|
568
|
+
// profile change BEFORE inference is normal: the vector and the evidence fingerprint are recorded for
|
|
569
|
+
// the profile captured here, so the stored fingerprint never stays pinned to an outdated profile.
|
|
570
|
+
writeKgVectorIfCurrent(captured, embedding) {
|
|
571
|
+
const rowid = safeRowid(captured.rowid);
|
|
572
|
+
const tx = this.db.transaction(() => {
|
|
573
|
+
const cur = this.db.prepare(`SELECT rowid, chunk_id, chunk_type, entity_id, relationship_id, text, metadata
|
|
574
|
+
FROM chunk_metadata WHERE rowid = ?`).get(rowid);
|
|
575
|
+
if (!cur || cur.text === null)
|
|
576
|
+
return false; // deleted during inference
|
|
577
|
+
if (cur.chunk_type !== 'entity' && cur.chunk_type !== 'relationship')
|
|
578
|
+
return false;
|
|
579
|
+
if (cur.chunk_id !== captured.chunk_id)
|
|
580
|
+
return false; // rowid reused by another chunk
|
|
581
|
+
if (sha256Utf8(cur.text) !== captured.text_sha256)
|
|
582
|
+
return false; // body changed
|
|
583
|
+
if (captured.cmp === null || kgComparisonValue(cur) !== captured.cmp)
|
|
584
|
+
return false; // inputs changed
|
|
585
|
+
if (this.currentProfileId !== captured.profile_id)
|
|
586
|
+
return false; // profile changed during inference
|
|
587
|
+
this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
|
|
588
|
+
this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
|
|
589
|
+
this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
590
|
+
.run(captured.text_sha256, captured.profile_id, rowid);
|
|
591
|
+
const meta = parseEvidenceMetadata(cur.metadata);
|
|
592
|
+
if (meta && meta.fingerprint_profile_id !== captured.profile_id) {
|
|
593
|
+
meta.input_fingerprint = fingerprintFromMetadata(meta, captured.profile_id);
|
|
594
|
+
meta.fingerprint_profile_id = captured.profile_id;
|
|
595
|
+
this.db.prepare(`UPDATE chunk_metadata SET metadata = ? WHERE rowid = ?`).run(JSON.stringify(meta), rowid);
|
|
596
|
+
}
|
|
597
|
+
this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
|
|
598
|
+
return true;
|
|
599
|
+
});
|
|
600
|
+
const written = tx();
|
|
601
|
+
this.coordinator?.invalidateCoverage();
|
|
602
|
+
return written;
|
|
603
|
+
}
|
|
546
604
|
// Backfill callback: re-embed one chunk and commit vector + provenance in a
|
|
547
605
|
// single transaction (§6a-2).
|
|
548
606
|
async reembedChunkByRowid(rowid) {
|
|
549
607
|
if (!this.db)
|
|
550
608
|
return false;
|
|
551
|
-
const row = this.db.prepare(`SELECT
|
|
552
|
-
|
|
609
|
+
const row = this.db.prepare(`SELECT rowid, chunk_id, chunk_type, entity_id, relationship_id, text, metadata
|
|
610
|
+
FROM chunk_metadata WHERE rowid = ?`).get(rowid);
|
|
553
611
|
if (!row || row.text === null)
|
|
554
612
|
return false;
|
|
613
|
+
if (row.chunk_type === 'entity' || row.chunk_type === 'relationship') {
|
|
614
|
+
// D2⑤: the text-only check below cannot tell a regenerated evidence chunk with the same body,
|
|
615
|
+
// or another chunk at a reused rowid with the same text, from the input we embedded.
|
|
616
|
+
const captured = captureKgEmbedInput(row, this.currentProfileId);
|
|
617
|
+
try {
|
|
618
|
+
const embedding = await this.generateEmbedding(row.text, 1024, false, 'backfill', `profile:${captured.profile_id}`);
|
|
619
|
+
return this.writeKgVectorIfCurrent(captured, embedding);
|
|
620
|
+
}
|
|
621
|
+
catch (e) {
|
|
622
|
+
if (e instanceof GateNotReadyError || e instanceof GateDisabledError)
|
|
623
|
+
return false;
|
|
624
|
+
throw e;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
555
627
|
try {
|
|
556
628
|
const embedding = await this.generateEmbedding(row.text, 1024, false, 'backfill');
|
|
557
629
|
const hash = createHash('sha256').update(row.text).digest('hex');
|
|
@@ -955,7 +1027,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
955
1027
|
// entity INSERT 도 같은 트랜잭션 안이다. 밖에 두면 lifecycle INSERT 가
|
|
956
1028
|
// 실패할 때 entity 행만 남는 split state 가 생긴다
|
|
957
1029
|
// (advisor 구현리뷰 r1 발견 2, 실행 재현).
|
|
958
|
-
const changed = this.mutateEntityAndInvalidate(entityId, () => {
|
|
1030
|
+
const changed = this.mutateEntityAndInvalidate(entityId, (kg) => {
|
|
959
1031
|
created = insertStmt.run(entityId, entity.name, entity.entityType, '{}').changes > 0;
|
|
960
1032
|
if (!created && entity.entityType && entity.entityType !== 'CONCEPT') {
|
|
961
1033
|
const cur = this.db.prepare(`SELECT entityType FROM entities WHERE id = ?`)
|
|
@@ -976,8 +1048,15 @@ export class RAGKnowledgeGraphManager {
|
|
|
976
1048
|
const dupId = activeByBare.get(bare);
|
|
977
1049
|
if (dupId) {
|
|
978
1050
|
// 같은 사실이 다른 출처에서 다시 왔다 = evidence 추가, 새 revision 아님.
|
|
979
|
-
|
|
980
|
-
|
|
1051
|
+
// D2③: a new source row changes the cited observation's source list (a fingerprint input):
|
|
1052
|
+
// collect before linking, drop only if a row was actually added.
|
|
1053
|
+
if (entity.sources?.length) {
|
|
1054
|
+
const cited = kg.collect([dupId]);
|
|
1055
|
+
const added = linkSources(this.db, dupId, entity.sources, ts);
|
|
1056
|
+
if (added > 0)
|
|
1057
|
+
kg.mark(cited);
|
|
1058
|
+
sourcesAdded += added;
|
|
1059
|
+
}
|
|
981
1060
|
ids.push(null);
|
|
982
1061
|
continue;
|
|
983
1062
|
}
|
|
@@ -1062,14 +1141,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
1062
1141
|
const ids = [];
|
|
1063
1142
|
const added = [];
|
|
1064
1143
|
let sourcesAdded = 0;
|
|
1065
|
-
const changed = this.mutateEntityAndInvalidate(entityId, () => {
|
|
1144
|
+
const changed = this.mutateEntityAndInvalidate(entityId, (kg) => {
|
|
1066
1145
|
for (const raw of obs.contents) {
|
|
1067
1146
|
const content = this._timestampObservation(raw);
|
|
1068
1147
|
const bare = stripDate(content);
|
|
1069
1148
|
const dupId = activeByBare.get(bare);
|
|
1070
1149
|
if (dupId) {
|
|
1071
|
-
if
|
|
1072
|
-
|
|
1150
|
+
// D2③: see createEntities — collect before linking, drop only if a source row was added.
|
|
1151
|
+
if (obs.sources?.length) {
|
|
1152
|
+
const cited = kg.collect([dupId]);
|
|
1153
|
+
const added = linkSources(this.db, dupId, obs.sources, ts);
|
|
1154
|
+
if (added > 0)
|
|
1155
|
+
kg.mark(cited);
|
|
1156
|
+
sourcesAdded += added;
|
|
1157
|
+
}
|
|
1073
1158
|
ids.push(null);
|
|
1074
1159
|
continue;
|
|
1075
1160
|
}
|
|
@@ -1104,11 +1189,13 @@ export class RAGKnowledgeGraphManager {
|
|
|
1104
1189
|
throw new Error(`observation ${observationId} not found`);
|
|
1105
1190
|
let newId = '';
|
|
1106
1191
|
const ts = new Date().toISOString();
|
|
1107
|
-
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
1192
|
+
this.mutateEntityAndInvalidate(row.entity_id, (kg) => {
|
|
1193
|
+
const cited = kg.collect([observationId]);
|
|
1108
1194
|
newId = correctRevision(this.db, {
|
|
1109
1195
|
observationId, content: this._timestampObservation(content),
|
|
1110
1196
|
changeKind, reason: reason ?? null, ts
|
|
1111
1197
|
});
|
|
1198
|
+
kg.mark(cited);
|
|
1112
1199
|
});
|
|
1113
1200
|
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
1114
1201
|
return newId;
|
|
@@ -1121,8 +1208,10 @@ export class RAGKnowledgeGraphManager {
|
|
|
1121
1208
|
if (!row)
|
|
1122
1209
|
throw new Error(`observation ${observationId} not found`);
|
|
1123
1210
|
const ts = new Date().toISOString();
|
|
1124
|
-
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
1211
|
+
this.mutateEntityAndInvalidate(row.entity_id, (kg) => {
|
|
1212
|
+
const cited = kg.collect([observationId]);
|
|
1125
1213
|
transitionStatus(this.db, { observationId, event, reason: reason ?? null, ts });
|
|
1214
|
+
kg.mark(cited);
|
|
1126
1215
|
});
|
|
1127
1216
|
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
1128
1217
|
}
|
|
@@ -1164,17 +1253,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
1164
1253
|
if (!row)
|
|
1165
1254
|
return { purged: 0 };
|
|
1166
1255
|
let purged = 0;
|
|
1167
|
-
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
1256
|
+
this.mutateEntityAndInvalidate(row.entity_id, (kg) => {
|
|
1168
1257
|
// newest-first so each DELETE has no successor referencing it
|
|
1169
1258
|
const victims = this.db.prepare(`SELECT observation_id FROM entity_observations
|
|
1170
1259
|
WHERE root_id = ? AND revision_no >= ?
|
|
1171
1260
|
ORDER BY revision_no DESC`).all(row.root_id, row.revision_no);
|
|
1261
|
+
const cited = kg.collect(victims.map(v => v.observation_id));
|
|
1172
1262
|
for (const v of victims) {
|
|
1173
1263
|
this.db.prepare(`DELETE FROM observation_events WHERE from_id = ? OR to_id = ?`)
|
|
1174
1264
|
.run(v.observation_id, v.observation_id);
|
|
1175
1265
|
purged += this.db.prepare(`DELETE FROM entity_observations WHERE observation_id = ?`)
|
|
1176
1266
|
.run(v.observation_id).changes;
|
|
1177
1267
|
}
|
|
1268
|
+
if (purged > 0)
|
|
1269
|
+
kg.mark(cited);
|
|
1178
1270
|
return purged > 0;
|
|
1179
1271
|
});
|
|
1180
1272
|
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
@@ -1190,6 +1282,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1190
1282
|
if (!this.db)
|
|
1191
1283
|
throw new Error('Database not initialized');
|
|
1192
1284
|
console.error(`🗑️ Deleting entities: ${entityNames.join(', ')}`);
|
|
1285
|
+
let anyDeleted = false;
|
|
1193
1286
|
for (const name of entityNames) {
|
|
1194
1287
|
const entityId = `entity_${name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
1195
1288
|
// One entity = one transaction. The four deletion steps are a single unit:
|
|
@@ -1205,6 +1298,9 @@ export class RAGKnowledgeGraphManager {
|
|
|
1205
1298
|
`).get(eid);
|
|
1206
1299
|
if (!entityExists)
|
|
1207
1300
|
return false;
|
|
1301
|
+
// D2③: evidence chunks of ANY relationship may cite this entity's observations; collect them
|
|
1302
|
+
// before the FK CASCADE takes the observation rows (step 3).
|
|
1303
|
+
const citedEvidence = collectEvidenceRowidsCitingOwnerEntity(this.db, eid);
|
|
1208
1304
|
// Step 0: Delete entity embeddings
|
|
1209
1305
|
const embeddingMetadata = this.db.prepare(`
|
|
1210
1306
|
SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
|
|
@@ -1247,6 +1343,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1247
1343
|
// deletion unless swept here (same fail-closed rationale as deleteStaleKgChunks).
|
|
1248
1344
|
deleteStaleKgChunks(this.db, eid);
|
|
1249
1345
|
deleteKgRelationshipChunks(this.db, relIds.map(r => r.id));
|
|
1346
|
+
deleteKgChunkRows(this.db, citedEvidence);
|
|
1250
1347
|
// Step 3: Finally delete the entity itself (FK CASCADE takes observation lifecycle rows)
|
|
1251
1348
|
const entity = this.db.prepare(`
|
|
1252
1349
|
DELETE FROM entities WHERE id = ?
|
|
@@ -1254,6 +1351,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1254
1351
|
return entity.changes > 0;
|
|
1255
1352
|
})(entityId);
|
|
1256
1353
|
if (deleted) {
|
|
1354
|
+
anyDeleted = true;
|
|
1257
1355
|
console.error(` └─ Deleted entity '${name}' successfully`);
|
|
1258
1356
|
}
|
|
1259
1357
|
else {
|
|
@@ -1265,6 +1363,10 @@ export class RAGKnowledgeGraphManager {
|
|
|
1265
1363
|
// Continue with other entities instead of failing completely
|
|
1266
1364
|
}
|
|
1267
1365
|
}
|
|
1366
|
+
// D2③ (final review A1): the committed deletes removed entity vectors and KG/evidence chunks, so the
|
|
1367
|
+
// cached coverage snapshot is stale — drop it like every other D2③ writer, once, after the commits.
|
|
1368
|
+
if (anyDeleted)
|
|
1369
|
+
this.coordinator?.invalidateCoverage();
|
|
1268
1370
|
console.error(`✅ Entity deletion process completed`);
|
|
1269
1371
|
}
|
|
1270
1372
|
// v3.6 (spec §5c, breaking): structured per-entity results + re-embedding.
|
|
@@ -1321,6 +1423,8 @@ export class RAGKnowledgeGraphManager {
|
|
|
1321
1423
|
if (touched.length > 0) {
|
|
1322
1424
|
const tx = this.db.transaction(() => {
|
|
1323
1425
|
for (const p of touched) {
|
|
1426
|
+
// D2③: this shim does not go through mutateEntityAndInvalidate, so it owes the evidence step itself.
|
|
1427
|
+
const citedEvidence = collectEvidenceRowidsCitingObservations(this.db, p.ids);
|
|
1324
1428
|
for (const id of p.ids) {
|
|
1325
1429
|
transitionStatus(this.db, { observationId: id, event: 'retract',
|
|
1326
1430
|
reason: 'deleteObservations (deprecated shim)', ts });
|
|
@@ -1334,6 +1438,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1334
1438
|
.run(p.entityId);
|
|
1335
1439
|
}
|
|
1336
1440
|
deleteStaleKgChunks(this.db, p.entityId);
|
|
1441
|
+
deleteKgChunkRows(this.db, citedEvidence);
|
|
1337
1442
|
}
|
|
1338
1443
|
});
|
|
1339
1444
|
tx();
|
|
@@ -1359,11 +1464,22 @@ export class RAGKnowledgeGraphManager {
|
|
|
1359
1464
|
for (const relation of relations) {
|
|
1360
1465
|
const sourceId = `entity_${relation.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
1361
1466
|
const targetId = `entity_${relation.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1467
|
+
// D2① (kg-relation-evidence): the KG chunk is keyed by relationship_id, so capture the ids
|
|
1468
|
+
// BEFORE the DELETE and remove chunk, vector and FTS row in the same transaction as the row.
|
|
1469
|
+
// 6.4.0 deleted only the row and the chunk stayed searchable (test/kg-delete-relation.test.mjs).
|
|
1470
|
+
this.db.transaction(() => {
|
|
1471
|
+
const ids = this.db.prepare(`
|
|
1472
|
+
SELECT id FROM relationships
|
|
1473
|
+
WHERE source_entity = ? AND target_entity = ? AND relationType = ?
|
|
1474
|
+
`).all(sourceId, targetId, relation.relationType);
|
|
1475
|
+
deleteKgRelationshipChunks(this.db, ids.map(r => r.id));
|
|
1476
|
+
this.db.prepare(`
|
|
1477
|
+
DELETE FROM relationships
|
|
1478
|
+
WHERE source_entity = ? AND target_entity = ? AND relationType = ?
|
|
1479
|
+
`).run(sourceId, targetId, relation.relationType);
|
|
1480
|
+
})();
|
|
1366
1481
|
}
|
|
1482
|
+
this.coordinator?.invalidateCoverage();
|
|
1367
1483
|
}
|
|
1368
1484
|
async updateRelations(updates) {
|
|
1369
1485
|
if (!this.db)
|
|
@@ -1397,9 +1513,15 @@ export class RAGKnowledgeGraphManager {
|
|
|
1397
1513
|
continue;
|
|
1398
1514
|
}
|
|
1399
1515
|
values.push(relationId);
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1516
|
+
// D2② (kg-relation-evidence): the copied display confidence in the KG chunk is refreshed in the
|
|
1517
|
+
// same transaction; body unchanged, so no re-embed (test/kg-confidence-copy.test.mjs).
|
|
1518
|
+
this.db.transaction(() => {
|
|
1519
|
+
this.db.prepare(`
|
|
1520
|
+
UPDATE relationships SET ${setClauses.join(', ')} WHERE id = ?
|
|
1521
|
+
`).run(...values);
|
|
1522
|
+
if (update.confidence !== undefined)
|
|
1523
|
+
refreshKgRelationshipConfidence(this.db, relationId);
|
|
1524
|
+
})();
|
|
1403
1525
|
updated++;
|
|
1404
1526
|
}
|
|
1405
1527
|
return { updated, notFound };
|
|
@@ -1970,34 +2092,17 @@ export class RAGKnowledgeGraphManager {
|
|
|
1970
2092
|
};
|
|
1971
2093
|
}
|
|
1972
2094
|
// NEW: Generate knowledge graph chunks for entities and relationships
|
|
2095
|
+
// D2⑤ (kg-relation-evidence): build every row first, then swap old -> new in ONE transaction. 6.4.0
|
|
2096
|
+
// deleted the old chunks and inserted row by row, so a failure in between published a partial KG.
|
|
2097
|
+
// Bodies, chunk IDs, metadata, offsets and counts are byte-for-byte 6.4.0's (C7.3) — including
|
|
2098
|
+
// end_pos = UTF-16 length, a pre-existing quirk kept on purpose (eval/kg-defaults/generate-snapshot.mjs).
|
|
1973
2099
|
async generateKnowledgeGraphChunks() {
|
|
1974
2100
|
if (!this.db)
|
|
1975
2101
|
throw new Error('Database not initialized');
|
|
1976
2102
|
console.error('🧠 Generating knowledge graph chunks...');
|
|
1977
|
-
// Clean up existing knowledge graph chunks
|
|
1978
|
-
await this.cleanupKnowledgeGraphChunks();
|
|
1979
|
-
let entityChunks = 0;
|
|
1980
|
-
let relationshipChunks = 0;
|
|
1981
|
-
// Generate entity chunks
|
|
1982
2103
|
const entities = this.db.prepare(`
|
|
1983
2104
|
SELECT id, name, entityType, observations FROM entities
|
|
1984
2105
|
`).all();
|
|
1985
|
-
for (const entity of entities) {
|
|
1986
|
-
const observations = JSON.parse(entity.observations);
|
|
1987
|
-
const chunkText = this.generateEntityChunkText(entity.name, entity.entityType, observations);
|
|
1988
|
-
const chunkId = `kg_entity_${entity.id}`;
|
|
1989
|
-
// Store chunk metadata
|
|
1990
|
-
this.db.prepare(`
|
|
1991
|
-
INSERT INTO chunk_metadata (
|
|
1992
|
-
chunk_id, chunk_type, entity_id, chunk_index, text, start_pos, end_pos, metadata
|
|
1993
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1994
|
-
`).run(chunkId, 'entity', entity.id, 0, chunkText, 0, chunkText.length, JSON.stringify({
|
|
1995
|
-
entity_name: entity.name,
|
|
1996
|
-
entity_type: entity.entityType
|
|
1997
|
-
}));
|
|
1998
|
-
entityChunks++;
|
|
1999
|
-
}
|
|
2000
|
-
// Generate relationship chunks
|
|
2001
2106
|
const relationships = this.db.prepare(`
|
|
2002
2107
|
SELECT
|
|
2003
2108
|
r.id,
|
|
@@ -2009,65 +2114,147 @@ export class RAGKnowledgeGraphManager {
|
|
|
2009
2114
|
JOIN entities e1 ON r.source_entity = e1.id
|
|
2010
2115
|
JOIN entities e2 ON r.target_entity = e2.id
|
|
2011
2116
|
`).all();
|
|
2117
|
+
const rows = [];
|
|
2118
|
+
for (const entity of entities) {
|
|
2119
|
+
const chunkText = this.generateEntityChunkText(entity.name, entity.entityType, JSON.parse(entity.observations));
|
|
2120
|
+
rows.push({ chunk_id: `kg_entity_${entity.id}`, chunk_type: 'entity', source: entity.id, text: chunkText,
|
|
2121
|
+
metadata: JSON.stringify({ entity_name: entity.name, entity_type: entity.entityType }) });
|
|
2122
|
+
}
|
|
2012
2123
|
for (const rel of relationships) {
|
|
2013
2124
|
const chunkText = this.generateRelationshipChunkText(rel.source_name, rel.target_name, rel.relationType);
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2125
|
+
rows.push({ chunk_id: `kg_relationship_${rel.id}`, chunk_type: 'relationship', source: rel.id, text: chunkText,
|
|
2126
|
+
metadata: JSON.stringify({ source_entity: rel.source_name, target_entity: rel.target_name, relation_type: rel.relationType, confidence: rel.confidence }) });
|
|
2127
|
+
}
|
|
2128
|
+
const insEntity = this.db.prepare(`
|
|
2129
|
+
INSERT INTO chunk_metadata (chunk_id, chunk_type, entity_id, chunk_index, text, start_pos, end_pos, metadata)
|
|
2130
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2131
|
+
const insRel = this.db.prepare(`
|
|
2132
|
+
INSERT INTO chunk_metadata (chunk_id, chunk_type, relationship_id, chunk_index, text, start_pos, end_pos, metadata)
|
|
2133
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2134
|
+
let entityChunks = 0;
|
|
2135
|
+
let relationshipChunks = 0;
|
|
2136
|
+
this.db.transaction(() => {
|
|
2137
|
+
this.deleteAllKgChunks();
|
|
2138
|
+
for (const r of rows) {
|
|
2139
|
+
(r.chunk_type === 'entity' ? insEntity : insRel).run(r.chunk_id, r.chunk_type, r.source, 0, r.text, 0, r.text.length, r.metadata);
|
|
2140
|
+
if (r.chunk_type === 'entity')
|
|
2141
|
+
entityChunks++;
|
|
2142
|
+
else
|
|
2143
|
+
relationshipChunks++;
|
|
2144
|
+
if (entityChunks + relationshipChunks === 1)
|
|
2145
|
+
__kgFaultHook?.('kg-generate-mid');
|
|
2146
|
+
}
|
|
2147
|
+
})();
|
|
2148
|
+
this.coordinator?.invalidateCoverage();
|
|
2028
2149
|
console.error(`✅ Knowledge graph chunks generated: ${entityChunks} entities, ${relationshipChunks} relationships`);
|
|
2029
2150
|
return { entityChunks, relationshipChunks };
|
|
2030
2151
|
}
|
|
2031
2152
|
// NEW: Embed knowledge graph chunks
|
|
2153
|
+
// D2⑤: capture before inference, re-check at write (writeKgVectorIfCurrent). 6.4.0 wrote whatever
|
|
2154
|
+
// came back, onto whatever row then held the rowid.
|
|
2032
2155
|
async embedKnowledgeGraphChunks() {
|
|
2033
2156
|
if (!this.db)
|
|
2034
2157
|
throw new Error('Database not initialized');
|
|
2035
2158
|
console.error('🔮 Embedding knowledge graph chunks...');
|
|
2036
|
-
// Get all knowledge graph chunks
|
|
2037
2159
|
const chunks = this.db.prepare(`
|
|
2038
|
-
SELECT rowid, chunk_id, text
|
|
2039
|
-
FROM chunk_metadata
|
|
2160
|
+
SELECT rowid, chunk_id, chunk_type, entity_id, relationship_id, text, metadata
|
|
2161
|
+
FROM chunk_metadata
|
|
2040
2162
|
WHERE chunk_type IN ('entity', 'relationship')
|
|
2041
2163
|
`).all();
|
|
2042
2164
|
let embeddedCount = 0;
|
|
2165
|
+
let discarded = 0;
|
|
2043
2166
|
const errors = [];
|
|
2044
2167
|
for (const chunk of chunks) {
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
const
|
|
2168
|
+
if (chunk.text === null)
|
|
2169
|
+
continue;
|
|
2170
|
+
const captured = captureKgEmbedInput(chunk, this.currentProfileId);
|
|
2171
|
+
const embedding = await this.generateEmbedding(chunk.text, 1024, false, 'bulk', `profile:${captured.profile_id}`);
|
|
2048
2172
|
try {
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
this.db.prepare(`
|
|
2054
|
-
INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
|
|
2055
|
-
`).run(Buffer.from(embedding.buffer));
|
|
2056
|
-
this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
2057
|
-
.run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, chunk.rowid);
|
|
2058
|
-
});
|
|
2059
|
-
tx();
|
|
2060
|
-
embeddedCount++;
|
|
2173
|
+
if (this.writeKgVectorIfCurrent(captured, embedding))
|
|
2174
|
+
embeddedCount++;
|
|
2175
|
+
else
|
|
2176
|
+
discarded++;
|
|
2061
2177
|
}
|
|
2062
2178
|
catch (error) {
|
|
2063
|
-
const errMsg = `chunk ${chunk.chunk_id} (rowid=${
|
|
2179
|
+
const errMsg = `chunk ${chunk.chunk_id} (rowid=${chunk.rowid}): ${error instanceof Error ? error.message : String(error)}`;
|
|
2064
2180
|
console.error(`Failed to embed ${errMsg}`);
|
|
2065
2181
|
errors.push(errMsg);
|
|
2066
2182
|
}
|
|
2067
2183
|
}
|
|
2068
2184
|
this.coordinator?.invalidateCoverage();
|
|
2069
|
-
console.error(`✅ Knowledge graph chunks embedded: ${embeddedCount}/${chunks.length}`);
|
|
2070
|
-
return {
|
|
2185
|
+
console.error(`✅ Knowledge graph chunks embedded: ${embeddedCount}/${chunks.length}${discarded > 0 ? ` (${discarded} discarded: input changed during inference)` : ''}`);
|
|
2186
|
+
return {
|
|
2187
|
+
embeddedChunks: embeddedCount, totalChunks: chunks.length,
|
|
2188
|
+
...(discarded > 0 && { discardedChunks: discarded }),
|
|
2189
|
+
...(errors.length > 0 && { errors: errors.slice(0, 5) }),
|
|
2190
|
+
};
|
|
2191
|
+
}
|
|
2192
|
+
// D1 (kg-relation-evidence): opt-in evidence-backed relationship chunks. Manager method only — not an
|
|
2193
|
+
// MCP tool (Q3). `items` is the COMPLETE set S of relationships that get a relationship chunk: a
|
|
2194
|
+
// successful call atomically replaces ALL relationship KG chunks (triplet or evidence) with the
|
|
2195
|
+
// evidence-v1 chunks of S; entity KG chunks, documents and relationship rows are untouched; [] is
|
|
2196
|
+
// valid and empties relationship KG chunks. Any invalid item rejects the whole call before any
|
|
2197
|
+
// write. Bodies are prepared first; inside the commit transaction every item is re-resolved from
|
|
2198
|
+
// the CURRENT rows (active status and chunk type/parent first, then the fingerprint) and must match
|
|
2199
|
+
// what was prepared; the re-resolved rows are what gets inserted. Vectors come later from
|
|
2200
|
+
// embedKnowledgeGraphChunks or the backfill.
|
|
2201
|
+
async generateRelationEvidenceChunks(items, options) {
|
|
2202
|
+
if (!this.db)
|
|
2203
|
+
throw new Error('Database not initialized');
|
|
2204
|
+
if (options?.formatVersion !== EVIDENCE_FORMAT) {
|
|
2205
|
+
throw new Error(`generateRelationEvidenceChunks: formatVersion must be '${EVIDENCE_FORMAT}'`);
|
|
2206
|
+
}
|
|
2207
|
+
if (!Array.isArray(items))
|
|
2208
|
+
throw new Error('generateRelationEvidenceChunks: items must be an array');
|
|
2209
|
+
const seen = new Set();
|
|
2210
|
+
items.forEach((it, i) => {
|
|
2211
|
+
const id = it?.relationship_id;
|
|
2212
|
+
if (typeof id !== 'string')
|
|
2213
|
+
return;
|
|
2214
|
+
if (seen.has(id))
|
|
2215
|
+
throw new EvidenceInputError('duplicate-relationship', i, id);
|
|
2216
|
+
seen.add(id);
|
|
2217
|
+
});
|
|
2218
|
+
const profileAtPrepare = this.currentProfileId;
|
|
2219
|
+
// Array.from, not map: map skips holes, so a hole in items reached the commit loop unprepared and
|
|
2220
|
+
// surfaced as a commit conflict instead of an input error (D1, same class as REVIEW-T5-2_ I1).
|
|
2221
|
+
const prepared = Array.from(items, (it, i) => resolveEvidenceItem(this.db, it, i, profileAtPrepare));
|
|
2222
|
+
__kgFaultHook?.('evidence-pre-commit');
|
|
2223
|
+
let removed = 0;
|
|
2224
|
+
this.db.transaction(() => {
|
|
2225
|
+
const db = this.db;
|
|
2226
|
+
// R1 (REVIEW-PLAN): what is published is the metadata re-resolved HERE, from the current rows —
|
|
2227
|
+
// not the prepared copy. A display-only change between preparation and commit (confidence is not a
|
|
2228
|
+
// fingerprint input) therefore succeeds with the current value instead of re-publishing the old one;
|
|
2229
|
+
// any fingerprint input change still fails the whole call.
|
|
2230
|
+
const current = [];
|
|
2231
|
+
for (let i = 0; i < items.length; i++) {
|
|
2232
|
+
let now;
|
|
2233
|
+
try {
|
|
2234
|
+
now = resolveEvidenceItem(db, items[i], i, this.currentProfileId);
|
|
2235
|
+
}
|
|
2236
|
+
catch (e) {
|
|
2237
|
+
throw new EvidenceCommitConflictError(i, e instanceof EvidenceInputError ? e.code : String(e));
|
|
2238
|
+
}
|
|
2239
|
+
if (now.metadata.input_fingerprint !== prepared[i].metadata.input_fingerprint) {
|
|
2240
|
+
throw new EvidenceCommitConflictError(i, 'fingerprint-changed');
|
|
2241
|
+
}
|
|
2242
|
+
current.push(now); // same fingerprint => same body (every body input is a fingerprint input)
|
|
2243
|
+
}
|
|
2244
|
+
const old = db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type = 'relationship'`).all();
|
|
2245
|
+
removed = deleteKgChunkRows(db, old.map(r => r.rowid));
|
|
2246
|
+
const ins = db.prepare(`
|
|
2247
|
+
INSERT INTO chunk_metadata (chunk_id, chunk_type, relationship_id, chunk_index, text, start_pos, end_pos, metadata)
|
|
2248
|
+
VALUES (?, 'relationship', ?, 0, ?, 0, ?, ?)`);
|
|
2249
|
+
current.forEach((p, i) => {
|
|
2250
|
+
ins.run(p.chunk_id, p.relationship_id, p.text, p.body_codepoints, JSON.stringify(p.metadata));
|
|
2251
|
+
if (i === 0)
|
|
2252
|
+
__kgFaultHook?.('evidence-mid-commit');
|
|
2253
|
+
});
|
|
2254
|
+
})();
|
|
2255
|
+
this.coordinator?.invalidateCoverage();
|
|
2256
|
+
console.error(`✅ Evidence relationship chunks: ${prepared.length} written, ${removed} relationship chunks replaced`);
|
|
2257
|
+
return { relationshipChunks: prepared.length, removedRelationshipChunks: removed, formatVersion: EVIDENCE_FORMAT };
|
|
2071
2258
|
}
|
|
2072
2259
|
// NEW: Generate textual representation for entity chunks
|
|
2073
2260
|
generateEntityChunkText(name, entityType, observations) {
|
|
@@ -2080,29 +2267,26 @@ export class RAGKnowledgeGraphManager {
|
|
|
2080
2267
|
const relationText = relationType.toLowerCase().replace(/_/g, ' ');
|
|
2081
2268
|
return `${sourceName} ${relationText} ${targetName}`;
|
|
2082
2269
|
}
|
|
2083
|
-
//
|
|
2084
|
-
|
|
2270
|
+
// Delete every KG chunk (vectors, links, metadata). Synchronous so it can run inside the swap
|
|
2271
|
+
// transaction of generateKnowledgeGraphChunks (D2⑤). Body = 6.4.0 cleanupKnowledgeGraphChunks.
|
|
2272
|
+
deleteAllKgChunks() {
|
|
2085
2273
|
if (!this.db)
|
|
2086
2274
|
return;
|
|
2087
2275
|
console.error('🧹 Cleaning up existing knowledge graph chunks...');
|
|
2088
|
-
// Get existing knowledge graph chunks
|
|
2089
2276
|
const existingChunks = this.db.prepare(`
|
|
2090
2277
|
SELECT rowid FROM chunk_metadata WHERE chunk_type IN ('entity', 'relationship')
|
|
2091
2278
|
`).all();
|
|
2092
2279
|
let deletedVectors = 0;
|
|
2093
2280
|
let deletedAssociations = 0;
|
|
2094
|
-
// Delete vectors and associations
|
|
2095
2281
|
for (const chunk of existingChunks) {
|
|
2096
2282
|
// Delete vector embeddings (vec0 needs literal integer, not parameterized)
|
|
2097
2283
|
this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
|
|
2098
2284
|
deletedVectors++;
|
|
2099
|
-
// Delete chunk-entity associations
|
|
2100
2285
|
const associations = this.db.prepare(`
|
|
2101
2286
|
DELETE FROM chunk_entities WHERE chunk_rowid = ?
|
|
2102
2287
|
`).run(chunk.rowid);
|
|
2103
2288
|
deletedAssociations += associations.changes;
|
|
2104
2289
|
}
|
|
2105
|
-
// Delete chunk metadata
|
|
2106
2290
|
const metadata = this.db.prepare(`
|
|
2107
2291
|
DELETE FROM chunk_metadata WHERE chunk_type IN ('entity', 'relationship')
|
|
2108
2292
|
`).run();
|
|
@@ -2300,9 +2484,13 @@ export class RAGKnowledgeGraphManager {
|
|
|
2300
2484
|
}
|
|
2301
2485
|
// Generate embeddings using sentence transformers
|
|
2302
2486
|
// isQuery: true for search queries (adds instruction prefix), false for documents/entities
|
|
2303
|
-
async generateEmbedding(text, dimensions = 1024, isQuery = false, priority = 'interactive') {
|
|
2487
|
+
async generateEmbedding(text, dimensions = 1024, isQuery = false, priority = 'interactive', cacheScope) {
|
|
2304
2488
|
// Check cache first (hash-based key to avoid collisions on long texts)
|
|
2305
|
-
|
|
2489
|
+
// kg-relation-evidence R5: the key has no profile, so a vector cached under profile P1 would be
|
|
2490
|
+
// returned — and stamped P2 by the KG CAS — after a switch to P2. The KG write paths pass
|
|
2491
|
+
// cacheScope = 'profile:<captured id>'; without a scope the key is exactly the 6.4.0 key, so
|
|
2492
|
+
// document and query caching is unchanged (their profile policy is not this change's scope).
|
|
2493
|
+
const cacheKey = createHash('md5').update(`${text}_${dimensions}_${isQuery}${cacheScope === undefined ? '' : `_${cacheScope}`}`).digest('hex');
|
|
2306
2494
|
const cached = this.embeddingCache.get(cacheKey);
|
|
2307
2495
|
if (cached)
|
|
2308
2496
|
return cached;
|
|
@@ -2431,12 +2619,16 @@ export class RAGKnowledgeGraphManager {
|
|
|
2431
2619
|
now.metadata === snap.metadata && now.chunking_signature === snap.chunking_signature);
|
|
2432
2620
|
if (!same)
|
|
2433
2621
|
throw new SyncCasConflictError(documentId);
|
|
2622
|
+
// D2③: collect evidence citing this document before its chunks are replaced; delete it below,
|
|
2623
|
+
// in this transaction (no separate deletion moment).
|
|
2624
|
+
const citedEvidence = collectEvidenceRowidsCitingDocument(db, documentId);
|
|
2434
2625
|
const existing = db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).all(documentId);
|
|
2435
2626
|
for (const ch of existing) {
|
|
2436
2627
|
db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(ch.rowid);
|
|
2437
2628
|
db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(ch.rowid)}`);
|
|
2438
2629
|
}
|
|
2439
2630
|
db.prepare(`DELETE FROM chunk_metadata WHERE document_id = ?`).run(documentId);
|
|
2631
|
+
deleteKgChunkRows(db, citedEvidence);
|
|
2440
2632
|
db.prepare(`DELETE FROM documents WHERE id = ?`).run(documentId);
|
|
2441
2633
|
db.prepare(`INSERT INTO documents (id, content, metadata, chunking_signature) VALUES (?, ?, ?, ?)`)
|
|
2442
2634
|
.run(documentId, content, JSON.stringify(metadata), signature);
|
|
@@ -2508,13 +2700,22 @@ export class RAGKnowledgeGraphManager {
|
|
|
2508
2700
|
// Decide `replaced` from the document row, not from the chunk count: a document stored but
|
|
2509
2701
|
// never chunked still gets overwritten here, and reporting that as a fresh write would be a lie.
|
|
2510
2702
|
const existed = this.db.prepare(`SELECT 1 FROM documents WHERE id = ?`).get(id) !== undefined;
|
|
2511
|
-
//
|
|
2512
|
-
|
|
2513
|
-
//
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2703
|
+
// D2③ + R-B (2_ RESULT-T7-TXN-2_): serialise first, then ONE synchronous transaction = cleanup
|
|
2704
|
+
// (collect citing evidence, delete chunks and evidence) + the documents REPLACE. With an await
|
|
2705
|
+
// between the two, another call could publish evidence on this document after the collection, and a
|
|
2706
|
+
// failing REPLACE left the chunks and evidence already deleted.
|
|
2707
|
+
const metadataJson = JSON.stringify(metadata);
|
|
2708
|
+
const cleaned = this.db.transaction(() => {
|
|
2709
|
+
const c = this.cleanupDocument(id);
|
|
2710
|
+
this.db.prepare(`
|
|
2711
|
+
INSERT OR REPLACE INTO documents (id, content, metadata)
|
|
2712
|
+
VALUES (?, ?, ?)
|
|
2713
|
+
`).run(id, content, metadataJson);
|
|
2714
|
+
return c;
|
|
2715
|
+
})();
|
|
2716
|
+
// R-B item 6 (review I1): only after the commit succeeded — chunks and evidence were removed, so the
|
|
2717
|
+
// cached coverage snapshot is stale. A rollback throws above and never reaches this.
|
|
2718
|
+
this.coordinator?.invalidateCoverage();
|
|
2518
2719
|
console.error(`✅ Document stored: ${id}`);
|
|
2519
2720
|
return { id, stored: true, replaced: existed, deletedChunks: cleaned.deletedChunks };
|
|
2520
2721
|
}
|
|
@@ -2530,32 +2731,38 @@ export class RAGKnowledgeGraphManager {
|
|
|
2530
2731
|
}
|
|
2531
2732
|
const { maxTokens } = this.validateChunkParams(options);
|
|
2532
2733
|
console.error(`🔪 Chunking document: ${documentId} (maxTokens: ${maxTokens}, chunker: c1)`);
|
|
2533
|
-
//
|
|
2534
|
-
|
|
2535
|
-
// Create chunks
|
|
2734
|
+
// Create chunks. D2③ + R-B (2_ RESULT-T7-TXN-2_): chunking is synchronous and runs BEFORE anything
|
|
2735
|
+
// is deleted, so a chunker failure deletes nothing.
|
|
2536
2736
|
const chunks = this.chunkStructured(document.content, maxTokens);
|
|
2537
2737
|
const resultChunks = [];
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2738
|
+
// ONE synchronous transaction = cleanup (collect evidence citing this document, delete its chunks and
|
|
2739
|
+
// that evidence) + every new chunk INSERT + the signature UPDATE. A re-chunk reuses the ordinal chunk
|
|
2740
|
+
// ids, so with an await between cleanup and the INSERTs another call could cite the new D_chunk_0
|
|
2741
|
+
// after the collection; and a failure after cleanup lost the old chunks for good.
|
|
2742
|
+
this.db.transaction(() => {
|
|
2743
|
+
this.cleanupDocument(documentId);
|
|
2744
|
+
for (const chunk of chunks) {
|
|
2745
|
+
const chunkId = `${documentId}_chunk_${chunk.chunk_index}`;
|
|
2746
|
+
// Store chunk metadata (no embedding yet)
|
|
2747
|
+
this.db.prepare(`
|
|
2748
|
+
INSERT INTO chunk_metadata (
|
|
2749
|
+
chunk_id, document_id, chunk_index, text, start_pos, end_pos, start_token, end_token
|
|
2750
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
2751
|
+
`).run(chunkId, documentId, chunk.chunk_index, chunk.text, chunk.start_pos, chunk.end_pos, chunk.start_token, chunk.end_token);
|
|
2752
|
+
resultChunks.push({
|
|
2753
|
+
id: chunkId,
|
|
2754
|
+
text: chunk.text,
|
|
2755
|
+
startPos: chunk.start_pos,
|
|
2756
|
+
endPos: chunk.end_pos,
|
|
2757
|
+
startToken: chunk.start_token,
|
|
2758
|
+
endToken: chunk.end_token
|
|
2759
|
+
});
|
|
2760
|
+
}
|
|
2761
|
+
// spec §7.1: 두 번째 chunk 생성 경로 — 스탬프를 안 박으면 §5.1 관측이 조용히 샌다.
|
|
2762
|
+
this.db.prepare(`UPDATE documents SET chunking_signature = ? WHERE id = ?`)
|
|
2763
|
+
.run(effectiveSignature(maxTokens), documentId);
|
|
2764
|
+
})();
|
|
2555
2765
|
console.error(`✅ Document chunked: ${chunks.length} chunks created`);
|
|
2556
|
-
// spec §7.1: 두 번째 chunk 생성 경로 — 스탬프를 안 박으면 §5.1 관측이 조용히 샌다.
|
|
2557
|
-
this.db.prepare(`UPDATE documents SET chunking_signature = ? WHERE id = ?`)
|
|
2558
|
-
.run(effectiveSignature(maxTokens), documentId);
|
|
2559
2766
|
// Indirect missing-row producer (spec §5): freshly chunked rows have no
|
|
2560
2767
|
// vectors yet — let the coordinator recover them without a restart.
|
|
2561
2768
|
this.coordinator?.invalidateCoverage();
|
|
@@ -3007,37 +3214,44 @@ export class RAGKnowledgeGraphManager {
|
|
|
3007
3214
|
// that replaces a document could not tell from the return value that anything was deleted
|
|
3008
3215
|
// (2026-08-05 field report from a deployed project: "{stored:true} came back and I did not know
|
|
3009
3216
|
// what I had just wiped"). Silent destruction is the defect; the numbers are free.
|
|
3010
|
-
|
|
3217
|
+
cleanupDocument(documentId) {
|
|
3011
3218
|
if (!this.db)
|
|
3012
3219
|
return { deletedChunks: 0, deletedAssociations: 0, deletedVectors: 0 };
|
|
3013
3220
|
console.error(`🧹 Cleaning up document: ${documentId}`);
|
|
3014
|
-
//
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
const
|
|
3024
|
-
|
|
3025
|
-
`).
|
|
3026
|
-
deletedAssociations
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3221
|
+
// D2③ (kg-relation-evidence): collect -> delete chunks -> invalidate citing evidence in ONE
|
|
3222
|
+
// transaction. 6.4.0 ran these statements one by one in autocommit. Used by storeDocument,
|
|
3223
|
+
// chunkDocument (re-chunk that reuses ordinal ids) and deleteDocument(s).
|
|
3224
|
+
// R-B (2_ RESULT-T7-TXN-2_): SYNCHRONOUS — results and exceptions reach the caller in the same
|
|
3225
|
+
// tick. Every caller runs it inside its own transaction together with the source change, so this
|
|
3226
|
+
// transaction nests as a savepoint and the caller's final SQL is inside the same rollback scope.
|
|
3227
|
+
return this.db.transaction(() => {
|
|
3228
|
+
const db = this.db;
|
|
3229
|
+
const citedEvidence = collectEvidenceRowidsCitingDocument(db, documentId);
|
|
3230
|
+
const existingChunks = db.prepare(`
|
|
3231
|
+
SELECT rowid FROM chunk_metadata WHERE document_id = ?
|
|
3232
|
+
`).all(documentId);
|
|
3233
|
+
let deletedAssociations = 0;
|
|
3234
|
+
let deletedVectors = 0;
|
|
3235
|
+
for (const chunk of existingChunks) {
|
|
3236
|
+
const associations = db.prepare(`
|
|
3237
|
+
DELETE FROM chunk_entities WHERE chunk_rowid = ?
|
|
3238
|
+
`).run(chunk.rowid);
|
|
3239
|
+
deletedAssociations += associations.changes;
|
|
3240
|
+
// Delete vector embeddings (vec0 needs literal integer, not parameterized)
|
|
3241
|
+
db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(chunk.rowid)}`);
|
|
3242
|
+
deletedVectors++;
|
|
3243
|
+
}
|
|
3244
|
+
const metadata = db.prepare(`
|
|
3245
|
+
DELETE FROM chunk_metadata WHERE document_id = ?
|
|
3246
|
+
`).run(documentId);
|
|
3247
|
+
deleteKgChunkRows(db, citedEvidence);
|
|
3248
|
+
if (existingChunks.length > 0) {
|
|
3249
|
+
console.error(` ├─ Deleted ${deletedAssociations} entity associations`);
|
|
3250
|
+
console.error(` ├─ Deleted ${deletedVectors} vector embeddings`);
|
|
3251
|
+
console.error(` └─ Deleted ${metadata.changes} chunk metadata records`);
|
|
3252
|
+
}
|
|
3253
|
+
return { deletedChunks: existingChunks.length, deletedAssociations, deletedVectors };
|
|
3254
|
+
})();
|
|
3041
3255
|
}
|
|
3042
3256
|
async deleteDocument(documentId) {
|
|
3043
3257
|
if (!this.db)
|
|
@@ -3052,12 +3266,18 @@ export class RAGKnowledgeGraphManager {
|
|
|
3052
3266
|
console.warn(`⚠️ Document '${documentId}' not found`);
|
|
3053
3267
|
return { documentId, deleted: false };
|
|
3054
3268
|
}
|
|
3055
|
-
//
|
|
3056
|
-
|
|
3057
|
-
//
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3269
|
+
// D2③ + R-B (2_ RESULT-T7-TXN-2_): cleanup (collect evidence citing this document, delete its
|
|
3270
|
+
// chunks and that evidence) and the document DELETE are ONE synchronous transaction — no await
|
|
3271
|
+
// between them, so no evidence can be published on this document after the collection, and a
|
|
3272
|
+
// failing DELETE rolls the cleanup back.
|
|
3273
|
+
const result = this.db.transaction(() => {
|
|
3274
|
+
this.cleanupDocument(documentId);
|
|
3275
|
+
return this.db.prepare(`
|
|
3276
|
+
DELETE FROM documents WHERE id = ?
|
|
3277
|
+
`).run(documentId);
|
|
3278
|
+
})();
|
|
3279
|
+
// R-B item 6 (review I1): after the successful commit only (a rollback throws above).
|
|
3280
|
+
this.coordinator?.invalidateCoverage();
|
|
3061
3281
|
if (result.changes > 0) {
|
|
3062
3282
|
console.error(`✅ Document '${documentId}' deleted successfully`);
|
|
3063
3283
|
return { documentId, deleted: true };
|
|
@@ -3217,6 +3437,26 @@ export class RAGKnowledgeGraphManager {
|
|
|
3217
3437
|
if (!this.db)
|
|
3218
3438
|
throw new Error('Database not initialized');
|
|
3219
3439
|
console.error(`📥 Importing knowledge graph (merge: ${options.merge !== false})...`);
|
|
3440
|
+
// D2③ / C3 (final review I1): the evidence dependency collectors compare string ids strictly, but
|
|
3441
|
+
// the INSERTs below bind whatever the dump holds, and SQLite stores a number bound to a TEXT id
|
|
3442
|
+
// column as its decimal text (7.5 -> '7.5'). A numeric id could therefore replace a document, or add
|
|
3443
|
+
// a source to an observation, that cited evidence names by string while the collector misses it.
|
|
3444
|
+
// One id contract for writing and collecting: these two ids must be strings, checked before any
|
|
3445
|
+
// write, so a violation refuses the whole import with nothing written. No numeric normalisation.
|
|
3446
|
+
if (Array.isArray(data.documents)) {
|
|
3447
|
+
data.documents.forEach((doc, i) => {
|
|
3448
|
+
if (typeof doc?.id !== 'string') {
|
|
3449
|
+
throw new Error(`importGraph: documents[${i}].id must be a string (got ${doc?.id === null ? 'null' : typeof doc?.id}); nothing was imported`);
|
|
3450
|
+
}
|
|
3451
|
+
});
|
|
3452
|
+
}
|
|
3453
|
+
if (Array.isArray(data.observation_sources)) {
|
|
3454
|
+
data.observation_sources.forEach((so, i) => {
|
|
3455
|
+
if (typeof so?.observation_id !== 'string') {
|
|
3456
|
+
throw new Error(`importGraph: observation_sources[${i}].observation_id must be a string (got ${so?.observation_id === null ? 'null' : typeof so?.observation_id}); nothing was imported`);
|
|
3457
|
+
}
|
|
3458
|
+
});
|
|
3459
|
+
}
|
|
3220
3460
|
const imported = { entities: 0, relations: 0, documents: 0 };
|
|
3221
3461
|
const skipped = { entities: 0, relations: 0, documents: 0 };
|
|
3222
3462
|
// merge 로 배열 위치가 재배정된 관찰. 조용히 순서를 바꾸면 호출자가 알 수 없으므로
|
|
@@ -3253,6 +3493,10 @@ export class RAGKnowledgeGraphManager {
|
|
|
3253
3493
|
this.db.exec(`DELETE FROM entity_embedding_metadata`);
|
|
3254
3494
|
console.error('🗑️ Cleared existing data for full import');
|
|
3255
3495
|
}
|
|
3496
|
+
// D2③ (kg-relation-evidence): import is an observation AND a document writer. Evidence citing an
|
|
3497
|
+
// observation that gains a source row, or a document this import replaces, is collected before the
|
|
3498
|
+
// row is written and deleted at the end of this transaction.
|
|
3499
|
+
const citedEvidence = new Set();
|
|
3256
3500
|
// Import entities using INSERT OR IGNORE
|
|
3257
3501
|
if (data.entities && Array.isArray(data.entities)) {
|
|
3258
3502
|
const stmt = this.db.prepare(`
|
|
@@ -3295,6 +3539,9 @@ export class RAGKnowledgeGraphManager {
|
|
|
3295
3539
|
VALUES (?, ?, ?, ?)
|
|
3296
3540
|
`);
|
|
3297
3541
|
for (const doc of data.documents) {
|
|
3542
|
+
// doc.id is a string here (checked above, before any write).
|
|
3543
|
+
for (const r of collectEvidenceRowidsCitingDocument(this.db, doc.id))
|
|
3544
|
+
citedEvidence.add(r);
|
|
3298
3545
|
const result = stmt.run(doc.id, doc.content, JSON.stringify(doc.metadata || {}), doc.created_at || new Date().toISOString());
|
|
3299
3546
|
if (result.changes > 0) {
|
|
3300
3547
|
imported.documents++;
|
|
@@ -3387,6 +3634,8 @@ export class RAGKnowledgeGraphManager {
|
|
|
3387
3634
|
`${so.observation_id}/${so.source_kind}/${so.source_ref} differs from the existing row`);
|
|
3388
3635
|
continue;
|
|
3389
3636
|
}
|
|
3637
|
+
for (const r of collectEvidenceRowidsCitingObservations(this.db, [so.observation_id]))
|
|
3638
|
+
citedEvidence.add(r);
|
|
3390
3639
|
this.db.prepare(`INSERT INTO observation_sources
|
|
3391
3640
|
(observation_id, source_kind, source_ref, source_hash, recorded_at) VALUES (?, ?, ?, ?, ?)`)
|
|
3392
3641
|
.run(so.observation_id, so.source_kind, so.source_ref, so.source_hash ?? null, so.recorded_at);
|
|
@@ -3427,6 +3676,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
3427
3676
|
});
|
|
3428
3677
|
tx();
|
|
3429
3678
|
}
|
|
3679
|
+
deleteKgChunkRows(this.db, [...citedEvidence]);
|
|
3430
3680
|
// projection 재합성 + 파생 상태 무효화.
|
|
3431
3681
|
// 무효화가 없으면 이미 있던 entity 를 덮어쓴 뒤에도 옛 벡터·옛 KG chunk 가
|
|
3432
3682
|
// 검색에 남는다. import 는 관찰을 바꾸는 writer 이므로 다른 writer 와 같은
|
|
@@ -3759,6 +4009,11 @@ export class RAGKnowledgeGraphManager {
|
|
|
3759
4009
|
for (const result of vectorResults) {
|
|
3760
4010
|
// Get entities associated with this chunk (for document chunks)
|
|
3761
4011
|
let chunkEntities = [];
|
|
4012
|
+
// D2④ (kg-relation-evidence): full_context_available = "getDetailedContext can resolve the stored
|
|
4013
|
+
// body". Document chunks: unchanged (true — the FK cascade removes a chunk with its document). KG
|
|
4014
|
+
// chunks: only while their entity / relationship row exists. It is NOT a verified-evidence signal;
|
|
4015
|
+
// that is evidence_status in getDetailedContext. 6.4.0 hard-coded true.
|
|
4016
|
+
let sourceResolved = result.chunk_type === 'document';
|
|
3762
4017
|
if (result.chunk_type === 'document') {
|
|
3763
4018
|
chunkEntities = this.db.prepare(`
|
|
3764
4019
|
SELECT e.name
|
|
@@ -3774,6 +4029,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
3774
4029
|
`).get(result.entity_id);
|
|
3775
4030
|
if (entity) {
|
|
3776
4031
|
chunkEntities = [entity.name];
|
|
4032
|
+
sourceResolved = true;
|
|
3777
4033
|
}
|
|
3778
4034
|
}
|
|
3779
4035
|
else if (result.chunk_type === 'relationship' && result.relationship_id) {
|
|
@@ -3787,6 +4043,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
3787
4043
|
`).get(result.relationship_id);
|
|
3788
4044
|
if (relEntities) {
|
|
3789
4045
|
chunkEntities = [relEntities.source_name, relEntities.target_name];
|
|
4046
|
+
sourceResolved = true;
|
|
3790
4047
|
}
|
|
3791
4048
|
}
|
|
3792
4049
|
// Enhanced graph boost calculation with decay + cap
|
|
@@ -3889,7 +4146,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
3889
4146
|
vector_similarity: vectorSimilarity,
|
|
3890
4147
|
graph_boost: (useGraph && !vectorDegraded) ? graphBoost : undefined,
|
|
3891
4148
|
fts_boost: ftsBoost > 0 ? ftsBoost : undefined,
|
|
3892
|
-
full_context_available:
|
|
4149
|
+
full_context_available: sourceResolved,
|
|
3893
4150
|
chunk_type: result.chunk_type,
|
|
3894
4151
|
source_id: sourceId
|
|
3895
4152
|
});
|
|
@@ -3937,6 +4194,13 @@ export class RAGKnowledgeGraphManager {
|
|
|
3937
4194
|
if (!this.db)
|
|
3938
4195
|
throw new Error('Database not initialized');
|
|
3939
4196
|
console.error(`📖 Getting detailed context for chunk: ${chunkId}`);
|
|
4197
|
+
// D2④ (kg-relation-evidence): KG chunks have no documents row, so the INNER JOIN below never found
|
|
4198
|
+
// them (6.4.0: every KG hit failed here). Branch on chunk_type first; the document branch below is
|
|
4199
|
+
// unchanged.
|
|
4200
|
+
const head = this.db.prepare(`SELECT chunk_type FROM chunk_metadata WHERE chunk_id = ?`).get(chunkId);
|
|
4201
|
+
if (head && (head.chunk_type === 'entity' || head.chunk_type === 'relationship')) {
|
|
4202
|
+
return this.getKgDetailedContext(chunkId);
|
|
4203
|
+
}
|
|
3940
4204
|
// Get the main chunk
|
|
3941
4205
|
const chunk = this.db.prepare(`
|
|
3942
4206
|
SELECT
|
|
@@ -4002,6 +4266,49 @@ export class RAGKnowledgeGraphManager {
|
|
|
4002
4266
|
metadata: metadata
|
|
4003
4267
|
};
|
|
4004
4268
|
}
|
|
4269
|
+
// D2④: entity -> the entity and its ACTIVE observations (history stays with getObservationHistory);
|
|
4270
|
+
// relationship -> the relationship row + evidence_status + per-reference evidence_refs. When the
|
|
4271
|
+
// entity / relationship row itself is gone the call fails as an unknown chunk does.
|
|
4272
|
+
getKgDetailedContext(chunkId) {
|
|
4273
|
+
const row = this.db.prepare(`SELECT chunk_id, chunk_type, entity_id, relationship_id, text, metadata FROM chunk_metadata WHERE chunk_id = ?`)
|
|
4274
|
+
.get(chunkId);
|
|
4275
|
+
let meta = {};
|
|
4276
|
+
try {
|
|
4277
|
+
const parsed = JSON.parse(row.metadata ?? '{}');
|
|
4278
|
+
if (parsed && typeof parsed === 'object')
|
|
4279
|
+
meta = parsed;
|
|
4280
|
+
}
|
|
4281
|
+
catch {
|
|
4282
|
+
meta = {};
|
|
4283
|
+
}
|
|
4284
|
+
const notFound = () => new Error(`Chunk with ID ${chunkId} not found`);
|
|
4285
|
+
if (row.chunk_type === 'entity') {
|
|
4286
|
+
const ent = row.entity_id
|
|
4287
|
+
? this.db.prepare(`SELECT id, name, entityType FROM entities WHERE id = ?`).get(row.entity_id)
|
|
4288
|
+
: undefined;
|
|
4289
|
+
if (!ent)
|
|
4290
|
+
throw notFound();
|
|
4291
|
+
const observations = this.db.prepare(`SELECT observation_id, revision_no, content FROM entity_observations
|
|
4292
|
+
WHERE entity_id = ? AND status = 'active' ORDER BY projection_order`).all(ent.id);
|
|
4293
|
+
return { chunk_id: row.chunk_id, document_id: null, document_title: null, source_kind: 'entity', source_id: ent.id,
|
|
4294
|
+
full_text: row.text ?? '', entities: [ent.name], metadata: meta,
|
|
4295
|
+
entity: { name: ent.name, entityType: ent.entityType, observations } };
|
|
4296
|
+
}
|
|
4297
|
+
const rel = row.relationship_id
|
|
4298
|
+
? this.db.prepare(`
|
|
4299
|
+
SELECT r.id, r.relationType, r.confidence, e1.name AS source_name, e2.name AS target_name
|
|
4300
|
+
FROM relationships r JOIN entities e1 ON e1.id = r.source_entity JOIN entities e2 ON e2.id = r.target_entity
|
|
4301
|
+
WHERE r.id = ?`).get(row.relationship_id)
|
|
4302
|
+
: undefined;
|
|
4303
|
+
if (!rel)
|
|
4304
|
+
throw notFound();
|
|
4305
|
+
const ev = resolveEvidenceStatus(this.db, meta);
|
|
4306
|
+
return { chunk_id: row.chunk_id, document_id: null, document_title: null, source_kind: 'relationship', source_id: rel.id,
|
|
4307
|
+
full_text: row.text ?? '', entities: [rel.source_name, rel.target_name], metadata: meta,
|
|
4308
|
+
relationship: { id: rel.id, source: rel.source_name, target: rel.target_name, relationType: rel.relationType, confidence: rel.confidence },
|
|
4309
|
+
evidence_status: ev.status, evidence_refs: ev.refs,
|
|
4310
|
+
support: meta.kg_format === EVIDENCE_FORMAT && meta.support ? meta.support : null };
|
|
4311
|
+
}
|
|
4005
4312
|
async getKnowledgeGraphStats() {
|
|
4006
4313
|
if (!this.db)
|
|
4007
4314
|
throw new Error('Database not initialized');
|