rag-memory-epf-mcp 3.5.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -4
- package/dist/index.d.ts +96 -9
- package/dist/index.js +1324 -270
- package/dist/src/backfillCoordinator.d.ts +59 -0
- package/dist/src/backfillCoordinator.js +552 -0
- package/dist/src/backup/preflight.d.ts +5 -0
- package/dist/src/backup/preflight.js +185 -0
- package/dist/src/embeddingGate.d.ts +68 -0
- package/dist/src/embeddingGate.js +227 -0
- package/dist/src/migrations/migrations.d.ts +2 -0
- package/dist/src/migrations/migrations.js +181 -0
- package/dist/src/modelCache.d.ts +33 -0
- package/dist/src/modelCache.js +235 -0
- package/dist/src/observations/history.d.ts +8 -0
- package/dist/src/observations/history.js +64 -0
- package/dist/src/observations/lifecycle.d.ts +45 -0
- package/dist/src/observations/lifecycle.js +101 -0
- package/dist/src/observations/projection.d.ts +3 -0
- package/dist/src/observations/projection.js +31 -0
- package/dist/src/observations/schema.d.ts +1 -0
- package/dist/src/observations/schema.js +115 -0
- package/dist/src/tools/graph-query-tools.js +29 -5
- package/dist/src/tools/knowledge-graph-tools.d.ts +14 -0
- package/dist/src/tools/knowledge-graph-tools.js +244 -5
- package/dist/src/tools/tool-registry.d.ts +7 -0
- package/dist/src/tools/tool-registry.js +43 -3
- package/dist/src/tools/types.d.ts +1 -0
- package/docs/UPDATING.md +178 -0
- package/package.json +9 -5
package/dist/index.js
CHANGED
|
@@ -21,13 +21,59 @@ import modularity from 'graphology-metrics/graph/modularity.js';
|
|
|
21
21
|
import { getAllMCPTools, validateToolArgs, getSystemInfo } from './src/tools/tool-registry.js';
|
|
22
22
|
// Import migration system
|
|
23
23
|
import { MigrationManager } from './src/migrations/migration-manager.js';
|
|
24
|
+
import { backupBeforeMigration } from './src/backup/preflight.js';
|
|
25
|
+
import { rebuildProjection, deleteStaleKgChunks } from './src/observations/projection.js';
|
|
26
|
+
import { addRevision, correctRevision, transitionStatus, linkSources, nextProjectionOrder } from './src/observations/lifecycle.js';
|
|
27
|
+
import { getObservationHistory } from './src/observations/history.js';
|
|
24
28
|
// Import chunk text algorithm (extracted for publish-time invariant testing)
|
|
25
29
|
import { chunkText as splitTextIntoChunks } from './src/chunkText.js';
|
|
26
30
|
import { migrations } from './src/migrations/migrations.js';
|
|
31
|
+
// v3.6 lite install: model lifecycle + version-independent cache (A′ boundary)
|
|
32
|
+
import { EmbeddingGate, GateNotReadyError, GateDisabledError, TerminalConfigError } from './src/embeddingGate.js';
|
|
33
|
+
import { resolveModelCacheDir, preflightCacheDir, artifactKey, ModelDownloadLock, handleLoaderFailure } from './src/modelCache.js';
|
|
34
|
+
import { BackfillCoordinator } from './src/backfillCoordinator.js';
|
|
35
|
+
import os from 'node:os';
|
|
27
36
|
import { createHash } from 'crypto';
|
|
28
37
|
import { createRequire } from 'module';
|
|
29
38
|
const require = createRequire(import.meta.url);
|
|
30
39
|
const PKG_VERSION = require('../package.json').version;
|
|
40
|
+
// v3.6: runtime Node floor (engines is advisory only under default npm config).
|
|
41
|
+
// Limitation: static native imports above may fail before this runs on very old
|
|
42
|
+
// Node — documented in docs/UPDATING.md.
|
|
43
|
+
const NODE_MAJOR = Number(process.versions.node.split('.')[0]);
|
|
44
|
+
function assertNodeVersion() {
|
|
45
|
+
if (NODE_MAJOR < 24) {
|
|
46
|
+
console.error(`❌ rag-memory-epf-mcp v${PKG_VERSION} requires Node >= 24 (current: ${process.versions.node}).`);
|
|
47
|
+
console.error(' See docs/UPDATING.md for the supported runtime matrix.');
|
|
48
|
+
process.exitCode = 1;
|
|
49
|
+
throw new Error('unsupported Node version');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
// v3.6: strip tokens / auth material / long URLs from operator-facing error text.
|
|
53
|
+
function sanitizeErrorMessage(msg) {
|
|
54
|
+
return msg
|
|
55
|
+
.replace(/(hf_|api[_-]?key=|authorization:\s*)\S+/gi, '$1[redacted]')
|
|
56
|
+
.replace(/https?:\/\/\S{60,}/g, '[url]')
|
|
57
|
+
.slice(0, 500);
|
|
58
|
+
}
|
|
59
|
+
// v3.6: startup self-report banner (version reliability — spec §8).
|
|
60
|
+
function printBanner(opts) {
|
|
61
|
+
console.error(`🚀 rag-memory-epf-mcp v${PKG_VERSION} | node v${process.versions.node} | model ${opts.model}@${opts.revision} (${opts.dtype}) | cache ${opts.cachePath} | db ${opts.dbPath}`);
|
|
62
|
+
}
|
|
63
|
+
// v3.6 (spec §5): ONE FTS5 literal-query compiler shared by chunk and entity
|
|
64
|
+
// search — raw user input can never produce MATCH syntax errors or trigger
|
|
65
|
+
// operators (every term is double-quoted; special characters stripped exactly
|
|
66
|
+
// as the pre-3.6 hybridSearch sanitizer did). Returns null when nothing
|
|
67
|
+
// searchable remains (contract: caller returns empty results + warning).
|
|
68
|
+
export function compileFtsLiteralQuery(q) {
|
|
69
|
+
const sanitized = q.replace(/["\*\(\)\-]/g, ' ').trim();
|
|
70
|
+
if (!sanitized)
|
|
71
|
+
return null;
|
|
72
|
+
const terms = sanitized.split(/\s+/).filter(t => t.length > 0);
|
|
73
|
+
if (terms.length === 0)
|
|
74
|
+
return null;
|
|
75
|
+
return terms.map(t => `"${t}"`).join(' OR ');
|
|
76
|
+
}
|
|
31
77
|
// Configure Hugging Face transformers for better compatibility
|
|
32
78
|
if (env.backends?.onnx?.wasm) {
|
|
33
79
|
env.backends.onnx.wasm.wasmPaths = './node_modules/@huggingface/transformers/dist/';
|
|
@@ -40,6 +86,21 @@ const DB_FILE_PATH = process.env.DB_FILE_PATH
|
|
|
40
86
|
: path.join(path.dirname(fileURLToPath(import.meta.url)), process.env.DB_FILE_PATH)
|
|
41
87
|
: defaultDbPath;
|
|
42
88
|
const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'Xenova/bge-m3';
|
|
89
|
+
// v3.5 default model config — grandfathering legacy vectors is only automatic
|
|
90
|
+
// when the current config matches this (spec §6b custom-model guard). An
|
|
91
|
+
// EXPLICIT `EMBEDDING_MODEL=Xenova/bge-m3` counts as default: same weights,
|
|
92
|
+
// same pin, same grandfather policy (beta 1R consistency fix).
|
|
93
|
+
const IS_DEFAULT_MODEL_CONFIG = !process.env.EMBEDDING_MODEL || process.env.EMBEDDING_MODEL === 'Xenova/bge-m3';
|
|
94
|
+
// Default model pinned to an upstream commit (spec §6c): a shared version-
|
|
95
|
+
// independent cache must never silently swap weights under 'main'. Verified
|
|
96
|
+
// 2026-07-18 via `git ls-remote https://huggingface.co/Xenova/bge-m3` — the
|
|
97
|
+
// same revision the local v3.5 cache was downloaded from. Custom models stay
|
|
98
|
+
// on 'main' (their vectors are never auto-grandfathered anyway).
|
|
99
|
+
const MODEL_REVISION = IS_DEFAULT_MODEL_CONFIG ? '4de13258303883538bd53b696b452bf8099f0858' : 'main';
|
|
100
|
+
const MODEL_DTYPE = 'fp16';
|
|
101
|
+
// Entity embedding text builder version — mixed into entity input hashes so a
|
|
102
|
+
// builder change re-backfills entities without touching chunk vectors (spec §6c).
|
|
103
|
+
const TEXT_BUILDER_VERSION = 'tb1';
|
|
43
104
|
// Safe rowid for vec0 virtual tables (require literal integer, not parameterized)
|
|
44
105
|
// Trim incomplete UTF-8 multi-byte sequences at chunk boundaries.
|
|
45
106
|
// Continuation bytes match 10xxxxxx (0x80-0xBF); lead bytes indicate how many
|
|
@@ -59,18 +120,26 @@ function safeRowid(value) {
|
|
|
59
120
|
export class RAGKnowledgeGraphManager {
|
|
60
121
|
db = null;
|
|
61
122
|
encoding = null;
|
|
62
|
-
|
|
63
|
-
|
|
123
|
+
gate;
|
|
124
|
+
embeddingsMode = 'lazy';
|
|
125
|
+
currentProfileId = 0;
|
|
126
|
+
// Automatic grandfathering of legacy vectors is only allowed under the v3.5
|
|
127
|
+
// default model config, or with the explicit trust opt-in (spec §6b guard).
|
|
128
|
+
grandfatherAllowed = IS_DEFAULT_MODEL_CONFIG || process.env.RAG_MEMORY_TRUST_LEGACY_VECTORS === '1';
|
|
129
|
+
coordinator = null;
|
|
64
130
|
embeddingCache = new Map();
|
|
65
131
|
EMBEDDING_CACHE_MAX = 500;
|
|
66
132
|
dictionaryCache = null;
|
|
133
|
+
// v3.6 (spec §3): initialize = DB + migrations + profile only. The embedding
|
|
134
|
+
// model is NEVER awaited here — main() connects the MCP server first and the
|
|
135
|
+
// gate loads in the background (lazy) or is awaited explicitly (eager).
|
|
136
|
+
// __testForceFkOff: 음성 대조군 전용. FK 게이트가 실제로 부팅을 막는지 시험한다.
|
|
137
|
+
// 이름에 __test 를 박아 둔 이유는 이것이 프로덕션 설정 표면이 아니라는 것을
|
|
138
|
+
// 호출부에서 읽히게 하기 위해서다.
|
|
67
139
|
async initialize(opts = {}) {
|
|
68
140
|
console.error('🚀 Initializing RAG Knowledge Graph MCP Server...');
|
|
69
|
-
// Initialize database
|
|
70
141
|
this.db = new Database(DB_FILE_PATH);
|
|
71
|
-
// Load sqlite-vec extension
|
|
72
142
|
sqliteVec.load(this.db);
|
|
73
|
-
// SQLite performance & safety optimizations
|
|
74
143
|
this.db.pragma('journal_mode = WAL');
|
|
75
144
|
this.db.pragma('synchronous = NORMAL');
|
|
76
145
|
this.db.pragma('busy_timeout = 5000');
|
|
@@ -78,37 +147,332 @@ export class RAGKnowledgeGraphManager {
|
|
|
78
147
|
this.db.pragma('temp_store = MEMORY');
|
|
79
148
|
this.db.pragma('mmap_size = 268435456');
|
|
80
149
|
this.db.pragma('foreign_keys = ON');
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
//
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
150
|
+
// spec §5.2: 관찰 lifecycle 의 무결성은 전부 FK CASCADE 를 전제한다 — root 를 지우면
|
|
151
|
+
// revision 이, revision 을 지우면 source 가 따라가야 history 가 고아로 남지 않는다.
|
|
152
|
+
// FK 가 꺼진 채 돌면 그 계약이 조용히 무효가 되고, 그게 최악이다. 스키마를 건드리기
|
|
153
|
+
// 전에(= runMigrations 앞에서) 멈춘다.
|
|
154
|
+
// 트랜잭션 내부에서는 이 pragma 가 no-op 이므로(실측 before=1·during=1·after=1)
|
|
155
|
+
// 부팅 시점 확인이 유일한 방어 지점이다.
|
|
156
|
+
//
|
|
157
|
+
// 음성 대조군 주입은 **인자로만** 받는다. 환경변수로 두면 프로덕션 경로에
|
|
158
|
+
// "부팅을 막는 스위치"가 상시 존재하게 되고, 오설정 한 줄로 서버가 안 뜬다
|
|
159
|
+
// (advisor beta 자기의심 2 = "더 나쁘다"). 테스트는 manager 를 직접 만들므로
|
|
160
|
+
// 인자 주입으로 충분하다.
|
|
161
|
+
if (opts.__testForceFkOff)
|
|
162
|
+
this.db.pragma('foreign_keys = OFF');
|
|
163
|
+
{
|
|
164
|
+
const fk = this.db.pragma('foreign_keys', { simple: true });
|
|
165
|
+
if (Number(fk) !== 1) {
|
|
166
|
+
throw new Error(`foreign_keys is ${fk}, expected 1. The observation lifecycle relies on FK CASCADE ` +
|
|
167
|
+
`for history integrity; refusing to run migrations without it.`);
|
|
168
|
+
}
|
|
89
169
|
}
|
|
90
|
-
|
|
170
|
+
this.encoding = get_encoding("cl100k_base");
|
|
91
171
|
await this.runMigrations();
|
|
92
|
-
|
|
93
|
-
|
|
172
|
+
this.currentProfileId = this.ensureCurrentProfile();
|
|
173
|
+
this.embeddingsMode = opts.skipModel
|
|
174
|
+
? 'off'
|
|
175
|
+
: (process.env.RAG_MEMORY_EMBEDDINGS || 'lazy');
|
|
176
|
+
if (!['lazy', 'eager', 'off'].includes(this.embeddingsMode))
|
|
177
|
+
this.embeddingsMode = 'lazy';
|
|
178
|
+
this.gate = opts.gate ?? new EmbeddingGate({
|
|
179
|
+
mode: this.embeddingsMode,
|
|
180
|
+
loadModel: () => this.buildRealLoader(),
|
|
181
|
+
onReady: () => this.coordinator?.kick(),
|
|
182
|
+
});
|
|
183
|
+
// Late-bound deps (closures): tests swap manager.gate / flip the guard.
|
|
184
|
+
this.coordinator = new BackfillCoordinator({
|
|
185
|
+
db: () => this.db,
|
|
186
|
+
gateIsReady: () => this.gate.isReady,
|
|
187
|
+
gateIsDisabled: () => this.gate.isDisabled,
|
|
188
|
+
mode: () => this.embeddingsMode,
|
|
189
|
+
grandfatherAllowed: () => this.grandfatherAllowed,
|
|
190
|
+
currentProfileId: () => this.currentProfileId,
|
|
191
|
+
buildEntityInputHash: (entityId) => this.entityInputHash(entityId),
|
|
192
|
+
hashEntityText: (text) => this.hashWithBuilderVersion(text),
|
|
193
|
+
chunkInputHash: (text) => createHash('sha256').update(text).digest('hex'),
|
|
194
|
+
reembedEntity: async (entityId) => this.embedEntity(entityId, 'backfill'),
|
|
195
|
+
reembedChunk: async (rowid) => this.reembedChunkByRowid(rowid),
|
|
196
|
+
});
|
|
197
|
+
console.error('✅ RAG-enabled knowledge graph initialized (embedding model deferred)');
|
|
94
198
|
const systemInfo = getSystemInfo();
|
|
95
199
|
console.error(`📊 System Info: ${systemInfo.toolCounts.total} tools available (${systemInfo.toolCounts.knowledgeGraph} knowledge graph, ${systemInfo.toolCounts.rag} RAG, ${systemInfo.toolCounts.graphQuery} query)`);
|
|
96
200
|
}
|
|
97
|
-
|
|
201
|
+
// Upsert the stored-vector compatibility profile (spec §6c layer 2) and record
|
|
202
|
+
// retrieval config in server_meta (layer 3 — never a backfill trigger).
|
|
203
|
+
ensureCurrentProfile() {
|
|
204
|
+
if (!this.db)
|
|
205
|
+
throw new Error('Database not initialized');
|
|
206
|
+
const dims = 1024;
|
|
207
|
+
if (dims !== 1024)
|
|
208
|
+
throw new Error('unsupported embedding dims (vec0 tables are fixed at 1024)'); // fail-fast contract
|
|
209
|
+
this.db.prepare(`INSERT OR IGNORE INTO embedding_profiles
|
|
210
|
+
(model_id, revision, dtype, dims, pooling, normalize) VALUES (?,?,?,?,?,?)`)
|
|
211
|
+
.run(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE, dims, 'cls', 1);
|
|
212
|
+
const row = this.db.prepare(`SELECT id FROM embedding_profiles
|
|
213
|
+
WHERE model_id=? AND revision=? AND dtype=? AND dims=? AND pooling=? AND normalize=?`)
|
|
214
|
+
.get(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE, dims, 'cls', 1);
|
|
215
|
+
this.db.prepare(`INSERT INTO server_meta(key,value) VALUES('current_profile_id',?)
|
|
216
|
+
ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(String(row.id));
|
|
217
|
+
this.db.prepare(`INSERT INTO server_meta(key,value) VALUES('query_prefix_version','1')
|
|
218
|
+
ON CONFLICT(key) DO NOTHING`).run();
|
|
219
|
+
return row.id;
|
|
220
|
+
}
|
|
221
|
+
// Real model loader used by the gate: version-independent cache dir with a
|
|
222
|
+
// cross-process download lock. Preflight failure throws (gate -> failed);
|
|
223
|
+
// silently falling back to the package-internal cache is forbidden (spec §7).
|
|
224
|
+
async buildRealLoader() {
|
|
225
|
+
const cacheDir = resolveModelCacheDir(process.env, process.platform, os.homedir());
|
|
226
|
+
const pf = preflightCacheDir(cacheDir);
|
|
227
|
+
if (!pf.ok)
|
|
228
|
+
throw new Error(`model cache dir not writable (${cacheDir}): ${pf.error}`);
|
|
229
|
+
const key = artifactKey(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE);
|
|
230
|
+
const lock = new ModelDownloadLock(cacheDir, key);
|
|
231
|
+
// Shutdown aborts the lock wait via the gate's AbortController (spec §3).
|
|
232
|
+
console.error('⏳ acquiring model download lock...'); // deterministic lock-wait marker (5R test residual)
|
|
233
|
+
const role = await lock.acquireOrWait({ timeoutMs: 10 * 60_000, signal: this.gate.abort.signal });
|
|
98
234
|
try {
|
|
99
|
-
|
|
100
|
-
// Configure environment to allow remote model downloads
|
|
235
|
+
this.gate.markDownloading();
|
|
101
236
|
env.allowRemoteModels = true;
|
|
102
237
|
env.allowLocalModels = true;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
238
|
+
console.error(`🤖 Loading embedding model: ${EMBEDDING_MODEL} (1024-dim, cache=${cacheDir})...`);
|
|
239
|
+
const model = await pipeline('feature-extraction', EMBEDDING_MODEL, { revision: MODEL_REVISION, dtype: MODEL_DTYPE, cache_dir: cacheDir });
|
|
240
|
+
// dims fail-fast (spec §2 / beta B8): probe the ACTUAL output length — a
|
|
241
|
+
// 384/768-dim custom model must fail here with a clear message, not at
|
|
242
|
+
// every subsequent vector write.
|
|
243
|
+
const probe = await model('dimension probe', { pooling: 'cls', normalize: true });
|
|
244
|
+
const actualDims = probe.data.length;
|
|
245
|
+
if (actualDims !== 1024) {
|
|
246
|
+
// Config incompatibility, NOT cache corruption (beta 2R B3): the
|
|
247
|
+
// download and load both succeeded — quarantining or retrying cannot
|
|
248
|
+
// change the model's dimensions.
|
|
249
|
+
if (role === 'owner')
|
|
250
|
+
lock.markComplete(); // cache itself is valid
|
|
251
|
+
throw new TerminalConfigError(`embedding model ${EMBEDDING_MODEL} outputs ${actualDims} dims — this engine's vec0 tables are fixed at 1024. Use a 1024-dim model.`);
|
|
252
|
+
}
|
|
253
|
+
if (role === 'owner')
|
|
254
|
+
lock.markComplete();
|
|
255
|
+
console.error(`✅ ${EMBEDDING_MODEL} model loaded (${MODEL_DTYPE})`);
|
|
256
|
+
return async (text, dims, isQuery) => {
|
|
257
|
+
const input = isQuery ? `Represent this sentence for searching relevant passages: ${text}` : text;
|
|
258
|
+
const r = await model(input, { pooling: 'cls', normalize: true });
|
|
259
|
+
return new Float32Array(r.data.slice(0, dims));
|
|
260
|
+
};
|
|
106
261
|
}
|
|
107
|
-
catch (
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
262
|
+
catch (e) {
|
|
263
|
+
// Cache policy by CAUSE and ROLE (beta 2R B3 -> 4R M1 -> 5R M1), unit-
|
|
264
|
+
// tested in modelCache: config errors touch nothing; integrity errors
|
|
265
|
+
// invalidate the marker, and only a lock-holding OWNER may quarantine
|
|
266
|
+
// (a ready-role process racing other readers never deletes shared
|
|
267
|
+
// files); OOM/network/unknown preserve everything.
|
|
268
|
+
const action = handleLoaderFailure({
|
|
269
|
+
role, error: e, lock, cacheDir, modelId: EMBEDDING_MODEL,
|
|
270
|
+
terminal: e instanceof TerminalConfigError,
|
|
271
|
+
});
|
|
272
|
+
if (action === 'quarantined')
|
|
273
|
+
console.error('🧹 cache-integrity failure (owner) — model cache quarantined');
|
|
274
|
+
else if (action === 'marker-invalidated')
|
|
275
|
+
console.error('… cache-integrity failure (reader) — marker dropped, next retry re-proves as locked owner');
|
|
276
|
+
else if (!(e instanceof TerminalConfigError))
|
|
277
|
+
console.error('… non-integrity load failure — model cache preserved');
|
|
278
|
+
throw e;
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
lock.release();
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Background provenance reconciliation (spec §6b). Runs in parallel with the
|
|
285
|
+
// model load; vector search and automatic backfill stay closed until it
|
|
286
|
+
// settles (eligibility barrier in the coordinator).
|
|
287
|
+
async startReconciliation() {
|
|
288
|
+
if (!this.coordinator)
|
|
289
|
+
return;
|
|
290
|
+
await this.coordinator.runReconciliation();
|
|
291
|
+
this.coordinator.sweepStart();
|
|
292
|
+
}
|
|
293
|
+
// sha256 with the entity text-builder version mixed in: a builder change
|
|
294
|
+
// re-backfills entities only, never chunks (spec §6c N2).
|
|
295
|
+
hashWithBuilderVersion(text) {
|
|
296
|
+
return createHash('sha256').update(`${TEXT_BUILDER_VERSION}\n${text}`).digest('hex');
|
|
297
|
+
}
|
|
298
|
+
// Rebuild the CURRENT embedding input hash for an entity. null = entity gone
|
|
299
|
+
// or malformed observations — reconciliation fail-closes to missing.
|
|
300
|
+
entityInputHash(entityId) {
|
|
301
|
+
try {
|
|
302
|
+
const entity = this.db.prepare(`SELECT name, entityType, observations FROM entities WHERE id = ?`)
|
|
303
|
+
.get(entityId);
|
|
304
|
+
if (!entity)
|
|
305
|
+
return null;
|
|
306
|
+
const built = this.buildEntityEmbeddingText({
|
|
307
|
+
name: entity.name,
|
|
308
|
+
entityType: entity.entityType,
|
|
309
|
+
observations: JSON.parse(entity.observations),
|
|
310
|
+
});
|
|
311
|
+
return this.hashWithBuilderVersion(built.text);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
return null;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
// Mutation-path embedding wrapper (spec §5): CRUD success never depends on
|
|
318
|
+
// model availability. On not-ready/disabled the stale vector is deleted in
|
|
319
|
+
// the same breath (dirty = missing, §6a-1) and the coordinator is kicked so
|
|
320
|
+
// the row is recovered without a restart (§5 kick column).
|
|
321
|
+
async tryEmbedEntity(entityId, priority = 'bulk') {
|
|
322
|
+
try {
|
|
323
|
+
const ok = await this.embedEntity(entityId, priority);
|
|
324
|
+
if (ok) {
|
|
325
|
+
// Success clears any stale backfill-failure record for this target.
|
|
326
|
+
this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'entity' AND target_id = ?`).run(entityId);
|
|
327
|
+
this.coordinator?.invalidateCoverage();
|
|
328
|
+
return 'embedded';
|
|
329
|
+
}
|
|
330
|
+
this.invalidateEntityVector(entityId);
|
|
331
|
+
this.coordinator?.kick();
|
|
332
|
+
return 'queued';
|
|
333
|
+
}
|
|
334
|
+
catch (e) {
|
|
335
|
+
// Any embedding-layer failure (not-ready, disabled, OR a ready-state
|
|
336
|
+
// inference error) must not fail the CRUD that already committed. The
|
|
337
|
+
// vector is invalidated (§6a-1) and recovery is owned by the backfill
|
|
338
|
+
// scanner with its attempts cap — never by rethrowing here (spec §5).
|
|
339
|
+
if (!(e instanceof GateNotReadyError) && !(e instanceof GateDisabledError)) {
|
|
340
|
+
console.error(`⚠️ embedding failed for ${entityId} (queued for backfill): ${e instanceof Error ? e.message : e}`);
|
|
341
|
+
}
|
|
342
|
+
this.invalidateEntityVector(entityId);
|
|
343
|
+
this.coordinator?.kick();
|
|
344
|
+
return e instanceof GateDisabledError ? 'disabled' : 'queued';
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
// §6a-1 (beta B2): the entity change and the stale-vector removal commit in
|
|
348
|
+
// ONE synchronous transaction, BEFORE any inference await. No window exists
|
|
349
|
+
// where another tool call can retrieve the pre-mutation vector, and a crash
|
|
350
|
+
// between mutation and re-embed leaves a clean missing state (backfill
|
|
351
|
+
// target), never a stale-searchable one.
|
|
352
|
+
// spec §4.5 단계 1: 관찰 변경 · projection 재합성 · entity vector 무효화 ·
|
|
353
|
+
// stale KG chunk 제거를 한 트랜잭션으로 묶는다. 하나만 되면 검색이 낡은
|
|
354
|
+
// 사실을 계속 반환한다.
|
|
355
|
+
// mutate 가 명시적으로 false 를 반환하면 "아무것도 바꾸지 않았다"는 뜻이고
|
|
356
|
+
// projection 재합성·벡터 무효화·KG 정리를 건너뛴다. 이 경로가 없으면
|
|
357
|
+
// 무변경 upsert 나 dedup-only add 가 **정상 벡터를 지우고 재임베딩도 안 해서**
|
|
358
|
+
// 검색 품질만 깎는다(advisor 구현리뷰 r1 발견 1, 실행 재현).
|
|
359
|
+
// 반환값 = 실제로 변경이 있었는가.
|
|
360
|
+
mutateEntityAndInvalidate(entityId, mutate) {
|
|
361
|
+
let changed = false;
|
|
362
|
+
const tx = this.db.transaction(() => {
|
|
363
|
+
changed = mutate() !== false;
|
|
364
|
+
if (!changed)
|
|
365
|
+
return;
|
|
366
|
+
rebuildProjection(this.db, entityId);
|
|
367
|
+
this.invalidateDerivedForEntity(entityId);
|
|
368
|
+
});
|
|
369
|
+
tx();
|
|
370
|
+
if (changed)
|
|
371
|
+
this.coordinator?.invalidateCoverage();
|
|
372
|
+
return changed;
|
|
373
|
+
}
|
|
374
|
+
// 관찰이 바뀐 entity 의 파생 상태를 무효화한다: entity vector + stale KG chunk.
|
|
375
|
+
// **트랜잭션을 열지 않는다** — 호출자가 이미 하나의 단위 안에 있다고 가정한다.
|
|
376
|
+
//
|
|
377
|
+
// importGraph 가 이 단계를 건너뛰고 있었다: projection 만 재합성하고 파생 상태를
|
|
378
|
+
// 그대로 둬서, 이미 존재하는 entity 를 import 로 덮으면 옛 벡터·옛 KG chunk 가
|
|
379
|
+
// 계속 검색에 나왔다(advisor beta 발견 2, hybridSearch 로 실측 재현).
|
|
380
|
+
// 그래서 "모든 관찰 변경은 mutateEntityAndInvalidate 를 통한다"는 규칙에
|
|
381
|
+
// 예외가 하나 있었고, 그 예외가 정확히 그 규칙이 막으려던 결함을 만들었다.
|
|
382
|
+
invalidateDerivedForEntity(entityId) {
|
|
383
|
+
const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
|
|
384
|
+
.get(entityId);
|
|
385
|
+
if (meta) {
|
|
386
|
+
this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
|
|
387
|
+
this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
|
|
388
|
+
}
|
|
389
|
+
deleteStaleKgChunks(this.db, entityId);
|
|
390
|
+
}
|
|
391
|
+
// §6a-1 invariant: when an entity's embedding input changed but re-embedding
|
|
392
|
+
// is unavailable, its old vector must not stay searchable.
|
|
393
|
+
invalidateEntityVector(entityId) {
|
|
394
|
+
if (!this.db)
|
|
395
|
+
return;
|
|
396
|
+
const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
|
|
397
|
+
.get(entityId);
|
|
398
|
+
if (!meta)
|
|
399
|
+
return;
|
|
400
|
+
const tx = this.db.transaction(() => {
|
|
401
|
+
this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
|
|
402
|
+
this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
|
|
403
|
+
});
|
|
404
|
+
tx();
|
|
405
|
+
this.coordinator?.invalidateCoverage();
|
|
406
|
+
}
|
|
407
|
+
// Backfill callback: re-embed one chunk and commit vector + provenance in a
|
|
408
|
+
// single transaction (§6a-2).
|
|
409
|
+
async reembedChunkByRowid(rowid) {
|
|
410
|
+
if (!this.db)
|
|
411
|
+
return false;
|
|
412
|
+
const row = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`)
|
|
413
|
+
.get(rowid);
|
|
414
|
+
if (!row || row.text === null)
|
|
415
|
+
return false;
|
|
416
|
+
try {
|
|
417
|
+
const embedding = await this.generateEmbedding(row.text, 1024, false, 'backfill');
|
|
418
|
+
const hash = createHash('sha256').update(row.text).digest('hex');
|
|
419
|
+
const safe = Number(rowid);
|
|
420
|
+
// Write-back CAS (beta 2R B1): the rowid may have been deleted and reused
|
|
421
|
+
// by a re-sync while inference ran — re-read the CURRENT text in the
|
|
422
|
+
// transaction and only write when it still matches what was embedded.
|
|
423
|
+
const tx = this.db.transaction(() => {
|
|
424
|
+
const cur = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`).get(rowid);
|
|
425
|
+
if (!cur || cur.text !== row.text)
|
|
426
|
+
return false; // superseded — discard
|
|
427
|
+
this.db.exec(`DELETE FROM chunks WHERE rowid = ${safe}`);
|
|
428
|
+
this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${safe}, ?)`).run(Buffer.from(embedding.buffer));
|
|
429
|
+
this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
430
|
+
.run(hash, this.currentProfileId, rowid);
|
|
431
|
+
this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
|
|
432
|
+
return true;
|
|
433
|
+
});
|
|
434
|
+
const written = tx();
|
|
435
|
+
this.coordinator?.invalidateCoverage();
|
|
436
|
+
return written;
|
|
437
|
+
}
|
|
438
|
+
catch (e) {
|
|
439
|
+
if (e instanceof GateNotReadyError || e instanceof GateDisabledError)
|
|
440
|
+
return false;
|
|
441
|
+
throw e;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
// spec §3 shutdown order (beta B1): block new batches -> settle coordinator
|
|
445
|
+
// (INCLUDING an in-flight reconciliation pass) -> settle gate (INCLUDING an
|
|
446
|
+
// in-flight model load, bounded) -> close DB -> natural exit.
|
|
447
|
+
//
|
|
448
|
+
// Bounded-exit rationale, re-derived after beta 1R: the exit decision is made
|
|
449
|
+
// AFTER the settle wait, not before — if the load completed during settling
|
|
450
|
+
// (ONNX session now exists) we take the natural-exit path. Only when the load
|
|
451
|
+
// is STILL pending after the deadline (dominant case: the 1.2GB download,
|
|
452
|
+
// which is un-abortable through transformers.js and would hold the event
|
|
453
|
+
// loop indefinitely) do we exit(). At that point the DB is already closed
|
|
454
|
+
// cleanly, so even the residual worst case — the load being inside ONNX
|
|
455
|
+
// session construction at exit — risks an ugly abort message, never data
|
|
456
|
+
// loss. Hanging forever is the alternative and is worse.
|
|
457
|
+
async shutdownAll() {
|
|
458
|
+
console.error('\n🧹 Cleaning up...');
|
|
459
|
+
try {
|
|
460
|
+
await this.coordinator?.shutdown(5000);
|
|
461
|
+
}
|
|
462
|
+
catch { /* settle best-effort */ }
|
|
463
|
+
try {
|
|
464
|
+
await this.gate?.shutdown(5000);
|
|
465
|
+
}
|
|
466
|
+
catch { /* settle best-effort */ }
|
|
467
|
+
const loadStillPending = this.gate?.loadInFlight ?? false;
|
|
468
|
+
try {
|
|
469
|
+
this.cleanup();
|
|
470
|
+
}
|
|
471
|
+
catch { /* DB close */ }
|
|
472
|
+
process.exitCode = process.exitCode ?? 0;
|
|
473
|
+
if (loadStillPending) {
|
|
474
|
+
console.error('… model load/download still in flight after settle deadline — bounded exit (DB already closed)');
|
|
475
|
+
process.exit(process.exitCode);
|
|
112
476
|
}
|
|
113
477
|
}
|
|
114
478
|
async runMigrations() {
|
|
@@ -123,6 +487,11 @@ export class RAGKnowledgeGraphManager {
|
|
|
123
487
|
});
|
|
124
488
|
// Get pending migrations before running them
|
|
125
489
|
const pendingBefore = migrationManager.getPendingMigrations();
|
|
490
|
+
// spec §5.1: 대기 중 마이그레이션이 있으면 먼저 일관 스냅샷을 남긴다.
|
|
491
|
+
// 실패는 throw = fail-closed (백업 없이 스키마를 바꾸지 않는다).
|
|
492
|
+
// await: 백업은 Online Backup API 를 쓰므로 비동기다. 여기서 await 를 빠뜨리면
|
|
493
|
+
// 백업이 끝나기 전에 마이그레이션이 시작한다 = 백업 없이 스키마를 바꾸는 것이다.
|
|
494
|
+
await backupBeforeMigration(this.db, DB_FILE_PATH, pendingBefore.map(m => m.version), migrationManager.getCurrentVersion());
|
|
126
495
|
// Run pending migrations
|
|
127
496
|
const result = await migrationManager.runMigrations();
|
|
128
497
|
console.error(`🔧 Database schema ready (version ${result.currentVersion}, ${result.applied} migrations applied)`);
|
|
@@ -140,11 +509,6 @@ export class RAGKnowledgeGraphManager {
|
|
|
140
509
|
this.encoding.free();
|
|
141
510
|
this.encoding = null;
|
|
142
511
|
}
|
|
143
|
-
if (this.embeddingModel) {
|
|
144
|
-
// Clean up the embedding model if it has cleanup methods
|
|
145
|
-
this.embeddingModel = null;
|
|
146
|
-
this.modelInitialized = false;
|
|
147
|
-
}
|
|
148
512
|
this.embeddingCache.clear();
|
|
149
513
|
if (this.db) {
|
|
150
514
|
this.db.close();
|
|
@@ -163,42 +527,73 @@ export class RAGKnowledgeGraphManager {
|
|
|
163
527
|
if (!this.db)
|
|
164
528
|
throw new Error('Database not initialized');
|
|
165
529
|
const result = [];
|
|
530
|
+
// v13: 관찰은 lifecycle 테이블이 정본이고 entities.observations 는 projection 이다.
|
|
531
|
+
// entity 행은 빈 배열로 만들고 rebuildProjection 이 채운다.
|
|
166
532
|
const insertStmt = this.db.prepare(`
|
|
167
533
|
INSERT OR IGNORE INTO entities (id, name, entityType, observations, metadata)
|
|
168
|
-
VALUES (?, ?, ?,
|
|
534
|
+
VALUES (?, ?, ?, '[]', ?)
|
|
169
535
|
`);
|
|
536
|
+
const stripDate = (s2) => s2.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
|
|
170
537
|
for (const entity of entities) {
|
|
171
538
|
const entityId = `entity_${entity.name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
const currentBare = new Set(currentObs.map(stripDate));
|
|
190
|
-
const newObs = timestamped.filter(o => !currentBare.has(stripDate(o)));
|
|
191
|
-
const needsTypeUpdate = entity.entityType && entity.entityType !== 'CONCEPT' && entity.entityType !== existing.entityType;
|
|
192
|
-
if (newObs.length > 0 || needsTypeUpdate) {
|
|
193
|
-
const mergedObs = [...currentObs, ...newObs];
|
|
194
|
-
const updatedType = needsTypeUpdate ? entity.entityType : existing.entityType;
|
|
195
|
-
this.db.prepare(`UPDATE entities SET observations = ?, entityType = ? WHERE id = ?`)
|
|
196
|
-
.run(JSON.stringify(mergedObs), updatedType, entityId);
|
|
197
|
-
console.error(`♻️ Upserted entity: ${entity.name} (+${newObs.length} obs${needsTypeUpdate ? ', type→' + updatedType : ''})`);
|
|
198
|
-
await this.embedEntity(entityId);
|
|
199
|
-
result.push({ ...entity, observations: mergedObs });
|
|
539
|
+
const ts = new Date().toISOString();
|
|
540
|
+
const ids = [];
|
|
541
|
+
let created = false;
|
|
542
|
+
let addedCount = 0;
|
|
543
|
+
let typeUpdated = false;
|
|
544
|
+
// entity INSERT 도 같은 트랜잭션 안이다. 밖에 두면 lifecycle INSERT 가
|
|
545
|
+
// 실패할 때 entity 행만 남는 split state 가 생긴다
|
|
546
|
+
// (advisor 구현리뷰 r1 발견 2, 실행 재현).
|
|
547
|
+
const changed = this.mutateEntityAndInvalidate(entityId, () => {
|
|
548
|
+
created = insertStmt.run(entityId, entity.name, entity.entityType, '{}').changes > 0;
|
|
549
|
+
if (!created && entity.entityType && entity.entityType !== 'CONCEPT') {
|
|
550
|
+
const cur = this.db.prepare(`SELECT entityType FROM entities WHERE id = ?`)
|
|
551
|
+
.get(entityId);
|
|
552
|
+
if (cur && cur.entityType !== entity.entityType) {
|
|
553
|
+
this.db.prepare(`UPDATE entities SET entityType = ? WHERE id = ?`)
|
|
554
|
+
.run(entity.entityType, entityId);
|
|
555
|
+
typeUpdated = true;
|
|
200
556
|
}
|
|
201
557
|
}
|
|
558
|
+
const activeRows = this.db.prepare(`SELECT observation_id, content FROM entity_observations
|
|
559
|
+
WHERE entity_id = ? AND status = 'active'`).all(entityId);
|
|
560
|
+
const activeByBare = new Map(activeRows.map(r => [stripDate(r.content), r.observation_id]));
|
|
561
|
+
let sourcesAdded = 0;
|
|
562
|
+
for (const raw of (entity.observations || [])) {
|
|
563
|
+
const content = this._timestampObservation(raw);
|
|
564
|
+
const bare = stripDate(content);
|
|
565
|
+
const dupId = activeByBare.get(bare);
|
|
566
|
+
if (dupId) {
|
|
567
|
+
// 같은 사실이 다른 출처에서 다시 왔다 = evidence 추가, 새 revision 아님.
|
|
568
|
+
if (entity.sources?.length)
|
|
569
|
+
sourcesAdded += linkSources(this.db, dupId, entity.sources, ts);
|
|
570
|
+
ids.push(null);
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
const id = addRevision(this.db, {
|
|
574
|
+
entityId, content, status: entity.status ?? 'active', sources: entity.sources, ts
|
|
575
|
+
});
|
|
576
|
+
activeByBare.set(bare, id);
|
|
577
|
+
ids.push(id);
|
|
578
|
+
addedCount++;
|
|
579
|
+
}
|
|
580
|
+
// 아무것도 안 바뀌었으면 projection·벡터·KG 를 건드리지 않는다.
|
|
581
|
+
return created || typeUpdated || addedCount > 0 || sourcesAdded > 0;
|
|
582
|
+
});
|
|
583
|
+
const projected = JSON.parse(this.db.prepare(`SELECT observations FROM entities WHERE id = ?`)
|
|
584
|
+
.get(entityId).observations);
|
|
585
|
+
// 재임베딩은 무효화가 실제로 일어났을 때만. 조건이 갈리면
|
|
586
|
+
// "벡터를 지우고 다시 만들지 않는" 창이 생긴다.
|
|
587
|
+
if (changed) {
|
|
588
|
+
console.error(created
|
|
589
|
+
? `🔮 Generating embedding for new entity: ${entity.name}`
|
|
590
|
+
: `♻️ Upserted entity: ${entity.name} (+${addedCount} obs${typeUpdated ? ', type→' + entity.entityType : ''})`);
|
|
591
|
+
const embedding_status = await this.tryEmbedEntity(entityId, 'bulk');
|
|
592
|
+
result.push({ ...entity, observations: projected, created,
|
|
593
|
+
observation_ids: ids, embedding_status });
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
result.push({ ...entity, observations: projected, created, observation_ids: ids });
|
|
202
597
|
}
|
|
203
598
|
}
|
|
204
599
|
return result;
|
|
@@ -208,24 +603,32 @@ export class RAGKnowledgeGraphManager {
|
|
|
208
603
|
throw new Error('Database not initialized');
|
|
209
604
|
const newRelations = [];
|
|
210
605
|
for (const relation of relations) {
|
|
211
|
-
// Ensure entities exist
|
|
212
|
-
|
|
606
|
+
// Ensure entities exist. v3.6 (spec §5c): auto-created endpoints may be
|
|
607
|
+
// embedded/queued/disabled independently — report per endpoint; 'n/a'
|
|
608
|
+
// means the endpoint already existed (no embedding work happened here).
|
|
609
|
+
const ensured = await this.createEntities([
|
|
213
610
|
{ name: relation.from, entityType: 'CONCEPT', observations: [] },
|
|
214
611
|
{ name: relation.to, entityType: 'CONCEPT', observations: [] }
|
|
215
612
|
]);
|
|
613
|
+
const statusOf = (name) => {
|
|
614
|
+
const hit = ensured.find(e => e.name === name);
|
|
615
|
+
return hit?.embedding_status ?? 'n/a';
|
|
616
|
+
};
|
|
617
|
+
const endpoint_embedding_status = { from: statusOf(relation.from), to: statusOf(relation.to) };
|
|
216
618
|
const sourceId = `entity_${relation.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
217
619
|
const targetId = `entity_${relation.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
218
620
|
const relationId = `rel_${sourceId}_${relation.relationType}_${targetId}`.toLowerCase();
|
|
219
621
|
const stmt = this.db.prepare(`
|
|
220
|
-
INSERT OR IGNORE INTO relationships
|
|
622
|
+
INSERT OR IGNORE INTO relationships
|
|
221
623
|
(id, source_entity, target_entity, relationType, confidence, metadata)
|
|
222
624
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
223
625
|
`);
|
|
224
626
|
const result = stmt.run(relationId, sourceId, targetId, relation.relationType, 1.0, '{}');
|
|
225
627
|
if (result.changes > 0) {
|
|
226
|
-
newRelations.push(relation);
|
|
628
|
+
newRelations.push({ ...relation, endpoint_embedding_status });
|
|
227
629
|
}
|
|
228
630
|
}
|
|
631
|
+
this.coordinator?.kick();
|
|
229
632
|
return newRelations;
|
|
230
633
|
}
|
|
231
634
|
async addObservations(observations) {
|
|
@@ -234,31 +637,144 @@ export class RAGKnowledgeGraphManager {
|
|
|
234
637
|
const results = [];
|
|
235
638
|
for (const obs of observations) {
|
|
236
639
|
const entityId = `entity_${obs.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
SELECT observations FROM entities WHERE id = ?
|
|
240
|
-
`).get(entityId);
|
|
241
|
-
if (!entity) {
|
|
640
|
+
const entity = this.db.prepare(`SELECT id FROM entities WHERE id = ?`).get(entityId);
|
|
641
|
+
if (!entity)
|
|
242
642
|
throw new Error(`Entity with name ${obs.entityName} not found`);
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
const
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
643
|
+
// dedup 기준은 v3.6 과 같다: 날짜 prefix 를 뗀 본문이 active 에 이미 있으면
|
|
644
|
+
// 새 revision 을 만들지 않는다. 다만 v13 에서는 같은 사실이 다른 출처에서 다시
|
|
645
|
+
// 온 것이므로 그 revision 에 source link 를 더한다(spec §8.3 T13).
|
|
646
|
+
const stripDate = (s2) => s2.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
|
|
647
|
+
const activeRows = this.db.prepare(`SELECT observation_id, content FROM entity_observations
|
|
648
|
+
WHERE entity_id = ? AND status = 'active'`).all(entityId);
|
|
649
|
+
const activeByBare = new Map(activeRows.map(r => [stripDate(r.content), r.observation_id]));
|
|
650
|
+
const ts = new Date().toISOString();
|
|
651
|
+
const ids = [];
|
|
652
|
+
const added = [];
|
|
653
|
+
let sourcesAdded = 0;
|
|
654
|
+
const changed = this.mutateEntityAndInvalidate(entityId, () => {
|
|
655
|
+
for (const raw of obs.contents) {
|
|
656
|
+
const content = this._timestampObservation(raw);
|
|
657
|
+
const bare = stripDate(content);
|
|
658
|
+
const dupId = activeByBare.get(bare);
|
|
659
|
+
if (dupId) {
|
|
660
|
+
if (obs.sources?.length)
|
|
661
|
+
sourcesAdded += linkSources(this.db, dupId, obs.sources, ts);
|
|
662
|
+
ids.push(null);
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
const id = addRevision(this.db, {
|
|
666
|
+
entityId, content, status: obs.status ?? 'active', sources: obs.sources, ts
|
|
667
|
+
});
|
|
668
|
+
activeByBare.set(bare, id);
|
|
669
|
+
ids.push(id);
|
|
670
|
+
added.push(content);
|
|
671
|
+
}
|
|
672
|
+
// 아무것도 안 바뀌었으면 projection·벡터·KG 를 건드리지 않는다.
|
|
673
|
+
// 이 반환이 없으면 빈 contents 나 dedup-only add 가 정상 벡터를
|
|
674
|
+
// 지우고 재임베딩도 안 한다(advisor 구현리뷰 r1 발견 1).
|
|
675
|
+
return added.length > 0 || sourcesAdded > 0;
|
|
676
|
+
});
|
|
677
|
+
let embedding_status;
|
|
678
|
+
if (changed) {
|
|
255
679
|
console.error(`🔮 Regenerating embedding for updated entity: ${obs.entityName}`);
|
|
256
|
-
await this.
|
|
680
|
+
embedding_status = await this.tryEmbedEntity(entityId, 'bulk');
|
|
257
681
|
}
|
|
258
|
-
results.push({ entityName: obs.entityName,
|
|
682
|
+
results.push({ entityName: obs.entityName, observation_ids: ids,
|
|
683
|
+
addedObservations: added, embedding_status });
|
|
259
684
|
}
|
|
260
685
|
return results;
|
|
261
686
|
}
|
|
687
|
+
async correctObservation(observationId, content, changeKind = 'correction', reason) {
|
|
688
|
+
if (!this.db)
|
|
689
|
+
throw new Error('Database not initialized');
|
|
690
|
+
const row = this.db.prepare(`SELECT entity_id FROM entity_observations WHERE observation_id = ?`)
|
|
691
|
+
.get(observationId);
|
|
692
|
+
if (!row)
|
|
693
|
+
throw new Error(`observation ${observationId} not found`);
|
|
694
|
+
let newId = '';
|
|
695
|
+
const ts = new Date().toISOString();
|
|
696
|
+
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
697
|
+
newId = correctRevision(this.db, {
|
|
698
|
+
observationId, content: this._timestampObservation(content),
|
|
699
|
+
changeKind, reason: reason ?? null, ts
|
|
700
|
+
});
|
|
701
|
+
});
|
|
702
|
+
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
703
|
+
return newId;
|
|
704
|
+
}
|
|
705
|
+
async _transition(observationId, event, reason) {
|
|
706
|
+
if (!this.db)
|
|
707
|
+
throw new Error('Database not initialized');
|
|
708
|
+
const row = this.db.prepare(`SELECT entity_id FROM entity_observations WHERE observation_id = ?`)
|
|
709
|
+
.get(observationId);
|
|
710
|
+
if (!row)
|
|
711
|
+
throw new Error(`observation ${observationId} not found`);
|
|
712
|
+
const ts = new Date().toISOString();
|
|
713
|
+
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
714
|
+
transitionStatus(this.db, { observationId, event, reason: reason ?? null, ts });
|
|
715
|
+
});
|
|
716
|
+
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
717
|
+
}
|
|
718
|
+
async retractObservation(observationId, reason) {
|
|
719
|
+
return this._transition(observationId, 'retract', reason);
|
|
720
|
+
}
|
|
721
|
+
async restoreObservation(observationId, reason) {
|
|
722
|
+
return this._transition(observationId, 'restore', reason);
|
|
723
|
+
}
|
|
724
|
+
async approveObservation(observationId, reason) {
|
|
725
|
+
return this._transition(observationId, 'approve', reason);
|
|
726
|
+
}
|
|
727
|
+
async declineObservation(observationId, reason) {
|
|
728
|
+
return this._transition(observationId, 'decline', reason);
|
|
729
|
+
}
|
|
730
|
+
// DESTRUCTIVE. Physically removes revisions (and their sources via CASCADE).
|
|
731
|
+
//
|
|
732
|
+
// Chain contract (advisor 구현리뷰 r1 발견 4): a revision chain is
|
|
733
|
+
// rev1 <- rev2 <- ... and purging a middle revision would either fail on the
|
|
734
|
+
// supersedes_id FK or leave a chain pointing at a deleted row, plus events
|
|
735
|
+
// whose from_id/to_id dangle. So purge is defined as **suffix purge from the
|
|
736
|
+
// target to the newest revision of that root**, newest-first:
|
|
737
|
+
// - purging the newest revision removes exactly it
|
|
738
|
+
// - purging rev2 of a 3-revision chain removes rev3 then rev2
|
|
739
|
+
// - purging rev1 removes the whole chain
|
|
740
|
+
// Events for purged revisions are removed too, so no event dangles.
|
|
741
|
+
// The root row is always kept: its projection_order stays reserved, because
|
|
742
|
+
// reusing an order would make a later restore/approve fail on the
|
|
743
|
+
// active-order index.
|
|
744
|
+
async purgeObservation(observationId, confirm) {
|
|
745
|
+
if (!this.db)
|
|
746
|
+
throw new Error('Database not initialized');
|
|
747
|
+
if (confirm !== 'PURGE') {
|
|
748
|
+
throw new Error(`purgeObservation refused: pass confirm='PURGE' to physically delete a revision. ` +
|
|
749
|
+
`This destroys history — retractObservation() is almost always what you want.`);
|
|
750
|
+
}
|
|
751
|
+
const row = this.db.prepare(`SELECT entity_id, root_id, revision_no FROM entity_observations WHERE observation_id = ?`)
|
|
752
|
+
.get(observationId);
|
|
753
|
+
if (!row)
|
|
754
|
+
return { purged: 0 };
|
|
755
|
+
let purged = 0;
|
|
756
|
+
this.mutateEntityAndInvalidate(row.entity_id, () => {
|
|
757
|
+
// newest-first so each DELETE has no successor referencing it
|
|
758
|
+
const victims = this.db.prepare(`SELECT observation_id FROM entity_observations
|
|
759
|
+
WHERE root_id = ? AND revision_no >= ?
|
|
760
|
+
ORDER BY revision_no DESC`).all(row.root_id, row.revision_no);
|
|
761
|
+
for (const v of victims) {
|
|
762
|
+
this.db.prepare(`DELETE FROM observation_events WHERE from_id = ? OR to_id = ?`)
|
|
763
|
+
.run(v.observation_id, v.observation_id);
|
|
764
|
+
purged += this.db.prepare(`DELETE FROM entity_observations WHERE observation_id = ?`)
|
|
765
|
+
.run(v.observation_id).changes;
|
|
766
|
+
}
|
|
767
|
+
return purged > 0;
|
|
768
|
+
});
|
|
769
|
+
await this.tryEmbedEntity(row.entity_id, 'bulk');
|
|
770
|
+
return { purged };
|
|
771
|
+
}
|
|
772
|
+
// spec §6.2: 과거 판본은 여기서만 나온다. 일반 검색은 active 만 반환한다.
|
|
773
|
+
async getObservationHistory(sel) {
|
|
774
|
+
if (!this.db)
|
|
775
|
+
throw new Error('Database not initialized');
|
|
776
|
+
return getObservationHistory(this.db, sel);
|
|
777
|
+
}
|
|
262
778
|
async deleteEntities(entityNames) {
|
|
263
779
|
if (!this.db)
|
|
264
780
|
throw new Error('Database not initialized');
|
|
@@ -322,22 +838,91 @@ export class RAGKnowledgeGraphManager {
|
|
|
322
838
|
}
|
|
323
839
|
console.error(`✅ Entity deletion process completed`);
|
|
324
840
|
}
|
|
841
|
+
// v3.6 (spec §5c, breaking): structured per-entity results + re-embedding.
|
|
842
|
+
// Pre-3.6 this method silently left STALE entity vectors behind (the input
|
|
843
|
+
// text changed but the vector was never regenerated) — fixed via
|
|
844
|
+
// tryEmbedEntity, which also covers the not-ready dirty contract.
|
|
845
|
+
// DEPRECATED shim (v13, one version only). Content-addressed deletion cannot
|
|
846
|
+
// express "which revision" — use retractObservation(observation_id) instead.
|
|
847
|
+
// Semantics: exact-string match against ACTIVE revisions -> soft retract.
|
|
848
|
+
//
|
|
849
|
+
// The whole call is one transaction and ambiguity is judged before any
|
|
850
|
+
// mutation: if any item matches 2+ active revisions the call aborts with 0
|
|
851
|
+
// mutations (spec §6.3). That is a deliberate change from v3.6, which deleted
|
|
852
|
+
// every duplicate and carried on — a machine cannot pick which revision was meant.
|
|
853
|
+
//
|
|
854
|
+
// Duplicate ids across items are collapsed. Without that, listing the same
|
|
855
|
+
// (entity, content) twice retracted it once and then failed on an illegal
|
|
856
|
+
// transition, returning an error *after* committing part of the batch
|
|
857
|
+
// (advisor 구현리뷰 r1 발견 3, 실행 재현). Embedding runs after the commit.
|
|
325
858
|
async deleteObservations(deletions) {
|
|
326
859
|
if (!this.db)
|
|
327
860
|
throw new Error('Database not initialized');
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
const
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
861
|
+
const plan = [];
|
|
862
|
+
const ambiguous = [];
|
|
863
|
+
const claimed = new Set(); // 항목 간 중복 id 흡수
|
|
864
|
+
for (const d of deletions) {
|
|
865
|
+
const entityId = `entity_${d.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
866
|
+
const ids = [];
|
|
867
|
+
for (const content of d.observations) {
|
|
868
|
+
const rows = this.db.prepare(`SELECT observation_id FROM entity_observations
|
|
869
|
+
WHERE entity_id = ? AND status = 'active' AND content = ?`).all(entityId, content);
|
|
870
|
+
if (rows.length > 1) {
|
|
871
|
+
ambiguous.push({ entityName: d.entityName, content, matches: rows.length });
|
|
872
|
+
continue;
|
|
873
|
+
}
|
|
874
|
+
if (rows.length === 1 && !claimed.has(rows[0].observation_id)) {
|
|
875
|
+
claimed.add(rows[0].observation_id);
|
|
876
|
+
ids.push(rows[0].observation_id);
|
|
877
|
+
}
|
|
878
|
+
// rows.length === 0 -> no-op (v3.6 behaviour, spec §6.3-3)
|
|
879
|
+
}
|
|
880
|
+
plan.push({ entityName: d.entityName, entityId, ids });
|
|
881
|
+
}
|
|
882
|
+
if (ambiguous.length > 0) {
|
|
883
|
+
throw new Error(`AMBIGUOUS_OBSERVATION_MATCH: ${ambiguous.length} item(s) matched multiple active ` +
|
|
884
|
+
`revisions; 0 mutations were applied. Use retractObservation(observation_id) instead. ` +
|
|
885
|
+
`Conflicts: ${JSON.stringify(ambiguous)}`);
|
|
886
|
+
}
|
|
887
|
+
// pass 2 — mutate everything in ONE transaction so a failure anywhere
|
|
888
|
+
// leaves zero mutations. Per-plan transactions plus an awaited embedding
|
|
889
|
+
// in between made a partial commit observable.
|
|
890
|
+
const touched = plan.filter(p => p.ids.length > 0);
|
|
891
|
+
const ts = new Date().toISOString();
|
|
892
|
+
if (touched.length > 0) {
|
|
893
|
+
const tx = this.db.transaction(() => {
|
|
894
|
+
for (const p of touched) {
|
|
895
|
+
for (const id of p.ids) {
|
|
896
|
+
transitionStatus(this.db, { observationId: id, event: 'retract',
|
|
897
|
+
reason: 'deleteObservations (deprecated shim)', ts });
|
|
898
|
+
}
|
|
899
|
+
rebuildProjection(this.db, p.entityId);
|
|
900
|
+
const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
|
|
901
|
+
.get(p.entityId);
|
|
902
|
+
if (meta) {
|
|
903
|
+
this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
|
|
904
|
+
this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`)
|
|
905
|
+
.run(p.entityId);
|
|
906
|
+
}
|
|
907
|
+
deleteStaleKgChunks(this.db, p.entityId);
|
|
908
|
+
}
|
|
909
|
+
});
|
|
910
|
+
tx();
|
|
911
|
+
this.coordinator?.invalidateCoverage();
|
|
912
|
+
}
|
|
913
|
+
// pass 3 — embedding after the commit
|
|
914
|
+
const results = [];
|
|
915
|
+
let total = 0;
|
|
916
|
+
for (const p of plan) {
|
|
917
|
+
if (p.ids.length === 0) {
|
|
918
|
+
results.push({ entityName: p.entityName, deleted: 0, embedding_status: 'n/a' });
|
|
919
|
+
continue;
|
|
339
920
|
}
|
|
921
|
+
const embedding_status = await this.tryEmbedEntity(p.entityId, 'bulk');
|
|
922
|
+
results.push({ entityName: p.entityName, deleted: p.ids.length, embedding_status });
|
|
923
|
+
total += p.ids.length;
|
|
340
924
|
}
|
|
925
|
+
return { results, total_deleted: total };
|
|
341
926
|
}
|
|
342
927
|
async deleteRelations(relations) {
|
|
343
928
|
if (!this.db)
|
|
@@ -524,10 +1109,24 @@ export class RAGKnowledgeGraphManager {
|
|
|
524
1109
|
console.error(`✅ getNeighbors: Found ${entities.length} entities, ${relations.length} relations, ${paths.length} paths (depth=${effectiveDepth})`);
|
|
525
1110
|
return { entities, relations, paths };
|
|
526
1111
|
}
|
|
1112
|
+
// v3.6 (spec §5·§5c, additive): FTS lexical fallback when vector search is
|
|
1113
|
+
// not eligible, hybrid-partial merge while backfill is catching up, and
|
|
1114
|
+
// top-level state fields on every response.
|
|
527
1115
|
async searchNodes(query, limit = 10, since, until) {
|
|
528
1116
|
if (!this.db)
|
|
529
1117
|
throw new Error('Database not initialized');
|
|
530
1118
|
console.error(`🔍 Semantic entity search: "${query}"`);
|
|
1119
|
+
const covS = this.coordinator?.coverage();
|
|
1120
|
+
const entityPct = covS && covS.entity.total > 0 ? Math.round((covS.entity.embedded / covS.entity.total) * 100) : 100;
|
|
1121
|
+
const stateFields = () => ({
|
|
1122
|
+
model_state: this.gate.status.state,
|
|
1123
|
+
coverage: { entity_pct: entityPct },
|
|
1124
|
+
});
|
|
1125
|
+
if (!(this.coordinator?.eligible ?? false)) {
|
|
1126
|
+
// No waiting on the model (spec §5) — lexical entities_fts fallback.
|
|
1127
|
+
return { ...this.searchNodesFts(query, limit, since, until), search_mode: 'fts-only',
|
|
1128
|
+
...stateFields(), degradation_reason: this.degradationReason() };
|
|
1129
|
+
}
|
|
531
1130
|
const queryVariants = this.buildCrossLingualVariants(query);
|
|
532
1131
|
if (queryVariants.length > 1) {
|
|
533
1132
|
console.error(`🌐 searchNodes variants: ${queryVariants.slice(1).join(' | ')}`);
|
|
@@ -551,16 +1150,26 @@ export class RAGKnowledgeGraphManager {
|
|
|
551
1150
|
`).all(Buffer.from(embedding.buffer), k);
|
|
552
1151
|
};
|
|
553
1152
|
const resultMap = new Map();
|
|
554
|
-
|
|
555
|
-
const
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
const
|
|
559
|
-
|
|
560
|
-
|
|
1153
|
+
try {
|
|
1154
|
+
for (const variant of queryVariants) {
|
|
1155
|
+
const embedding = await this.generateEmbedding(variant, 1024, true);
|
|
1156
|
+
const variantResults = searchEntities(embedding, limit * 2);
|
|
1157
|
+
for (const result of variantResults) {
|
|
1158
|
+
const existing = resultMap.get(result.entity_id);
|
|
1159
|
+
if (!existing || result.distance < existing.distance) {
|
|
1160
|
+
resultMap.set(result.entity_id, result);
|
|
1161
|
+
}
|
|
561
1162
|
}
|
|
562
1163
|
}
|
|
563
1164
|
}
|
|
1165
|
+
catch (embErr) {
|
|
1166
|
+
// Ready-state inference failure degrades to FTS instead of failing the
|
|
1167
|
+
// tool (beta B6) — same contract as hybridSearch. The gate's own
|
|
1168
|
+
// consecutive-failure counter handles the systemic transition.
|
|
1169
|
+
console.error(`⚠️ searchNodes vector path failed — FTS fallback:`, embErr instanceof Error ? embErr.message : embErr);
|
|
1170
|
+
return { ...this.searchNodesFts(query, limit, since, until), search_mode: 'fts-only',
|
|
1171
|
+
...stateFields(), degradation_reason: this.degradationReason() ?? 'inference_error' };
|
|
1172
|
+
}
|
|
564
1173
|
const entityResults = Array.from(resultMap.values()).sort((a, b) => a.distance - b.distance).slice(0, limit);
|
|
565
1174
|
// Filter by temporal range if specified
|
|
566
1175
|
let filteredResults = entityResults;
|
|
@@ -576,35 +1185,79 @@ export class RAGKnowledgeGraphManager {
|
|
|
576
1185
|
return true;
|
|
577
1186
|
});
|
|
578
1187
|
}
|
|
579
|
-
if (filteredResults.length === 0) {
|
|
580
|
-
console.error(`ℹ️ No semantic matches found for "${query}"`);
|
|
581
|
-
return { entities: [], relations: [] };
|
|
582
|
-
}
|
|
583
1188
|
const entities = filteredResults.map(result => ({
|
|
584
1189
|
name: result.name,
|
|
585
1190
|
entityType: result.entityType,
|
|
586
1191
|
observations: JSON.parse(result.observations),
|
|
587
1192
|
similarity: Math.max(0, 1 - result.distance / 2) // Convert cosine distance (0-2) to similarity (1-0)
|
|
588
1193
|
}));
|
|
589
|
-
//
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
1194
|
+
// hybrid-partial (spec §4): entities without vectors must not vanish from
|
|
1195
|
+
// search while backfill catches up — merge lexical FTS hits for the gap.
|
|
1196
|
+
let search_mode = 'hybrid';
|
|
1197
|
+
if (entityPct < 100) {
|
|
1198
|
+
search_mode = 'hybrid-partial';
|
|
1199
|
+
const seen = new Set(entities.map(e => e.name));
|
|
1200
|
+
const ftsExtra = this.searchNodesFts(query, limit, since, until);
|
|
1201
|
+
for (const e of ftsExtra.entities) {
|
|
1202
|
+
if (entities.length >= limit)
|
|
1203
|
+
break;
|
|
1204
|
+
if (!seen.has(e.name)) {
|
|
1205
|
+
seen.add(e.name);
|
|
1206
|
+
entities.push(e);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
if (entities.length === 0) {
|
|
1211
|
+
console.error(`ℹ️ No semantic matches found for "${query}"`);
|
|
1212
|
+
return { entities: [], relations: [], search_mode, ...stateFields() };
|
|
1213
|
+
}
|
|
1214
|
+
const relations = this.relationsAmong(entities.map(e => e.name));
|
|
1215
|
+
console.error(`✅ Found ${entities.length} semantically similar entities with ${relations.length} relationships`);
|
|
1216
|
+
return { entities, relations, search_mode, ...stateFields() };
|
|
1217
|
+
}
|
|
1218
|
+
// Lexical entity search over entities_fts (spec §5 contract: name /
|
|
1219
|
+
// observations / entityType lexical match — no semantic-equivalence claim).
|
|
1220
|
+
// Temporal filters apply in SQL so LIMIT is not distorted.
|
|
1221
|
+
searchNodesFts(query, limit, since, until) {
|
|
1222
|
+
const expr = compileFtsLiteralQuery(query);
|
|
1223
|
+
if (expr === null) {
|
|
1224
|
+
return { entities: [], relations: [], warning: 'query has no searchable terms' };
|
|
1225
|
+
}
|
|
1226
|
+
const rows = this.db.prepare(`
|
|
1227
|
+
SELECT e.name, e.entityType, e.observations
|
|
1228
|
+
FROM entities_fts f
|
|
1229
|
+
JOIN entities e ON f.rowid = e.rowid
|
|
1230
|
+
WHERE entities_fts MATCH @expr
|
|
1231
|
+
${since ? 'AND e.created_at >= @since' : ''}
|
|
1232
|
+
${until ? 'AND e.created_at <= @until' : ''}
|
|
1233
|
+
ORDER BY bm25(entities_fts)
|
|
1234
|
+
LIMIT @limit
|
|
1235
|
+
`).all({ expr, since, until, limit });
|
|
1236
|
+
const entities = rows.map(r => ({
|
|
1237
|
+
name: r.name,
|
|
1238
|
+
entityType: r.entityType,
|
|
1239
|
+
observations: JSON.parse(r.observations),
|
|
1240
|
+
}));
|
|
1241
|
+
return { entities, relations: this.relationsAmong(entities.map(e => e.name)) };
|
|
1242
|
+
}
|
|
1243
|
+
relationsAmong(entityNames) {
|
|
1244
|
+
if (entityNames.length === 0)
|
|
1245
|
+
return [];
|
|
1246
|
+
return this.db.prepare(`
|
|
1247
|
+
SELECT
|
|
593
1248
|
e1.name as from_name,
|
|
594
1249
|
e2.name as to_name,
|
|
595
1250
|
r.relationType
|
|
596
1251
|
FROM relationships r
|
|
597
1252
|
JOIN entities e1 ON r.source_entity = e1.id
|
|
598
1253
|
JOIN entities e2 ON r.target_entity = e2.id
|
|
599
|
-
WHERE e1.name IN (${entityNames.map(() => '?').join(',')})
|
|
1254
|
+
WHERE e1.name IN (${entityNames.map(() => '?').join(',')})
|
|
600
1255
|
AND e2.name IN (${entityNames.map(() => '?').join(',')})
|
|
601
1256
|
`).all(...entityNames, ...entityNames).map((row) => ({
|
|
602
1257
|
from: row.from_name,
|
|
603
1258
|
to: row.to_name,
|
|
604
1259
|
relationType: row.relationType
|
|
605
1260
|
}));
|
|
606
|
-
console.error(`✅ Found ${entities.length} semantically similar entities with ${relations.length} relationships`);
|
|
607
|
-
return { entities, relations };
|
|
608
1261
|
}
|
|
609
1262
|
async openNodes(names) {
|
|
610
1263
|
if (!this.db)
|
|
@@ -800,7 +1453,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
800
1453
|
};
|
|
801
1454
|
}
|
|
802
1455
|
// Generate and store embedding for a single entity
|
|
803
|
-
async embedEntity(entityId) {
|
|
1456
|
+
async embedEntity(entityId, priority = 'bulk') {
|
|
804
1457
|
if (!this.db)
|
|
805
1458
|
throw new Error('Database not initialized');
|
|
806
1459
|
// Get entity data
|
|
@@ -822,28 +1475,43 @@ export class RAGKnowledgeGraphManager {
|
|
|
822
1475
|
// char size (identity excluded), and embed duration. `capped` = some observation chars dropped.
|
|
823
1476
|
const capped = built.cappedObsChars < built.filteredObsChars;
|
|
824
1477
|
const embedStart = Date.now();
|
|
825
|
-
const embedding = await this.generateEmbedding(embeddingText);
|
|
1478
|
+
const embedding = await this.generateEmbedding(embeddingText, 1024, false, priority);
|
|
826
1479
|
const embedMs = Date.now() - embedStart;
|
|
827
1480
|
console.error(`[embed] ${entity.name}: ${built.selectedObsCount}/${built.totalObsCount} obs, ${built.filteredObsChars}ch -> ${built.cappedObsChars}ch${capped ? ' (capped)' : ''}, ${embedMs}ms`);
|
|
828
1481
|
try {
|
|
829
|
-
//
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
1482
|
+
// v3.6 (§6a-2): vector replace + provenance stamp commit atomically.
|
|
1483
|
+
// Write-back CAS (beta 2R B1): the entity may have been mutated again
|
|
1484
|
+
// while THIS inference was in flight — a late writer must never
|
|
1485
|
+
// re-insert a vector for superseded content as 'verified'. Inside the
|
|
1486
|
+
// write transaction the CURRENT entity text is rebuilt and hashed; on
|
|
1487
|
+
// mismatch the result is discarded and the row stays missing/queued for
|
|
1488
|
+
// the backfill pass that the newer mutation already kicked.
|
|
1489
|
+
const inputHash = this.hashWithBuilderVersion(embeddingText);
|
|
1490
|
+
const writeTx = this.db.transaction(() => {
|
|
1491
|
+
const currentHash = this.entityInputHash(entityId);
|
|
1492
|
+
if (currentHash !== inputHash)
|
|
1493
|
+
return false; // superseded — discard
|
|
1494
|
+
const existingMetadata = this.db.prepare(`
|
|
1495
|
+
SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
|
|
1496
|
+
`).get(entityId);
|
|
1497
|
+
if (existingMetadata) {
|
|
1498
|
+
this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(existingMetadata.rowid)}`);
|
|
1499
|
+
this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
|
|
1500
|
+
}
|
|
1501
|
+
const result = this.db.prepare(`
|
|
1502
|
+
INSERT INTO entity_embeddings (embedding) VALUES (?)
|
|
1503
|
+
`).run(Buffer.from(embedding.buffer));
|
|
1504
|
+
this.db.prepare(`
|
|
1505
|
+
INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text, input_hash, profile_id, provenance_state)
|
|
1506
|
+
VALUES (?, ?, ?, ?, ?, 'verified')
|
|
1507
|
+
`).run(result.lastInsertRowid, entityId, embeddingText, inputHash, this.currentProfileId);
|
|
1508
|
+
return true;
|
|
1509
|
+
});
|
|
1510
|
+
const written = writeTx();
|
|
1511
|
+
if (!written) {
|
|
1512
|
+
console.error(`⏭️ discarded superseded embedding for ${entityId} (entity changed during inference)`);
|
|
836
1513
|
}
|
|
837
|
-
|
|
838
|
-
const result = this.db.prepare(`
|
|
839
|
-
INSERT INTO entity_embeddings (embedding) VALUES (?)
|
|
840
|
-
`).run(Buffer.from(embedding.buffer));
|
|
841
|
-
// Store metadata
|
|
842
|
-
this.db.prepare(`
|
|
843
|
-
INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text)
|
|
844
|
-
VALUES (?, ?, ?)
|
|
845
|
-
`).run(result.lastInsertRowid, entityId, embeddingText);
|
|
846
|
-
return true;
|
|
1514
|
+
return written;
|
|
847
1515
|
}
|
|
848
1516
|
catch (error) {
|
|
849
1517
|
console.error(`Failed to embed entity ${entityId}:`, error);
|
|
@@ -866,6 +1534,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
866
1534
|
embeddedCount += results.filter(Boolean).length;
|
|
867
1535
|
}
|
|
868
1536
|
console.error(`✅ Entity embeddings completed: ${embeddedCount}/${entities.length} entities embedded`);
|
|
1537
|
+
this.coordinator?.invalidateCoverage();
|
|
869
1538
|
return {
|
|
870
1539
|
totalEntities: entities.length,
|
|
871
1540
|
embeddedEntities: embeddedCount
|
|
@@ -945,15 +1614,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
945
1614
|
const errors = [];
|
|
946
1615
|
for (const chunk of chunks) {
|
|
947
1616
|
// Generate embedding
|
|
948
|
-
const embedding = await this.generateEmbedding(chunk.text);
|
|
1617
|
+
const embedding = await this.generateEmbedding(chunk.text, 1024, false, 'bulk');
|
|
949
1618
|
const rowid = safeRowid(chunk.rowid);
|
|
950
1619
|
try {
|
|
951
|
-
//
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
1620
|
+
// vector + verified provenance in one transaction (§6a-2) — KG chunks
|
|
1621
|
+
// must never become vector-bearing provenance-NULL rows post-recon.
|
|
1622
|
+
const tx = this.db.transaction(() => {
|
|
1623
|
+
this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
|
|
1624
|
+
this.db.prepare(`
|
|
1625
|
+
INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
|
|
1626
|
+
`).run(Buffer.from(embedding.buffer));
|
|
1627
|
+
this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
1628
|
+
.run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, chunk.rowid);
|
|
1629
|
+
});
|
|
1630
|
+
tx();
|
|
957
1631
|
embeddedCount++;
|
|
958
1632
|
}
|
|
959
1633
|
catch (error) {
|
|
@@ -962,6 +1636,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
962
1636
|
errors.push(errMsg);
|
|
963
1637
|
}
|
|
964
1638
|
}
|
|
1639
|
+
this.coordinator?.invalidateCoverage();
|
|
965
1640
|
console.error(`✅ Knowledge graph chunks embedded: ${embeddedCount}/${chunks.length}`);
|
|
966
1641
|
return { embeddedChunks: embeddedCount, totalChunks: chunks.length, ...(errors.length > 0 && { errors: errors.slice(0, 5) }) };
|
|
967
1642
|
}
|
|
@@ -1185,37 +1860,23 @@ export class RAGKnowledgeGraphManager {
|
|
|
1185
1860
|
}
|
|
1186
1861
|
// Generate embeddings using sentence transformers
|
|
1187
1862
|
// isQuery: true for search queries (adds instruction prefix), false for documents/entities
|
|
1188
|
-
async generateEmbedding(text, dimensions = 1024, isQuery = false) {
|
|
1863
|
+
async generateEmbedding(text, dimensions = 1024, isQuery = false, priority = 'interactive') {
|
|
1189
1864
|
// Check cache first (hash-based key to avoid collisions on long texts)
|
|
1190
1865
|
const cacheKey = createHash('md5').update(`${text}_${dimensions}_${isQuery}`).digest('hex');
|
|
1191
1866
|
const cached = this.embeddingCache.get(cacheKey);
|
|
1192
1867
|
if (cached)
|
|
1193
1868
|
return cached;
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
|
|
1206
|
-
const firstKey = this.embeddingCache.keys().next().value;
|
|
1207
|
-
if (firstKey)
|
|
1208
|
-
this.embeddingCache.delete(firstKey);
|
|
1209
|
-
}
|
|
1210
|
-
this.embeddingCache.set(cacheKey, modelResult);
|
|
1211
|
-
return modelResult;
|
|
1212
|
-
}
|
|
1213
|
-
catch (error) {
|
|
1214
|
-
console.error(`⚠️ Embedding model failed for text "${text.slice(0, 50)}...":`, error instanceof Error ? error.message : error);
|
|
1215
|
-
throw new Error(`Embedding model not available. Ensure the model is loaded. Original error: ${error instanceof Error ? error.message : error}`);
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
throw new Error('Embedding model not initialized. The server may still be loading the model — retry in a few seconds.');
|
|
1869
|
+
// v3.6: all inference goes through the gate — state check + execution in one
|
|
1870
|
+
// atomic boundary (TOCTOU-safe). GateNotReadyError / GateDisabledError
|
|
1871
|
+
// propagate so each consumer honors its own not-ready contract (spec §5).
|
|
1872
|
+
const modelResult = await this.gate.embed(text, { dims: dimensions, isQuery, priority });
|
|
1873
|
+
if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
|
|
1874
|
+
const firstKey = this.embeddingCache.keys().next().value;
|
|
1875
|
+
if (firstKey)
|
|
1876
|
+
this.embeddingCache.delete(firstKey);
|
|
1877
|
+
}
|
|
1878
|
+
this.embeddingCache.set(cacheKey, modelResult);
|
|
1879
|
+
return modelResult;
|
|
1219
1880
|
}
|
|
1220
1881
|
// === NEW SEPARATE TOOLS ===
|
|
1221
1882
|
async syncDocumentFromFile(filePath, documentId, options = {}) {
|
|
@@ -1243,31 +1904,56 @@ export class RAGKnowledgeGraphManager {
|
|
|
1243
1904
|
catch { /* ignore */ }
|
|
1244
1905
|
if (existingHash === contentHash) {
|
|
1245
1906
|
const cmCount = this.db.prepare(`SELECT count(*) AS n FROM chunk_metadata WHERE document_id = ?`).get(documentId).n;
|
|
1907
|
+
// "Embedded" for dedup completeness = vector exists AND its profile is
|
|
1908
|
+
// current (or legacy-NULL awaiting grandfather). Raw vector counts
|
|
1909
|
+
// would misjudge old-profile rows as complete (beta 1R supplement).
|
|
1246
1910
|
const embCount = this.db.prepare(`
|
|
1247
|
-
SELECT count(*) AS n FROM chunks c JOIN chunk_metadata m ON c.rowid = m.rowid
|
|
1911
|
+
SELECT count(*) AS n FROM chunks c JOIN chunk_metadata m ON c.rowid = m.rowid
|
|
1912
|
+
WHERE m.document_id = ? AND (m.provenance_state IS NULL OR m.profile_id = ?)
|
|
1913
|
+
`).get(documentId, this.currentProfileId).n;
|
|
1914
|
+
const linked = this.db.prepare(`
|
|
1915
|
+
SELECT count(DISTINCT ce.entity_id) AS n FROM chunk_entities ce
|
|
1916
|
+
JOIN chunk_metadata m ON ce.chunk_rowid = m.rowid WHERE m.document_id = ?
|
|
1248
1917
|
`).get(documentId).n;
|
|
1249
1918
|
if (cmCount > 0 && cmCount === embCount) {
|
|
1250
|
-
const linked = this.db.prepare(`
|
|
1251
|
-
SELECT count(DISTINCT ce.entity_id) AS n FROM chunk_entities ce
|
|
1252
|
-
JOIN chunk_metadata m ON ce.chunk_rowid = m.rowid WHERE m.document_id = ?
|
|
1253
|
-
`).get(documentId).n;
|
|
1254
1919
|
console.error(`⏭️ syncDocumentFromFile: ${documentId} unchanged (hash match, ${cmCount} chunks embedded) — skipped`);
|
|
1255
1920
|
return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged' };
|
|
1256
1921
|
}
|
|
1922
|
+
if (cmCount > 0 && embCount < cmCount) {
|
|
1923
|
+
// v3.6 (spec §5b M12): identical content with incomplete/stale vectors
|
|
1924
|
+
// keeps the document, chunks, rowids and entity links — only the
|
|
1925
|
+
// missing vectors are re-queued via the coordinator. Full re-chunking
|
|
1926
|
+
// here would churn rowids and links for no content change.
|
|
1927
|
+
console.error(`♻️ syncDocumentFromFile: ${documentId} unchanged but ${cmCount - embCount} vectors missing — re-queued (chunks preserved)`);
|
|
1928
|
+
this.coordinator?.kick();
|
|
1929
|
+
return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged-revectorizing', embedding_status: this.gate.isDisabled ? 'disabled' : 'queued' };
|
|
1930
|
+
}
|
|
1257
1931
|
}
|
|
1258
1932
|
}
|
|
1259
1933
|
console.error(`🔄 syncDocumentFromFile: ${documentId} <- ${filePath} (${bytes} bytes)`);
|
|
1260
|
-
// 3.
|
|
1261
|
-
//
|
|
1934
|
+
// 3. Two contracts (spec §5b):
|
|
1935
|
+
// ready — pre-compute ALL embeddings BEFORE any DB mutation; if
|
|
1936
|
+
// inference throws mid-way the old document stays intact
|
|
1937
|
+
// (v3.5.0 atomicity, unchanged).
|
|
1938
|
+
// not-ready — intentional lazy sync: store document + chunks + FTS in
|
|
1939
|
+
// one transaction with NO vectors (embedding_status:
|
|
1940
|
+
// queued); the backfill coordinator recovers them.
|
|
1262
1941
|
const { maxTokens = 800, overlap = 160 } = options.chunkParams || {};
|
|
1263
1942
|
const segments = this.chunkText(content, maxTokens, overlap);
|
|
1943
|
+
const lazySync = !this.gate.isReady;
|
|
1264
1944
|
const embedded = [];
|
|
1265
|
-
|
|
1266
|
-
const
|
|
1267
|
-
|
|
1945
|
+
if (lazySync) {
|
|
1946
|
+
for (const seg of segments)
|
|
1947
|
+
embedded.push({ seg, embedding: null });
|
|
1948
|
+
}
|
|
1949
|
+
else {
|
|
1950
|
+
for (const seg of segments) {
|
|
1951
|
+
const embedding = await this.generateEmbedding(seg.text, 1024, false, 'bulk');
|
|
1952
|
+
embedded.push({ seg, embedding });
|
|
1953
|
+
}
|
|
1268
1954
|
}
|
|
1269
|
-
// 4. Atomic swap: delete old -> insert doc -> insert chunks + embeddings
|
|
1270
|
-
//
|
|
1955
|
+
// 4. Atomic swap: delete old -> insert doc -> insert chunks (+ embeddings
|
|
1956
|
+
// with verified provenance when ready), one synchronous transaction.
|
|
1271
1957
|
const applyTx = this.db.transaction(() => {
|
|
1272
1958
|
const db = this.db;
|
|
1273
1959
|
// 4a. cleanup old doc (inlined sync version of cleanupDocument).
|
|
@@ -1281,7 +1967,8 @@ export class RAGKnowledgeGraphManager {
|
|
|
1281
1967
|
// 4b. insert document.
|
|
1282
1968
|
db.prepare(`INSERT INTO documents (id, content, metadata) VALUES (?, ?, ?)`)
|
|
1283
1969
|
.run(documentId, content, JSON.stringify(metadata));
|
|
1284
|
-
// 4c. insert chunk_metadata (FTS5 chunks_fts auto-filled by trigger)
|
|
1970
|
+
// 4c. insert chunk_metadata (FTS5 chunks_fts auto-filled by trigger);
|
|
1971
|
+
// vectors + provenance only on the ready path (§6a-2).
|
|
1285
1972
|
for (const { seg, embedding } of embedded) {
|
|
1286
1973
|
const chunkId = `${documentId}_chunk_${seg.chunk_index}`;
|
|
1287
1974
|
const info = db.prepare(`
|
|
@@ -1289,11 +1976,18 @@ export class RAGKnowledgeGraphManager {
|
|
|
1289
1976
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1290
1977
|
`).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
|
|
1291
1978
|
const rowid = Number(info.lastInsertRowid);
|
|
1292
|
-
|
|
1979
|
+
if (embedding) {
|
|
1980
|
+
db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
|
|
1981
|
+
db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
1982
|
+
.run(createHash('sha256').update(seg.text).digest('hex'), this.currentProfileId, rowid);
|
|
1983
|
+
}
|
|
1293
1984
|
}
|
|
1294
1985
|
});
|
|
1295
1986
|
applyTx();
|
|
1296
|
-
|
|
1987
|
+
this.coordinator?.invalidateCoverage();
|
|
1988
|
+
if (lazySync)
|
|
1989
|
+
this.coordinator?.kick();
|
|
1990
|
+
const embeddedChunks = lazySync ? 0 : embedded.length;
|
|
1297
1991
|
// 5. Entity linking AFTER commit. Non-destructive + idempotent (INSERT OR
|
|
1298
1992
|
// IGNORE), so a linking failure cannot corrupt the doc/embeddings.
|
|
1299
1993
|
const linkedEntities = await this.autoLinkEntities(documentId);
|
|
@@ -1309,6 +2003,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1309
2003
|
chunks: segments.length,
|
|
1310
2004
|
embeddedChunks,
|
|
1311
2005
|
linkedEntities,
|
|
2006
|
+
embedding_status: lazySync ? (this.gate.isDisabled ? 'disabled' : 'queued') : 'embedded',
|
|
1312
2007
|
...(explicitlyLinked !== undefined ? { explicitlyLinked } : {}),
|
|
1313
2008
|
};
|
|
1314
2009
|
if (linkedEntities === 0 && explicitlyLinked === undefined) {
|
|
@@ -1366,6 +2061,10 @@ export class RAGKnowledgeGraphManager {
|
|
|
1366
2061
|
});
|
|
1367
2062
|
}
|
|
1368
2063
|
console.error(`✅ Document chunked: ${chunks.length} chunks created`);
|
|
2064
|
+
// Indirect missing-row producer (spec §5): freshly chunked rows have no
|
|
2065
|
+
// vectors yet — let the coordinator recover them without a restart.
|
|
2066
|
+
this.coordinator?.invalidateCoverage();
|
|
2067
|
+
this.coordinator?.kick();
|
|
1369
2068
|
return { documentId, chunks: resultChunks };
|
|
1370
2069
|
}
|
|
1371
2070
|
async embedChunks(documentId) {
|
|
@@ -1382,18 +2081,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
1382
2081
|
let embeddedCount = 0;
|
|
1383
2082
|
const errors = [];
|
|
1384
2083
|
for (const chunk of chunks) {
|
|
1385
|
-
// Generate embedding
|
|
1386
|
-
const embedding = await this.generateEmbedding(chunk.text);
|
|
2084
|
+
// Generate embedding (foreground-bulk priority)
|
|
2085
|
+
const embedding = await this.generateEmbedding(chunk.text, 1024, false, 'bulk');
|
|
1387
2086
|
const rowid = Number(chunk.rowid);
|
|
1388
|
-
// Store in vector table
|
|
2087
|
+
// Store in vector table (+ verified provenance, §6a-2 atomic)
|
|
1389
2088
|
try {
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
2089
|
+
const tx = this.db.transaction(() => {
|
|
2090
|
+
this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(rowid)}`);
|
|
2091
|
+
this.db.prepare(`
|
|
2092
|
+
INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
|
|
2093
|
+
`).run(Buffer.from(embedding.buffer));
|
|
2094
|
+
this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
|
|
2095
|
+
.run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, rowid);
|
|
2096
|
+
});
|
|
2097
|
+
tx();
|
|
1397
2098
|
embeddedCount++;
|
|
1398
2099
|
}
|
|
1399
2100
|
catch (error) {
|
|
@@ -1403,6 +2104,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
1403
2104
|
}
|
|
1404
2105
|
}
|
|
1405
2106
|
console.error(`✅ Chunks embedded: ${embeddedCount}/${chunks.length}`);
|
|
2107
|
+
this.coordinator?.invalidateCoverage();
|
|
1406
2108
|
// Auto-link entities to document after embedding
|
|
1407
2109
|
const linkedCount = await this.autoLinkEntities(documentId);
|
|
1408
2110
|
return { documentId, embeddedChunks: embeddedCount, totalChunks: chunks.length, linkedEntities: linkedCount, ...(errors.length > 0 && { errors: errors.slice(0, 5) }) };
|
|
@@ -1736,10 +2438,20 @@ export class RAGKnowledgeGraphManager {
|
|
|
1736
2438
|
created_at: row.created_at
|
|
1737
2439
|
}));
|
|
1738
2440
|
console.error(`✅ Export completed: ${entities.length} entities, ${relations.length} relations, ${documents.length} documents`);
|
|
2441
|
+
// spec §6.4: lifecycle 정본을 함께 내보낸다. 이게 없으면 export->import 뒤
|
|
2442
|
+
// 관찰의 신원·출처·이력이 사라지고 projection 만 남는다.
|
|
2443
|
+
const observation_roots = this.db.prepare(`SELECT * FROM observation_roots ORDER BY entity_id, projection_order`).all();
|
|
2444
|
+
const entity_observations = this.db.prepare(`SELECT * FROM entity_observations ORDER BY root_id, revision_no`).all();
|
|
2445
|
+
const observation_sources = this.db.prepare(`SELECT * FROM observation_sources ORDER BY observation_id, source_kind, source_ref`).all();
|
|
2446
|
+
const observation_events = this.db.prepare(`SELECT * FROM observation_events ORDER BY root_id, recorded_at, event_id`).all();
|
|
1739
2447
|
return {
|
|
1740
2448
|
entities,
|
|
1741
2449
|
relations,
|
|
1742
2450
|
documents,
|
|
2451
|
+
observation_roots,
|
|
2452
|
+
entity_observations,
|
|
2453
|
+
observation_sources,
|
|
2454
|
+
observation_events,
|
|
1743
2455
|
metadata: {
|
|
1744
2456
|
exportedAt: new Date().toISOString(),
|
|
1745
2457
|
version: PKG_VERSION,
|
|
@@ -1755,63 +2467,245 @@ export class RAGKnowledgeGraphManager {
|
|
|
1755
2467
|
console.error(`📥 Importing knowledge graph (merge: ${options.merge !== false})...`);
|
|
1756
2468
|
const imported = { entities: 0, relations: 0, documents: 0 };
|
|
1757
2469
|
const skipped = { entities: 0, relations: 0, documents: 0 };
|
|
1758
|
-
//
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
2470
|
+
// merge 로 배열 위치가 재배정된 관찰. 조용히 순서를 바꾸면 호출자가 알 수 없으므로
|
|
2471
|
+
// 응답으로 내보낸다(advisor beta r3 발견 3).
|
|
2472
|
+
const remapReport = [];
|
|
2473
|
+
// spec §6.4: abort 는 0 mutation 이다. lifecycle 만 트랜잭션으로 감싸면
|
|
2474
|
+
// 충돌로 throw 할 때 그 앞에서 넣은 entity·relation·document 가 살아남는다
|
|
2475
|
+
// (T17b 가 ghost entity 로 실증). import 전체가 한 단위여야 한다.
|
|
2476
|
+
// 내부 transaction() 호출은 better-sqlite3 에서 savepoint 로 중첩된다.
|
|
2477
|
+
const importAll = this.db.transaction(() => {
|
|
2478
|
+
// If merge=false, clear existing data first
|
|
2479
|
+
if (options.merge === false) {
|
|
2480
|
+
this.db.exec(`DELETE FROM relationships`);
|
|
2481
|
+
// entities 삭제가 FK CASCADE 로 lifecycle 4테이블을 지우지만, 순서를 계약으로
|
|
2482
|
+
// 두어 FK 가 꺼진 환경에서도 잔존 행이 남지 않게 한다.
|
|
2483
|
+
this.db.exec(`DELETE FROM observation_events`);
|
|
2484
|
+
this.db.exec(`DELETE FROM observation_sources`);
|
|
2485
|
+
this.db.exec(`DELETE FROM entity_observations`);
|
|
2486
|
+
this.db.exec(`DELETE FROM observation_roots`);
|
|
2487
|
+
this.db.exec(`DELETE FROM entities`);
|
|
2488
|
+
this.db.exec(`DELETE FROM documents`);
|
|
2489
|
+
// entities 를 지워도 파생 데이터는 따라오지 않는다: chunk_metadata 에는
|
|
2490
|
+
// entities 로 가는 FK 가 없고 entity_embedding_metadata.entity_id 는 UNIQUE 일
|
|
2491
|
+
// 뿐이다. 그래서 replace-import 뒤에 **사라진 entity 의 벡터와 KG chunk 가
|
|
2492
|
+
// 검색에 남았다**(advisor beta 발견 2). document chunk 는 documents 의
|
|
2493
|
+
// CASCADE 로 이미 정리되므로 여기서는 entity·relationship chunk 만 지운다.
|
|
2494
|
+
const orphanChunks = this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type IN ('entity','relationship')`)
|
|
2495
|
+
.all();
|
|
2496
|
+
for (const c of orphanChunks) {
|
|
2497
|
+
this.db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
|
|
2498
|
+
this.db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
|
|
2499
|
+
}
|
|
2500
|
+
this.db.exec(`DELETE FROM entity_embeddings WHERE rowid IN (SELECT rowid FROM entity_embedding_metadata)`);
|
|
2501
|
+
this.db.exec(`DELETE FROM entity_embedding_metadata`);
|
|
2502
|
+
console.error('🗑️ Cleared existing data for full import');
|
|
2503
|
+
}
|
|
2504
|
+
// Import entities using INSERT OR IGNORE
|
|
2505
|
+
if (data.entities && Array.isArray(data.entities)) {
|
|
2506
|
+
const stmt = this.db.prepare(`
|
|
1768
2507
|
INSERT OR IGNORE INTO entities (id, name, entityType, observations, metadata, created_at)
|
|
1769
2508
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
1770
2509
|
`);
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2510
|
+
for (const entity of data.entities) {
|
|
2511
|
+
const result = stmt.run(entity.id, entity.name, entity.entityType || 'CONCEPT',
|
|
2512
|
+
// v13: observations 는 projection 이다. lifecycle 행을 넣은 뒤
|
|
2513
|
+
// rebuildProjection 이 채운다 — 여기서 배열을 심으면 정본과 갈라진다.
|
|
2514
|
+
'[]', JSON.stringify(entity.metadata || {}), entity.created_at || new Date().toISOString());
|
|
2515
|
+
if (result.changes > 0) {
|
|
2516
|
+
imported.entities++;
|
|
2517
|
+
}
|
|
2518
|
+
else {
|
|
2519
|
+
skipped.entities++;
|
|
2520
|
+
}
|
|
1778
2521
|
}
|
|
1779
2522
|
}
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
const stmt = this.db.prepare(`
|
|
2523
|
+
// Import relations using INSERT OR IGNORE
|
|
2524
|
+
if (data.relations && Array.isArray(data.relations)) {
|
|
2525
|
+
const stmt = this.db.prepare(`
|
|
1784
2526
|
INSERT OR IGNORE INTO relationships (id, source_entity, target_entity, relationType, confidence, metadata, created_at)
|
|
1785
2527
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1786
2528
|
`);
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
2529
|
+
for (const relation of data.relations) {
|
|
2530
|
+
const result = stmt.run(relation.id, relation.source_entity, relation.target_entity, relation.relationType, relation.confidence ?? 1.0, JSON.stringify(relation.metadata || {}), relation.created_at || new Date().toISOString());
|
|
2531
|
+
if (result.changes > 0) {
|
|
2532
|
+
imported.relations++;
|
|
2533
|
+
}
|
|
2534
|
+
else {
|
|
2535
|
+
skipped.relations++;
|
|
2536
|
+
}
|
|
1794
2537
|
}
|
|
1795
2538
|
}
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
const stmt = this.db.prepare(`
|
|
2539
|
+
// Import documents using INSERT OR REPLACE
|
|
2540
|
+
if (data.documents && Array.isArray(data.documents)) {
|
|
2541
|
+
const stmt = this.db.prepare(`
|
|
1800
2542
|
INSERT OR REPLACE INTO documents (id, content, metadata, created_at)
|
|
1801
2543
|
VALUES (?, ?, ?, ?)
|
|
1802
2544
|
`);
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
2545
|
+
for (const doc of data.documents) {
|
|
2546
|
+
const result = stmt.run(doc.id, doc.content, JSON.stringify(doc.metadata || {}), doc.created_at || new Date().toISOString());
|
|
2547
|
+
if (result.changes > 0) {
|
|
2548
|
+
imported.documents++;
|
|
2549
|
+
}
|
|
2550
|
+
else {
|
|
2551
|
+
skipped.documents++;
|
|
2552
|
+
}
|
|
1810
2553
|
}
|
|
1811
2554
|
}
|
|
1812
|
-
|
|
2555
|
+
// ---- spec §6.4: lifecycle import ----
|
|
2556
|
+
// 순서가 계약이다: entities -> roots -> revisions(root별 revision_no ↑)
|
|
2557
|
+
// -> sources/events. §4.1 트리거가 root 선행과 체인 연속성을 요구하므로
|
|
2558
|
+
// importer 는 입력 순서와 무관하게 재정렬한다 (역순 export 를 그대로
|
|
2559
|
+
// 스트리밍하면 'immediately preceding revision' 으로 죽는다).
|
|
2560
|
+
const sameRow = (a, b, cols) => cols.every(c => (a[c] ?? null) === (b[c] ?? null));
|
|
2561
|
+
const hasLifecycle = Array.isArray(data.observation_roots);
|
|
2562
|
+
if (hasLifecycle) {
|
|
2563
|
+
const tx = this.db.transaction(() => {
|
|
2564
|
+
// 새 root 가 이미 점유된 (entity_id, projection_order) 슬롯을 요구할 수 있다:
|
|
2565
|
+
// 두 DB 가 같은 entity 이름을 갖고 서로 다른 관찰을 배열 0번에 두면 그렇다.
|
|
2566
|
+
// 이건 §6.4 의 "같은 키 다른 값" 충돌이 아니라 **슬롯 충돌**이고, 규칙이 없어서
|
|
2567
|
+
// raw UNIQUE 오류로 터졌다(내 MCP 왕복 테스트가 잡았다). merge 의 뜻은
|
|
2568
|
+
// "더한다"이므로 들어오는 root 에 다음 빈 순번을 준다 — 남의 관찰을 덮지 않고,
|
|
2569
|
+
// 배열 끝에 붙는다. remap 은 그 root 의 revision 들에도 그대로 적용해야 한다
|
|
2570
|
+
// (trg_obs_matches_root 가 둘의 일치를 요구한다).
|
|
2571
|
+
// 입력 순서에 결과가 의존하면 같은 dump 를 두 번 넣었을 때 배열 순서가 달라진다.
|
|
2572
|
+
// (entity_id, projection_order, root_id) 로 정렬해 결정론을 만든다.
|
|
2573
|
+
const incomingRoots = [...(data.observation_roots ?? [])].sort((a, b) => String(a.entity_id).localeCompare(String(b.entity_id)) ||
|
|
2574
|
+
(a.projection_order - b.projection_order) ||
|
|
2575
|
+
String(a.root_id).localeCompare(String(b.root_id)));
|
|
2576
|
+
const remappedOrder = new Map();
|
|
2577
|
+
for (const r of incomingRoots) {
|
|
2578
|
+
const cur = this.db.prepare(`SELECT * FROM observation_roots WHERE root_id = ?`)
|
|
2579
|
+
.get(r.root_id);
|
|
2580
|
+
if (cur) {
|
|
2581
|
+
// projection_order 는 **target-local** 속성이다: merge 는 배열 위치를
|
|
2582
|
+
// 이 DB 기준으로 재배정하므로, 이미 remap 된 root 를 같은 dump 로 다시
|
|
2583
|
+
// 넣으면 dump 의 옛 순번과 다를 수밖에 없다. 그걸 충돌로 보면 동일
|
|
2584
|
+
// 재수입이 실패한다(advisor beta r3 발견 3, 실행 재현).
|
|
2585
|
+
if (!sameRow(cur, r, ['entity_id', 'created_at']))
|
|
2586
|
+
throw new Error(`import conflict: observation_roots ${r.root_id} differs from the existing row`);
|
|
2587
|
+
remappedOrder.set(r.root_id, cur.projection_order);
|
|
2588
|
+
continue;
|
|
2589
|
+
}
|
|
2590
|
+
let order = r.projection_order;
|
|
2591
|
+
const taken = this.db.prepare(`SELECT root_id FROM observation_roots WHERE entity_id = ? AND projection_order = ?`)
|
|
2592
|
+
.get(r.entity_id, order);
|
|
2593
|
+
if (taken) {
|
|
2594
|
+
order = nextProjectionOrder(this.db, r.entity_id);
|
|
2595
|
+
remappedOrder.set(r.root_id, order);
|
|
2596
|
+
remapReport.push({ root_id: r.root_id, entity_id: r.entity_id,
|
|
2597
|
+
from: r.projection_order, to: order });
|
|
2598
|
+
console.error(` ├─ ↪️ import: ${r.entity_id} position ${r.projection_order} is held by ` +
|
|
2599
|
+
`${taken.root_id}; appending imported observation at ${order}`);
|
|
2600
|
+
}
|
|
2601
|
+
this.db.prepare(`INSERT INTO observation_roots
|
|
2602
|
+
(root_id, entity_id, projection_order, created_at) VALUES (?, ?, ?, ?)`)
|
|
2603
|
+
.run(r.root_id, r.entity_id, order, r.created_at);
|
|
2604
|
+
}
|
|
2605
|
+
// projection_order 는 root 와 같은 이유로 비교 대상이 아니다(target-local).
|
|
2606
|
+
const revCols = ['root_id', 'entity_id', 'revision_no', 'content',
|
|
2607
|
+
'status', 'supersedes_id', 'recorded_at', 'superseded_at'];
|
|
2608
|
+
const revs = [...(data.entity_observations ?? [])]
|
|
2609
|
+
.sort((a, b) => a.root_id === b.root_id
|
|
2610
|
+
? a.revision_no - b.revision_no
|
|
2611
|
+
: String(a.root_id).localeCompare(String(b.root_id)));
|
|
2612
|
+
for (const v of revs) {
|
|
2613
|
+
const cur = this.db.prepare(`SELECT * FROM entity_observations WHERE observation_id = ?`)
|
|
2614
|
+
.get(v.observation_id);
|
|
2615
|
+
if (cur) {
|
|
2616
|
+
if (!sameRow(cur, v, revCols))
|
|
2617
|
+
throw new Error(`import conflict: entity_observations ${v.observation_id} differs from the existing row`);
|
|
2618
|
+
continue;
|
|
2619
|
+
}
|
|
2620
|
+
const order = remappedOrder.has(v.root_id)
|
|
2621
|
+
? remappedOrder.get(v.root_id) : v.projection_order;
|
|
2622
|
+
this.db.prepare(`INSERT INTO entity_observations
|
|
2623
|
+
(observation_id, root_id, entity_id, revision_no, projection_order,
|
|
2624
|
+
content, status, supersedes_id, recorded_at, superseded_at)
|
|
2625
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2626
|
+
.run(v.observation_id, v.root_id, v.entity_id, v.revision_no, order, v.content, v.status, v.supersedes_id ?? null, v.recorded_at, v.superseded_at ?? null);
|
|
2627
|
+
}
|
|
2628
|
+
for (const so of (data.observation_sources ?? [])) {
|
|
2629
|
+
const cur = this.db.prepare(`SELECT * FROM observation_sources
|
|
2630
|
+
WHERE observation_id=? AND source_kind=? AND source_ref=?`)
|
|
2631
|
+
.get(so.observation_id, so.source_kind, so.source_ref);
|
|
2632
|
+
if (cur) {
|
|
2633
|
+
if (!sameRow(cur, so, ['source_hash', 'recorded_at']))
|
|
2634
|
+
throw new Error(`import conflict: observation_sources ` +
|
|
2635
|
+
`${so.observation_id}/${so.source_kind}/${so.source_ref} differs from the existing row`);
|
|
2636
|
+
continue;
|
|
2637
|
+
}
|
|
2638
|
+
this.db.prepare(`INSERT INTO observation_sources
|
|
2639
|
+
(observation_id, source_kind, source_ref, source_hash, recorded_at) VALUES (?, ?, ?, ?, ?)`)
|
|
2640
|
+
.run(so.observation_id, so.source_kind, so.source_ref, so.source_hash ?? null, so.recorded_at);
|
|
2641
|
+
}
|
|
2642
|
+
const evCols = ['root_id', 'from_id', 'to_id', 'event', 'change_kind', 'reason', 'actor', 'batch_id', 'recorded_at'];
|
|
2643
|
+
for (const e of (data.observation_events ?? [])) {
|
|
2644
|
+
const cur = this.db.prepare(`SELECT * FROM observation_events WHERE event_id = ?`)
|
|
2645
|
+
.get(e.event_id);
|
|
2646
|
+
if (cur) {
|
|
2647
|
+
if (!sameRow(cur, e, evCols))
|
|
2648
|
+
throw new Error(`import conflict: observation_events ${e.event_id} differs from the existing row`);
|
|
2649
|
+
continue;
|
|
2650
|
+
}
|
|
2651
|
+
this.db.prepare(`INSERT INTO observation_events
|
|
2652
|
+
(event_id, root_id, from_id, to_id, event, change_kind, reason, actor, batch_id, recorded_at)
|
|
2653
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
2654
|
+
.run(e.event_id, e.root_id, e.from_id ?? null, e.to_id ?? null, e.event, e.change_kind ?? null, e.reason ?? null, e.actor ?? null, e.batch_id ?? null, e.recorded_at);
|
|
2655
|
+
}
|
|
2656
|
+
});
|
|
2657
|
+
tx();
|
|
2658
|
+
}
|
|
2659
|
+
else {
|
|
2660
|
+
// 구(舊) 형식 export: lifecycle 필드가 없으므로 entities.observations 를
|
|
2661
|
+
// 신규 root 로 승격한다. legacy import 필수 필드값 = spec §6.4.
|
|
2662
|
+
const ts = new Date().toISOString();
|
|
2663
|
+
const tx = this.db.transaction(() => {
|
|
2664
|
+
for (const ent of (data.entities ?? [])) {
|
|
2665
|
+
const entityId = ent.id ??
|
|
2666
|
+
`entity_${String(ent.name).toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
|
|
2667
|
+
for (const content of (ent.observations ?? [])) {
|
|
2668
|
+
addRevision(this.db, {
|
|
2669
|
+
entityId, content, status: 'active',
|
|
2670
|
+
sources: [{ source_kind: 'import', source_ref: 'legacy-export', source_hash: null }],
|
|
2671
|
+
actor: 'import', ts, event: 'import'
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
});
|
|
2676
|
+
tx();
|
|
2677
|
+
}
|
|
2678
|
+
// projection 재합성 + 파생 상태 무효화.
|
|
2679
|
+
// 무효화가 없으면 이미 있던 entity 를 덮어쓴 뒤에도 옛 벡터·옛 KG chunk 가
|
|
2680
|
+
// 검색에 남는다. import 는 관찰을 바꾸는 writer 이므로 다른 writer 와 같은
|
|
2681
|
+
// 계약을 져야 한다(advisor beta 발견 2).
|
|
2682
|
+
//
|
|
2683
|
+
// 대상은 `data.entities` 가 아니라 **영향받은 entity 전부**다. lifecycle import 는
|
|
2684
|
+
// observation_roots 만 있어도 활성화되므로, entities 없이 lifecycle 배열만 보내면
|
|
2685
|
+
// revision 은 들어가는데 projection 이 갱신되지 않아 새 사실이 reader 에 안 보이고
|
|
2686
|
+
// 옛 벡터가 남는다(advisor beta r3 발견 2, 실행 재현).
|
|
2687
|
+
const affected = new Set();
|
|
2688
|
+
for (const ent of (data.entities ?? [])) {
|
|
2689
|
+
affected.add(ent.id ??
|
|
2690
|
+
`entity_${String(ent.name).toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`);
|
|
2691
|
+
}
|
|
2692
|
+
for (const r of (data.observation_roots ?? []))
|
|
2693
|
+
if (r.entity_id)
|
|
2694
|
+
affected.add(r.entity_id);
|
|
2695
|
+
for (const v of (data.entity_observations ?? []))
|
|
2696
|
+
if (v.entity_id)
|
|
2697
|
+
affected.add(v.entity_id);
|
|
2698
|
+
for (const entityId of affected) {
|
|
2699
|
+
rebuildProjection(this.db, entityId);
|
|
2700
|
+
this.invalidateDerivedForEntity(entityId);
|
|
2701
|
+
}
|
|
2702
|
+
});
|
|
2703
|
+
importAll();
|
|
1813
2704
|
console.error(`✅ Import completed: ${imported.entities} entities, ${imported.relations} relations, ${imported.documents} documents imported`);
|
|
1814
|
-
|
|
2705
|
+
// Indirect missing-row producer (spec §5): imported rows may lack vectors.
|
|
2706
|
+
this.coordinator?.invalidateCoverage();
|
|
2707
|
+
this.coordinator?.kick();
|
|
2708
|
+
return { imported, skipped, observation_order_remap: remapReport };
|
|
1815
2709
|
}
|
|
1816
2710
|
async hybridSearch(query, limit = 5, useGraph = true) {
|
|
1817
2711
|
if (!this.db)
|
|
@@ -1819,6 +2713,9 @@ export class RAGKnowledgeGraphManager {
|
|
|
1819
2713
|
if (!this.encoding)
|
|
1820
2714
|
throw new Error('Tokenizer not initialized');
|
|
1821
2715
|
console.error(`🔍 Enhanced hybrid search: "${query}"`);
|
|
2716
|
+
// Parity with searchNodes (beta 1R supplement): an unsearchable query gets
|
|
2717
|
+
// an explicit warning instead of a silent empty envelope.
|
|
2718
|
+
const ftsUnsearchable = compileFtsLiteralQuery(query) === null;
|
|
1822
2719
|
const queryVariants = this.buildCrossLingualVariants(query);
|
|
1823
2720
|
if (queryVariants.length > 1) {
|
|
1824
2721
|
console.error(`🌐 Cross-lingual variants: ${queryVariants.slice(1).join(' | ')}`);
|
|
@@ -1853,37 +2750,47 @@ export class RAGKnowledgeGraphManager {
|
|
|
1853
2750
|
`).all(Buffer.from(embedding.buffer), k);
|
|
1854
2751
|
};
|
|
1855
2752
|
// Search original query plus cross-lingual expansions and keep best match per chunk.
|
|
2753
|
+
// v3.6 eligibility gate (spec §3): vector usage requires model_ready AND
|
|
2754
|
+
// reconciliation settled — otherwise FTS5-only, no waiting.
|
|
1856
2755
|
const resultMap = new Map();
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
2756
|
+
let degradationReason;
|
|
2757
|
+
if (!(this.coordinator?.eligible ?? false)) {
|
|
2758
|
+
vectorDegraded = true;
|
|
2759
|
+
degradationReason = this.degradationReason();
|
|
2760
|
+
console.error(`ℹ️ vector search not eligible (${degradationReason ?? 'unknown'}) — FTS5-only`);
|
|
2761
|
+
}
|
|
2762
|
+
else {
|
|
2763
|
+
try {
|
|
2764
|
+
primaryQueryEmbedding = await this.generateEmbedding(queryVariants[0], 1024, true);
|
|
2765
|
+
for (const variant of queryVariants) {
|
|
2766
|
+
const embedding = await this.generateEmbedding(variant, 1024, true);
|
|
2767
|
+
const variantResults = searchChunks(embedding, limit * 3);
|
|
2768
|
+
for (const r of variantResults) {
|
|
2769
|
+
const existing = resultMap.get(r.chunk_id);
|
|
2770
|
+
if (!existing || r.distance < existing.distance) {
|
|
2771
|
+
resultMap.set(r.chunk_id, r);
|
|
2772
|
+
}
|
|
1866
2773
|
}
|
|
1867
2774
|
}
|
|
1868
2775
|
}
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
2776
|
+
catch (embErr) {
|
|
2777
|
+
vectorDegraded = true;
|
|
2778
|
+
// 'inference_error' (not 'model_not_ready'): model_state may still read
|
|
2779
|
+
// 'ready' here — a contradictory reason pair confused callers (beta B6).
|
|
2780
|
+
degradationReason = this.degradationReason() ?? 'inference_error';
|
|
2781
|
+
console.error(`⚠️ Vector search unavailable — degrading to FTS5-only:`, embErr instanceof Error ? embErr.message : embErr);
|
|
2782
|
+
}
|
|
1873
2783
|
}
|
|
1874
2784
|
const vectorResults = Array.from(resultMap.values()).sort((a, b) => a.distance - b.distance);
|
|
1875
2785
|
// FTS5 full-text search as additional signal (Reciprocal Rank Fusion)
|
|
1876
2786
|
const ftsBoostMap = new Map();
|
|
1877
2787
|
try {
|
|
1878
2788
|
const ftsSearchQuery = (q) => {
|
|
1879
|
-
//
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
const terms = sanitized.split(/\s+/).filter(t => t.length > 0);
|
|
1884
|
-
if (terms.length === 0)
|
|
2789
|
+
// Shared compiler (spec §5) — same sanitize rules as pre-3.6, extracted
|
|
2790
|
+
// so entity FTS fallback uses identical MATCH-safety guarantees.
|
|
2791
|
+
const ftsExpr = compileFtsLiteralQuery(q);
|
|
2792
|
+
if (ftsExpr === null)
|
|
1885
2793
|
return [];
|
|
1886
|
-
const ftsExpr = terms.map(t => `"${t}"`).join(' OR ');
|
|
1887
2794
|
return this.db.prepare(`
|
|
1888
2795
|
SELECT cm.rowid, cm.chunk_id, bm25(chunks_fts) as fts_score
|
|
1889
2796
|
FROM chunks_fts
|
|
@@ -1953,7 +2860,19 @@ export class RAGKnowledgeGraphManager {
|
|
|
1953
2860
|
}
|
|
1954
2861
|
if (vectorResults.length === 0) {
|
|
1955
2862
|
console.error(`ℹ️ No vector or FTS5 matches found for "${query}"`);
|
|
1956
|
-
|
|
2863
|
+
// Empty results still carry state (spec §5c: envelope exists so callers
|
|
2864
|
+
// can distinguish "nothing matched" from "vector search was degraded").
|
|
2865
|
+
const covE = this.coordinator?.coverage();
|
|
2866
|
+
const chunkPctE = covE && covE.chunk.total > 0 ? Math.round((covE.chunk.embedded / covE.chunk.total) * 100) : 100;
|
|
2867
|
+
const graphPctE = covE && covE.entity.total > 0 ? Math.round((covE.entity.embedded / covE.entity.total) * 100) : 100;
|
|
2868
|
+
return {
|
|
2869
|
+
results: [],
|
|
2870
|
+
search_mode: vectorDegraded ? 'fts-only' : (chunkPctE < 100 ? 'hybrid-partial' : 'hybrid'),
|
|
2871
|
+
model_state: this.gate.status.state,
|
|
2872
|
+
coverage: { chunk_pct: chunkPctE, graph_coverage_pct: graphPctE },
|
|
2873
|
+
...(degradationReason ? { degradation_reason: degradationReason } : {}),
|
|
2874
|
+
...(ftsUnsearchable ? { warning: 'query has no searchable terms for FTS' } : {}),
|
|
2875
|
+
};
|
|
1957
2876
|
}
|
|
1958
2877
|
// Get entity information for graph enhancement via vector similarity
|
|
1959
2878
|
let connectedEntities = new Set();
|
|
@@ -2157,8 +3076,7 @@ export class RAGKnowledgeGraphManager {
|
|
|
2157
3076
|
fts_boost: ftsBoost > 0 ? ftsBoost : undefined,
|
|
2158
3077
|
full_context_available: true,
|
|
2159
3078
|
chunk_type: result.chunk_type,
|
|
2160
|
-
source_id: sourceId
|
|
2161
|
-
search_mode: vectorDegraded ? 'fts-only' : 'hybrid'
|
|
3079
|
+
source_id: sourceId
|
|
2162
3080
|
});
|
|
2163
3081
|
}
|
|
2164
3082
|
// Sort by relevance and return top results
|
|
@@ -2170,7 +3088,34 @@ export class RAGKnowledgeGraphManager {
|
|
|
2170
3088
|
const entityResults = finalResults.filter(r => r.chunk_type === 'entity').length;
|
|
2171
3089
|
const relResults = finalResults.filter(r => r.chunk_type === 'relationship').length;
|
|
2172
3090
|
console.error(`✅ Enhanced hybrid search completed: ${finalResults.length} results (${docResults} docs, ${entityResults} entities, ${relResults} relationships)`);
|
|
2173
|
-
|
|
3091
|
+
// v3.6 envelope (spec §5c, breaking): search_mode moved from per-item to
|
|
3092
|
+
// top-level so state is visible even on empty results; coverage tells the
|
|
3093
|
+
// caller how much of the corpus is actually vector-searchable.
|
|
3094
|
+
const cov = this.coordinator?.coverage();
|
|
3095
|
+
const chunkPct = cov && cov.chunk.total > 0 ? Math.round((cov.chunk.embedded / cov.chunk.total) * 100) : 100;
|
|
3096
|
+
const graphPct = cov && cov.entity.total > 0 ? Math.round((cov.entity.embedded / cov.entity.total) * 100) : 100;
|
|
3097
|
+
const search_mode = vectorDegraded ? 'fts-only' : (chunkPct < 100 ? 'hybrid-partial' : 'hybrid');
|
|
3098
|
+
return {
|
|
3099
|
+
results: finalResults,
|
|
3100
|
+
search_mode,
|
|
3101
|
+
model_state: this.gate.status.state,
|
|
3102
|
+
coverage: { chunk_pct: chunkPct, graph_coverage_pct: graphPct },
|
|
3103
|
+
...(degradationReason ? { degradation_reason: degradationReason } : {}),
|
|
3104
|
+
...(ftsUnsearchable ? { warning: 'query has no searchable terms for FTS' } : {}),
|
|
3105
|
+
};
|
|
3106
|
+
}
|
|
3107
|
+
// v3.6 (spec §5c / 6R note 2): why is vector search degraded right now?
|
|
3108
|
+
degradationReason() {
|
|
3109
|
+
if (this.gate.isDisabled)
|
|
3110
|
+
return 'disabled';
|
|
3111
|
+
if (!this.gate.isReady)
|
|
3112
|
+
return 'model_not_ready';
|
|
3113
|
+
const rs = this.coordinator?.reconState;
|
|
3114
|
+
if (rs === 'failed')
|
|
3115
|
+
return 'reconciliation_failed';
|
|
3116
|
+
if (rs && rs !== 'complete' && rs !== 'n/a')
|
|
3117
|
+
return 'reconciling';
|
|
3118
|
+
return undefined;
|
|
2174
3119
|
}
|
|
2175
3120
|
// NEW: Get detailed context for a specific chunk
|
|
2176
3121
|
async getDetailedContext(chunkId, includeSurrounding = true) {
|
|
@@ -2261,6 +3206,10 @@ export class RAGKnowledgeGraphManager {
|
|
|
2261
3206
|
const chunkCount = this.db.prepare(`
|
|
2262
3207
|
SELECT COUNT(*) as count FROM chunk_metadata
|
|
2263
3208
|
`).get();
|
|
3209
|
+
// v3.6 (spec §8-2, additive): server self-report — the framework's /start
|
|
3210
|
+
// reads version, model/reconciliation state, and provenance coverage here.
|
|
3211
|
+
const gs = this.gate.status;
|
|
3212
|
+
const cov = this.coordinator?.coverage();
|
|
2264
3213
|
return {
|
|
2265
3214
|
entities: {
|
|
2266
3215
|
total: entityStats.reduce((sum, stat) => sum + stat.count, 0),
|
|
@@ -2271,7 +3220,23 @@ export class RAGKnowledgeGraphManager {
|
|
|
2271
3220
|
by_type: Object.fromEntries(relationshipStats.map(s => [s.relationType, s.count]))
|
|
2272
3221
|
},
|
|
2273
3222
|
documents: documentCount.count,
|
|
2274
|
-
chunks: chunkCount.count
|
|
3223
|
+
chunks: chunkCount.count,
|
|
3224
|
+
server: {
|
|
3225
|
+
version: PKG_VERSION,
|
|
3226
|
+
node: process.versions.node,
|
|
3227
|
+
embeddings_mode: this.embeddingsMode,
|
|
3228
|
+
model: `${EMBEDDING_MODEL}@${MODEL_REVISION}`,
|
|
3229
|
+
model_state: gs.state,
|
|
3230
|
+
ready_since: gs.readySince ?? null,
|
|
3231
|
+
last_error: gs.lastError ? sanitizeErrorMessage(gs.lastError) : null,
|
|
3232
|
+
retry_at: gs.retryAt ?? null,
|
|
3233
|
+
reconciliation_state: this.coordinator?.reconState ?? 'n/a',
|
|
3234
|
+
reconciliation_last_error: this.coordinator?.reconLastError ?? null,
|
|
3235
|
+
coverage: cov ? {
|
|
3236
|
+
chunk: { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded },
|
|
3237
|
+
entity: { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded },
|
|
3238
|
+
} : null,
|
|
3239
|
+
}
|
|
2275
3240
|
};
|
|
2276
3241
|
}
|
|
2277
3242
|
// === GRAPH ANALYTICS TOOLS (graphology) ===
|
|
@@ -2595,13 +3560,39 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2595
3560
|
case "createRelations":
|
|
2596
3561
|
return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.createRelations(validatedArgs.relations), null, 2) }] };
|
|
2597
3562
|
case "addObservations":
|
|
3563
|
+
// v13: status·sources 를 그대로 넘긴다. 여기서 떨어뜨리면 스키마가 받아도
|
|
3564
|
+
// 엔진에 도달하지 않아 provenance 가 조용히 사라진다.
|
|
2598
3565
|
return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.addObservations(validatedArgs.observations), null, 2) }] };
|
|
3566
|
+
// v13 observation lifecycle (spec §6.1 / §6.2)
|
|
3567
|
+
case "correctObservation":
|
|
3568
|
+
return { content: [{ type: "text", text: JSON.stringify({ observation_id: await ragKgManager.correctObservation(validatedArgs.observation_id, validatedArgs.content, validatedArgs.change_kind ?? 'correction', validatedArgs.reason) }, null, 2) }] };
|
|
3569
|
+
case "retractObservation":
|
|
3570
|
+
await ragKgManager.retractObservation(validatedArgs.observation_id, validatedArgs.reason);
|
|
3571
|
+
return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'retracted' }, null, 2) }] };
|
|
3572
|
+
case "restoreObservation":
|
|
3573
|
+
await ragKgManager.restoreObservation(validatedArgs.observation_id, validatedArgs.reason);
|
|
3574
|
+
return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'active' }, null, 2) }] };
|
|
3575
|
+
case "approveObservation":
|
|
3576
|
+
await ragKgManager.approveObservation(validatedArgs.observation_id, validatedArgs.reason);
|
|
3577
|
+
return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'active' }, null, 2) }] };
|
|
3578
|
+
case "declineObservation":
|
|
3579
|
+
await ragKgManager.declineObservation(validatedArgs.observation_id, validatedArgs.reason);
|
|
3580
|
+
return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'retracted' }, null, 2) }] };
|
|
3581
|
+
case "purgeObservation":
|
|
3582
|
+
return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.purgeObservation(validatedArgs.observation_id, validatedArgs.confirm), null, 2) }] };
|
|
3583
|
+
case "getObservationHistory":
|
|
3584
|
+
return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.getObservationHistory({
|
|
3585
|
+
entity_name: validatedArgs.entity_name,
|
|
3586
|
+
observation_id: validatedArgs.observation_id,
|
|
3587
|
+
root_id: validatedArgs.root_id,
|
|
3588
|
+
}), null, 2) }] };
|
|
2599
3589
|
case "deleteEntities":
|
|
2600
3590
|
await ragKgManager.deleteEntities(validatedArgs.entityNames);
|
|
2601
3591
|
return { content: [{ type: "text", text: "Entities deleted successfully" }] };
|
|
2602
3592
|
case "deleteObservations":
|
|
2603
|
-
|
|
2604
|
-
|
|
3593
|
+
// v3.6 (spec §5c, breaking): structured per-entity results replace the
|
|
3594
|
+
// bare success string — mixed embedded/queued/no-op states are visible.
|
|
3595
|
+
return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.deleteObservations(validatedArgs.deletions), null, 2) }] };
|
|
2605
3596
|
case "deleteRelations":
|
|
2606
3597
|
await ragKgManager.deleteRelations(validatedArgs.relations);
|
|
2607
3598
|
return { content: [{ type: "text", text: "Relations deleted successfully" }] };
|
|
@@ -2672,35 +3663,95 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2672
3663
|
}
|
|
2673
3664
|
}
|
|
2674
3665
|
catch (error) {
|
|
3666
|
+
// v3.6 (spec §5c): machine-distinguishable failures. Embedding-gate errors
|
|
3667
|
+
// become structured retryable/terminal payloads; every error response now
|
|
3668
|
+
// sets isError so clients stop parsing "Error: ..." strings.
|
|
3669
|
+
if (error instanceof GateNotReadyError) {
|
|
3670
|
+
return { isError: true, content: [{ type: "text", text: JSON.stringify({
|
|
3671
|
+
code: error.code, state: error.state,
|
|
3672
|
+
...(error.retryAfterMs !== undefined ? { retry_after_ms: error.retryAfterMs } : {}),
|
|
3673
|
+
message: error.message
|
|
3674
|
+
}) }] };
|
|
3675
|
+
}
|
|
3676
|
+
if (error instanceof GateDisabledError) {
|
|
3677
|
+
return { isError: true, content: [{ type: "text", text: JSON.stringify({
|
|
3678
|
+
code: error.code, state: error.state, message: error.message
|
|
3679
|
+
}) }] };
|
|
3680
|
+
}
|
|
2675
3681
|
if (error instanceof Error) {
|
|
2676
3682
|
console.error(`❌ Tool execution error for ${name}:`, error.message);
|
|
2677
|
-
return { content: [{ type: "text", text: `Error: ${error.message}` }] };
|
|
3683
|
+
return { isError: true, content: [{ type: "text", text: `Error: ${error.message}` }] };
|
|
2678
3684
|
}
|
|
2679
3685
|
throw error;
|
|
2680
3686
|
}
|
|
2681
3687
|
});
|
|
2682
3688
|
async function main() {
|
|
2683
3689
|
try {
|
|
3690
|
+
assertNodeVersion();
|
|
2684
3691
|
await ragKgManager.initialize();
|
|
3692
|
+
printBanner({
|
|
3693
|
+
model: EMBEDDING_MODEL, revision: MODEL_REVISION, dtype: MODEL_DTYPE,
|
|
3694
|
+
cachePath: resolveModelCacheDir(process.env, process.platform, os.homedir()),
|
|
3695
|
+
dbPath: DB_FILE_PATH,
|
|
3696
|
+
});
|
|
3697
|
+
if (ragKgManager.embeddingsMode === 'eager') {
|
|
3698
|
+
// eager = wait for BOTH the first model load attempt and reconciliation to
|
|
3699
|
+
// settle (success or failure) before connecting — v3.5-equivalent boot
|
|
3700
|
+
// extended to legacy DBs (spec §9). Failures fall back to background retry.
|
|
3701
|
+
await Promise.allSettled([ragKgManager.gate.start(), ragKgManager.startReconciliation()]);
|
|
3702
|
+
}
|
|
2685
3703
|
const transport = new StdioServerTransport();
|
|
2686
3704
|
await server.connect(transport);
|
|
2687
3705
|
console.error("🚀 Enhanced RAG Knowledge Graph MCP Server running on stdio");
|
|
2688
|
-
|
|
3706
|
+
if (ragKgManager.embeddingsMode === 'lazy') {
|
|
3707
|
+
// Background: model load + provenance reconciliation run in parallel.
|
|
3708
|
+
// Failures surface via gate/coordinator state, never as rejections.
|
|
3709
|
+
void ragKgManager.gate.start().catch(() => { });
|
|
3710
|
+
void ragKgManager.startReconciliation().catch(() => { });
|
|
3711
|
+
}
|
|
3712
|
+
else if (ragKgManager.embeddingsMode === 'off') {
|
|
3713
|
+
// off mode still CLASSIFIES reconciliation state (deferred vs n/a) so
|
|
3714
|
+
// stats honor the mode matrix — no sanitation, no inference (beta B7).
|
|
3715
|
+
void ragKgManager.startReconciliation().catch(() => { });
|
|
3716
|
+
}
|
|
3717
|
+
// Graceful shutdown (spec §3 order, beta-2R-amended) — transport close
|
|
3718
|
+
// FIRST so stdin stops holding the event loop, then settle coordinator and
|
|
3719
|
+
// gate, then DB close. process.exit is forbidden EXCEPT the one spec'd
|
|
3720
|
+
// case: a model load/download still pending after the settle deadline
|
|
3721
|
+
// (un-abortable fetch would hold the loop forever) — see shutdownAll.
|
|
3722
|
+
let shuttingDown = false;
|
|
2689
3723
|
const shutdown = () => {
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
3724
|
+
if (shuttingDown)
|
|
3725
|
+
return;
|
|
3726
|
+
shuttingDown = true;
|
|
3727
|
+
void (async () => {
|
|
3728
|
+
try {
|
|
3729
|
+
await server.close();
|
|
3730
|
+
}
|
|
3731
|
+
catch { /* transport already gone */ }
|
|
3732
|
+
try {
|
|
3733
|
+
process.stdin.pause();
|
|
3734
|
+
process.stdin.unref?.();
|
|
3735
|
+
}
|
|
3736
|
+
catch { /* best-effort */ }
|
|
3737
|
+
await ragKgManager.shutdownAll();
|
|
3738
|
+
})();
|
|
2695
3739
|
};
|
|
2696
3740
|
process.on('SIGINT', shutdown);
|
|
2697
3741
|
process.on('SIGTERM', shutdown);
|
|
2698
|
-
process.on('exit',
|
|
3742
|
+
process.on('exit', () => { try {
|
|
3743
|
+
ragKgManager.cleanup();
|
|
3744
|
+
}
|
|
3745
|
+
catch { /* idempotent */ } });
|
|
3746
|
+
console.error('🛡️ shutdown handlers registered'); // deterministic handler-ready marker (5R test residual)
|
|
2699
3747
|
}
|
|
2700
3748
|
catch (error) {
|
|
2701
3749
|
console.error("Failed to initialize server:", error);
|
|
2702
|
-
|
|
2703
|
-
|
|
3750
|
+
try {
|
|
3751
|
+
ragKgManager.cleanup();
|
|
3752
|
+
}
|
|
3753
|
+
catch { /* already down */ }
|
|
3754
|
+
process.exitCode = 1;
|
|
2704
3755
|
}
|
|
2705
3756
|
}
|
|
2706
3757
|
// Boot the server unless explicitly suppressed. Tests import this module with
|
|
@@ -2711,7 +3762,10 @@ async function main() {
|
|
|
2711
3762
|
if (process.env.RAG_MEMORY_NO_AUTOSTART !== '1') {
|
|
2712
3763
|
main().catch((error) => {
|
|
2713
3764
|
console.error("Fatal error in main():", error);
|
|
2714
|
-
|
|
2715
|
-
|
|
3765
|
+
try {
|
|
3766
|
+
ragKgManager.cleanup();
|
|
3767
|
+
}
|
|
3768
|
+
catch { /* already down */ }
|
|
3769
|
+
process.exitCode = 1;
|
|
2716
3770
|
});
|
|
2717
3771
|
}
|