rag-memory-epf-mcp 3.5.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -21,13 +21,59 @@ import modularity from 'graphology-metrics/graph/modularity.js';
21
21
  import { getAllMCPTools, validateToolArgs, getSystemInfo } from './src/tools/tool-registry.js';
22
22
  // Import migration system
23
23
  import { MigrationManager } from './src/migrations/migration-manager.js';
24
+ import { backupBeforeMigration } from './src/backup/preflight.js';
25
+ import { rebuildProjection, deleteStaleKgChunks } from './src/observations/projection.js';
26
+ import { addRevision, correctRevision, transitionStatus, linkSources, nextProjectionOrder } from './src/observations/lifecycle.js';
27
+ import { getObservationHistory } from './src/observations/history.js';
24
28
  // Import chunk text algorithm (extracted for publish-time invariant testing)
25
29
  import { chunkText as splitTextIntoChunks } from './src/chunkText.js';
26
30
  import { migrations } from './src/migrations/migrations.js';
31
+ // v3.6 lite install: model lifecycle + version-independent cache (A′ boundary)
32
+ import { EmbeddingGate, GateNotReadyError, GateDisabledError, TerminalConfigError } from './src/embeddingGate.js';
33
+ import { resolveModelCacheDir, preflightCacheDir, artifactKey, ModelDownloadLock, handleLoaderFailure } from './src/modelCache.js';
34
+ import { BackfillCoordinator } from './src/backfillCoordinator.js';
35
+ import os from 'node:os';
27
36
  import { createHash } from 'crypto';
28
37
  import { createRequire } from 'module';
29
38
  const require = createRequire(import.meta.url);
30
39
  const PKG_VERSION = require('../package.json').version;
40
+ // v3.6: runtime Node floor (engines is advisory only under default npm config).
41
+ // Limitation: static native imports above may fail before this runs on very old
42
+ // Node — documented in docs/UPDATING.md.
43
+ const NODE_MAJOR = Number(process.versions.node.split('.')[0]);
44
+ function assertNodeVersion() {
45
+ if (NODE_MAJOR < 24) {
46
+ console.error(`❌ rag-memory-epf-mcp v${PKG_VERSION} requires Node >= 24 (current: ${process.versions.node}).`);
47
+ console.error(' See docs/UPDATING.md for the supported runtime matrix.');
48
+ process.exitCode = 1;
49
+ throw new Error('unsupported Node version');
50
+ }
51
+ }
52
+ // v3.6: strip tokens / auth material / long URLs from operator-facing error text.
53
+ function sanitizeErrorMessage(msg) {
54
+ return msg
55
+ .replace(/(hf_|api[_-]?key=|authorization:\s*)\S+/gi, '$1[redacted]')
56
+ .replace(/https?:\/\/\S{60,}/g, '[url]')
57
+ .slice(0, 500);
58
+ }
59
+ // v3.6: startup self-report banner (version reliability — spec §8).
60
+ function printBanner(opts) {
61
+ console.error(`🚀 rag-memory-epf-mcp v${PKG_VERSION} | node v${process.versions.node} | model ${opts.model}@${opts.revision} (${opts.dtype}) | cache ${opts.cachePath} | db ${opts.dbPath}`);
62
+ }
63
+ // v3.6 (spec §5): ONE FTS5 literal-query compiler shared by chunk and entity
64
+ // search — raw user input can never produce MATCH syntax errors or trigger
65
+ // operators (every term is double-quoted; special characters stripped exactly
66
+ // as the pre-3.6 hybridSearch sanitizer did). Returns null when nothing
67
+ // searchable remains (contract: caller returns empty results + warning).
68
+ export function compileFtsLiteralQuery(q) {
69
+ const sanitized = q.replace(/["\*\(\)\-]/g, ' ').trim();
70
+ if (!sanitized)
71
+ return null;
72
+ const terms = sanitized.split(/\s+/).filter(t => t.length > 0);
73
+ if (terms.length === 0)
74
+ return null;
75
+ return terms.map(t => `"${t}"`).join(' OR ');
76
+ }
31
77
  // Configure Hugging Face transformers for better compatibility
32
78
  if (env.backends?.onnx?.wasm) {
33
79
  env.backends.onnx.wasm.wasmPaths = './node_modules/@huggingface/transformers/dist/';
@@ -40,6 +86,21 @@ const DB_FILE_PATH = process.env.DB_FILE_PATH
40
86
  : path.join(path.dirname(fileURLToPath(import.meta.url)), process.env.DB_FILE_PATH)
41
87
  : defaultDbPath;
42
88
  const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'Xenova/bge-m3';
89
+ // v3.5 default model config — grandfathering legacy vectors is only automatic
90
+ // when the current config matches this (spec §6b custom-model guard). An
91
+ // EXPLICIT `EMBEDDING_MODEL=Xenova/bge-m3` counts as default: same weights,
92
+ // same pin, same grandfather policy (beta 1R consistency fix).
93
+ const IS_DEFAULT_MODEL_CONFIG = !process.env.EMBEDDING_MODEL || process.env.EMBEDDING_MODEL === 'Xenova/bge-m3';
94
+ // Default model pinned to an upstream commit (spec §6c): a shared version-
95
+ // independent cache must never silently swap weights under 'main'. Verified
96
+ // 2026-07-18 via `git ls-remote https://huggingface.co/Xenova/bge-m3` — the
97
+ // same revision the local v3.5 cache was downloaded from. Custom models stay
98
+ // on 'main' (their vectors are never auto-grandfathered anyway).
99
+ const MODEL_REVISION = IS_DEFAULT_MODEL_CONFIG ? '4de13258303883538bd53b696b452bf8099f0858' : 'main';
100
+ const MODEL_DTYPE = 'fp16';
101
+ // Entity embedding text builder version — mixed into entity input hashes so a
102
+ // builder change re-backfills entities without touching chunk vectors (spec §6c).
103
+ const TEXT_BUILDER_VERSION = 'tb1';
43
104
  // Safe rowid for vec0 virtual tables (require literal integer, not parameterized)
44
105
  // Trim incomplete UTF-8 multi-byte sequences at chunk boundaries.
45
106
  // Continuation bytes match 10xxxxxx (0x80-0xBF); lead bytes indicate how many
@@ -59,18 +120,26 @@ function safeRowid(value) {
59
120
  export class RAGKnowledgeGraphManager {
60
121
  db = null;
61
122
  encoding = null;
62
- embeddingModel = null;
63
- modelInitialized = false;
123
+ gate;
124
+ embeddingsMode = 'lazy';
125
+ currentProfileId = 0;
126
+ // Automatic grandfathering of legacy vectors is only allowed under the v3.5
127
+ // default model config, or with the explicit trust opt-in (spec §6b guard).
128
+ grandfatherAllowed = IS_DEFAULT_MODEL_CONFIG || process.env.RAG_MEMORY_TRUST_LEGACY_VECTORS === '1';
129
+ coordinator = null;
64
130
  embeddingCache = new Map();
65
131
  EMBEDDING_CACHE_MAX = 500;
66
132
  dictionaryCache = null;
133
+ // v3.6 (spec §3): initialize = DB + migrations + profile only. The embedding
134
+ // model is NEVER awaited here — main() connects the MCP server first and the
135
+ // gate loads in the background (lazy) or is awaited explicitly (eager).
136
+ // __testForceFkOff: 음성 대조군 전용. FK 게이트가 실제로 부팅을 막는지 시험한다.
137
+ // 이름에 __test 를 박아 둔 이유는 이것이 프로덕션 설정 표면이 아니라는 것을
138
+ // 호출부에서 읽히게 하기 위해서다.
67
139
  async initialize(opts = {}) {
68
140
  console.error('🚀 Initializing RAG Knowledge Graph MCP Server...');
69
- // Initialize database
70
141
  this.db = new Database(DB_FILE_PATH);
71
- // Load sqlite-vec extension
72
142
  sqliteVec.load(this.db);
73
- // SQLite performance & safety optimizations
74
143
  this.db.pragma('journal_mode = WAL');
75
144
  this.db.pragma('synchronous = NORMAL');
76
145
  this.db.pragma('busy_timeout = 5000');
@@ -78,37 +147,332 @@ export class RAGKnowledgeGraphManager {
78
147
  this.db.pragma('temp_store = MEMORY');
79
148
  this.db.pragma('mmap_size = 268435456');
80
149
  this.db.pragma('foreign_keys = ON');
81
- // Initialize tiktoken
82
- this.encoding = get_encoding("cl100k_base");
83
- // Initialize embedding model (skippable for tests / FTS-only environments)
84
- if (!opts.skipModel) {
85
- await this.initializeEmbeddingModel();
86
- }
87
- else {
88
- console.error('⏭️ Skipping embedding model load (skipModel=true)');
150
+ // spec §5.2: 관찰 lifecycle 의 무결성은 전부 FK CASCADE 를 전제한다 — root 를 지우면
151
+ // revision 이, revision 을 지우면 source 가 따라가야 history 가 고아로 남지 않는다.
152
+ // FK 가 꺼진 채 돌면 그 계약이 조용히 무효가 되고, 그게 최악이다. 스키마를 건드리기
153
+ // 전에(= runMigrations 앞에서) 멈춘다.
154
+ // 트랜잭션 내부에서는 이 pragma 가 no-op 이므로(실측 before=1·during=1·after=1)
155
+ // 부팅 시점 확인이 유일한 방어 지점이다.
156
+ //
157
+ // 음성 대조군 주입은 **인자로만** 받는다. 환경변수로 두면 프로덕션 경로에
158
+ // "부팅을 막는 스위치"가 상시 존재하게 되고, 오설정 한 줄로 서버가 안 뜬다
159
+ // (advisor beta 자기의심 2 = "더 나쁘다"). 테스트는 manager 를 직접 만들므로
160
+ // 인자 주입으로 충분하다.
161
+ if (opts.__testForceFkOff)
162
+ this.db.pragma('foreign_keys = OFF');
163
+ {
164
+ const fk = this.db.pragma('foreign_keys', { simple: true });
165
+ if (Number(fk) !== 1) {
166
+ throw new Error(`foreign_keys is ${fk}, expected 1. The observation lifecycle relies on FK CASCADE ` +
167
+ `for history integrity; refusing to run migrations without it.`);
168
+ }
89
169
  }
90
- // Run database migrations
170
+ this.encoding = get_encoding("cl100k_base");
91
171
  await this.runMigrations();
92
- console.error('✅ RAG-enabled knowledge graph initialized');
93
- // Log system info
172
+ this.currentProfileId = this.ensureCurrentProfile();
173
+ this.embeddingsMode = opts.skipModel
174
+ ? 'off'
175
+ : (process.env.RAG_MEMORY_EMBEDDINGS || 'lazy');
176
+ if (!['lazy', 'eager', 'off'].includes(this.embeddingsMode))
177
+ this.embeddingsMode = 'lazy';
178
+ this.gate = opts.gate ?? new EmbeddingGate({
179
+ mode: this.embeddingsMode,
180
+ loadModel: () => this.buildRealLoader(),
181
+ onReady: () => this.coordinator?.kick(),
182
+ });
183
+ // Late-bound deps (closures): tests swap manager.gate / flip the guard.
184
+ this.coordinator = new BackfillCoordinator({
185
+ db: () => this.db,
186
+ gateIsReady: () => this.gate.isReady,
187
+ gateIsDisabled: () => this.gate.isDisabled,
188
+ mode: () => this.embeddingsMode,
189
+ grandfatherAllowed: () => this.grandfatherAllowed,
190
+ currentProfileId: () => this.currentProfileId,
191
+ buildEntityInputHash: (entityId) => this.entityInputHash(entityId),
192
+ hashEntityText: (text) => this.hashWithBuilderVersion(text),
193
+ chunkInputHash: (text) => createHash('sha256').update(text).digest('hex'),
194
+ reembedEntity: async (entityId) => this.embedEntity(entityId, 'backfill'),
195
+ reembedChunk: async (rowid) => this.reembedChunkByRowid(rowid),
196
+ });
197
+ console.error('✅ RAG-enabled knowledge graph initialized (embedding model deferred)');
94
198
  const systemInfo = getSystemInfo();
95
199
  console.error(`📊 System Info: ${systemInfo.toolCounts.total} tools available (${systemInfo.toolCounts.knowledgeGraph} knowledge graph, ${systemInfo.toolCounts.rag} RAG, ${systemInfo.toolCounts.graphQuery} query)`);
96
200
  }
97
- async initializeEmbeddingModel() {
201
+ // Upsert the stored-vector compatibility profile (spec §6c layer 2) and record
202
+ // retrieval config in server_meta (layer 3 — never a backfill trigger).
203
+ ensureCurrentProfile() {
204
+ if (!this.db)
205
+ throw new Error('Database not initialized');
206
+ const dims = 1024;
207
+ if (dims !== 1024)
208
+ throw new Error('unsupported embedding dims (vec0 tables are fixed at 1024)'); // fail-fast contract
209
+ this.db.prepare(`INSERT OR IGNORE INTO embedding_profiles
210
+ (model_id, revision, dtype, dims, pooling, normalize) VALUES (?,?,?,?,?,?)`)
211
+ .run(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE, dims, 'cls', 1);
212
+ const row = this.db.prepare(`SELECT id FROM embedding_profiles
213
+ WHERE model_id=? AND revision=? AND dtype=? AND dims=? AND pooling=? AND normalize=?`)
214
+ .get(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE, dims, 'cls', 1);
215
+ this.db.prepare(`INSERT INTO server_meta(key,value) VALUES('current_profile_id',?)
216
+ ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(String(row.id));
217
+ this.db.prepare(`INSERT INTO server_meta(key,value) VALUES('query_prefix_version','1')
218
+ ON CONFLICT(key) DO NOTHING`).run();
219
+ return row.id;
220
+ }
221
+ // Real model loader used by the gate: version-independent cache dir with a
222
+ // cross-process download lock. Preflight failure throws (gate -> failed);
223
+ // silently falling back to the package-internal cache is forbidden (spec §7).
224
+ async buildRealLoader() {
225
+ const cacheDir = resolveModelCacheDir(process.env, process.platform, os.homedir());
226
+ const pf = preflightCacheDir(cacheDir);
227
+ if (!pf.ok)
228
+ throw new Error(`model cache dir not writable (${cacheDir}): ${pf.error}`);
229
+ const key = artifactKey(EMBEDDING_MODEL, MODEL_REVISION, MODEL_DTYPE);
230
+ const lock = new ModelDownloadLock(cacheDir, key);
231
+ // Shutdown aborts the lock wait via the gate's AbortController (spec §3).
232
+ console.error('⏳ acquiring model download lock...'); // deterministic lock-wait marker (5R test residual)
233
+ const role = await lock.acquireOrWait({ timeoutMs: 10 * 60_000, signal: this.gate.abort.signal });
98
234
  try {
99
- console.error(`🤖 Loading embedding model: ${EMBEDDING_MODEL} (1024-dim, 100+ languages)...`);
100
- // Configure environment to allow remote model downloads
235
+ this.gate.markDownloading();
101
236
  env.allowRemoteModels = true;
102
237
  env.allowLocalModels = true;
103
- this.embeddingModel = await pipeline('feature-extraction', EMBEDDING_MODEL, { revision: 'main', dtype: 'fp16' });
104
- this.modelInitialized = true;
105
- console.error(`✅ ${EMBEDDING_MODEL} model loaded successfully (fp16)`);
238
+ console.error(`🤖 Loading embedding model: ${EMBEDDING_MODEL} (1024-dim, cache=${cacheDir})...`);
239
+ const model = await pipeline('feature-extraction', EMBEDDING_MODEL, { revision: MODEL_REVISION, dtype: MODEL_DTYPE, cache_dir: cacheDir });
240
+ // dims fail-fast (spec §2 / beta B8): probe the ACTUAL output length — a
241
+ // 384/768-dim custom model must fail here with a clear message, not at
242
+ // every subsequent vector write.
243
+ const probe = await model('dimension probe', { pooling: 'cls', normalize: true });
244
+ const actualDims = probe.data.length;
245
+ if (actualDims !== 1024) {
246
+ // Config incompatibility, NOT cache corruption (beta 2R B3): the
247
+ // download and load both succeeded — quarantining or retrying cannot
248
+ // change the model's dimensions.
249
+ if (role === 'owner')
250
+ lock.markComplete(); // cache itself is valid
251
+ throw new TerminalConfigError(`embedding model ${EMBEDDING_MODEL} outputs ${actualDims} dims — this engine's vec0 tables are fixed at 1024. Use a 1024-dim model.`);
252
+ }
253
+ if (role === 'owner')
254
+ lock.markComplete();
255
+ console.error(`✅ ${EMBEDDING_MODEL} model loaded (${MODEL_DTYPE})`);
256
+ return async (text, dims, isQuery) => {
257
+ const input = isQuery ? `Represent this sentence for searching relevant passages: ${text}` : text;
258
+ const r = await model(input, { pooling: 'cls', normalize: true });
259
+ return new Float32Array(r.data.slice(0, dims));
260
+ };
106
261
  }
107
- catch (error) {
108
- console.error('❌ Failed to load embedding model:', error instanceof Error ? error.message : error);
109
- console.error('⚠️ Semantic search will be unavailable. Other tools (CRUD, graph queries) still work.');
110
- console.error('💡 Fix: Ensure ONNX model cache is accessible. Try: rm -rf ~/.npm/_onnx_models && restart.');
111
- this.modelInitialized = false;
262
+ catch (e) {
263
+ // Cache policy by CAUSE and ROLE (beta 2R B3 -> 4R M1 -> 5R M1), unit-
264
+ // tested in modelCache: config errors touch nothing; integrity errors
265
+ // invalidate the marker, and only a lock-holding OWNER may quarantine
266
+ // (a ready-role process racing other readers never deletes shared
267
+ // files); OOM/network/unknown preserve everything.
268
+ const action = handleLoaderFailure({
269
+ role, error: e, lock, cacheDir, modelId: EMBEDDING_MODEL,
270
+ terminal: e instanceof TerminalConfigError,
271
+ });
272
+ if (action === 'quarantined')
273
+ console.error('🧹 cache-integrity failure (owner) — model cache quarantined');
274
+ else if (action === 'marker-invalidated')
275
+ console.error('… cache-integrity failure (reader) — marker dropped, next retry re-proves as locked owner');
276
+ else if (!(e instanceof TerminalConfigError))
277
+ console.error('… non-integrity load failure — model cache preserved');
278
+ throw e;
279
+ }
280
+ finally {
281
+ lock.release();
282
+ }
283
+ }
284
+ // Background provenance reconciliation (spec §6b). Runs in parallel with the
285
+ // model load; vector search and automatic backfill stay closed until it
286
+ // settles (eligibility barrier in the coordinator).
287
+ async startReconciliation() {
288
+ if (!this.coordinator)
289
+ return;
290
+ await this.coordinator.runReconciliation();
291
+ this.coordinator.sweepStart();
292
+ }
293
+ // sha256 with the entity text-builder version mixed in: a builder change
294
+ // re-backfills entities only, never chunks (spec §6c N2).
295
+ hashWithBuilderVersion(text) {
296
+ return createHash('sha256').update(`${TEXT_BUILDER_VERSION}\n${text}`).digest('hex');
297
+ }
298
+ // Rebuild the CURRENT embedding input hash for an entity. null = entity gone
299
+ // or malformed observations — reconciliation fail-closes to missing.
300
+ entityInputHash(entityId) {
301
+ try {
302
+ const entity = this.db.prepare(`SELECT name, entityType, observations FROM entities WHERE id = ?`)
303
+ .get(entityId);
304
+ if (!entity)
305
+ return null;
306
+ const built = this.buildEntityEmbeddingText({
307
+ name: entity.name,
308
+ entityType: entity.entityType,
309
+ observations: JSON.parse(entity.observations),
310
+ });
311
+ return this.hashWithBuilderVersion(built.text);
312
+ }
313
+ catch {
314
+ return null;
315
+ }
316
+ }
317
+ // Mutation-path embedding wrapper (spec §5): CRUD success never depends on
318
+ // model availability. On not-ready/disabled the stale vector is deleted in
319
+ // the same breath (dirty = missing, §6a-1) and the coordinator is kicked so
320
+ // the row is recovered without a restart (§5 kick column).
321
+ async tryEmbedEntity(entityId, priority = 'bulk') {
322
+ try {
323
+ const ok = await this.embedEntity(entityId, priority);
324
+ if (ok) {
325
+ // Success clears any stale backfill-failure record for this target.
326
+ this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'entity' AND target_id = ?`).run(entityId);
327
+ this.coordinator?.invalidateCoverage();
328
+ return 'embedded';
329
+ }
330
+ this.invalidateEntityVector(entityId);
331
+ this.coordinator?.kick();
332
+ return 'queued';
333
+ }
334
+ catch (e) {
335
+ // Any embedding-layer failure (not-ready, disabled, OR a ready-state
336
+ // inference error) must not fail the CRUD that already committed. The
337
+ // vector is invalidated (§6a-1) and recovery is owned by the backfill
338
+ // scanner with its attempts cap — never by rethrowing here (spec §5).
339
+ if (!(e instanceof GateNotReadyError) && !(e instanceof GateDisabledError)) {
340
+ console.error(`⚠️ embedding failed for ${entityId} (queued for backfill): ${e instanceof Error ? e.message : e}`);
341
+ }
342
+ this.invalidateEntityVector(entityId);
343
+ this.coordinator?.kick();
344
+ return e instanceof GateDisabledError ? 'disabled' : 'queued';
345
+ }
346
+ }
347
+ // §6a-1 (beta B2): the entity change and the stale-vector removal commit in
348
+ // ONE synchronous transaction, BEFORE any inference await. No window exists
349
+ // where another tool call can retrieve the pre-mutation vector, and a crash
350
+ // between mutation and re-embed leaves a clean missing state (backfill
351
+ // target), never a stale-searchable one.
352
+ // spec §4.5 단계 1: 관찰 변경 · projection 재합성 · entity vector 무효화 ·
353
+ // stale KG chunk 제거를 한 트랜잭션으로 묶는다. 하나만 되면 검색이 낡은
354
+ // 사실을 계속 반환한다.
355
+ // mutate 가 명시적으로 false 를 반환하면 "아무것도 바꾸지 않았다"는 뜻이고
356
+ // projection 재합성·벡터 무효화·KG 정리를 건너뛴다. 이 경로가 없으면
357
+ // 무변경 upsert 나 dedup-only add 가 **정상 벡터를 지우고 재임베딩도 안 해서**
358
+ // 검색 품질만 깎는다(advisor 구현리뷰 r1 발견 1, 실행 재현).
359
+ // 반환값 = 실제로 변경이 있었는가.
360
+ mutateEntityAndInvalidate(entityId, mutate) {
361
+ let changed = false;
362
+ const tx = this.db.transaction(() => {
363
+ changed = mutate() !== false;
364
+ if (!changed)
365
+ return;
366
+ rebuildProjection(this.db, entityId);
367
+ this.invalidateDerivedForEntity(entityId);
368
+ });
369
+ tx();
370
+ if (changed)
371
+ this.coordinator?.invalidateCoverage();
372
+ return changed;
373
+ }
374
+ // 관찰이 바뀐 entity 의 파생 상태를 무효화한다: entity vector + stale KG chunk.
375
+ // **트랜잭션을 열지 않는다** — 호출자가 이미 하나의 단위 안에 있다고 가정한다.
376
+ //
377
+ // importGraph 가 이 단계를 건너뛰고 있었다: projection 만 재합성하고 파생 상태를
378
+ // 그대로 둬서, 이미 존재하는 entity 를 import 로 덮으면 옛 벡터·옛 KG chunk 가
379
+ // 계속 검색에 나왔다(advisor beta 발견 2, hybridSearch 로 실측 재현).
380
+ // 그래서 "모든 관찰 변경은 mutateEntityAndInvalidate 를 통한다"는 규칙에
381
+ // 예외가 하나 있었고, 그 예외가 정확히 그 규칙이 막으려던 결함을 만들었다.
382
+ invalidateDerivedForEntity(entityId) {
383
+ const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
384
+ .get(entityId);
385
+ if (meta) {
386
+ this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
387
+ this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
388
+ }
389
+ deleteStaleKgChunks(this.db, entityId);
390
+ }
391
+ // §6a-1 invariant: when an entity's embedding input changed but re-embedding
392
+ // is unavailable, its old vector must not stay searchable.
393
+ invalidateEntityVector(entityId) {
394
+ if (!this.db)
395
+ return;
396
+ const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
397
+ .get(entityId);
398
+ if (!meta)
399
+ return;
400
+ const tx = this.db.transaction(() => {
401
+ this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
402
+ this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
403
+ });
404
+ tx();
405
+ this.coordinator?.invalidateCoverage();
406
+ }
407
+ // Backfill callback: re-embed one chunk and commit vector + provenance in a
408
+ // single transaction (§6a-2).
409
+ async reembedChunkByRowid(rowid) {
410
+ if (!this.db)
411
+ return false;
412
+ const row = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`)
413
+ .get(rowid);
414
+ if (!row || row.text === null)
415
+ return false;
416
+ try {
417
+ const embedding = await this.generateEmbedding(row.text, 1024, false, 'backfill');
418
+ const hash = createHash('sha256').update(row.text).digest('hex');
419
+ const safe = Number(rowid);
420
+ // Write-back CAS (beta 2R B1): the rowid may have been deleted and reused
421
+ // by a re-sync while inference ran — re-read the CURRENT text in the
422
+ // transaction and only write when it still matches what was embedded.
423
+ const tx = this.db.transaction(() => {
424
+ const cur = this.db.prepare(`SELECT text FROM chunk_metadata WHERE rowid = ?`).get(rowid);
425
+ if (!cur || cur.text !== row.text)
426
+ return false; // superseded — discard
427
+ this.db.exec(`DELETE FROM chunks WHERE rowid = ${safe}`);
428
+ this.db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${safe}, ?)`).run(Buffer.from(embedding.buffer));
429
+ this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
430
+ .run(hash, this.currentProfileId, rowid);
431
+ this.db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id = ?`).run(String(rowid));
432
+ return true;
433
+ });
434
+ const written = tx();
435
+ this.coordinator?.invalidateCoverage();
436
+ return written;
437
+ }
438
+ catch (e) {
439
+ if (e instanceof GateNotReadyError || e instanceof GateDisabledError)
440
+ return false;
441
+ throw e;
442
+ }
443
+ }
444
+ // spec §3 shutdown order (beta B1): block new batches -> settle coordinator
445
+ // (INCLUDING an in-flight reconciliation pass) -> settle gate (INCLUDING an
446
+ // in-flight model load, bounded) -> close DB -> natural exit.
447
+ //
448
+ // Bounded-exit rationale, re-derived after beta 1R: the exit decision is made
449
+ // AFTER the settle wait, not before — if the load completed during settling
450
+ // (ONNX session now exists) we take the natural-exit path. Only when the load
451
+ // is STILL pending after the deadline (dominant case: the 1.2GB download,
452
+ // which is un-abortable through transformers.js and would hold the event
453
+ // loop indefinitely) do we exit(). At that point the DB is already closed
454
+ // cleanly, so even the residual worst case — the load being inside ONNX
455
+ // session construction at exit — risks an ugly abort message, never data
456
+ // loss. Hanging forever is the alternative and is worse.
457
+ async shutdownAll() {
458
+ console.error('\n🧹 Cleaning up...');
459
+ try {
460
+ await this.coordinator?.shutdown(5000);
461
+ }
462
+ catch { /* settle best-effort */ }
463
+ try {
464
+ await this.gate?.shutdown(5000);
465
+ }
466
+ catch { /* settle best-effort */ }
467
+ const loadStillPending = this.gate?.loadInFlight ?? false;
468
+ try {
469
+ this.cleanup();
470
+ }
471
+ catch { /* DB close */ }
472
+ process.exitCode = process.exitCode ?? 0;
473
+ if (loadStillPending) {
474
+ console.error('… model load/download still in flight after settle deadline — bounded exit (DB already closed)');
475
+ process.exit(process.exitCode);
112
476
  }
113
477
  }
114
478
  async runMigrations() {
@@ -123,6 +487,11 @@ export class RAGKnowledgeGraphManager {
123
487
  });
124
488
  // Get pending migrations before running them
125
489
  const pendingBefore = migrationManager.getPendingMigrations();
490
+ // spec §5.1: 대기 중 마이그레이션이 있으면 먼저 일관 스냅샷을 남긴다.
491
+ // 실패는 throw = fail-closed (백업 없이 스키마를 바꾸지 않는다).
492
+ // await: 백업은 Online Backup API 를 쓰므로 비동기다. 여기서 await 를 빠뜨리면
493
+ // 백업이 끝나기 전에 마이그레이션이 시작한다 = 백업 없이 스키마를 바꾸는 것이다.
494
+ await backupBeforeMigration(this.db, DB_FILE_PATH, pendingBefore.map(m => m.version), migrationManager.getCurrentVersion());
126
495
  // Run pending migrations
127
496
  const result = await migrationManager.runMigrations();
128
497
  console.error(`🔧 Database schema ready (version ${result.currentVersion}, ${result.applied} migrations applied)`);
@@ -140,11 +509,6 @@ export class RAGKnowledgeGraphManager {
140
509
  this.encoding.free();
141
510
  this.encoding = null;
142
511
  }
143
- if (this.embeddingModel) {
144
- // Clean up the embedding model if it has cleanup methods
145
- this.embeddingModel = null;
146
- this.modelInitialized = false;
147
- }
148
512
  this.embeddingCache.clear();
149
513
  if (this.db) {
150
514
  this.db.close();
@@ -163,42 +527,73 @@ export class RAGKnowledgeGraphManager {
163
527
  if (!this.db)
164
528
  throw new Error('Database not initialized');
165
529
  const result = [];
530
+ // v13: 관찰은 lifecycle 테이블이 정본이고 entities.observations 는 projection 이다.
531
+ // entity 행은 빈 배열로 만들고 rebuildProjection 이 채운다.
166
532
  const insertStmt = this.db.prepare(`
167
533
  INSERT OR IGNORE INTO entities (id, name, entityType, observations, metadata)
168
- VALUES (?, ?, ?, ?, ?)
534
+ VALUES (?, ?, ?, '[]', ?)
169
535
  `);
536
+ const stripDate = (s2) => s2.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
170
537
  for (const entity of entities) {
171
538
  const entityId = `entity_${entity.name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
172
- const timestamped = (entity.observations || []).map(o => this._timestampObservation(o));
173
- // Try insert first
174
- const insertResult = insertStmt.run(entityId, entity.name, entity.entityType, JSON.stringify(timestamped), '{}');
175
- if (insertResult.changes > 0) {
176
- // New entity created
177
- result.push({ ...entity, observations: timestamped });
178
- console.error(`🔮 Generating embedding for new entity: ${entity.name}`);
179
- await this.embedEntity(entityId);
180
- }
181
- else {
182
- // Entity already exists — upsert: merge observations and update entityType
183
- const existing = this.db.prepare(`SELECT observations, entityType FROM entities WHERE id = ?`)
184
- .get(entityId);
185
- if (existing) {
186
- const currentObs = JSON.parse(existing.observations);
187
- // Strip date prefix for dedup comparison
188
- const stripDate = (s) => s.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
189
- const currentBare = new Set(currentObs.map(stripDate));
190
- const newObs = timestamped.filter(o => !currentBare.has(stripDate(o)));
191
- const needsTypeUpdate = entity.entityType && entity.entityType !== 'CONCEPT' && entity.entityType !== existing.entityType;
192
- if (newObs.length > 0 || needsTypeUpdate) {
193
- const mergedObs = [...currentObs, ...newObs];
194
- const updatedType = needsTypeUpdate ? entity.entityType : existing.entityType;
195
- this.db.prepare(`UPDATE entities SET observations = ?, entityType = ? WHERE id = ?`)
196
- .run(JSON.stringify(mergedObs), updatedType, entityId);
197
- console.error(`♻️ Upserted entity: ${entity.name} (+${newObs.length} obs${needsTypeUpdate ? ', type→' + updatedType : ''})`);
198
- await this.embedEntity(entityId);
199
- result.push({ ...entity, observations: mergedObs });
539
+ const ts = new Date().toISOString();
540
+ const ids = [];
541
+ let created = false;
542
+ let addedCount = 0;
543
+ let typeUpdated = false;
544
+ // entity INSERT 도 같은 트랜잭션 안이다. 밖에 두면 lifecycle INSERT 가
545
+ // 실패할 때 entity 행만 남는 split state 가 생긴다
546
+ // (advisor 구현리뷰 r1 발견 2, 실행 재현).
547
+ const changed = this.mutateEntityAndInvalidate(entityId, () => {
548
+ created = insertStmt.run(entityId, entity.name, entity.entityType, '{}').changes > 0;
549
+ if (!created && entity.entityType && entity.entityType !== 'CONCEPT') {
550
+ const cur = this.db.prepare(`SELECT entityType FROM entities WHERE id = ?`)
551
+ .get(entityId);
552
+ if (cur && cur.entityType !== entity.entityType) {
553
+ this.db.prepare(`UPDATE entities SET entityType = ? WHERE id = ?`)
554
+ .run(entity.entityType, entityId);
555
+ typeUpdated = true;
200
556
  }
201
557
  }
558
+ const activeRows = this.db.prepare(`SELECT observation_id, content FROM entity_observations
559
+ WHERE entity_id = ? AND status = 'active'`).all(entityId);
560
+ const activeByBare = new Map(activeRows.map(r => [stripDate(r.content), r.observation_id]));
561
+ let sourcesAdded = 0;
562
+ for (const raw of (entity.observations || [])) {
563
+ const content = this._timestampObservation(raw);
564
+ const bare = stripDate(content);
565
+ const dupId = activeByBare.get(bare);
566
+ if (dupId) {
567
+ // 같은 사실이 다른 출처에서 다시 왔다 = evidence 추가, 새 revision 아님.
568
+ if (entity.sources?.length)
569
+ sourcesAdded += linkSources(this.db, dupId, entity.sources, ts);
570
+ ids.push(null);
571
+ continue;
572
+ }
573
+ const id = addRevision(this.db, {
574
+ entityId, content, status: entity.status ?? 'active', sources: entity.sources, ts
575
+ });
576
+ activeByBare.set(bare, id);
577
+ ids.push(id);
578
+ addedCount++;
579
+ }
580
+ // 아무것도 안 바뀌었으면 projection·벡터·KG 를 건드리지 않는다.
581
+ return created || typeUpdated || addedCount > 0 || sourcesAdded > 0;
582
+ });
583
+ const projected = JSON.parse(this.db.prepare(`SELECT observations FROM entities WHERE id = ?`)
584
+ .get(entityId).observations);
585
+ // 재임베딩은 무효화가 실제로 일어났을 때만. 조건이 갈리면
586
+ // "벡터를 지우고 다시 만들지 않는" 창이 생긴다.
587
+ if (changed) {
588
+ console.error(created
589
+ ? `🔮 Generating embedding for new entity: ${entity.name}`
590
+ : `♻️ Upserted entity: ${entity.name} (+${addedCount} obs${typeUpdated ? ', type→' + entity.entityType : ''})`);
591
+ const embedding_status = await this.tryEmbedEntity(entityId, 'bulk');
592
+ result.push({ ...entity, observations: projected, created,
593
+ observation_ids: ids, embedding_status });
594
+ }
595
+ else {
596
+ result.push({ ...entity, observations: projected, created, observation_ids: ids });
202
597
  }
203
598
  }
204
599
  return result;
@@ -208,24 +603,32 @@ export class RAGKnowledgeGraphManager {
208
603
  throw new Error('Database not initialized');
209
604
  const newRelations = [];
210
605
  for (const relation of relations) {
211
- // Ensure entities exist
212
- await this.createEntities([
606
+ // Ensure entities exist. v3.6 (spec §5c): auto-created endpoints may be
607
+ // embedded/queued/disabled independently — report per endpoint; 'n/a'
608
+ // means the endpoint already existed (no embedding work happened here).
609
+ const ensured = await this.createEntities([
213
610
  { name: relation.from, entityType: 'CONCEPT', observations: [] },
214
611
  { name: relation.to, entityType: 'CONCEPT', observations: [] }
215
612
  ]);
613
+ const statusOf = (name) => {
614
+ const hit = ensured.find(e => e.name === name);
615
+ return hit?.embedding_status ?? 'n/a';
616
+ };
617
+ const endpoint_embedding_status = { from: statusOf(relation.from), to: statusOf(relation.to) };
216
618
  const sourceId = `entity_${relation.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
217
619
  const targetId = `entity_${relation.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
218
620
  const relationId = `rel_${sourceId}_${relation.relationType}_${targetId}`.toLowerCase();
219
621
  const stmt = this.db.prepare(`
220
- INSERT OR IGNORE INTO relationships
622
+ INSERT OR IGNORE INTO relationships
221
623
  (id, source_entity, target_entity, relationType, confidence, metadata)
222
624
  VALUES (?, ?, ?, ?, ?, ?)
223
625
  `);
224
626
  const result = stmt.run(relationId, sourceId, targetId, relation.relationType, 1.0, '{}');
225
627
  if (result.changes > 0) {
226
- newRelations.push(relation);
628
+ newRelations.push({ ...relation, endpoint_embedding_status });
227
629
  }
228
630
  }
631
+ this.coordinator?.kick();
229
632
  return newRelations;
230
633
  }
231
634
  async addObservations(observations) {
@@ -234,31 +637,144 @@ export class RAGKnowledgeGraphManager {
234
637
  const results = [];
235
638
  for (const obs of observations) {
236
639
  const entityId = `entity_${obs.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
237
- // Get current observations
238
- const entity = this.db.prepare(`
239
- SELECT observations FROM entities WHERE id = ?
240
- `).get(entityId);
241
- if (!entity) {
640
+ const entity = this.db.prepare(`SELECT id FROM entities WHERE id = ?`).get(entityId);
641
+ if (!entity)
242
642
  throw new Error(`Entity with name ${obs.entityName} not found`);
243
- }
244
- const currentObservations = JSON.parse(entity.observations);
245
- const stripDate = (s) => s.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
246
- const currentBare = new Set(currentObservations.map(stripDate));
247
- const timestamped = obs.contents.map(c => this._timestampObservation(c));
248
- const newObservations = timestamped.filter(c => !currentBare.has(stripDate(c)));
249
- if (newObservations.length > 0) {
250
- const updatedObservations = [...currentObservations, ...newObservations];
251
- this.db.prepare(`
252
- UPDATE entities SET observations = ? WHERE id = ?
253
- `).run(JSON.stringify(updatedObservations), entityId);
254
- // Regenerate embedding for the updated entity
643
+ // dedup 기준은 v3.6 과 같다: 날짜 prefix 를 뗀 본문이 active 에 이미 있으면
644
+ // 새 revision 을 만들지 않는다. 다만 v13 에서는 같은 사실이 다른 출처에서 다시
645
+ // 온 것이므로 그 revision 에 source link 를 더한다(spec §8.3 T13).
646
+ const stripDate = (s2) => s2.replace(/^\[\d{4}-\d{2}-\d{2}\]\s*/, '');
647
+ const activeRows = this.db.prepare(`SELECT observation_id, content FROM entity_observations
648
+ WHERE entity_id = ? AND status = 'active'`).all(entityId);
649
+ const activeByBare = new Map(activeRows.map(r => [stripDate(r.content), r.observation_id]));
650
+ const ts = new Date().toISOString();
651
+ const ids = [];
652
+ const added = [];
653
+ let sourcesAdded = 0;
654
+ const changed = this.mutateEntityAndInvalidate(entityId, () => {
655
+ for (const raw of obs.contents) {
656
+ const content = this._timestampObservation(raw);
657
+ const bare = stripDate(content);
658
+ const dupId = activeByBare.get(bare);
659
+ if (dupId) {
660
+ if (obs.sources?.length)
661
+ sourcesAdded += linkSources(this.db, dupId, obs.sources, ts);
662
+ ids.push(null);
663
+ continue;
664
+ }
665
+ const id = addRevision(this.db, {
666
+ entityId, content, status: obs.status ?? 'active', sources: obs.sources, ts
667
+ });
668
+ activeByBare.set(bare, id);
669
+ ids.push(id);
670
+ added.push(content);
671
+ }
672
+ // 아무것도 안 바뀌었으면 projection·벡터·KG 를 건드리지 않는다.
673
+ // 이 반환이 없으면 빈 contents 나 dedup-only add 가 정상 벡터를
674
+ // 지우고 재임베딩도 안 한다(advisor 구현리뷰 r1 발견 1).
675
+ return added.length > 0 || sourcesAdded > 0;
676
+ });
677
+ let embedding_status;
678
+ if (changed) {
255
679
  console.error(`🔮 Regenerating embedding for updated entity: ${obs.entityName}`);
256
- await this.embedEntity(entityId);
680
+ embedding_status = await this.tryEmbedEntity(entityId, 'bulk');
257
681
  }
258
- results.push({ entityName: obs.entityName, addedObservations: newObservations });
682
+ results.push({ entityName: obs.entityName, observation_ids: ids,
683
+ addedObservations: added, embedding_status });
259
684
  }
260
685
  return results;
261
686
  }
687
+ async correctObservation(observationId, content, changeKind = 'correction', reason) {
688
+ if (!this.db)
689
+ throw new Error('Database not initialized');
690
+ const row = this.db.prepare(`SELECT entity_id FROM entity_observations WHERE observation_id = ?`)
691
+ .get(observationId);
692
+ if (!row)
693
+ throw new Error(`observation ${observationId} not found`);
694
+ let newId = '';
695
+ const ts = new Date().toISOString();
696
+ this.mutateEntityAndInvalidate(row.entity_id, () => {
697
+ newId = correctRevision(this.db, {
698
+ observationId, content: this._timestampObservation(content),
699
+ changeKind, reason: reason ?? null, ts
700
+ });
701
+ });
702
+ await this.tryEmbedEntity(row.entity_id, 'bulk');
703
+ return newId;
704
+ }
705
+ async _transition(observationId, event, reason) {
706
+ if (!this.db)
707
+ throw new Error('Database not initialized');
708
+ const row = this.db.prepare(`SELECT entity_id FROM entity_observations WHERE observation_id = ?`)
709
+ .get(observationId);
710
+ if (!row)
711
+ throw new Error(`observation ${observationId} not found`);
712
+ const ts = new Date().toISOString();
713
+ this.mutateEntityAndInvalidate(row.entity_id, () => {
714
+ transitionStatus(this.db, { observationId, event, reason: reason ?? null, ts });
715
+ });
716
+ await this.tryEmbedEntity(row.entity_id, 'bulk');
717
+ }
718
+ async retractObservation(observationId, reason) {
719
+ return this._transition(observationId, 'retract', reason);
720
+ }
721
+ async restoreObservation(observationId, reason) {
722
+ return this._transition(observationId, 'restore', reason);
723
+ }
724
+ async approveObservation(observationId, reason) {
725
+ return this._transition(observationId, 'approve', reason);
726
+ }
727
+ async declineObservation(observationId, reason) {
728
+ return this._transition(observationId, 'decline', reason);
729
+ }
730
+ // DESTRUCTIVE. Physically removes revisions (and their sources via CASCADE).
731
+ //
732
+ // Chain contract (advisor 구현리뷰 r1 발견 4): a revision chain is
733
+ // rev1 <- rev2 <- ... and purging a middle revision would either fail on the
734
+ // supersedes_id FK or leave a chain pointing at a deleted row, plus events
735
+ // whose from_id/to_id dangle. So purge is defined as **suffix purge from the
736
+ // target to the newest revision of that root**, newest-first:
737
+ // - purging the newest revision removes exactly it
738
+ // - purging rev2 of a 3-revision chain removes rev3 then rev2
739
+ // - purging rev1 removes the whole chain
740
+ // Events for purged revisions are removed too, so no event dangles.
741
+ // The root row is always kept: its projection_order stays reserved, because
742
+ // reusing an order would make a later restore/approve fail on the
743
+ // active-order index.
744
+ async purgeObservation(observationId, confirm) {
745
+ if (!this.db)
746
+ throw new Error('Database not initialized');
747
+ if (confirm !== 'PURGE') {
748
+ throw new Error(`purgeObservation refused: pass confirm='PURGE' to physically delete a revision. ` +
749
+ `This destroys history — retractObservation() is almost always what you want.`);
750
+ }
751
+ const row = this.db.prepare(`SELECT entity_id, root_id, revision_no FROM entity_observations WHERE observation_id = ?`)
752
+ .get(observationId);
753
+ if (!row)
754
+ return { purged: 0 };
755
+ let purged = 0;
756
+ this.mutateEntityAndInvalidate(row.entity_id, () => {
757
+ // newest-first so each DELETE has no successor referencing it
758
+ const victims = this.db.prepare(`SELECT observation_id FROM entity_observations
759
+ WHERE root_id = ? AND revision_no >= ?
760
+ ORDER BY revision_no DESC`).all(row.root_id, row.revision_no);
761
+ for (const v of victims) {
762
+ this.db.prepare(`DELETE FROM observation_events WHERE from_id = ? OR to_id = ?`)
763
+ .run(v.observation_id, v.observation_id);
764
+ purged += this.db.prepare(`DELETE FROM entity_observations WHERE observation_id = ?`)
765
+ .run(v.observation_id).changes;
766
+ }
767
+ return purged > 0;
768
+ });
769
+ await this.tryEmbedEntity(row.entity_id, 'bulk');
770
+ return { purged };
771
+ }
772
+ // spec §6.2: 과거 판본은 여기서만 나온다. 일반 검색은 active 만 반환한다.
773
+ async getObservationHistory(sel) {
774
+ if (!this.db)
775
+ throw new Error('Database not initialized');
776
+ return getObservationHistory(this.db, sel);
777
+ }
262
778
  async deleteEntities(entityNames) {
263
779
  if (!this.db)
264
780
  throw new Error('Database not initialized');
@@ -322,22 +838,91 @@ export class RAGKnowledgeGraphManager {
322
838
  }
323
839
  console.error(`✅ Entity deletion process completed`);
324
840
  }
841
+ // v3.6 (spec §5c, breaking): structured per-entity results + re-embedding.
842
+ // Pre-3.6 this method silently left STALE entity vectors behind (the input
843
+ // text changed but the vector was never regenerated) — fixed via
844
+ // tryEmbedEntity, which also covers the not-ready dirty contract.
845
+ // DEPRECATED shim (v13, one version only). Content-addressed deletion cannot
846
+ // express "which revision" — use retractObservation(observation_id) instead.
847
+ // Semantics: exact-string match against ACTIVE revisions -> soft retract.
848
+ //
849
+ // The whole call is one transaction and ambiguity is judged before any
850
+ // mutation: if any item matches 2+ active revisions the call aborts with 0
851
+ // mutations (spec §6.3). That is a deliberate change from v3.6, which deleted
852
+ // every duplicate and carried on — a machine cannot pick which revision was meant.
853
+ //
854
+ // Duplicate ids across items are collapsed. Without that, listing the same
855
+ // (entity, content) twice retracted it once and then failed on an illegal
856
+ // transition, returning an error *after* committing part of the batch
857
+ // (advisor 구현리뷰 r1 발견 3, 실행 재현). Embedding runs after the commit.
325
858
  async deleteObservations(deletions) {
326
859
  if (!this.db)
327
860
  throw new Error('Database not initialized');
328
- for (const deletion of deletions) {
329
- const entityId = `entity_${deletion.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
330
- const entity = this.db.prepare(`
331
- SELECT observations FROM entities WHERE id = ?
332
- `).get(entityId);
333
- if (entity) {
334
- const currentObservations = JSON.parse(entity.observations);
335
- const filteredObservations = currentObservations.filter((obs) => !deletion.observations.includes(obs));
336
- this.db.prepare(`
337
- UPDATE entities SET observations = ? WHERE id = ?
338
- `).run(JSON.stringify(filteredObservations), entityId);
861
+ const plan = [];
862
+ const ambiguous = [];
863
+ const claimed = new Set(); // 항목 간 중복 id 흡수
864
+ for (const d of deletions) {
865
+ const entityId = `entity_${d.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
866
+ const ids = [];
867
+ for (const content of d.observations) {
868
+ const rows = this.db.prepare(`SELECT observation_id FROM entity_observations
869
+ WHERE entity_id = ? AND status = 'active' AND content = ?`).all(entityId, content);
870
+ if (rows.length > 1) {
871
+ ambiguous.push({ entityName: d.entityName, content, matches: rows.length });
872
+ continue;
873
+ }
874
+ if (rows.length === 1 && !claimed.has(rows[0].observation_id)) {
875
+ claimed.add(rows[0].observation_id);
876
+ ids.push(rows[0].observation_id);
877
+ }
878
+ // rows.length === 0 -> no-op (v3.6 behaviour, spec §6.3-3)
879
+ }
880
+ plan.push({ entityName: d.entityName, entityId, ids });
881
+ }
882
+ if (ambiguous.length > 0) {
883
+ throw new Error(`AMBIGUOUS_OBSERVATION_MATCH: ${ambiguous.length} item(s) matched multiple active ` +
884
+ `revisions; 0 mutations were applied. Use retractObservation(observation_id) instead. ` +
885
+ `Conflicts: ${JSON.stringify(ambiguous)}`);
886
+ }
887
+ // pass 2 — mutate everything in ONE transaction so a failure anywhere
888
+ // leaves zero mutations. Per-plan transactions plus an awaited embedding
889
+ // in between made a partial commit observable.
890
+ const touched = plan.filter(p => p.ids.length > 0);
891
+ const ts = new Date().toISOString();
892
+ if (touched.length > 0) {
893
+ const tx = this.db.transaction(() => {
894
+ for (const p of touched) {
895
+ for (const id of p.ids) {
896
+ transitionStatus(this.db, { observationId: id, event: 'retract',
897
+ reason: 'deleteObservations (deprecated shim)', ts });
898
+ }
899
+ rebuildProjection(this.db, p.entityId);
900
+ const meta = this.db.prepare(`SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?`)
901
+ .get(p.entityId);
902
+ if (meta) {
903
+ this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(meta.rowid)}`);
904
+ this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`)
905
+ .run(p.entityId);
906
+ }
907
+ deleteStaleKgChunks(this.db, p.entityId);
908
+ }
909
+ });
910
+ tx();
911
+ this.coordinator?.invalidateCoverage();
912
+ }
913
+ // pass 3 — embedding after the commit
914
+ const results = [];
915
+ let total = 0;
916
+ for (const p of plan) {
917
+ if (p.ids.length === 0) {
918
+ results.push({ entityName: p.entityName, deleted: 0, embedding_status: 'n/a' });
919
+ continue;
339
920
  }
921
+ const embedding_status = await this.tryEmbedEntity(p.entityId, 'bulk');
922
+ results.push({ entityName: p.entityName, deleted: p.ids.length, embedding_status });
923
+ total += p.ids.length;
340
924
  }
925
+ return { results, total_deleted: total };
341
926
  }
342
927
  async deleteRelations(relations) {
343
928
  if (!this.db)
@@ -524,10 +1109,24 @@ export class RAGKnowledgeGraphManager {
524
1109
  console.error(`✅ getNeighbors: Found ${entities.length} entities, ${relations.length} relations, ${paths.length} paths (depth=${effectiveDepth})`);
525
1110
  return { entities, relations, paths };
526
1111
  }
1112
+ // v3.6 (spec §5·§5c, additive): FTS lexical fallback when vector search is
1113
+ // not eligible, hybrid-partial merge while backfill is catching up, and
1114
+ // top-level state fields on every response.
527
1115
  async searchNodes(query, limit = 10, since, until) {
528
1116
  if (!this.db)
529
1117
  throw new Error('Database not initialized');
530
1118
  console.error(`🔍 Semantic entity search: "${query}"`);
1119
+ const covS = this.coordinator?.coverage();
1120
+ const entityPct = covS && covS.entity.total > 0 ? Math.round((covS.entity.embedded / covS.entity.total) * 100) : 100;
1121
+ const stateFields = () => ({
1122
+ model_state: this.gate.status.state,
1123
+ coverage: { entity_pct: entityPct },
1124
+ });
1125
+ if (!(this.coordinator?.eligible ?? false)) {
1126
+ // No waiting on the model (spec §5) — lexical entities_fts fallback.
1127
+ return { ...this.searchNodesFts(query, limit, since, until), search_mode: 'fts-only',
1128
+ ...stateFields(), degradation_reason: this.degradationReason() };
1129
+ }
531
1130
  const queryVariants = this.buildCrossLingualVariants(query);
532
1131
  if (queryVariants.length > 1) {
533
1132
  console.error(`🌐 searchNodes variants: ${queryVariants.slice(1).join(' | ')}`);
@@ -551,16 +1150,26 @@ export class RAGKnowledgeGraphManager {
551
1150
  `).all(Buffer.from(embedding.buffer), k);
552
1151
  };
553
1152
  const resultMap = new Map();
554
- for (const variant of queryVariants) {
555
- const embedding = await this.generateEmbedding(variant, 1024, true);
556
- const variantResults = searchEntities(embedding, limit * 2);
557
- for (const result of variantResults) {
558
- const existing = resultMap.get(result.entity_id);
559
- if (!existing || result.distance < existing.distance) {
560
- resultMap.set(result.entity_id, result);
1153
+ try {
1154
+ for (const variant of queryVariants) {
1155
+ const embedding = await this.generateEmbedding(variant, 1024, true);
1156
+ const variantResults = searchEntities(embedding, limit * 2);
1157
+ for (const result of variantResults) {
1158
+ const existing = resultMap.get(result.entity_id);
1159
+ if (!existing || result.distance < existing.distance) {
1160
+ resultMap.set(result.entity_id, result);
1161
+ }
561
1162
  }
562
1163
  }
563
1164
  }
1165
+ catch (embErr) {
1166
+ // Ready-state inference failure degrades to FTS instead of failing the
1167
+ // tool (beta B6) — same contract as hybridSearch. The gate's own
1168
+ // consecutive-failure counter handles the systemic transition.
1169
+ console.error(`⚠️ searchNodes vector path failed — FTS fallback:`, embErr instanceof Error ? embErr.message : embErr);
1170
+ return { ...this.searchNodesFts(query, limit, since, until), search_mode: 'fts-only',
1171
+ ...stateFields(), degradation_reason: this.degradationReason() ?? 'inference_error' };
1172
+ }
564
1173
  const entityResults = Array.from(resultMap.values()).sort((a, b) => a.distance - b.distance).slice(0, limit);
565
1174
  // Filter by temporal range if specified
566
1175
  let filteredResults = entityResults;
@@ -576,35 +1185,79 @@ export class RAGKnowledgeGraphManager {
576
1185
  return true;
577
1186
  });
578
1187
  }
579
- if (filteredResults.length === 0) {
580
- console.error(`ℹ️ No semantic matches found for "${query}"`);
581
- return { entities: [], relations: [] };
582
- }
583
1188
  const entities = filteredResults.map(result => ({
584
1189
  name: result.name,
585
1190
  entityType: result.entityType,
586
1191
  observations: JSON.parse(result.observations),
587
1192
  similarity: Math.max(0, 1 - result.distance / 2) // Convert cosine distance (0-2) to similarity (1-0)
588
1193
  }));
589
- // Get relationships between the found entities
590
- const entityNames = entities.map(e => e.name);
591
- const relations = this.db.prepare(`
592
- SELECT
1194
+ // hybrid-partial (spec §4): entities without vectors must not vanish from
1195
+ // search while backfill catches up — merge lexical FTS hits for the gap.
1196
+ let search_mode = 'hybrid';
1197
+ if (entityPct < 100) {
1198
+ search_mode = 'hybrid-partial';
1199
+ const seen = new Set(entities.map(e => e.name));
1200
+ const ftsExtra = this.searchNodesFts(query, limit, since, until);
1201
+ for (const e of ftsExtra.entities) {
1202
+ if (entities.length >= limit)
1203
+ break;
1204
+ if (!seen.has(e.name)) {
1205
+ seen.add(e.name);
1206
+ entities.push(e);
1207
+ }
1208
+ }
1209
+ }
1210
+ if (entities.length === 0) {
1211
+ console.error(`ℹ️ No semantic matches found for "${query}"`);
1212
+ return { entities: [], relations: [], search_mode, ...stateFields() };
1213
+ }
1214
+ const relations = this.relationsAmong(entities.map(e => e.name));
1215
+ console.error(`✅ Found ${entities.length} semantically similar entities with ${relations.length} relationships`);
1216
+ return { entities, relations, search_mode, ...stateFields() };
1217
+ }
1218
+ // Lexical entity search over entities_fts (spec §5 contract: name /
1219
+ // observations / entityType lexical match — no semantic-equivalence claim).
1220
+ // Temporal filters apply in SQL so LIMIT is not distorted.
1221
+ searchNodesFts(query, limit, since, until) {
1222
+ const expr = compileFtsLiteralQuery(query);
1223
+ if (expr === null) {
1224
+ return { entities: [], relations: [], warning: 'query has no searchable terms' };
1225
+ }
1226
+ const rows = this.db.prepare(`
1227
+ SELECT e.name, e.entityType, e.observations
1228
+ FROM entities_fts f
1229
+ JOIN entities e ON f.rowid = e.rowid
1230
+ WHERE entities_fts MATCH @expr
1231
+ ${since ? 'AND e.created_at >= @since' : ''}
1232
+ ${until ? 'AND e.created_at <= @until' : ''}
1233
+ ORDER BY bm25(entities_fts)
1234
+ LIMIT @limit
1235
+ `).all({ expr, since, until, limit });
1236
+ const entities = rows.map(r => ({
1237
+ name: r.name,
1238
+ entityType: r.entityType,
1239
+ observations: JSON.parse(r.observations),
1240
+ }));
1241
+ return { entities, relations: this.relationsAmong(entities.map(e => e.name)) };
1242
+ }
1243
+ relationsAmong(entityNames) {
1244
+ if (entityNames.length === 0)
1245
+ return [];
1246
+ return this.db.prepare(`
1247
+ SELECT
593
1248
  e1.name as from_name,
594
1249
  e2.name as to_name,
595
1250
  r.relationType
596
1251
  FROM relationships r
597
1252
  JOIN entities e1 ON r.source_entity = e1.id
598
1253
  JOIN entities e2 ON r.target_entity = e2.id
599
- WHERE e1.name IN (${entityNames.map(() => '?').join(',')})
1254
+ WHERE e1.name IN (${entityNames.map(() => '?').join(',')})
600
1255
  AND e2.name IN (${entityNames.map(() => '?').join(',')})
601
1256
  `).all(...entityNames, ...entityNames).map((row) => ({
602
1257
  from: row.from_name,
603
1258
  to: row.to_name,
604
1259
  relationType: row.relationType
605
1260
  }));
606
- console.error(`✅ Found ${entities.length} semantically similar entities with ${relations.length} relationships`);
607
- return { entities, relations };
608
1261
  }
609
1262
  async openNodes(names) {
610
1263
  if (!this.db)
@@ -800,7 +1453,7 @@ export class RAGKnowledgeGraphManager {
800
1453
  };
801
1454
  }
802
1455
  // Generate and store embedding for a single entity
803
- async embedEntity(entityId) {
1456
+ async embedEntity(entityId, priority = 'bulk') {
804
1457
  if (!this.db)
805
1458
  throw new Error('Database not initialized');
806
1459
  // Get entity data
@@ -822,28 +1475,43 @@ export class RAGKnowledgeGraphManager {
822
1475
  // char size (identity excluded), and embed duration. `capped` = some observation chars dropped.
823
1476
  const capped = built.cappedObsChars < built.filteredObsChars;
824
1477
  const embedStart = Date.now();
825
- const embedding = await this.generateEmbedding(embeddingText);
1478
+ const embedding = await this.generateEmbedding(embeddingText, 1024, false, priority);
826
1479
  const embedMs = Date.now() - embedStart;
827
1480
  console.error(`[embed] ${entity.name}: ${built.selectedObsCount}/${built.totalObsCount} obs, ${built.filteredObsChars}ch -> ${built.cappedObsChars}ch${capped ? ' (capped)' : ''}, ${embedMs}ms`);
828
1481
  try {
829
- // Delete existing embedding if any
830
- const existingMetadata = this.db.prepare(`
831
- SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
832
- `).get(entityId);
833
- if (existingMetadata) {
834
- this.db.prepare(`DELETE FROM entity_embeddings WHERE rowid = ?`).run(existingMetadata.rowid);
835
- this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
1482
+ // v3.6 (§6a-2): vector replace + provenance stamp commit atomically.
1483
+ // Write-back CAS (beta 2R B1): the entity may have been mutated again
1484
+ // while THIS inference was in flight — a late writer must never
1485
+ // re-insert a vector for superseded content as 'verified'. Inside the
1486
+ // write transaction the CURRENT entity text is rebuilt and hashed; on
1487
+ // mismatch the result is discarded and the row stays missing/queued for
1488
+ // the backfill pass that the newer mutation already kicked.
1489
+ const inputHash = this.hashWithBuilderVersion(embeddingText);
1490
+ const writeTx = this.db.transaction(() => {
1491
+ const currentHash = this.entityInputHash(entityId);
1492
+ if (currentHash !== inputHash)
1493
+ return false; // superseded — discard
1494
+ const existingMetadata = this.db.prepare(`
1495
+ SELECT rowid FROM entity_embedding_metadata WHERE entity_id = ?
1496
+ `).get(entityId);
1497
+ if (existingMetadata) {
1498
+ this.db.exec(`DELETE FROM entity_embeddings WHERE rowid = ${Number(existingMetadata.rowid)}`);
1499
+ this.db.prepare(`DELETE FROM entity_embedding_metadata WHERE entity_id = ?`).run(entityId);
1500
+ }
1501
+ const result = this.db.prepare(`
1502
+ INSERT INTO entity_embeddings (embedding) VALUES (?)
1503
+ `).run(Buffer.from(embedding.buffer));
1504
+ this.db.prepare(`
1505
+ INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text, input_hash, profile_id, provenance_state)
1506
+ VALUES (?, ?, ?, ?, ?, 'verified')
1507
+ `).run(result.lastInsertRowid, entityId, embeddingText, inputHash, this.currentProfileId);
1508
+ return true;
1509
+ });
1510
+ const written = writeTx();
1511
+ if (!written) {
1512
+ console.error(`⏭️ discarded superseded embedding for ${entityId} (entity changed during inference)`);
836
1513
  }
837
- // Insert new embedding
838
- const result = this.db.prepare(`
839
- INSERT INTO entity_embeddings (embedding) VALUES (?)
840
- `).run(Buffer.from(embedding.buffer));
841
- // Store metadata
842
- this.db.prepare(`
843
- INSERT INTO entity_embedding_metadata (rowid, entity_id, embedding_text)
844
- VALUES (?, ?, ?)
845
- `).run(result.lastInsertRowid, entityId, embeddingText);
846
- return true;
1514
+ return written;
847
1515
  }
848
1516
  catch (error) {
849
1517
  console.error(`Failed to embed entity ${entityId}:`, error);
@@ -866,6 +1534,7 @@ export class RAGKnowledgeGraphManager {
866
1534
  embeddedCount += results.filter(Boolean).length;
867
1535
  }
868
1536
  console.error(`✅ Entity embeddings completed: ${embeddedCount}/${entities.length} entities embedded`);
1537
+ this.coordinator?.invalidateCoverage();
869
1538
  return {
870
1539
  totalEntities: entities.length,
871
1540
  embeddedEntities: embeddedCount
@@ -945,15 +1614,20 @@ export class RAGKnowledgeGraphManager {
945
1614
  const errors = [];
946
1615
  for (const chunk of chunks) {
947
1616
  // Generate embedding
948
- const embedding = await this.generateEmbedding(chunk.text);
1617
+ const embedding = await this.generateEmbedding(chunk.text, 1024, false, 'bulk');
949
1618
  const rowid = safeRowid(chunk.rowid);
950
1619
  try {
951
- // Delete existing embedding if any
952
- this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
953
- // Insert new embedding - rowid as literal integer for vec0 compatibility
954
- this.db.prepare(`
955
- INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
956
- `).run(Buffer.from(embedding.buffer));
1620
+ // vector + verified provenance in one transaction (§6a-2) — KG chunks
1621
+ // must never become vector-bearing provenance-NULL rows post-recon.
1622
+ const tx = this.db.transaction(() => {
1623
+ this.db.exec(`DELETE FROM chunks WHERE rowid = ${rowid}`);
1624
+ this.db.prepare(`
1625
+ INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
1626
+ `).run(Buffer.from(embedding.buffer));
1627
+ this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
1628
+ .run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, chunk.rowid);
1629
+ });
1630
+ tx();
957
1631
  embeddedCount++;
958
1632
  }
959
1633
  catch (error) {
@@ -962,6 +1636,7 @@ export class RAGKnowledgeGraphManager {
962
1636
  errors.push(errMsg);
963
1637
  }
964
1638
  }
1639
+ this.coordinator?.invalidateCoverage();
965
1640
  console.error(`✅ Knowledge graph chunks embedded: ${embeddedCount}/${chunks.length}`);
966
1641
  return { embeddedChunks: embeddedCount, totalChunks: chunks.length, ...(errors.length > 0 && { errors: errors.slice(0, 5) }) };
967
1642
  }
@@ -1185,37 +1860,23 @@ export class RAGKnowledgeGraphManager {
1185
1860
  }
1186
1861
  // Generate embeddings using sentence transformers
1187
1862
  // isQuery: true for search queries (adds instruction prefix), false for documents/entities
1188
- async generateEmbedding(text, dimensions = 1024, isQuery = false) {
1863
+ async generateEmbedding(text, dimensions = 1024, isQuery = false, priority = 'interactive') {
1189
1864
  // Check cache first (hash-based key to avoid collisions on long texts)
1190
1865
  const cacheKey = createHash('md5').update(`${text}_${dimensions}_${isQuery}`).digest('hex');
1191
1866
  const cached = this.embeddingCache.get(cacheKey);
1192
1867
  if (cached)
1193
1868
  return cached;
1194
- if (this.modelInitialized && this.embeddingModel) {
1195
- try {
1196
- // BGE-M3: instruction prefix improves retrieval quality for queries
1197
- const inputText = isQuery
1198
- ? `Represent this sentence for searching relevant passages: ${text}`
1199
- : text;
1200
- const result = await this.embeddingModel(inputText, { pooling: 'cls', normalize: true });
1201
- // Extract the embedding array and convert to Float32Array
1202
- const embedding = result.data;
1203
- const modelResult = new Float32Array(embedding.slice(0, dimensions));
1204
- // Cache the result (LRU: evict oldest if full)
1205
- if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
1206
- const firstKey = this.embeddingCache.keys().next().value;
1207
- if (firstKey)
1208
- this.embeddingCache.delete(firstKey);
1209
- }
1210
- this.embeddingCache.set(cacheKey, modelResult);
1211
- return modelResult;
1212
- }
1213
- catch (error) {
1214
- console.error(`⚠️ Embedding model failed for text "${text.slice(0, 50)}...":`, error instanceof Error ? error.message : error);
1215
- throw new Error(`Embedding model not available. Ensure the model is loaded. Original error: ${error instanceof Error ? error.message : error}`);
1216
- }
1217
- }
1218
- throw new Error('Embedding model not initialized. The server may still be loading the model — retry in a few seconds.');
1869
+ // v3.6: all inference goes through the gate — state check + execution in one
1870
+ // atomic boundary (TOCTOU-safe). GateNotReadyError / GateDisabledError
1871
+ // propagate so each consumer honors its own not-ready contract (spec §5).
1872
+ const modelResult = await this.gate.embed(text, { dims: dimensions, isQuery, priority });
1873
+ if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
1874
+ const firstKey = this.embeddingCache.keys().next().value;
1875
+ if (firstKey)
1876
+ this.embeddingCache.delete(firstKey);
1877
+ }
1878
+ this.embeddingCache.set(cacheKey, modelResult);
1879
+ return modelResult;
1219
1880
  }
1220
1881
  // === NEW SEPARATE TOOLS ===
1221
1882
  async syncDocumentFromFile(filePath, documentId, options = {}) {
@@ -1243,31 +1904,56 @@ export class RAGKnowledgeGraphManager {
1243
1904
  catch { /* ignore */ }
1244
1905
  if (existingHash === contentHash) {
1245
1906
  const cmCount = this.db.prepare(`SELECT count(*) AS n FROM chunk_metadata WHERE document_id = ?`).get(documentId).n;
1907
+ // "Embedded" for dedup completeness = vector exists AND its profile is
1908
+ // current (or legacy-NULL awaiting grandfather). Raw vector counts
1909
+ // would misjudge old-profile rows as complete (beta 1R supplement).
1246
1910
  const embCount = this.db.prepare(`
1247
- SELECT count(*) AS n FROM chunks c JOIN chunk_metadata m ON c.rowid = m.rowid WHERE m.document_id = ?
1911
+ SELECT count(*) AS n FROM chunks c JOIN chunk_metadata m ON c.rowid = m.rowid
1912
+ WHERE m.document_id = ? AND (m.provenance_state IS NULL OR m.profile_id = ?)
1913
+ `).get(documentId, this.currentProfileId).n;
1914
+ const linked = this.db.prepare(`
1915
+ SELECT count(DISTINCT ce.entity_id) AS n FROM chunk_entities ce
1916
+ JOIN chunk_metadata m ON ce.chunk_rowid = m.rowid WHERE m.document_id = ?
1248
1917
  `).get(documentId).n;
1249
1918
  if (cmCount > 0 && cmCount === embCount) {
1250
- const linked = this.db.prepare(`
1251
- SELECT count(DISTINCT ce.entity_id) AS n FROM chunk_entities ce
1252
- JOIN chunk_metadata m ON ce.chunk_rowid = m.rowid WHERE m.document_id = ?
1253
- `).get(documentId).n;
1254
1919
  console.error(`⏭️ syncDocumentFromFile: ${documentId} unchanged (hash match, ${cmCount} chunks embedded) — skipped`);
1255
1920
  return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged' };
1256
1921
  }
1922
+ if (cmCount > 0 && embCount < cmCount) {
1923
+ // v3.6 (spec §5b M12): identical content with incomplete/stale vectors
1924
+ // keeps the document, chunks, rowids and entity links — only the
1925
+ // missing vectors are re-queued via the coordinator. Full re-chunking
1926
+ // here would churn rowids and links for no content change.
1927
+ console.error(`♻️ syncDocumentFromFile: ${documentId} unchanged but ${cmCount - embCount} vectors missing — re-queued (chunks preserved)`);
1928
+ this.coordinator?.kick();
1929
+ return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged-revectorizing', embedding_status: this.gate.isDisabled ? 'disabled' : 'queued' };
1930
+ }
1257
1931
  }
1258
1932
  }
1259
1933
  console.error(`🔄 syncDocumentFromFile: ${documentId} <- ${filePath} (${bytes} bytes)`);
1260
- // 3. Pre-compute chunks + embeddings BEFORE any DB mutation. If embedding
1261
- // throws (model down), the existing document is left completely intact.
1934
+ // 3. Two contracts (spec §5b):
1935
+ // ready — pre-compute ALL embeddings BEFORE any DB mutation; if
1936
+ // inference throws mid-way the old document stays intact
1937
+ // (v3.5.0 atomicity, unchanged).
1938
+ // not-ready — intentional lazy sync: store document + chunks + FTS in
1939
+ // one transaction with NO vectors (embedding_status:
1940
+ // queued); the backfill coordinator recovers them.
1262
1941
  const { maxTokens = 800, overlap = 160 } = options.chunkParams || {};
1263
1942
  const segments = this.chunkText(content, maxTokens, overlap);
1943
+ const lazySync = !this.gate.isReady;
1264
1944
  const embedded = [];
1265
- for (const seg of segments) {
1266
- const embedding = await this.generateEmbedding(seg.text);
1267
- embedded.push({ seg, embedding });
1945
+ if (lazySync) {
1946
+ for (const seg of segments)
1947
+ embedded.push({ seg, embedding: null });
1948
+ }
1949
+ else {
1950
+ for (const seg of segments) {
1951
+ const embedding = await this.generateEmbedding(seg.text, 1024, false, 'bulk');
1952
+ embedded.push({ seg, embedding });
1953
+ }
1268
1954
  }
1269
- // 4. Atomic swap: delete old -> insert doc -> insert chunks + embeddings,
1270
- // all in a single synchronous better-sqlite3 transaction (all-or-nothing).
1955
+ // 4. Atomic swap: delete old -> insert doc -> insert chunks (+ embeddings
1956
+ // with verified provenance when ready), one synchronous transaction.
1271
1957
  const applyTx = this.db.transaction(() => {
1272
1958
  const db = this.db;
1273
1959
  // 4a. cleanup old doc (inlined sync version of cleanupDocument).
@@ -1281,7 +1967,8 @@ export class RAGKnowledgeGraphManager {
1281
1967
  // 4b. insert document.
1282
1968
  db.prepare(`INSERT INTO documents (id, content, metadata) VALUES (?, ?, ?)`)
1283
1969
  .run(documentId, content, JSON.stringify(metadata));
1284
- // 4c. insert chunk_metadata (FTS5 chunks_fts auto-filled by trigger) + embeddings.
1970
+ // 4c. insert chunk_metadata (FTS5 chunks_fts auto-filled by trigger);
1971
+ // vectors + provenance only on the ready path (§6a-2).
1285
1972
  for (const { seg, embedding } of embedded) {
1286
1973
  const chunkId = `${documentId}_chunk_${seg.chunk_index}`;
1287
1974
  const info = db.prepare(`
@@ -1289,11 +1976,18 @@ export class RAGKnowledgeGraphManager {
1289
1976
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1290
1977
  `).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
1291
1978
  const rowid = Number(info.lastInsertRowid);
1292
- db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
1979
+ if (embedding) {
1980
+ db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
1981
+ db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
1982
+ .run(createHash('sha256').update(seg.text).digest('hex'), this.currentProfileId, rowid);
1983
+ }
1293
1984
  }
1294
1985
  });
1295
1986
  applyTx();
1296
- const embeddedChunks = embedded.length;
1987
+ this.coordinator?.invalidateCoverage();
1988
+ if (lazySync)
1989
+ this.coordinator?.kick();
1990
+ const embeddedChunks = lazySync ? 0 : embedded.length;
1297
1991
  // 5. Entity linking AFTER commit. Non-destructive + idempotent (INSERT OR
1298
1992
  // IGNORE), so a linking failure cannot corrupt the doc/embeddings.
1299
1993
  const linkedEntities = await this.autoLinkEntities(documentId);
@@ -1309,6 +2003,7 @@ export class RAGKnowledgeGraphManager {
1309
2003
  chunks: segments.length,
1310
2004
  embeddedChunks,
1311
2005
  linkedEntities,
2006
+ embedding_status: lazySync ? (this.gate.isDisabled ? 'disabled' : 'queued') : 'embedded',
1312
2007
  ...(explicitlyLinked !== undefined ? { explicitlyLinked } : {}),
1313
2008
  };
1314
2009
  if (linkedEntities === 0 && explicitlyLinked === undefined) {
@@ -1366,6 +2061,10 @@ export class RAGKnowledgeGraphManager {
1366
2061
  });
1367
2062
  }
1368
2063
  console.error(`✅ Document chunked: ${chunks.length} chunks created`);
2064
+ // Indirect missing-row producer (spec §5): freshly chunked rows have no
2065
+ // vectors yet — let the coordinator recover them without a restart.
2066
+ this.coordinator?.invalidateCoverage();
2067
+ this.coordinator?.kick();
1369
2068
  return { documentId, chunks: resultChunks };
1370
2069
  }
1371
2070
  async embedChunks(documentId) {
@@ -1382,18 +2081,20 @@ export class RAGKnowledgeGraphManager {
1382
2081
  let embeddedCount = 0;
1383
2082
  const errors = [];
1384
2083
  for (const chunk of chunks) {
1385
- // Generate embedding
1386
- const embedding = await this.generateEmbedding(chunk.text);
2084
+ // Generate embedding (foreground-bulk priority)
2085
+ const embedding = await this.generateEmbedding(chunk.text, 1024, false, 'bulk');
1387
2086
  const rowid = Number(chunk.rowid);
1388
- // Store in vector table
2087
+ // Store in vector table (+ verified provenance, §6a-2 atomic)
1389
2088
  try {
1390
- // First, delete any existing embedding for this rowid
1391
- this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(rowid)}`);
1392
- // Insert new embedding with explicit rowid to match chunk_metadata
1393
- // Use parameterized only for embedding blob, rowid as literal integer
1394
- this.db.prepare(`
1395
- INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
1396
- `).run(Buffer.from(embedding.buffer));
2089
+ const tx = this.db.transaction(() => {
2090
+ this.db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(rowid)}`);
2091
+ this.db.prepare(`
2092
+ INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)
2093
+ `).run(Buffer.from(embedding.buffer));
2094
+ this.db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
2095
+ .run(createHash('sha256').update(chunk.text).digest('hex'), this.currentProfileId, rowid);
2096
+ });
2097
+ tx();
1397
2098
  embeddedCount++;
1398
2099
  }
1399
2100
  catch (error) {
@@ -1403,6 +2104,7 @@ export class RAGKnowledgeGraphManager {
1403
2104
  }
1404
2105
  }
1405
2106
  console.error(`✅ Chunks embedded: ${embeddedCount}/${chunks.length}`);
2107
+ this.coordinator?.invalidateCoverage();
1406
2108
  // Auto-link entities to document after embedding
1407
2109
  const linkedCount = await this.autoLinkEntities(documentId);
1408
2110
  return { documentId, embeddedChunks: embeddedCount, totalChunks: chunks.length, linkedEntities: linkedCount, ...(errors.length > 0 && { errors: errors.slice(0, 5) }) };
@@ -1736,10 +2438,20 @@ export class RAGKnowledgeGraphManager {
1736
2438
  created_at: row.created_at
1737
2439
  }));
1738
2440
  console.error(`✅ Export completed: ${entities.length} entities, ${relations.length} relations, ${documents.length} documents`);
2441
+ // spec §6.4: lifecycle 정본을 함께 내보낸다. 이게 없으면 export->import 뒤
2442
+ // 관찰의 신원·출처·이력이 사라지고 projection 만 남는다.
2443
+ const observation_roots = this.db.prepare(`SELECT * FROM observation_roots ORDER BY entity_id, projection_order`).all();
2444
+ const entity_observations = this.db.prepare(`SELECT * FROM entity_observations ORDER BY root_id, revision_no`).all();
2445
+ const observation_sources = this.db.prepare(`SELECT * FROM observation_sources ORDER BY observation_id, source_kind, source_ref`).all();
2446
+ const observation_events = this.db.prepare(`SELECT * FROM observation_events ORDER BY root_id, recorded_at, event_id`).all();
1739
2447
  return {
1740
2448
  entities,
1741
2449
  relations,
1742
2450
  documents,
2451
+ observation_roots,
2452
+ entity_observations,
2453
+ observation_sources,
2454
+ observation_events,
1743
2455
  metadata: {
1744
2456
  exportedAt: new Date().toISOString(),
1745
2457
  version: PKG_VERSION,
@@ -1755,63 +2467,245 @@ export class RAGKnowledgeGraphManager {
1755
2467
  console.error(`📥 Importing knowledge graph (merge: ${options.merge !== false})...`);
1756
2468
  const imported = { entities: 0, relations: 0, documents: 0 };
1757
2469
  const skipped = { entities: 0, relations: 0, documents: 0 };
1758
- // If merge=false, clear existing data first
1759
- if (options.merge === false) {
1760
- this.db.exec(`DELETE FROM relationships`);
1761
- this.db.exec(`DELETE FROM entities`);
1762
- this.db.exec(`DELETE FROM documents`);
1763
- console.error('🗑️ Cleared existing data for full import');
1764
- }
1765
- // Import entities using INSERT OR IGNORE
1766
- if (data.entities && Array.isArray(data.entities)) {
1767
- const stmt = this.db.prepare(`
2470
+ // merge 로 배열 위치가 재배정된 관찰. 조용히 순서를 바꾸면 호출자가 알 수 없으므로
2471
+ // 응답으로 내보낸다(advisor beta r3 발견 3).
2472
+ const remapReport = [];
2473
+ // spec §6.4: abort 는 0 mutation 이다. lifecycle 만 트랜잭션으로 감싸면
2474
+ // 충돌로 throw 할 때 그 앞에서 넣은 entity·relation·document 가 살아남는다
2475
+ // (T17b 가 ghost entity 로 실증). import 전체가 한 단위여야 한다.
2476
+ // 내부 transaction() 호출은 better-sqlite3 에서 savepoint 로 중첩된다.
2477
+ const importAll = this.db.transaction(() => {
2478
+ // If merge=false, clear existing data first
2479
+ if (options.merge === false) {
2480
+ this.db.exec(`DELETE FROM relationships`);
2481
+ // entities 삭제가 FK CASCADE 로 lifecycle 4테이블을 지우지만, 순서를 계약으로
2482
+ // 두어 FK 가 꺼진 환경에서도 잔존 행이 남지 않게 한다.
2483
+ this.db.exec(`DELETE FROM observation_events`);
2484
+ this.db.exec(`DELETE FROM observation_sources`);
2485
+ this.db.exec(`DELETE FROM entity_observations`);
2486
+ this.db.exec(`DELETE FROM observation_roots`);
2487
+ this.db.exec(`DELETE FROM entities`);
2488
+ this.db.exec(`DELETE FROM documents`);
2489
+ // entities 를 지워도 파생 데이터는 따라오지 않는다: chunk_metadata 에는
2490
+ // entities 로 가는 FK 가 없고 entity_embedding_metadata.entity_id 는 UNIQUE 일
2491
+ // 뿐이다. 그래서 replace-import 뒤에 **사라진 entity 의 벡터와 KG chunk 가
2492
+ // 검색에 남았다**(advisor beta 발견 2). document chunk 는 documents 의
2493
+ // CASCADE 로 이미 정리되므로 여기서는 entity·relationship chunk 만 지운다.
2494
+ const orphanChunks = this.db.prepare(`SELECT rowid FROM chunk_metadata WHERE chunk_type IN ('entity','relationship')`)
2495
+ .all();
2496
+ for (const c of orphanChunks) {
2497
+ this.db.exec(`DELETE FROM chunks WHERE rowid = ${Number(c.rowid)}`);
2498
+ this.db.prepare(`DELETE FROM chunk_metadata WHERE rowid = ?`).run(c.rowid);
2499
+ }
2500
+ this.db.exec(`DELETE FROM entity_embeddings WHERE rowid IN (SELECT rowid FROM entity_embedding_metadata)`);
2501
+ this.db.exec(`DELETE FROM entity_embedding_metadata`);
2502
+ console.error('🗑️ Cleared existing data for full import');
2503
+ }
2504
+ // Import entities using INSERT OR IGNORE
2505
+ if (data.entities && Array.isArray(data.entities)) {
2506
+ const stmt = this.db.prepare(`
1768
2507
  INSERT OR IGNORE INTO entities (id, name, entityType, observations, metadata, created_at)
1769
2508
  VALUES (?, ?, ?, ?, ?, ?)
1770
2509
  `);
1771
- for (const entity of data.entities) {
1772
- const result = stmt.run(entity.id, entity.name, entity.entityType || 'CONCEPT', JSON.stringify(entity.observations || []), JSON.stringify(entity.metadata || {}), entity.created_at || new Date().toISOString());
1773
- if (result.changes > 0) {
1774
- imported.entities++;
1775
- }
1776
- else {
1777
- skipped.entities++;
2510
+ for (const entity of data.entities) {
2511
+ const result = stmt.run(entity.id, entity.name, entity.entityType || 'CONCEPT',
2512
+ // v13: observations 는 projection 이다. lifecycle 행을 넣은 뒤
2513
+ // rebuildProjection 이 채운다 — 여기서 배열을 심으면 정본과 갈라진다.
2514
+ '[]', JSON.stringify(entity.metadata || {}), entity.created_at || new Date().toISOString());
2515
+ if (result.changes > 0) {
2516
+ imported.entities++;
2517
+ }
2518
+ else {
2519
+ skipped.entities++;
2520
+ }
1778
2521
  }
1779
2522
  }
1780
- }
1781
- // Import relations using INSERT OR IGNORE
1782
- if (data.relations && Array.isArray(data.relations)) {
1783
- const stmt = this.db.prepare(`
2523
+ // Import relations using INSERT OR IGNORE
2524
+ if (data.relations && Array.isArray(data.relations)) {
2525
+ const stmt = this.db.prepare(`
1784
2526
  INSERT OR IGNORE INTO relationships (id, source_entity, target_entity, relationType, confidence, metadata, created_at)
1785
2527
  VALUES (?, ?, ?, ?, ?, ?, ?)
1786
2528
  `);
1787
- for (const relation of data.relations) {
1788
- const result = stmt.run(relation.id, relation.source_entity, relation.target_entity, relation.relationType, relation.confidence ?? 1.0, JSON.stringify(relation.metadata || {}), relation.created_at || new Date().toISOString());
1789
- if (result.changes > 0) {
1790
- imported.relations++;
1791
- }
1792
- else {
1793
- skipped.relations++;
2529
+ for (const relation of data.relations) {
2530
+ const result = stmt.run(relation.id, relation.source_entity, relation.target_entity, relation.relationType, relation.confidence ?? 1.0, JSON.stringify(relation.metadata || {}), relation.created_at || new Date().toISOString());
2531
+ if (result.changes > 0) {
2532
+ imported.relations++;
2533
+ }
2534
+ else {
2535
+ skipped.relations++;
2536
+ }
1794
2537
  }
1795
2538
  }
1796
- }
1797
- // Import documents using INSERT OR REPLACE
1798
- if (data.documents && Array.isArray(data.documents)) {
1799
- const stmt = this.db.prepare(`
2539
+ // Import documents using INSERT OR REPLACE
2540
+ if (data.documents && Array.isArray(data.documents)) {
2541
+ const stmt = this.db.prepare(`
1800
2542
  INSERT OR REPLACE INTO documents (id, content, metadata, created_at)
1801
2543
  VALUES (?, ?, ?, ?)
1802
2544
  `);
1803
- for (const doc of data.documents) {
1804
- const result = stmt.run(doc.id, doc.content, JSON.stringify(doc.metadata || {}), doc.created_at || new Date().toISOString());
1805
- if (result.changes > 0) {
1806
- imported.documents++;
1807
- }
1808
- else {
1809
- skipped.documents++;
2545
+ for (const doc of data.documents) {
2546
+ const result = stmt.run(doc.id, doc.content, JSON.stringify(doc.metadata || {}), doc.created_at || new Date().toISOString());
2547
+ if (result.changes > 0) {
2548
+ imported.documents++;
2549
+ }
2550
+ else {
2551
+ skipped.documents++;
2552
+ }
1810
2553
  }
1811
2554
  }
1812
- }
2555
+ // ---- spec §6.4: lifecycle import ----
2556
+ // 순서가 계약이다: entities -> roots -> revisions(root별 revision_no ↑)
2557
+ // -> sources/events. §4.1 트리거가 root 선행과 체인 연속성을 요구하므로
2558
+ // importer 는 입력 순서와 무관하게 재정렬한다 (역순 export 를 그대로
2559
+ // 스트리밍하면 'immediately preceding revision' 으로 죽는다).
2560
+ const sameRow = (a, b, cols) => cols.every(c => (a[c] ?? null) === (b[c] ?? null));
2561
+ const hasLifecycle = Array.isArray(data.observation_roots);
2562
+ if (hasLifecycle) {
2563
+ const tx = this.db.transaction(() => {
2564
+ // 새 root 가 이미 점유된 (entity_id, projection_order) 슬롯을 요구할 수 있다:
2565
+ // 두 DB 가 같은 entity 이름을 갖고 서로 다른 관찰을 배열 0번에 두면 그렇다.
2566
+ // 이건 §6.4 의 "같은 키 다른 값" 충돌이 아니라 **슬롯 충돌**이고, 규칙이 없어서
2567
+ // raw UNIQUE 오류로 터졌다(내 MCP 왕복 테스트가 잡았다). merge 의 뜻은
2568
+ // "더한다"이므로 들어오는 root 에 다음 빈 순번을 준다 — 남의 관찰을 덮지 않고,
2569
+ // 배열 끝에 붙는다. remap 은 그 root 의 revision 들에도 그대로 적용해야 한다
2570
+ // (trg_obs_matches_root 가 둘의 일치를 요구한다).
2571
+ // 입력 순서에 결과가 의존하면 같은 dump 를 두 번 넣었을 때 배열 순서가 달라진다.
2572
+ // (entity_id, projection_order, root_id) 로 정렬해 결정론을 만든다.
2573
+ const incomingRoots = [...(data.observation_roots ?? [])].sort((a, b) => String(a.entity_id).localeCompare(String(b.entity_id)) ||
2574
+ (a.projection_order - b.projection_order) ||
2575
+ String(a.root_id).localeCompare(String(b.root_id)));
2576
+ const remappedOrder = new Map();
2577
+ for (const r of incomingRoots) {
2578
+ const cur = this.db.prepare(`SELECT * FROM observation_roots WHERE root_id = ?`)
2579
+ .get(r.root_id);
2580
+ if (cur) {
2581
+ // projection_order 는 **target-local** 속성이다: merge 는 배열 위치를
2582
+ // 이 DB 기준으로 재배정하므로, 이미 remap 된 root 를 같은 dump 로 다시
2583
+ // 넣으면 dump 의 옛 순번과 다를 수밖에 없다. 그걸 충돌로 보면 동일
2584
+ // 재수입이 실패한다(advisor beta r3 발견 3, 실행 재현).
2585
+ if (!sameRow(cur, r, ['entity_id', 'created_at']))
2586
+ throw new Error(`import conflict: observation_roots ${r.root_id} differs from the existing row`);
2587
+ remappedOrder.set(r.root_id, cur.projection_order);
2588
+ continue;
2589
+ }
2590
+ let order = r.projection_order;
2591
+ const taken = this.db.prepare(`SELECT root_id FROM observation_roots WHERE entity_id = ? AND projection_order = ?`)
2592
+ .get(r.entity_id, order);
2593
+ if (taken) {
2594
+ order = nextProjectionOrder(this.db, r.entity_id);
2595
+ remappedOrder.set(r.root_id, order);
2596
+ remapReport.push({ root_id: r.root_id, entity_id: r.entity_id,
2597
+ from: r.projection_order, to: order });
2598
+ console.error(` ├─ ↪️ import: ${r.entity_id} position ${r.projection_order} is held by ` +
2599
+ `${taken.root_id}; appending imported observation at ${order}`);
2600
+ }
2601
+ this.db.prepare(`INSERT INTO observation_roots
2602
+ (root_id, entity_id, projection_order, created_at) VALUES (?, ?, ?, ?)`)
2603
+ .run(r.root_id, r.entity_id, order, r.created_at);
2604
+ }
2605
+ // projection_order 는 root 와 같은 이유로 비교 대상이 아니다(target-local).
2606
+ const revCols = ['root_id', 'entity_id', 'revision_no', 'content',
2607
+ 'status', 'supersedes_id', 'recorded_at', 'superseded_at'];
2608
+ const revs = [...(data.entity_observations ?? [])]
2609
+ .sort((a, b) => a.root_id === b.root_id
2610
+ ? a.revision_no - b.revision_no
2611
+ : String(a.root_id).localeCompare(String(b.root_id)));
2612
+ for (const v of revs) {
2613
+ const cur = this.db.prepare(`SELECT * FROM entity_observations WHERE observation_id = ?`)
2614
+ .get(v.observation_id);
2615
+ if (cur) {
2616
+ if (!sameRow(cur, v, revCols))
2617
+ throw new Error(`import conflict: entity_observations ${v.observation_id} differs from the existing row`);
2618
+ continue;
2619
+ }
2620
+ const order = remappedOrder.has(v.root_id)
2621
+ ? remappedOrder.get(v.root_id) : v.projection_order;
2622
+ this.db.prepare(`INSERT INTO entity_observations
2623
+ (observation_id, root_id, entity_id, revision_no, projection_order,
2624
+ content, status, supersedes_id, recorded_at, superseded_at)
2625
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2626
+ .run(v.observation_id, v.root_id, v.entity_id, v.revision_no, order, v.content, v.status, v.supersedes_id ?? null, v.recorded_at, v.superseded_at ?? null);
2627
+ }
2628
+ for (const so of (data.observation_sources ?? [])) {
2629
+ const cur = this.db.prepare(`SELECT * FROM observation_sources
2630
+ WHERE observation_id=? AND source_kind=? AND source_ref=?`)
2631
+ .get(so.observation_id, so.source_kind, so.source_ref);
2632
+ if (cur) {
2633
+ if (!sameRow(cur, so, ['source_hash', 'recorded_at']))
2634
+ throw new Error(`import conflict: observation_sources ` +
2635
+ `${so.observation_id}/${so.source_kind}/${so.source_ref} differs from the existing row`);
2636
+ continue;
2637
+ }
2638
+ this.db.prepare(`INSERT INTO observation_sources
2639
+ (observation_id, source_kind, source_ref, source_hash, recorded_at) VALUES (?, ?, ?, ?, ?)`)
2640
+ .run(so.observation_id, so.source_kind, so.source_ref, so.source_hash ?? null, so.recorded_at);
2641
+ }
2642
+ const evCols = ['root_id', 'from_id', 'to_id', 'event', 'change_kind', 'reason', 'actor', 'batch_id', 'recorded_at'];
2643
+ for (const e of (data.observation_events ?? [])) {
2644
+ const cur = this.db.prepare(`SELECT * FROM observation_events WHERE event_id = ?`)
2645
+ .get(e.event_id);
2646
+ if (cur) {
2647
+ if (!sameRow(cur, e, evCols))
2648
+ throw new Error(`import conflict: observation_events ${e.event_id} differs from the existing row`);
2649
+ continue;
2650
+ }
2651
+ this.db.prepare(`INSERT INTO observation_events
2652
+ (event_id, root_id, from_id, to_id, event, change_kind, reason, actor, batch_id, recorded_at)
2653
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
2654
+ .run(e.event_id, e.root_id, e.from_id ?? null, e.to_id ?? null, e.event, e.change_kind ?? null, e.reason ?? null, e.actor ?? null, e.batch_id ?? null, e.recorded_at);
2655
+ }
2656
+ });
2657
+ tx();
2658
+ }
2659
+ else {
2660
+ // 구(舊) 형식 export: lifecycle 필드가 없으므로 entities.observations 를
2661
+ // 신규 root 로 승격한다. legacy import 필수 필드값 = spec §6.4.
2662
+ const ts = new Date().toISOString();
2663
+ const tx = this.db.transaction(() => {
2664
+ for (const ent of (data.entities ?? [])) {
2665
+ const entityId = ent.id ??
2666
+ `entity_${String(ent.name).toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
2667
+ for (const content of (ent.observations ?? [])) {
2668
+ addRevision(this.db, {
2669
+ entityId, content, status: 'active',
2670
+ sources: [{ source_kind: 'import', source_ref: 'legacy-export', source_hash: null }],
2671
+ actor: 'import', ts, event: 'import'
2672
+ });
2673
+ }
2674
+ }
2675
+ });
2676
+ tx();
2677
+ }
2678
+ // projection 재합성 + 파생 상태 무효화.
2679
+ // 무효화가 없으면 이미 있던 entity 를 덮어쓴 뒤에도 옛 벡터·옛 KG chunk 가
2680
+ // 검색에 남는다. import 는 관찰을 바꾸는 writer 이므로 다른 writer 와 같은
2681
+ // 계약을 져야 한다(advisor beta 발견 2).
2682
+ //
2683
+ // 대상은 `data.entities` 가 아니라 **영향받은 entity 전부**다. lifecycle import 는
2684
+ // observation_roots 만 있어도 활성화되므로, entities 없이 lifecycle 배열만 보내면
2685
+ // revision 은 들어가는데 projection 이 갱신되지 않아 새 사실이 reader 에 안 보이고
2686
+ // 옛 벡터가 남는다(advisor beta r3 발견 2, 실행 재현).
2687
+ const affected = new Set();
2688
+ for (const ent of (data.entities ?? [])) {
2689
+ affected.add(ent.id ??
2690
+ `entity_${String(ent.name).toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`);
2691
+ }
2692
+ for (const r of (data.observation_roots ?? []))
2693
+ if (r.entity_id)
2694
+ affected.add(r.entity_id);
2695
+ for (const v of (data.entity_observations ?? []))
2696
+ if (v.entity_id)
2697
+ affected.add(v.entity_id);
2698
+ for (const entityId of affected) {
2699
+ rebuildProjection(this.db, entityId);
2700
+ this.invalidateDerivedForEntity(entityId);
2701
+ }
2702
+ });
2703
+ importAll();
1813
2704
  console.error(`✅ Import completed: ${imported.entities} entities, ${imported.relations} relations, ${imported.documents} documents imported`);
1814
- return { imported, skipped };
2705
+ // Indirect missing-row producer (spec §5): imported rows may lack vectors.
2706
+ this.coordinator?.invalidateCoverage();
2707
+ this.coordinator?.kick();
2708
+ return { imported, skipped, observation_order_remap: remapReport };
1815
2709
  }
1816
2710
  async hybridSearch(query, limit = 5, useGraph = true) {
1817
2711
  if (!this.db)
@@ -1819,6 +2713,9 @@ export class RAGKnowledgeGraphManager {
1819
2713
  if (!this.encoding)
1820
2714
  throw new Error('Tokenizer not initialized');
1821
2715
  console.error(`🔍 Enhanced hybrid search: "${query}"`);
2716
+ // Parity with searchNodes (beta 1R supplement): an unsearchable query gets
2717
+ // an explicit warning instead of a silent empty envelope.
2718
+ const ftsUnsearchable = compileFtsLiteralQuery(query) === null;
1822
2719
  const queryVariants = this.buildCrossLingualVariants(query);
1823
2720
  if (queryVariants.length > 1) {
1824
2721
  console.error(`🌐 Cross-lingual variants: ${queryVariants.slice(1).join(' | ')}`);
@@ -1853,37 +2750,47 @@ export class RAGKnowledgeGraphManager {
1853
2750
  `).all(Buffer.from(embedding.buffer), k);
1854
2751
  };
1855
2752
  // Search original query plus cross-lingual expansions and keep best match per chunk.
2753
+ // v3.6 eligibility gate (spec §3): vector usage requires model_ready AND
2754
+ // reconciliation settled — otherwise FTS5-only, no waiting.
1856
2755
  const resultMap = new Map();
1857
- try {
1858
- primaryQueryEmbedding = await this.generateEmbedding(queryVariants[0], 1024, true);
1859
- for (const variant of queryVariants) {
1860
- const embedding = await this.generateEmbedding(variant, 1024, true);
1861
- const variantResults = searchChunks(embedding, limit * 3);
1862
- for (const r of variantResults) {
1863
- const existing = resultMap.get(r.chunk_id);
1864
- if (!existing || r.distance < existing.distance) {
1865
- resultMap.set(r.chunk_id, r);
2756
+ let degradationReason;
2757
+ if (!(this.coordinator?.eligible ?? false)) {
2758
+ vectorDegraded = true;
2759
+ degradationReason = this.degradationReason();
2760
+ console.error(`ℹ️ vector search not eligible (${degradationReason ?? 'unknown'}) — FTS5-only`);
2761
+ }
2762
+ else {
2763
+ try {
2764
+ primaryQueryEmbedding = await this.generateEmbedding(queryVariants[0], 1024, true);
2765
+ for (const variant of queryVariants) {
2766
+ const embedding = await this.generateEmbedding(variant, 1024, true);
2767
+ const variantResults = searchChunks(embedding, limit * 3);
2768
+ for (const r of variantResults) {
2769
+ const existing = resultMap.get(r.chunk_id);
2770
+ if (!existing || r.distance < existing.distance) {
2771
+ resultMap.set(r.chunk_id, r);
2772
+ }
1866
2773
  }
1867
2774
  }
1868
2775
  }
1869
- }
1870
- catch (embErr) {
1871
- vectorDegraded = true;
1872
- console.error(`⚠️ Vector search unavailable (embedding model down) — degrading to FTS5-only:`, embErr instanceof Error ? embErr.message : embErr);
2776
+ catch (embErr) {
2777
+ vectorDegraded = true;
2778
+ // 'inference_error' (not 'model_not_ready'): model_state may still read
2779
+ // 'ready' here — a contradictory reason pair confused callers (beta B6).
2780
+ degradationReason = this.degradationReason() ?? 'inference_error';
2781
+ console.error(`⚠️ Vector search unavailable — degrading to FTS5-only:`, embErr instanceof Error ? embErr.message : embErr);
2782
+ }
1873
2783
  }
1874
2784
  const vectorResults = Array.from(resultMap.values()).sort((a, b) => a.distance - b.distance);
1875
2785
  // FTS5 full-text search as additional signal (Reciprocal Rank Fusion)
1876
2786
  const ftsBoostMap = new Map();
1877
2787
  try {
1878
2788
  const ftsSearchQuery = (q) => {
1879
- // Escape FTS5 special characters and build a query with OR between terms
1880
- const sanitized = q.replace(/["\*\(\)\-]/g, ' ').trim();
1881
- if (!sanitized)
1882
- return [];
1883
- const terms = sanitized.split(/\s+/).filter(t => t.length > 0);
1884
- if (terms.length === 0)
2789
+ // Shared compiler (spec §5) — same sanitize rules as pre-3.6, extracted
2790
+ // so entity FTS fallback uses identical MATCH-safety guarantees.
2791
+ const ftsExpr = compileFtsLiteralQuery(q);
2792
+ if (ftsExpr === null)
1885
2793
  return [];
1886
- const ftsExpr = terms.map(t => `"${t}"`).join(' OR ');
1887
2794
  return this.db.prepare(`
1888
2795
  SELECT cm.rowid, cm.chunk_id, bm25(chunks_fts) as fts_score
1889
2796
  FROM chunks_fts
@@ -1953,7 +2860,19 @@ export class RAGKnowledgeGraphManager {
1953
2860
  }
1954
2861
  if (vectorResults.length === 0) {
1955
2862
  console.error(`ℹ️ No vector or FTS5 matches found for "${query}"`);
1956
- return [];
2863
+ // Empty results still carry state (spec §5c: envelope exists so callers
2864
+ // can distinguish "nothing matched" from "vector search was degraded").
2865
+ const covE = this.coordinator?.coverage();
2866
+ const chunkPctE = covE && covE.chunk.total > 0 ? Math.round((covE.chunk.embedded / covE.chunk.total) * 100) : 100;
2867
+ const graphPctE = covE && covE.entity.total > 0 ? Math.round((covE.entity.embedded / covE.entity.total) * 100) : 100;
2868
+ return {
2869
+ results: [],
2870
+ search_mode: vectorDegraded ? 'fts-only' : (chunkPctE < 100 ? 'hybrid-partial' : 'hybrid'),
2871
+ model_state: this.gate.status.state,
2872
+ coverage: { chunk_pct: chunkPctE, graph_coverage_pct: graphPctE },
2873
+ ...(degradationReason ? { degradation_reason: degradationReason } : {}),
2874
+ ...(ftsUnsearchable ? { warning: 'query has no searchable terms for FTS' } : {}),
2875
+ };
1957
2876
  }
1958
2877
  // Get entity information for graph enhancement via vector similarity
1959
2878
  let connectedEntities = new Set();
@@ -2157,8 +3076,7 @@ export class RAGKnowledgeGraphManager {
2157
3076
  fts_boost: ftsBoost > 0 ? ftsBoost : undefined,
2158
3077
  full_context_available: true,
2159
3078
  chunk_type: result.chunk_type,
2160
- source_id: sourceId,
2161
- search_mode: vectorDegraded ? 'fts-only' : 'hybrid'
3079
+ source_id: sourceId
2162
3080
  });
2163
3081
  }
2164
3082
  // Sort by relevance and return top results
@@ -2170,7 +3088,34 @@ export class RAGKnowledgeGraphManager {
2170
3088
  const entityResults = finalResults.filter(r => r.chunk_type === 'entity').length;
2171
3089
  const relResults = finalResults.filter(r => r.chunk_type === 'relationship').length;
2172
3090
  console.error(`✅ Enhanced hybrid search completed: ${finalResults.length} results (${docResults} docs, ${entityResults} entities, ${relResults} relationships)`);
2173
- return finalResults;
3091
+ // v3.6 envelope (spec §5c, breaking): search_mode moved from per-item to
3092
+ // top-level so state is visible even on empty results; coverage tells the
3093
+ // caller how much of the corpus is actually vector-searchable.
3094
+ const cov = this.coordinator?.coverage();
3095
+ const chunkPct = cov && cov.chunk.total > 0 ? Math.round((cov.chunk.embedded / cov.chunk.total) * 100) : 100;
3096
+ const graphPct = cov && cov.entity.total > 0 ? Math.round((cov.entity.embedded / cov.entity.total) * 100) : 100;
3097
+ const search_mode = vectorDegraded ? 'fts-only' : (chunkPct < 100 ? 'hybrid-partial' : 'hybrid');
3098
+ return {
3099
+ results: finalResults,
3100
+ search_mode,
3101
+ model_state: this.gate.status.state,
3102
+ coverage: { chunk_pct: chunkPct, graph_coverage_pct: graphPct },
3103
+ ...(degradationReason ? { degradation_reason: degradationReason } : {}),
3104
+ ...(ftsUnsearchable ? { warning: 'query has no searchable terms for FTS' } : {}),
3105
+ };
3106
+ }
3107
+ // v3.6 (spec §5c / 6R note 2): why is vector search degraded right now?
3108
+ degradationReason() {
3109
+ if (this.gate.isDisabled)
3110
+ return 'disabled';
3111
+ if (!this.gate.isReady)
3112
+ return 'model_not_ready';
3113
+ const rs = this.coordinator?.reconState;
3114
+ if (rs === 'failed')
3115
+ return 'reconciliation_failed';
3116
+ if (rs && rs !== 'complete' && rs !== 'n/a')
3117
+ return 'reconciling';
3118
+ return undefined;
2174
3119
  }
2175
3120
  // NEW: Get detailed context for a specific chunk
2176
3121
  async getDetailedContext(chunkId, includeSurrounding = true) {
@@ -2261,6 +3206,10 @@ export class RAGKnowledgeGraphManager {
2261
3206
  const chunkCount = this.db.prepare(`
2262
3207
  SELECT COUNT(*) as count FROM chunk_metadata
2263
3208
  `).get();
3209
+ // v3.6 (spec §8-2, additive): server self-report — the framework's /start
3210
+ // reads version, model/reconciliation state, and provenance coverage here.
3211
+ const gs = this.gate.status;
3212
+ const cov = this.coordinator?.coverage();
2264
3213
  return {
2265
3214
  entities: {
2266
3215
  total: entityStats.reduce((sum, stat) => sum + stat.count, 0),
@@ -2271,7 +3220,23 @@ export class RAGKnowledgeGraphManager {
2271
3220
  by_type: Object.fromEntries(relationshipStats.map(s => [s.relationType, s.count]))
2272
3221
  },
2273
3222
  documents: documentCount.count,
2274
- chunks: chunkCount.count
3223
+ chunks: chunkCount.count,
3224
+ server: {
3225
+ version: PKG_VERSION,
3226
+ node: process.versions.node,
3227
+ embeddings_mode: this.embeddingsMode,
3228
+ model: `${EMBEDDING_MODEL}@${MODEL_REVISION}`,
3229
+ model_state: gs.state,
3230
+ ready_since: gs.readySince ?? null,
3231
+ last_error: gs.lastError ? sanitizeErrorMessage(gs.lastError) : null,
3232
+ retry_at: gs.retryAt ?? null,
3233
+ reconciliation_state: this.coordinator?.reconState ?? 'n/a',
3234
+ reconciliation_last_error: this.coordinator?.reconLastError ?? null,
3235
+ coverage: cov ? {
3236
+ chunk: { total: cov.chunk.total, embedded: cov.chunk.embedded, verified: cov.chunk.verified, legacy_assumed: cov.chunk.legacy_assumed, missing: cov.chunk.total - cov.chunk.embedded },
3237
+ entity: { total: cov.entity.total, embedded: cov.entity.embedded, verified: cov.entity.verified, legacy_assumed: cov.entity.legacy_assumed, missing: cov.entity.total - cov.entity.embedded },
3238
+ } : null,
3239
+ }
2275
3240
  };
2276
3241
  }
2277
3242
  // === GRAPH ANALYTICS TOOLS (graphology) ===
@@ -2595,13 +3560,39 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
2595
3560
  case "createRelations":
2596
3561
  return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.createRelations(validatedArgs.relations), null, 2) }] };
2597
3562
  case "addObservations":
3563
+ // v13: status·sources 를 그대로 넘긴다. 여기서 떨어뜨리면 스키마가 받아도
3564
+ // 엔진에 도달하지 않아 provenance 가 조용히 사라진다.
2598
3565
  return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.addObservations(validatedArgs.observations), null, 2) }] };
3566
+ // v13 observation lifecycle (spec §6.1 / §6.2)
3567
+ case "correctObservation":
3568
+ return { content: [{ type: "text", text: JSON.stringify({ observation_id: await ragKgManager.correctObservation(validatedArgs.observation_id, validatedArgs.content, validatedArgs.change_kind ?? 'correction', validatedArgs.reason) }, null, 2) }] };
3569
+ case "retractObservation":
3570
+ await ragKgManager.retractObservation(validatedArgs.observation_id, validatedArgs.reason);
3571
+ return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'retracted' }, null, 2) }] };
3572
+ case "restoreObservation":
3573
+ await ragKgManager.restoreObservation(validatedArgs.observation_id, validatedArgs.reason);
3574
+ return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'active' }, null, 2) }] };
3575
+ case "approveObservation":
3576
+ await ragKgManager.approveObservation(validatedArgs.observation_id, validatedArgs.reason);
3577
+ return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'active' }, null, 2) }] };
3578
+ case "declineObservation":
3579
+ await ragKgManager.declineObservation(validatedArgs.observation_id, validatedArgs.reason);
3580
+ return { content: [{ type: "text", text: JSON.stringify({ observation_id: validatedArgs.observation_id, status: 'retracted' }, null, 2) }] };
3581
+ case "purgeObservation":
3582
+ return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.purgeObservation(validatedArgs.observation_id, validatedArgs.confirm), null, 2) }] };
3583
+ case "getObservationHistory":
3584
+ return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.getObservationHistory({
3585
+ entity_name: validatedArgs.entity_name,
3586
+ observation_id: validatedArgs.observation_id,
3587
+ root_id: validatedArgs.root_id,
3588
+ }), null, 2) }] };
2599
3589
  case "deleteEntities":
2600
3590
  await ragKgManager.deleteEntities(validatedArgs.entityNames);
2601
3591
  return { content: [{ type: "text", text: "Entities deleted successfully" }] };
2602
3592
  case "deleteObservations":
2603
- await ragKgManager.deleteObservations(validatedArgs.deletions);
2604
- return { content: [{ type: "text", text: "Observations deleted successfully" }] };
3593
+ // v3.6 (spec §5c, breaking): structured per-entity results replace the
3594
+ // bare success string — mixed embedded/queued/no-op states are visible.
3595
+ return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.deleteObservations(validatedArgs.deletions), null, 2) }] };
2605
3596
  case "deleteRelations":
2606
3597
  await ragKgManager.deleteRelations(validatedArgs.relations);
2607
3598
  return { content: [{ type: "text", text: "Relations deleted successfully" }] };
@@ -2672,35 +3663,95 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
2672
3663
  }
2673
3664
  }
2674
3665
  catch (error) {
3666
+ // v3.6 (spec §5c): machine-distinguishable failures. Embedding-gate errors
3667
+ // become structured retryable/terminal payloads; every error response now
3668
+ // sets isError so clients stop parsing "Error: ..." strings.
3669
+ if (error instanceof GateNotReadyError) {
3670
+ return { isError: true, content: [{ type: "text", text: JSON.stringify({
3671
+ code: error.code, state: error.state,
3672
+ ...(error.retryAfterMs !== undefined ? { retry_after_ms: error.retryAfterMs } : {}),
3673
+ message: error.message
3674
+ }) }] };
3675
+ }
3676
+ if (error instanceof GateDisabledError) {
3677
+ return { isError: true, content: [{ type: "text", text: JSON.stringify({
3678
+ code: error.code, state: error.state, message: error.message
3679
+ }) }] };
3680
+ }
2675
3681
  if (error instanceof Error) {
2676
3682
  console.error(`❌ Tool execution error for ${name}:`, error.message);
2677
- return { content: [{ type: "text", text: `Error: ${error.message}` }] };
3683
+ return { isError: true, content: [{ type: "text", text: `Error: ${error.message}` }] };
2678
3684
  }
2679
3685
  throw error;
2680
3686
  }
2681
3687
  });
2682
3688
  async function main() {
2683
3689
  try {
3690
+ assertNodeVersion();
2684
3691
  await ragKgManager.initialize();
3692
+ printBanner({
3693
+ model: EMBEDDING_MODEL, revision: MODEL_REVISION, dtype: MODEL_DTYPE,
3694
+ cachePath: resolveModelCacheDir(process.env, process.platform, os.homedir()),
3695
+ dbPath: DB_FILE_PATH,
3696
+ });
3697
+ if (ragKgManager.embeddingsMode === 'eager') {
3698
+ // eager = wait for BOTH the first model load attempt and reconciliation to
3699
+ // settle (success or failure) before connecting — v3.5-equivalent boot
3700
+ // extended to legacy DBs (spec §9). Failures fall back to background retry.
3701
+ await Promise.allSettled([ragKgManager.gate.start(), ragKgManager.startReconciliation()]);
3702
+ }
2685
3703
  const transport = new StdioServerTransport();
2686
3704
  await server.connect(transport);
2687
3705
  console.error("🚀 Enhanced RAG Knowledge Graph MCP Server running on stdio");
2688
- // Cleanup on exit — avoid process.exit() to prevent ONNX runtime mutex crash
3706
+ if (ragKgManager.embeddingsMode === 'lazy') {
3707
+ // Background: model load + provenance reconciliation run in parallel.
3708
+ // Failures surface via gate/coordinator state, never as rejections.
3709
+ void ragKgManager.gate.start().catch(() => { });
3710
+ void ragKgManager.startReconciliation().catch(() => { });
3711
+ }
3712
+ else if (ragKgManager.embeddingsMode === 'off') {
3713
+ // off mode still CLASSIFIES reconciliation state (deferred vs n/a) so
3714
+ // stats honor the mode matrix — no sanitation, no inference (beta B7).
3715
+ void ragKgManager.startReconciliation().catch(() => { });
3716
+ }
3717
+ // Graceful shutdown (spec §3 order, beta-2R-amended) — transport close
3718
+ // FIRST so stdin stops holding the event loop, then settle coordinator and
3719
+ // gate, then DB close. process.exit is forbidden EXCEPT the one spec'd
3720
+ // case: a model load/download still pending after the settle deadline
3721
+ // (un-abortable fetch would hold the loop forever) — see shutdownAll.
3722
+ let shuttingDown = false;
2689
3723
  const shutdown = () => {
2690
- console.error('\n🧹 Cleaning up...');
2691
- try {
2692
- ragKgManager.cleanup();
2693
- }
2694
- catch { }
3724
+ if (shuttingDown)
3725
+ return;
3726
+ shuttingDown = true;
3727
+ void (async () => {
3728
+ try {
3729
+ await server.close();
3730
+ }
3731
+ catch { /* transport already gone */ }
3732
+ try {
3733
+ process.stdin.pause();
3734
+ process.stdin.unref?.();
3735
+ }
3736
+ catch { /* best-effort */ }
3737
+ await ragKgManager.shutdownAll();
3738
+ })();
2695
3739
  };
2696
3740
  process.on('SIGINT', shutdown);
2697
3741
  process.on('SIGTERM', shutdown);
2698
- process.on('exit', shutdown);
3742
+ process.on('exit', () => { try {
3743
+ ragKgManager.cleanup();
3744
+ }
3745
+ catch { /* idempotent */ } });
3746
+ console.error('🛡️ shutdown handlers registered'); // deterministic handler-ready marker (5R test residual)
2699
3747
  }
2700
3748
  catch (error) {
2701
3749
  console.error("Failed to initialize server:", error);
2702
- ragKgManager.cleanup();
2703
- process.exit(1);
3750
+ try {
3751
+ ragKgManager.cleanup();
3752
+ }
3753
+ catch { /* already down */ }
3754
+ process.exitCode = 1;
2704
3755
  }
2705
3756
  }
2706
3757
  // Boot the server unless explicitly suppressed. Tests import this module with
@@ -2711,7 +3762,10 @@ async function main() {
2711
3762
  if (process.env.RAG_MEMORY_NO_AUTOSTART !== '1') {
2712
3763
  main().catch((error) => {
2713
3764
  console.error("Fatal error in main():", error);
2714
- ragKgManager.cleanup();
2715
- process.exit(1);
3765
+ try {
3766
+ ragKgManager.cleanup();
3767
+ }
3768
+ catch { /* already down */ }
3769
+ process.exitCode = 1;
2716
3770
  });
2717
3771
  }