rag-memory-epf-mcp 3.5.2 → 4.0.0

package/README.md

@@ -13,7 +13,8 @@ A **project-local RAG memory** MCP server — knowledge graph + multilingual vec
 - **3-signal hybrid search** — vector similarity (bge-m3, 1024-dim) + FTS5 BM25 keyword matching + knowledge graph re-ranking, combined via Reciprocal Rank Fusion
 - **100+ languages** — Korean, Chinese, Japanese, Arabic, and more. Cross-lingual search works out of the box.
 - **Graph-aware scoring** — per-entity geometric decay (0.5^i) with hard cap prevents any single document from dominating results
-- **31 MCP tools** — knowledge graph CRUD, document pipeline, hybrid search, multi-hop traversal, graph analytics (centrality / community detection / structure), export/import, temporal queries
+- **38 MCP tools** — knowledge graph CRUD, observation lifecycle (correct / retract / history), document pipeline, hybrid search, multi-hop traversal, graph analytics (centrality / community detection / structure), export/import, temporal queries
+- **Observations that hold their history** — corrections supersede instead of overwrite, search returns only current facts, and every revision keeps its provenance
 - **Codepoint-safe chunking** — chunk offsets are Unicode codepoints, language-neutral across SQL `substr`, Python slicing, and JS `[...str]` iteration. Korean/CJK/emoji documents stay aligned. Verified by a publish-time invariant test.
 - **SQLite optimized** — WAL mode, 32MB cache, 256MB mmap, FTS5 triggers, 7 indexes
 - **MCP SDK 1.27.1** — Tool Annotations (readOnly/destructive/idempotent), latest protocol 2025-11-25
@@ -36,7 +37,7 @@ A **project-local RAG memory** MCP server — knowledge graph + multilingual vec
 
 Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each project maintains completely isolated memory.
 
-## Tools (31)
+## Tools (38)
 
 ### Knowledge Graph (7)
 | Tool | Description | Annotation |
@@ -47,7 +48,18 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 | `updateRelations` | Update relationship confidence and metadata | idempotent |
 | `deleteEntities` | Remove entities and relationships | destructive |
 | `deleteRelations` | Remove specific relationships | destructive |
-| `deleteObservations` | Remove specific observations | destructive |
+| `deleteObservations` | **Deprecated** soft-retract shim — see Observation Lifecycle | destructive |
+
+### Observation Lifecycle (7)
+| Tool | Description | Annotation |
+|------|------------|------------|
+| `correctObservation` | Supersede a revision with corrected text, keeping the old one | |
+| `retractObservation` | `active` → `retracted` (hidden from search, kept in history) | |
+| `restoreObservation` | `retracted` → `active` | |
+| `approveObservation` | `provisional` → `active` | |
+| `declineObservation` | `provisional` → `retracted` (reason required) | |
+| `purgeObservation` | Physically delete a revision and its successors (`confirm='PURGE'`) | destructive |
+| `getObservationHistory` | Every revision, status, provenance and event | read-only |
 
 ### Document Pipeline (9)
 | Tool | Description | Annotation |
@@ -121,7 +133,8 @@ storeDocument(id, content, metadata)
 │  │  ├── chunks (sqlite-vec, 1024-dim)     │  │
 │  │  ├── entity_embeddings (sqlite-vec)    │  │
 │  │  ├── entities_fts + chunks_fts (FTS5)  │  │
-│  │  └── 11 migrations (auto-applied)      │  │
+│  │  ├── observation lifecycle (4 tables)  │  │
+│  │  └── 13 migrations (auto-applied)      │  │
 │  └────────────────────────────────────────┘  │
 │                                              │
 │  bge-m3 (ONNX, 100+ langs)                   │
@@ -134,9 +147,58 @@ storeDocument(id, content, metadata)
 |----------|---------|-------------|
 | `DB_FILE_PATH` | `rag-memory.db` (server dir) | Path to project-local SQLite database |
 | `EMBEDDING_MODEL` | `Xenova/bge-m3` | HuggingFace model ID for embeddings |
+| `RAG_MEMORY_EMBEDDINGS` | `lazy` | Boot mode: `lazy` (connect instantly, model loads in background), `eager` (wait for model + reconciliation, pre-3.6 behavior), `off` (never load the model — FTS5-only, zero download) |
+| `RAG_MEMORY_MODEL_CACHE_DIR` | OS user cache | Version-independent model cache location (see `docs/UPDATING.md`) |
+| `RAG_MEMORY_TRUST_LEGACY_VECTORS` | unset | Set `1` to grandfather pre-existing vectors under a **custom** `EMBEDDING_MODEL` (default model configs grandfather automatically) |
 
 ## Changelog
 
+### v4.0.0
+
+**Observation lifecycle (schema v13).** Observations used to be a JSON array of strings on the
+entity row. A correction overwrote a string, so the fact that it *was* a correction disappeared —
+and if you deleted the wrong duplicate, nothing recorded that either. Observations now have stable
+ids, provenance, and a status, and `entities.observations` becomes a projection synthesised from
+the `active` revisions.
+
+- **Corrections keep the previous revision.** `correctObservation(observation_id, content, change_kind, reason)`
+  marks the old revision `superseded` and inserts a new one that inherits its position in the array,
+  so a correction does not reorder anything.
+- **Search returns `active` revisions only.** A retracted or superseded fact stops coming back from
+  `openNodes` / `searchNodes` / `readGraph` / `getNeighbors` without being destroyed.
+- **`getObservationHistory({entity_name | observation_id | root_id})` is the only history surface.**
+  It always returns `{ roots: [...] }`, one root per logical observation, revisions oldest-first.
+- **State transitions are a table, not a guess**: `retract` / `restore` / `approve` / `decline`.
+  `superseded` is terminal. Anything outside the table is rejected.
+- **Provenance**: `addObservations` and `createEntities` accept `sources: [{source_kind, source_ref, source_hash?}]`.
+  Repeated content from a *new* source adds evidence to the existing revision instead of a duplicate.
+  Unknown provenance is zero source rows — the engine does not invent one.
+- **`observation_ids`**: `addObservations` and `createEntities` return ids aligned 1:1 with the input,
+  `null` where no revision was created (dedup or source-only).
+- **`purgeObservation(observation_id, 'PURGE')`** physically deletes, as a suffix purge from the
+  target to the newest revision of that root. It is separate, explicit, and almost never what you want.
+- **⚠️ BREAKING**: `deleteObservations` is deprecated. It now performs a soft **retract** instead of
+  a delete, and a batch where any item matches two or more active revisions **aborts with zero
+  mutations** — v3.6 deleted every duplicate and carried on, but a machine cannot tell which
+  revision was meant. Use `retractObservation(observation_id)` to say which one.
+- Migration to v13 writes a recovery point first (`<db>.v12.bak`) using the SQLite Online Backup API,
+  and verifies it — `quick_check` plus FTS5's own `integrity-check`, because a snapshot can be
+  structurally valid and still have a broken full-text index. **An existing one is never
+  overwritten**: the next attempt writes `.bak.1`, then `.bak.2`. Every file in that rotation was
+  taken before any schema change, so each is a valid pre-migration snapshot on its own and nothing
+  has to prove which matches the live database. Slots are bounded and a full set refuses to migrate.
+  Conversion runs in one transaction with two gates: `PRAGMA foreign_key_check`, then a byte-exact
+  comparison of the rebuilt projection against the original array. `foreign_keys` is checked at boot
+  and the server refuses to migrate without it. Restore and slot-exhaustion runbook:
+  `docs/UPDATING.md`.
+
+### v3.6.0
+- **Lite install / lazy boot**: the MCP server connects immediately — FTS5 search, knowledge graph and CRUD work from the first second, while the bge-m3 model (~1.2GB) loads or downloads in the background. Hybrid search switches on automatically. Requires Node **>= 24**.
+- **Version-independent model cache** with a cross-process download lock: engine version bumps no longer re-download the model, and concurrent servers on one machine never corrupt a download. Cleaning the npx cache no longer deletes the model.
+- **Embedding provenance + automatic backfill**: every vector records its input hash and model profile; anything missing or stale (including rows written while the model was unavailable) is re-embedded automatically with a per-target retry cap. Fixes a long-standing defect where `deleteObservations` left stale entity vectors behind.
+- **Search state transparency**: responses report `search_mode` (`hybrid` / `hybrid-partial` / `fts-only`), model state, provenance coverage and a `degradation_reason`; `searchNodes` gains a lexical FTS fallback so entities never disappear from search while embeddings catch up. `getKnowledgeGraphStats` gains a `server` block (version, node, states, coverage) for update-reliability checks.
+- **⚠️ BREAKING**: (1) `hybridSearch` now returns an envelope `{results, search_mode, model_state, coverage, degradation_reason?}` instead of a bare array (per-item `search_mode` removed). (2) `deleteObservations` returns `{results: [{entityName, deleted, embedding_status}], total_deleted}` instead of a success string. Error responses now set `isError: true`. Migration/rollback and fleet-rollout guidance: `docs/UPDATING.md`.
+
 ### v3.5.0
 - **Atomic `syncDocumentFromFile`**: embeddings are computed before any DB write, then applied in a single synchronous transaction, so a failed embedding (e.g. model still loading) leaves the existing document fully intact instead of a half-deleted or partially-embedded state.
 - **`content_hash` dedup**: unchanged files short-circuit the delete/chunk/embed pipeline (`skipped: true`), with a completeness gate that still re-processes a partially-embedded document.

package/dist/index.d.ts

@@ -1,4 +1,9 @@
 #!/usr/bin/env node
+import { type SourceInput } from './src/observations/lifecycle.js';
+import { EmbeddingGate } from './src/embeddingGate.js';
+import type { EmbedPriority } from './src/embeddingGate.js';
+import { BackfillCoordinator } from './src/backfillCoordinator.js';
+export declare function compileFtsLiteralQuery(q: string): string | null;
 interface Entity {
     name: string;
     entityType: string;
@@ -44,15 +49,30 @@ interface DetailedContext {
 export declare class RAGKnowledgeGraphManager {
     private db;
     private encoding;
-    private embeddingModel;
-    private modelInitialized;
+    gate: EmbeddingGate;
+    embeddingsMode: 'lazy' | 'eager' | 'off';
+    currentProfileId: number;
+    grandfatherAllowed: boolean;
+    coordinator: BackfillCoordinator | null;
     private embeddingCache;
     private readonly EMBEDDING_CACHE_MAX;
     private dictionaryCache;
     initialize(opts?: {
         skipModel?: boolean;
+        gate?: EmbeddingGate;
+        __testForceFkOff?: boolean;
     }): Promise<void>;
-    private initializeEmbeddingModel;
+    private ensureCurrentProfile;
+    private buildRealLoader;
+    startReconciliation(): Promise<void>;
+    hashWithBuilderVersion(text: string): string;
+    entityInputHash(entityId: string): string | null;
+    tryEmbedEntity(entityId: string, priority?: EmbedPriority): Promise<'embedded' | 'queued' | 'disabled'>;
+    private mutateEntityAndInvalidate;
+    private invalidateDerivedForEntity;
+    invalidateEntityVector(entityId: string): void;
+    reembedChunkByRowid(rowid: number): Promise<boolean>;
+    shutdownAll(): Promise<void>;
     runMigrations(): Promise<{
         applied: number;
         currentVersion: number;
@@ -63,20 +83,53 @@ export declare class RAGKnowledgeGraphManager {
     }>;
     cleanup(): void;
     private _timestampObservation;
-    createEntities(entities: Entity[]): Promise<Entity[]>;
+    createEntities(entities: Array<Entity & {
+        status?: 'active' | 'provisional';
+        sources?: SourceInput[];
+    }>): Promise<Array<Entity & {
+        created?: boolean;
+        observation_ids?: (string | null)[];
+    }>>;
     createRelations(relations: Relation[]): Promise<Relation[]>;
     addObservations(observations: {
         entityName: string;
         contents: string[];
-    }[]): Promise<{
+        status?: 'active' | 'provisional';
+        sources?: SourceInput[];
+    }[]): Promise<Array<{
         entityName: string;
+        observation_ids: (string | null)[];
         addedObservations: string[];
-    }[]>;
+        embedding_status?: string;
+    }>>;
+    correctObservation(observationId: string, content: string, changeKind?: 'correction' | 'world_change', reason?: string): Promise<string>;
+    private _transition;
+    retractObservation(observationId: string, reason?: string): Promise<void>;
+    restoreObservation(observationId: string, reason?: string): Promise<void>;
+    approveObservation(observationId: string, reason?: string): Promise<void>;
+    declineObservation(observationId: string, reason: string): Promise<void>;
+    purgeObservation(observationId: string, confirm: string): Promise<{
+        purged: number;
+    }>;
+    getObservationHistory(sel: {
+        entity_name?: string;
+        observation_id?: string;
+        root_id?: string;
+    }): Promise<{
+        roots: any[];
+    }>;
     deleteEntities(entityNames: string[]): Promise<void>;
     deleteObservations(deletions: {
         entityName: string;
         observations: string[];
-    }[]): Promise<void>;
+    }[]): Promise<{
+        results: Array<{
+            entityName: string;
+            deleted: number;
+            embedding_status: string;
+        }>;
+        total_deleted: number;
+    }>;
     deleteRelations(relations: Relation[]): Promise<void>;
     updateRelations(updates: {
         from: string;
@@ -108,7 +161,17 @@ export declare class RAGKnowledgeGraphManager {
             path: string[];
         }>;
     }>;
-    searchNodes(query: string, limit?: number, since?: string, until?: string): Promise<KnowledgeGraph>;
+    searchNodes(query: string, limit?: number, since?: string, until?: string): Promise<KnowledgeGraph & {
+        search_mode?: string;
+        model_state?: string;
+        coverage?: {
+            entity_pct: number;
+        };
+        degradation_reason?: string;
+        warning?: string;
+    }>;
+    private searchNodesFts;
+    private relationsAmong;
     openNodes(names: string[]): Promise<KnowledgeGraph>;
     private generateEntityEmbeddingText;
     private buildEntityEmbeddingText;
@@ -241,6 +304,10 @@ export declare class RAGKnowledgeGraphManager {
         entities: any[];
         relations: any[];
         documents: any[];
+        observation_roots: any[];
+        entity_observations: any[];
+        observation_sources: any[];
+        observation_events: any[];
         metadata: {
             exportedAt: string;
             version: string;
@@ -253,6 +320,10 @@ export declare class RAGKnowledgeGraphManager {
         entities?: any[];
         relations?: any[];
         documents?: any[];
+        observation_roots?: any[];
+        entity_observations?: any[];
+        observation_sources?: any[];
+        observation_events?: any[];
     }, options?: {
         merge?: boolean;
     }): Promise<{
@@ -266,8 +337,24 @@ export declare class RAGKnowledgeGraphManager {
             relations: number;
             documents: number;
         };
+        observation_order_remap: Array<{
+            root_id: string;
+            entity_id: string;
+            from: number;
+            to: number;
+        }>;
+    }>;
+    hybridSearch(query: string, limit?: number, useGraph?: boolean): Promise<{
+        results: EnhancedSearchResult[];
+        search_mode: 'hybrid' | 'hybrid-partial' | 'fts-only';
+        model_state: string;
+        coverage: {
+            chunk_pct: number;
+            graph_coverage_pct: number;
+        };
+        degradation_reason?: string;
     }>;
-    hybridSearch(query: string, limit?: number, useGraph?: boolean): Promise<EnhancedSearchResult[]>;
+    degradationReason(): 'disabled' | 'model_not_ready' | 'reconciling' | 'reconciliation_failed' | undefined;
     getDetailedContext(chunkId: string, includeSurrounding?: boolean): Promise<DetailedContext>;
     getKnowledgeGraphStats(): Promise<any>;
     private _buildGraphologyGraph;