rag-memory-epf-mcp 3.6.0 → 5.0.0

package/README.md

@@ -13,8 +13,9 @@ A **project-local RAG memory** MCP server — knowledge graph + multilingual vec
 - **3-signal hybrid search** — vector similarity (bge-m3, 1024-dim) + FTS5 BM25 keyword matching + knowledge graph re-ranking, combined via Reciprocal Rank Fusion
 - **100+ languages** — Korean, Chinese, Japanese, Arabic, and more. Cross-lingual search works out of the box.
 - **Graph-aware scoring** — per-entity geometric decay (0.5^i) with hard cap prevents any single document from dominating results
-- **31 MCP tools** — knowledge graph CRUD, document pipeline, hybrid search, multi-hop traversal, graph analytics (centrality / community detection / structure), export/import, temporal queries
-- **Codepoint-safe chunking** — chunk offsets are Unicode codepoints, language-neutral across SQL `substr`, Python slicing, and JS `[...str]` iteration. Korean/CJK/emoji documents stay aligned. Verified by a publish-time invariant test.
+- **38 MCP tools** — knowledge graph CRUD, observation lifecycle (correct / retract / history), document pipeline, hybrid search, multi-hop traversal, graph analytics (centrality / community detection / structure), export/import, temporal queries
+- **Observations that hold their history** — corrections supersede instead of overwrite, search returns only current facts, and every revision keeps its provenance
+- **Structure-anchored chunking (c1, v5)** — boundaries anchor to markdown structure (fence-aware, H1–H4 first, block-greedy, exact-token fallback), so editing the top of a file no longer re-embeds the whole document: unchanged text reuses its stored vectors at sync time. Chunk offsets are Unicode codepoints, language-neutral across SQL `substr`, Python slicing, and JS `[...str]` iteration; a publish-time invariant gate locks the gap-free partition. `overlap` is retired (omit or 0).
 - **SQLite optimized** — WAL mode, 32MB cache, 256MB mmap, FTS5 triggers, 7 indexes
 - **MCP SDK 1.27.1** — Tool Annotations (readOnly/destructive/idempotent), latest protocol 2025-11-25
 
@@ -36,7 +37,7 @@ A **project-local RAG memory** MCP server — knowledge graph + multilingual vec
 
 Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each project maintains completely isolated memory.
 
-## Tools (31)
+## Tools (38)
 
 ### Knowledge Graph (7)
 | Tool | Description | Annotation |
@@ -47,7 +48,18 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 | `updateRelations` | Update relationship confidence and metadata | idempotent |
 | `deleteEntities` | Remove entities and relationships | destructive |
 | `deleteRelations` | Remove specific relationships | destructive |
-| `deleteObservations` | Remove specific observations | destructive |
+| `deleteObservations` | **Deprecated** soft-retract shim — see Observation Lifecycle | destructive |
+
+### Observation Lifecycle (7)
+| Tool | Description | Annotation |
+|------|------------|------------|
+| `correctObservation` | Supersede a revision with corrected text, keeping the old one | |
+| `retractObservation` | `active` → `retracted` (hidden from search, kept in history) | |
+| `restoreObservation` | `retracted` → `active` | |
+| `approveObservation` | `provisional` → `active` | |
+| `declineObservation` | `provisional` → `retracted` (reason required) | |
+| `purgeObservation` | Physically delete a revision and its successors (`confirm='PURGE'`) | destructive |
+| `getObservationHistory` | Every revision, status, provenance and event | read-only |
 
 ### Document Pipeline (9)
 | Tool | Description | Annotation |
@@ -93,7 +105,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 
 ```
 storeDocument(id, content, metadata)
-  → chunkDocument(documentId, maxTokens, overlap)
+  → chunkDocument(documentId, maxTokens)   # overlap retired in v5 (omit or 0)
     → embedChunks(documentId)
        ├── generates vector embeddings for each chunk
        ├── auto-links entities to chunks (word boundary + CJK aware)
@@ -121,7 +133,8 @@ storeDocument(id, content, metadata)
 │  │  ├── chunks (sqlite-vec, 1024-dim)     │  │
 │  │  ├── entity_embeddings (sqlite-vec)    │  │
 │  │  ├── entities_fts + chunks_fts (FTS5)  │  │
-│  │  └── 11 migrations (auto-applied)      │  │
+│  │  ├── observation lifecycle (4 tables)  │  │
+│  │  └── 13 migrations (auto-applied)      │  │
 │  └────────────────────────────────────────┘  │
 │                                              │
 │  bge-m3 (ONNX, 100+ langs)                   │
@@ -140,6 +153,45 @@ storeDocument(id, content, metadata)
 
 ## Changelog
 
+### v4.0.0
+
+**Observation lifecycle (schema v13).** Observations used to be a JSON array of strings on the
+entity row. A correction overwrote a string, so the fact that it *was* a correction disappeared —
+and if you deleted the wrong duplicate, nothing recorded that either. Observations now have stable
+ids, provenance, and a status, and `entities.observations` becomes a projection synthesised from
+the `active` revisions.
+
+- **Corrections keep the previous revision.** `correctObservation(observation_id, content, change_kind, reason)`
+  marks the old revision `superseded` and inserts a new one that inherits its position in the array,
+  so a correction does not reorder anything.
+- **Search returns `active` revisions only.** A retracted or superseded fact stops coming back from
+  `openNodes` / `searchNodes` / `readGraph` / `getNeighbors` without being destroyed.
+- **`getObservationHistory({entity_name | observation_id | root_id})` is the only history surface.**
+  It always returns `{ roots: [...] }`, one root per logical observation, revisions oldest-first.
+- **State transitions are a table, not a guess**: `retract` / `restore` / `approve` / `decline`.
+  `superseded` is terminal. Anything outside the table is rejected.
+- **Provenance**: `addObservations` and `createEntities` accept `sources: [{source_kind, source_ref, source_hash?}]`.
+  Repeated content from a *new* source adds evidence to the existing revision instead of a duplicate.
+  Unknown provenance is zero source rows — the engine does not invent one.
+- **`observation_ids`**: `addObservations` and `createEntities` return ids aligned 1:1 with the input,
+  `null` where no revision was created (dedup or source-only).
+- **`purgeObservation(observation_id, 'PURGE')`** physically deletes, as a suffix purge from the
+  target to the newest revision of that root. It is separate, explicit, and almost never what you want.
+- **⚠️ BREAKING**: `deleteObservations` is deprecated. It now performs a soft **retract** instead of
+  a delete, and a batch where any item matches two or more active revisions **aborts with zero
+  mutations** — v3.6 deleted every duplicate and carried on, but a machine cannot tell which
+  revision was meant. Use `retractObservation(observation_id)` to say which one.
+- Migration to v13 writes a recovery point first (`<db>.v12.bak`) using the SQLite Online Backup API,
+  and verifies it — `quick_check` plus FTS5's own `integrity-check`, because a snapshot can be
+  structurally valid and still have a broken full-text index. **An existing one is never
+  overwritten**: the next attempt writes `.bak.1`, then `.bak.2`. Every file in that rotation was
+  taken before any schema change, so each is a valid pre-migration snapshot on its own and nothing
+  has to prove which matches the live database. Slots are bounded and a full set refuses to migrate.
+  Conversion runs in one transaction with two gates: `PRAGMA foreign_key_check`, then a byte-exact
+  comparison of the rebuilt projection against the original array. `foreign_keys` is checked at boot
+  and the server refuses to migrate without it. Restore and slot-exhaustion runbook:
+  `docs/UPDATING.md`.
+
 ### v3.6.0
 - **Lite install / lazy boot**: the MCP server connects immediately — FTS5 search, knowledge graph and CRUD work from the first second, while the bge-m3 model (~1.2GB) loads or downloads in the background. Hybrid search switches on automatically. Requires Node **>= 24**.
 - **Version-independent model cache** with a cross-process download lock: engine version bumps no longer re-download the model, and concurrent servers on one machine never corrupt a download. Cleaning the npx cache no longer deletes the model.

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { type SourceInput } from './src/observations/lifecycle.js';
 import { EmbeddingGate } from './src/embeddingGate.js';
 import type { EmbedPriority } from './src/embeddingGate.js';
 import { BackfillCoordinator } from './src/backfillCoordinator.js';
@@ -45,6 +46,22 @@ interface DetailedContext {
     entities: string[];
     metadata: Record<string, any>;
 }
+export declare class SyncCasConflictError extends Error {
+    constructor(documentId: string);
+}
+export declare function setSyncFaultPoint(point: string | null, fn: (() => void) | null): void;
+export interface ReuseCandidate {
+    rowid: number;
+    text: string;
+    input_hash: string | null;
+    profile_id: number | null;
+    provenance_state: string | null;
+    embedding: Buffer | null;
+}
+export declare function selectReusableVector(candidates: ReuseCandidate[], text: string, currentProfileId: number, sha256hex: (s: string) => string): {
+    vec: Buffer;
+    provenance: 'verified' | 'legacy_assumed';
+} | null;
 export declare class RAGKnowledgeGraphManager {
     private db;
     private encoding;
@@ -59,6 +76,7 @@ export declare class RAGKnowledgeGraphManager {
     initialize(opts?: {
         skipModel?: boolean;
         gate?: EmbeddingGate;
+        __testForceFkOff?: boolean;
     }): Promise<void>;
     private ensureCurrentProfile;
     private buildRealLoader;
@@ -67,6 +85,7 @@ export declare class RAGKnowledgeGraphManager {
     entityInputHash(entityId: string): string | null;
     tryEmbedEntity(entityId: string, priority?: EmbedPriority): Promise<'embedded' | 'queued' | 'disabled'>;
     private mutateEntityAndInvalidate;
+    private invalidateDerivedForEntity;
     invalidateEntityVector(entityId: string): void;
     reembedChunkByRowid(rowid: number): Promise<boolean>;
     shutdownAll(): Promise<void>;
@@ -80,15 +99,41 @@ export declare class RAGKnowledgeGraphManager {
     }>;
     cleanup(): void;
     private _timestampObservation;
-    createEntities(entities: Entity[]): Promise<Entity[]>;
+    createEntities(entities: Array<Entity & {
+        status?: 'active' | 'provisional';
+        sources?: SourceInput[];
+    }>): Promise<Array<Entity & {
+        created?: boolean;
+        observation_ids?: (string | null)[];
+    }>>;
     createRelations(relations: Relation[]): Promise<Relation[]>;
     addObservations(observations: {
         entityName: string;
         contents: string[];
-    }[]): Promise<{
+        status?: 'active' | 'provisional';
+        sources?: SourceInput[];
+    }[]): Promise<Array<{
         entityName: string;
+        observation_ids: (string | null)[];
         addedObservations: string[];
-    }[]>;
+        embedding_status?: string;
+    }>>;
+    correctObservation(observationId: string, content: string, changeKind?: 'correction' | 'world_change', reason?: string): Promise<string>;
+    private _transition;
+    retractObservation(observationId: string, reason?: string): Promise<void>;
+    restoreObservation(observationId: string, reason?: string): Promise<void>;
+    approveObservation(observationId: string, reason?: string): Promise<void>;
+    declineObservation(observationId: string, reason: string): Promise<void>;
+    purgeObservation(observationId: string, confirm: string): Promise<{
+        purged: number;
+    }>;
+    getObservationHistory(sel: {
+        entity_name?: string;
+        observation_id?: string;
+        root_id?: string;
+    }): Promise<{
+        roots: any[];
+    }>;
     deleteEntities(entityNames: string[]): Promise<void>;
     deleteObservations(deletions: {
         entityName: string;
@@ -97,7 +142,7 @@ export declare class RAGKnowledgeGraphManager {
         results: Array<{
             entityName: string;
             deleted: number;
-            embedding_status: 'embedded' | 'queued' | 'disabled' | 'n/a';
+            embedding_status: string;
         }>;
         total_deleted: number;
     }>;
@@ -176,7 +221,8 @@ export declare class RAGKnowledgeGraphManager {
     private translateQueryWithMap;
     private buildCrossLingualVariants;
     private extractTermsFromText;
-    private chunkText;
+    private chunkStructured;
+    private validateChunkParams;
     private generateEmbedding;
     syncDocumentFromFile(filePath: string, documentId: string, options?: {
         metadata?: Record<string, any>;
@@ -196,6 +242,12 @@ export declare class RAGKnowledgeGraphManager {
         warning?: string;
         skipped?: boolean;
         reason?: string;
+        embedding_status?: string;
+        reusedChunks: number;
+        newlyEmbeddedChunks: number;
+        queuedChunks: number;
+        deletedChunks: number;
+        chunkerTransitioned: boolean;
     }>;
     storeDocument(id: string, content: string, metadata?: Record<string, any>): Promise<{
         id: string;
@@ -223,6 +275,7 @@ export declare class RAGKnowledgeGraphManager {
         errors?: string[];
     }>;
     private hasCJK;
+    private buildEntityRangeFinder;
     private buildEntityMatcher;
     private autoLinkEntities;
     extractTerms(documentId: string, options?: {
@@ -275,6 +328,10 @@ export declare class RAGKnowledgeGraphManager {
         entities: any[];
         relations: any[];
         documents: any[];
+        observation_roots: any[];
+        entity_observations: any[];
+        observation_sources: any[];
+        observation_events: any[];
         metadata: {
             exportedAt: string;
             version: string;
@@ -287,6 +344,10 @@ export declare class RAGKnowledgeGraphManager {
         entities?: any[];
         relations?: any[];
         documents?: any[];
+        observation_roots?: any[];
+        entity_observations?: any[];
+        observation_sources?: any[];
+        observation_events?: any[];
     }, options?: {
         merge?: boolean;
     }): Promise<{
@@ -300,6 +361,12 @@ export declare class RAGKnowledgeGraphManager {
             relations: number;
             documents: number;
         };
+        observation_order_remap: Array<{
+            root_id: string;
+            entity_id: string;
+            from: number;
+            to: number;
+        }>;
     }>;
     hybridSearch(query: string, limit?: number, useGraph?: boolean): Promise<{
         results: EnhancedSearchResult[];
@@ -335,6 +402,8 @@ export declare class RAGKnowledgeGraphManager {
             version: number;
             description: string;
         }>;
+        semanticRollback?: boolean;
+        warning?: string;
     }>;
 }
 export {};