rag-memory-epf-mcp 5.0.0 → 5.1.0

package/README.md

@@ -64,7 +64,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 ### Document Pipeline (9)
 | Tool | Description | Annotation |
 |------|------------|------------|
-| `storeDocument` | Store documents with metadata | idempotent |
+| `storeDocument` | Store documents with metadata. Replacing an existing document reports what it destroyed: `{ replaced, deletedChunks }` | idempotent |
 | `chunkDocument` | Create text chunks with configurable parameters | — |
 | `embedChunks` | Generate 1024-dim embeddings + auto-link entities | idempotent |
 | `embedAllEntities` | Batch embed all entities (32 parallel) | idempotent |
@@ -72,7 +72,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 | `linkEntitiesToDocument` | Link entities to chunks where they actually appear (text-matched) | idempotent |
 | `deleteDocuments` | Remove documents and associated data | destructive |
 | `listDocuments` | View all stored documents | readOnly |
-| `syncDocumentFromFile` | One-call server-side sync: reads file + delete/store/chunk/embed/link, content stays off model context. Atomic (embed-first transaction swap) + `content_hash` dedup (skips unchanged files) | idempotent |
+| `syncDocumentFromFile` | One-call server-side sync: reads file + delete/store/chunk/embed/link, content stays off model context. Atomic (embed-first transaction swap) + `content_hash` dedup (skips unchanged files). `excludePattern` strips regions before indexing, and the hash follows the stripped text so changing the pattern re-indexes | idempotent |
 
 ### Search & Retrieval (9)
 | Tool | Description | Annotation |

package/dist/index.d.ts

@@ -227,6 +227,7 @@ export declare class RAGKnowledgeGraphManager {
     syncDocumentFromFile(filePath: string, documentId: string, options?: {
         metadata?: Record<string, any>;
         content?: string;
+        excludePattern?: string | string[];
         entityNames?: string[];
         chunkParams?: {
             maxTokens?: number;
@@ -252,6 +253,8 @@ export declare class RAGKnowledgeGraphManager {
     storeDocument(id: string, content: string, metadata?: Record<string, any>): Promise<{
         id: string;
         stored: boolean;
+        replaced: boolean;
+        deletedChunks: number;
     }>;
     chunkDocument(documentId: string, options?: {
         maxTokens?: number;

package/dist/index.js

@@ -146,6 +146,28 @@ function safeRowid(value) {
     }
     return n;
 }
+// Remove regions the caller does not want indexed. Compiled with `s` because the intended use is
+// spanning a marked block (`<!-- SECRET -->…<!-- /SECRET -->`) and JS has no inline (?s) flag —
+// without it every such pattern would silently match nothing.
+// A malformed pattern throws rather than degrading to "no exclusion": indexing is a disclosure
+// path, so believing you excluded something you did not is worse than a failed sync.
+function applyExcludePatterns(text, pattern) {
+    if (pattern === undefined)
+        return text;
+    const patterns = Array.isArray(pattern) ? pattern : [pattern];
+    let out = text;
+    for (const p of patterns) {
+        let re;
+        try {
+            re = new RegExp(p, 'gs');
+        }
+        catch (e) {
+            throw new Error(`excludePattern is not a valid regular expression: ${JSON.stringify(p)} (${e.message})`);
+        }
+        out = out.replace(re, '');
+    }
+    return out;
+}
 // Enhanced RAG-enabled Knowledge Graph Manager
 export class RAGKnowledgeGraphManager {
     db = null;
@@ -1935,7 +1957,11 @@ export class RAGKnowledgeGraphManager {
         const zero = { reusedChunks: 0, newlyEmbeddedChunks: 0, queuedChunks: 0, deletedChunks: 0, chunkerTransitioned: false };
         for (let attempt = 1; attempt <= 3; attempt++) {
             // r6-3: CAS 재시작 = 처음부터 — 파일 읽기·hash·metadata 도 attempt 안에서 재계산한다.
-            const content = options.content !== undefined ? options.content : fsSync.readFileSync(filePath, 'utf-8');
+            // Strip excluded regions before anything else looks at the text. Everything downstream —
+            // content_hash, bytes, chunking — then describes what was actually indexed, so changing the
+            // pattern alone still invalidates the dedup gate below. Hashing the raw file instead would
+            // report `unchanged` for a different exclusion, which is the silent-wrong case.
+            const content = applyExcludePatterns(options.content !== undefined ? options.content : fsSync.readFileSync(filePath, 'utf-8'), options.excludePattern);
             const bytes = Buffer.byteLength(content, 'utf-8');
             const today = new Date().toISOString().slice(0, 10);
             const contentHash = shaHex(content);
@@ -2104,15 +2130,18 @@ export class RAGKnowledgeGraphManager {
         if (!this.db)
             throw new Error('Database not initialized');
         console.error(`📄 Storing document: ${id}`);
+        // Decide `replaced` from the document row, not from the chunk count: a document stored but
+        // never chunked still gets overwritten here, and reporting that as a fresh write would be a lie.
+        const existed = this.db.prepare(`SELECT 1 FROM documents WHERE id = ?`).get(id) !== undefined;
         // Clean up existing document
-        await this.cleanupDocument(id);
+        const cleaned = await this.cleanupDocument(id);
         // Store document
         this.db.prepare(`
       INSERT OR REPLACE INTO documents (id, content, metadata)
       VALUES (?, ?, ?)
     `).run(id, content, JSON.stringify(metadata));
         console.error(`✅ Document stored: ${id}`);
-        return { id, stored: true };
+        return { id, stored: true, replaced: existed, deletedChunks: cleaned.deletedChunks };
     }
     async chunkDocument(documentId, options = {}) {
         if (!this.db)
@@ -2430,9 +2459,13 @@ export class RAGKnowledgeGraphManager {
         console.error(`✅ Entities linked: ${linkedCount} entities linked to document`);
         return { documentId, linkedEntities: linkedCount };
     }
+    // Report what was destroyed. The counts were already computed here and thrown away, so a caller
+    // that replaces a document could not tell from the return value that anything was deleted
+    // (2026-08-05 field report from a deployed project: "{stored:true} came back and I did not know
+    // what I had just wiped"). Silent destruction is the defect; the numbers are free.
     async cleanupDocument(documentId) {
         if (!this.db)
-            return;
+            return { deletedChunks: 0, deletedAssociations: 0, deletedVectors: 0 };
         console.error(`🧹 Cleaning up document: ${documentId}`);
         // Get existing chunks
         const existingChunks = this.db.prepare(`
@@ -2460,6 +2493,7 @@ export class RAGKnowledgeGraphManager {
             console.error(`  ├─ Deleted ${deletedVectors} vector embeddings`);
             console.error(`  └─ Deleted ${metadata.changes} chunk metadata records`);
         }
+        return { deletedChunks: existingChunks.length, deletedAssociations, deletedVectors };
     }
     async deleteDocument(documentId) {
         if (!this.db)
@@ -3839,6 +3873,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.syncDocumentFromFile(validatedArgs.path, validatedArgs.documentId, {
                                 metadata: validatedArgs.metadata,
                                 content: validatedArgs.content,
+                                excludePattern: validatedArgs.excludePattern,
                                 entityNames: validatedArgs.entityNames,
                                 chunkParams: validatedArgs.chunkParams,
                             }), null, 2) }] };

package/dist/src/tools/rag-tools.js

@@ -594,6 +594,7 @@ const syncDocumentFromFileSchema = {
     documentId: z.string().describe('RAG document ID to (re)create from the file'),
     metadata: z.record(z.any()).optional().describe('Metadata merged into the stored document'),
     content: z.string().optional().describe('Optional content override; stored instead of reading the file'),
+    excludePattern: z.union([z.string(), z.array(z.string())]).optional().describe('Regular expression(s) whose matches are stripped before indexing. Applied to the file (or to `content`) first, so hash, byte count and chunking all describe what was actually indexed. Compiled with the dotAll flag, so a pattern may span lines to drop a marked block. An invalid expression fails the call rather than indexing the whole file.'),
     entityNames: z.array(z.string()).optional().describe('Optional entities to explicitly link'),
     chunkParams: z.record(z.any()).optional().describe('Optional chunking parameters { maxTokens }. overlap: omit or 0 only (rejected otherwise since v5.0.0)'),
 };

package/docs/UPDATING.md

@@ -108,6 +108,41 @@ path and holder pid (e.g. `.download-<key>.lock`). Verify the holder process
 is genuinely gone or hung (`ps -p <pid>`), then remove the lock file manually;
 the next start becomes a clean download owner.
 
+## v5.1.0 (schema v14, unchanged): destructive-replace reporting + `excludePattern`
+
+**What changes on upgrade**: nothing you have to do. No migration, no schema
+change, no re-embedding. Both changes are additive — existing calls keep their
+arguments and keep working, and the new response fields are extra keys.
+
+**`storeDocument` now says what it destroyed.** It has always deleted the
+previous document's chunks, vectors and entity links before writing, but the
+response was `{ id, stored: true }`, so a caller replacing a document could not
+tell from the return value that anything was removed. It now returns
+`{ id, stored, replaced, deletedChunks }`, matching what `syncDocumentFromFile`
+already reported. `replaced` is decided by the document row, not the chunk
+count — a document that was stored but never chunked still gets overwritten,
+and reporting that as a fresh write would be wrong.
+
+**`syncDocumentFromFile` accepts `excludePattern`** (string or array of
+strings): regions matching these regular expressions are stripped before
+indexing. Previously the only way to leave part of a file out was to read it
+yourself and pass the whole edited text through `content`, which defeats the
+point of a tool that reads server-side to keep content off the model context.
+
+Three properties worth knowing:
+
+1. **The exclusion happens first**, before hashing and chunking, so
+   `content_hash`, the reported `bytes` and the chunk boundaries all describe
+   what was actually indexed. Changing only the pattern therefore invalidates
+   the dedup gate and re-indexes; it does not silently return `unchanged`.
+2. **Patterns are compiled with the dotAll flag**, so one pattern can span
+   lines to drop a marked block (`<!-- SECRET -->[\s\S]*?<!-- /SECRET -->`).
+   JavaScript has no inline `(?s)`, so without this every such pattern would
+   quietly match nothing.
+3. **An invalid expression fails the call.** Degrading to "no exclusion" would
+   index the whole file while the caller believes it was filtered, and an index
+   is a disclosure path — a failed sync is the safer error.
+
 ## v5.0.0 (schema v14): chunker c1 + vector reuse
 
 **Breaking**: `chunkParams.overlap` is rejected on BOTH public paths

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "engines": {
     "node": ">=24"
   },
-  "description": "Project-local RAG memory MCP server \u2014 knowledge graph + multilingual vector + FTS5 in a single SQLite file. Per-project isolation, 38 MCP tools, codepoint-safe chunking (Korean/CJK/emoji).",
+  "description": "Project-local RAG memory MCP server — knowledge graph + multilingual vector + FTS5 in a single SQLite file. Per-project isolation, 38 MCP tools, codepoint-safe chunking (Korean/CJK/emoji).",
   "keywords": [
     "mcp",
     "model-context-protocol",
@@ -45,7 +45,7 @@
     "prepare": "npm run build",
     "watch": "tsc --watch",
     "verify:invariants": "node test/chunk-invariants.test.mjs",
-    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs",
+    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/document-return-contracts.test.mjs",
     "test": "npm run build && npm run verify:invariants && npm run verify:engine",
     "prepublishOnly": "npm run build && npm run verify:invariants && npm run verify:engine"
   },