rag-memory-epf-mcp 4.0.0 → 5.1.0

package/README.md

@@ -15,7 +15,7 @@ A **project-local RAG memory** MCP server — knowledge graph + multilingual vec
 - **Graph-aware scoring** — per-entity geometric decay (0.5^i) with hard cap prevents any single document from dominating results
 - **38 MCP tools** — knowledge graph CRUD, observation lifecycle (correct / retract / history), document pipeline, hybrid search, multi-hop traversal, graph analytics (centrality / community detection / structure), export/import, temporal queries
 - **Observations that hold their history** — corrections supersede instead of overwrite, search returns only current facts, and every revision keeps its provenance
-- **Codepoint-safe chunking** — chunk offsets are Unicode codepoints, language-neutral across SQL `substr`, Python slicing, and JS `[...str]` iteration. Korean/CJK/emoji documents stay aligned. Verified by a publish-time invariant test.
+- **Structure-anchored chunking (c1, v5)** — boundaries anchor to markdown structure (fence-aware, H1–H4 first, block-greedy, exact-token fallback), so editing the top of a file no longer re-embeds the whole document: unchanged text reuses its stored vectors at sync time. Chunk offsets are Unicode codepoints, language-neutral across SQL `substr`, Python slicing, and JS `[...str]` iteration; a publish-time invariant gate locks the gap-free partition. `overlap` is retired (omit or 0).
 - **SQLite optimized** — WAL mode, 32MB cache, 256MB mmap, FTS5 triggers, 7 indexes
 - **MCP SDK 1.27.1** — Tool Annotations (readOnly/destructive/idempotent), latest protocol 2025-11-25
 
@@ -64,7 +64,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 ### Document Pipeline (9)
 | Tool | Description | Annotation |
 |------|------------|------------|
-| `storeDocument` | Store documents with metadata | idempotent |
+| `storeDocument` | Store documents with metadata. Replacing an existing document reports what it destroyed: `{ replaced, deletedChunks }` | idempotent |
 | `chunkDocument` | Create text chunks with configurable parameters | — |
 | `embedChunks` | Generate 1024-dim embeddings + auto-link entities | idempotent |
 | `embedAllEntities` | Batch embed all entities (32 parallel) | idempotent |
@@ -72,7 +72,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 | `linkEntitiesToDocument` | Link entities to chunks where they actually appear (text-matched) | idempotent |
 | `deleteDocuments` | Remove documents and associated data | destructive |
 | `listDocuments` | View all stored documents | readOnly |
-| `syncDocumentFromFile` | One-call server-side sync: reads file + delete/store/chunk/embed/link, content stays off model context. Atomic (embed-first transaction swap) + `content_hash` dedup (skips unchanged files) | idempotent |
+| `syncDocumentFromFile` | One-call server-side sync: reads file + delete/store/chunk/embed/link, content stays off model context. Atomic (embed-first transaction swap) + `content_hash` dedup (skips unchanged files). `excludePattern` strips regions before indexing, and the hash follows the stripped text so changing the pattern re-indexes | idempotent |
 
 ### Search & Retrieval (9)
 | Tool | Description | Annotation |
@@ -105,7 +105,7 @@ Place this `.mcp.json` in each project folder with its own `DB_FILE_PATH`. Each
 
 ```
 storeDocument(id, content, metadata)
-  → chunkDocument(documentId, maxTokens, overlap)
+  → chunkDocument(documentId, maxTokens)   # overlap retired in v5 (omit or 0)
     → embedChunks(documentId)
        ├── generates vector embeddings for each chunk
        ├── auto-links entities to chunks (word boundary + CJK aware)

package/dist/index.d.ts

@@ -46,6 +46,22 @@ interface DetailedContext {
     entities: string[];
     metadata: Record<string, any>;
 }
+export declare class SyncCasConflictError extends Error {
+    constructor(documentId: string);
+}
+export declare function setSyncFaultPoint(point: string | null, fn: (() => void) | null): void;
+export interface ReuseCandidate {
+    rowid: number;
+    text: string;
+    input_hash: string | null;
+    profile_id: number | null;
+    provenance_state: string | null;
+    embedding: Buffer | null;
+}
+export declare function selectReusableVector(candidates: ReuseCandidate[], text: string, currentProfileId: number, sha256hex: (s: string) => string): {
+    vec: Buffer;
+    provenance: 'verified' | 'legacy_assumed';
+} | null;
 export declare class RAGKnowledgeGraphManager {
     private db;
     private encoding;
@@ -205,11 +221,13 @@ export declare class RAGKnowledgeGraphManager {
     private translateQueryWithMap;
     private buildCrossLingualVariants;
     private extractTermsFromText;
-    private chunkText;
+    private chunkStructured;
+    private validateChunkParams;
     private generateEmbedding;
     syncDocumentFromFile(filePath: string, documentId: string, options?: {
         metadata?: Record<string, any>;
         content?: string;
+        excludePattern?: string | string[];
         entityNames?: string[];
         chunkParams?: {
             maxTokens?: number;
@@ -225,10 +243,18 @@ export declare class RAGKnowledgeGraphManager {
         warning?: string;
         skipped?: boolean;
         reason?: string;
+        embedding_status?: string;
+        reusedChunks: number;
+        newlyEmbeddedChunks: number;
+        queuedChunks: number;
+        deletedChunks: number;
+        chunkerTransitioned: boolean;
     }>;
     storeDocument(id: string, content: string, metadata?: Record<string, any>): Promise<{
         id: string;
         stored: boolean;
+        replaced: boolean;
+        deletedChunks: number;
     }>;
     chunkDocument(documentId: string, options?: {
         maxTokens?: number;
@@ -252,6 +278,7 @@ export declare class RAGKnowledgeGraphManager {
         errors?: string[];
     }>;
     private hasCJK;
+    private buildEntityRangeFinder;
     private buildEntityMatcher;
     private autoLinkEntities;
     extractTerms(documentId: string, options?: {
@@ -378,6 +405,8 @@ export declare class RAGKnowledgeGraphManager {
             version: number;
             description: string;
         }>;
+        semanticRollback?: boolean;
+        warning?: string;
     }>;
 }
 export {};

package/dist/index.js

@@ -26,7 +26,7 @@ import { rebuildProjection, deleteStaleKgChunks } from './src/observations/proje
 import { addRevision, correctRevision, transitionStatus, linkSources, nextProjectionOrder } from './src/observations/lifecycle.js';
 import { getObservationHistory } from './src/observations/history.js';
 // Import chunk text algorithm (extracted for publish-time invariant testing)
-import { chunkText as splitTextIntoChunks } from './src/chunkText.js';
+import { chunkStructured as chunkStructuredText, effectiveSignature, isCurrentFormatSignature, LEGACY_SIGNATURE, DEFAULT_MAX_TOKENS } from './src/chunkerC.js';
 import { migrations } from './src/migrations/migrations.js';
 // v3.6 lite install: model lifecycle + version-independent cache (A′ boundary)
 import { EmbeddingGate, GateNotReadyError, GateDisabledError, TerminalConfigError } from './src/embeddingGate.js';
@@ -109,6 +109,36 @@ const TEXT_BUILDER_VERSION = 'tb1';
 // edge belongs to an adjacent chunk and must be removed so TextDecoder does
 // not emit U+FFFD. Pass trimHead/trimTail=false to preserve head/tail bytes.
 // (Implementation moved to src/chunkText.ts for testability.)
+export class SyncCasConflictError extends Error {
+    constructor(documentId) { super(`sync CAS conflict on ${documentId}`); this.name = 'SyncCasConflictError'; }
+}
+// Test-only fault hook (v13 setMigrationFaultPoint 선례 — 환경변수 금지: 상시 스위치는
+// 오설정 한 줄로 sync 를 깬다).
+let __syncFaultHook = null;
+export function setSyncFaultPoint(point, fn) {
+    __syncFaultHook = point && fn ? (p) => { if (p === point)
+        fn(); } : null;
+}
+// Pure vector-reuse decision (spec §5.2 조건 1~5). Returns an OWNED Buffer copy:
+// a Buffer read back from SQLite has no byteOffset-0 guarantee, and inserting
+// `.buffer` of a subarray would write the wrong 4,096 bytes (advisor r5-9).
+export function selectReusableVector(candidates, text, currentProfileId, sha256hex) {
+    const h = sha256hex(text);
+    for (const r of candidates) {
+        if (!r.embedding)
+            continue; // 조건 1: 벡터 실존
+        if (r.input_hash !== h)
+            continue; // 조건 2: input_hash 일치
+        if (r.text !== text)
+            continue; // 조건 3: exact text 최종판정
+        if (r.profile_id !== currentProfileId)
+            continue; // 조건 4: 현행 프로필
+        if (r.provenance_state !== 'verified' && r.provenance_state !== 'legacy_assumed')
+            continue; // 조건 5 (NULL 제외)
+        return { vec: Buffer.from(r.embedding), provenance: r.provenance_state }; // owned copy
+    }
+    return null;
+}
 function safeRowid(value) {
     const n = Number(value);
     if (!Number.isInteger(n) || n < 0) {
@@ -116,6 +146,28 @@ function safeRowid(value) {
     }
     return n;
 }
+// Remove regions the caller does not want indexed. Compiled with `s` because the intended use is
+// spanning a marked block (`<!-- SECRET -->…<!-- /SECRET -->`) and JS has no inline (?s) flag —
+// without it every such pattern would silently match nothing.
+// A malformed pattern throws rather than degrading to "no exclusion": indexing is a disclosure
+// path, so believing you excluded something you did not is worse than a failed sync.
+function applyExcludePatterns(text, pattern) {
+    if (pattern === undefined)
+        return text;
+    const patterns = Array.isArray(pattern) ? pattern : [pattern];
+    let out = text;
+    for (const p of patterns) {
+        let re;
+        try {
+            re = new RegExp(p, 'gs');
+        }
+        catch (e) {
+            throw new Error(`excludePattern is not a valid regular expression: ${JSON.stringify(p)} (${e.message})`);
+        }
+        out = out.replace(re, '');
+    }
+    return out;
+}
 // Enhanced RAG-enabled Knowledge Graph Manager
 export class RAGKnowledgeGraphManager {
     db = null;
@@ -170,6 +222,11 @@ export class RAGKnowledgeGraphManager {
         this.encoding = get_encoding("cl100k_base");
         await this.runMigrations();
         this.currentProfileId = this.ensureCurrentProfile();
+        // v14 (spec §7.2): 런타임이 기본 chunker 의 SSOT — 마이그레이션의 리터럴은 동결된
+        // 역사이고, 기본값이 진화하면(c2 등) 이 upsert 가 부팅마다 현재값을 기록한다.
+        this.db.prepare(`INSERT INTO server_meta (key, value) VALUES ('current_default_chunker', ?)
+                     ON CONFLICT(key) DO UPDATE SET value = excluded.value`)
+            .run(effectiveSignature(DEFAULT_MAX_TOKENS));
         this.embeddingsMode = opts.skipModel
             ? 'off'
             : (process.env.RAG_MEMORY_EMBEDDINGS || 'lazy');
@@ -1843,21 +1900,32 @@ export class RAGKnowledgeGraphManager {
     // CJK), so the function maintains parallel UTF-16 and codepoint cursors and
     // reports codepoint offsets. On a coincidental indexOf miss the char offsets
     // are NULL.
-    chunkText(text, maxTokens = 800, overlap = 160) {
+    chunkStructured(text, maxTokens = DEFAULT_MAX_TOKENS) {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
-        const segments = splitTextIntoChunks(text, this.encoding, maxTokens, overlap);
-        return segments.map((seg, idx) => ({
+        return chunkStructuredText(text, this.encoding, maxTokens).map((seg, idx) => ({
             id: '',
             document_id: '',
             chunk_index: idx,
             text: seg.text,
             start_pos: seg.start_pos,
             end_pos: seg.end_pos,
-            start_token: seg.start_token,
-            end_token: seg.end_token
+            // c1 has no token-space offsets (spec §4.3, r4 D4). Legacy rows keep theirs.
+            start_token: null,
+            end_token: null
         }));
     }
+    // spec §7.1 (r4·r5-8): overlap is rejected on BOTH public paths, BEFORE any
+    // content/dedup judgment — silently accepting it on unchanged content would
+    // void the contract. maxTokens must be a positive integer.
+    validateChunkParams(params) {
+        const { maxTokens = DEFAULT_MAX_TOKENS, overlap = 0 } = params || {};
+        if (!Number.isInteger(maxTokens) || maxTokens <= 0)
+            throw new Error(`chunkParams.maxTokens must be a positive integer (got ${maxTokens})`);
+        if (overlap !== 0)
+            throw new Error(`chunkParams.overlap is no longer supported (chunker c1 has no overlap); omit it or pass 0 (got ${overlap})`);
+        return { maxTokens };
+    }
     // Generate embeddings using sentence transformers
     // isQuery: true for search queries (adds instruction prefix), false for documents/entities
     async generateEmbedding(text, dimensions = 1024, isQuery = false, priority = 'interactive') {
@@ -1882,31 +1950,36 @@ export class RAGKnowledgeGraphManager {
     async syncDocumentFromFile(filePath, documentId, options = {}) {
         if (!this.db)
             throw new Error('Database not initialized');
-        // 1. Resolve content: raw file verbatim (default) or explicit override.
-        //    Content is read on the server and never routed through the model context.
-        const content = options.content !== undefined
-            ? options.content
-            : fsSync.readFileSync(filePath, 'utf-8');
-        const bytes = Buffer.byteLength(content, 'utf-8');
-        // 2. Metadata: default source=path, updated=today, content_hash; caller can override.
-        const today = new Date().toISOString().slice(0, 10);
-        const contentHash = createHash('sha256').update(content).digest('hex');
-        const metadata = { source: filePath, updated: today, content_hash: contentHash, ...(options.metadata || {}) };
-        // 2b. Dedup gate: skip the full delete/store/chunk/embed pipeline when the
-        //     file is unchanged AND the existing document is fully embedded. The
-        //     completeness check avoids wrongly skipping a partial/failed prior sync.
-        const existingDoc = this.db.prepare(`SELECT metadata FROM documents WHERE id = ?`).get(documentId);
-        if (existingDoc) {
+        // spec §7.1 + r5-8: 검증은 content 해석·dedup 판정보다 앞 (첫 실행문).
+        const { maxTokens } = this.validateChunkParams(options.chunkParams);
+        const signature = effectiveSignature(maxTokens);
+        const shaHex = (t) => createHash('sha256').update(t).digest('hex');
+        const zero = { reusedChunks: 0, newlyEmbeddedChunks: 0, queuedChunks: 0, deletedChunks: 0, chunkerTransitioned: false };
+        for (let attempt = 1; attempt <= 3; attempt++) {
+            // r6-3: CAS 재시작 = 처음부터 — 파일 읽기·hash·metadata 도 attempt 안에서 재계산한다.
+            // Strip excluded regions before anything else looks at the text. Everything downstream —
+            // content_hash, bytes, chunking — then describes what was actually indexed, so changing the
+            // pattern alone still invalidates the dedup gate below. Hashing the raw file instead would
+            // report `unchanged` for a different exclusion, which is the silent-wrong case.
+            const content = applyExcludePatterns(options.content !== undefined ? options.content : fsSync.readFileSync(filePath, 'utf-8'), options.excludePattern);
+            const bytes = Buffer.byteLength(content, 'utf-8');
+            const today = new Date().toISOString().slice(0, 10);
+            const contentHash = shaHex(content);
+            // spec §5.1: content_hash 는 system-owned — user metadata 뒤에 쓴다 (r1: spread 가 덮어쓸 수 있었다).
+            const metadata = { source: filePath, updated: today, ...(options.metadata || {}), content_hash: contentHash };
+            const snap = this.db.prepare(`SELECT content, metadata, chunking_signature FROM documents WHERE id = ?`)
+                .get(documentId);
             let existingHash;
-            try {
-                existingHash = JSON.parse(existingDoc.metadata)?.content_hash;
+            if (snap) {
+                try {
+                    existingHash = JSON.parse(snap.metadata)?.content_hash;
+                }
+                catch { /* hash 없으면 full 경로 */ }
             }
-            catch { /* ignore */ }
-            if (existingHash === contentHash) {
+            // dedup gate — spec §5.1 그대로: "content_hash 동일" 만 (r6-8: content=== 확장 금지.
+            // hash 가 없거나 낡은 문서는 full 경로로 가서 hash 가 복구된다). signature 무관.
+            if (snap && existingHash === contentHash) {
                 const cmCount = this.db.prepare(`SELECT count(*) AS n FROM chunk_metadata WHERE document_id = ?`).get(documentId).n;
-                // "Embedded" for dedup completeness = vector exists AND its profile is
-                // current (or legacy-NULL awaiting grandfather). Raw vector counts
-                // would misjudge old-profile rows as complete (beta 1R supplement).
                 const embCount = this.db.prepare(`
           SELECT count(*) AS n FROM chunks c JOIN chunk_metadata m ON c.rowid = m.rowid
           WHERE m.document_id = ? AND (m.provenance_state IS NULL OR m.profile_id = ?)
@@ -1917,114 +1990,158 @@ export class RAGKnowledgeGraphManager {
         `).get(documentId).n;
                 if (cmCount > 0 && cmCount === embCount) {
                     console.error(`⏭️  syncDocumentFromFile: ${documentId} unchanged (hash match, ${cmCount} chunks embedded) — skipped`);
-                    return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged' };
+                    return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked,
+                        skipped: true, reason: 'unchanged', ...zero };
                 }
                 if (cmCount > 0 && embCount < cmCount) {
-                    // v3.6 (spec §5b M12): identical content with incomplete/stale vectors
-                    // keeps the document, chunks, rowids and entity links — only the
-                    // missing vectors are re-queued via the coordinator. Full re-chunking
-                    // here would churn rowids and links for no content change.
+                    // v3.6 (spec §5b M12): identical content with incomplete/stale vectors keeps the
+                    // document, chunks, rowids and entity links — only missing vectors are re-queued.
                     console.error(`♻️ syncDocumentFromFile: ${documentId} unchanged but ${cmCount - embCount} vectors missing — re-queued (chunks preserved)`);
                     this.coordinator?.kick();
-                    return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked, skipped: true, reason: 'unchanged-revectorizing', embedding_status: this.gate.isDisabled ? 'disabled' : 'queued' };
+                    return { documentId, bytes, chunks: cmCount, embeddedChunks: embCount, linkedEntities: linked,
+                        skipped: true, reason: 'unchanged-revectorizing',
+                        embedding_status: this.gate.isDisabled ? 'disabled' : 'queued',
+                        ...zero, queuedChunks: cmCount - embCount };
                 }
+                // cmCount === 0 이면 아래 full 경로로 계속 (최초 생성).
+            }
+            console.error(`🔄 syncDocumentFromFile: ${documentId} <- ${filePath} (${bytes} bytes)`);
+            const segments = this.chunkStructured(content, maxTokens);
+            // spec §5.2-2: 옛 행을 트랜잭션 밖에서 읽는다 (벡터 재사용 후보).
+            const oldRows = this.db.prepare(`
+        SELECT m.rowid, m.text, m.input_hash, m.profile_id, m.provenance_state, c.embedding
+        FROM chunk_metadata m LEFT JOIN chunks c ON c.rowid = m.rowid
+        WHERE m.document_id = ?`).all(documentId);
+            const oldRowids = oldRows.map(r => r.rowid);
+            const byHash = new Map();
+            for (const r of oldRows) {
+                if (!r.input_hash)
+                    continue;
+                const arr = byHash.get(r.input_hash);
+                if (arr)
+                    arr.push(r);
+                else
+                    byHash.set(r.input_hash, [r]);
+            }
+            // 임베딩/재사용 — 트랜잭션 밖, ready 경로 한정 (N2: not-ready 계약 불변).
+            const lazySync = !this.gate.isReady;
+            const slots = [];
+            let reusedChunks = 0, newlyEmbeddedChunks = 0;
+            if (lazySync) {
+                for (const seg of segments)
+                    slots.push({ seg, vec: null, provenance: null });
             }
-        }
-        console.error(`🔄 syncDocumentFromFile: ${documentId} <- ${filePath} (${bytes} bytes)`);
-        // 3. Two contracts (spec §5b):
-        //    ready       — pre-compute ALL embeddings BEFORE any DB mutation; if
-        //                  inference throws mid-way the old document stays intact
-        //                  (v3.5.0 atomicity, unchanged).
-        //    not-ready   — intentional lazy sync: store document + chunks + FTS in
-        //                  one transaction with NO vectors (embedding_status:
-        //                  queued); the backfill coordinator recovers them.
-        const { maxTokens = 800, overlap = 160 } = options.chunkParams || {};
-        const segments = this.chunkText(content, maxTokens, overlap);
-        const lazySync = !this.gate.isReady;
-        const embedded = [];
-        if (lazySync) {
-            for (const seg of segments)
-                embedded.push({ seg, embedding: null });
-        }
-        else {
-            for (const seg of segments) {
-                const embedding = await this.generateEmbedding(seg.text, 1024, false, 'bulk');
-                embedded.push({ seg, embedding });
-            }
-        }
-        // 4. Atomic swap: delete old -> insert doc -> insert chunks (+ embeddings
-        //    with verified provenance when ready), one synchronous transaction.
-        const applyTx = this.db.transaction(() => {
-            const db = this.db;
-            // 4a. cleanup old doc (inlined sync version of cleanupDocument).
-            const existing = db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).all(documentId);
-            for (const ch of existing) {
-                db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(ch.rowid);
-                db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(ch.rowid)}`);
-            }
-            db.prepare(`DELETE FROM chunk_metadata WHERE document_id = ?`).run(documentId);
-            db.prepare(`DELETE FROM documents WHERE id = ?`).run(documentId);
-            // 4b. insert document.
-            db.prepare(`INSERT INTO documents (id, content, metadata) VALUES (?, ?, ?)`)
-                .run(documentId, content, JSON.stringify(metadata));
-            // 4c. insert chunk_metadata (FTS5 chunks_fts auto-filled by trigger);
-            //     vectors + provenance only on the ready path (§6a-2).
-            for (const { seg, embedding } of embedded) {
-                const chunkId = `${documentId}_chunk_${seg.chunk_index}`;
-                const info = db.prepare(`
-          INSERT INTO chunk_metadata (chunk_id, document_id, chunk_index, text, start_pos, end_pos, start_token, end_token)
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        `).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
-                const rowid = Number(info.lastInsertRowid);
-                if (embedding) {
-                    db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(Buffer.from(embedding.buffer));
-                    db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = 'verified' WHERE rowid = ?`)
-                        .run(createHash('sha256').update(seg.text).digest('hex'), this.currentProfileId, rowid);
+            else {
+                for (const seg of segments) {
+                    const hit = selectReusableVector(byHash.get(shaHex(seg.text)) ?? [], seg.text, this.currentProfileId, shaHex);
+                    if (hit) {
+                        slots.push({ seg, vec: hit.vec, provenance: hit.provenance });
+                        reusedChunks++;
+                    }
+                    else {
+                        const embedding = await this.generateEmbedding(seg.text, 1024, false, 'bulk');
+                        slots.push({ seg, vec: Buffer.from(embedding.buffer), provenance: 'verified' });
+                        newlyEmbeddedChunks++;
+                    }
                 }
             }
-        });
-        applyTx();
-        this.coordinator?.invalidateCoverage();
-        if (lazySync)
-            this.coordinator?.kick();
-        const embeddedChunks = lazySync ? 0 : embedded.length;
-        // 5. Entity linking AFTER commit. Non-destructive + idempotent (INSERT OR
-        //    IGNORE), so a linking failure cannot corrupt the doc/embeddings.
-        const linkedEntities = await this.autoLinkEntities(documentId);
-        let explicitlyLinked;
-        if (options.entityNames && options.entityNames.length > 0) {
-            const linkResult = await this.linkEntitiesToDocument(documentId, options.entityNames);
-            explicitlyLinked = linkResult.linkedEntities;
-        }
-        // 6. Terse summary only (no chunk text / content echo) to keep caller context flat.
-        const result = {
-            documentId,
-            bytes,
-            chunks: segments.length,
-            embeddedChunks,
-            linkedEntities,
-            embedding_status: lazySync ? (this.gate.isDisabled ? 'disabled' : 'queued') : 'embedded',
-            ...(explicitlyLinked !== undefined ? { explicitlyLinked } : {}),
-        };
-        if (linkedEntities === 0 && explicitlyLinked === undefined) {
-            result.warning = 'linkedEntities=0: ensure the file content contains entity-name literals (e.g. a wiki anchor line "RAG entity: ...") so term-matching can link entities.';
+            __syncFaultHook?.('pre-transaction');
+            // 한 트랜잭션: CAS 첫 문장 -> full delete/insert -> failure 정리 (spec §5.2-4·5).
+            const applyTx = this.db.transaction(() => {
+                const db = this.db;
+                const now = db.prepare(`SELECT content, metadata, chunking_signature FROM documents WHERE id = ?`)
+                    .get(documentId);
+                const same = (snap === undefined && now === undefined) ||
+                    (snap !== undefined && now !== undefined && now.content === snap.content &&
+                        now.metadata === snap.metadata && now.chunking_signature === snap.chunking_signature);
+                if (!same)
+                    throw new SyncCasConflictError(documentId);
+                const existing = db.prepare(`SELECT rowid FROM chunk_metadata WHERE document_id = ?`).all(documentId);
+                for (const ch of existing) {
+                    db.prepare(`DELETE FROM chunk_entities WHERE chunk_rowid = ?`).run(ch.rowid);
+                    db.exec(`DELETE FROM chunks WHERE rowid = ${safeRowid(ch.rowid)}`);
+                }
+                db.prepare(`DELETE FROM chunk_metadata WHERE document_id = ?`).run(documentId);
+                db.prepare(`DELETE FROM documents WHERE id = ?`).run(documentId);
+                db.prepare(`INSERT INTO documents (id, content, metadata, chunking_signature) VALUES (?, ?, ?, ?)`)
+                    .run(documentId, content, JSON.stringify(metadata), signature);
+                for (const { seg, vec, provenance } of slots) {
+                    const chunkId = `${documentId}_chunk_${seg.chunk_index}`;
+                    const info = db.prepare(`
+            INSERT INTO chunk_metadata (chunk_id, document_id, chunk_index, text, start_pos, end_pos, start_token, end_token)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+          `).run(chunkId, documentId, seg.chunk_index, seg.text, seg.start_pos, seg.end_pos, seg.start_token, seg.end_token);
+                    const rowid = Number(info.lastInsertRowid);
+                    if (vec) {
+                        db.prepare(`INSERT INTO chunks (rowid, embedding) VALUES (${rowid}, ?)`).run(vec);
+                        db.prepare(`UPDATE chunk_metadata SET input_hash = ?, profile_id = ?, provenance_state = ? WHERE rowid = ?`)
+                            .run(shaHex(seg.text), this.currentProfileId, provenance, rowid);
+                    }
+                }
+                if (oldRowids.length > 0) {
+                    // r5-9: 키는 (kind, target_id) — kind 조건 없이 지우면 같은 숫자 ID 의 entity failure 까지 지운다.
+                    const ph = oldRowids.map(() => '?').join(',');
+                    db.prepare(`DELETE FROM embedding_backfill_failures WHERE kind = 'chunk' AND target_id IN (${ph})`)
+                        .run(...oldRowids.map(String));
+                }
+            });
+            try {
+                applyTx();
+            }
+            catch (e) {
+                if (e instanceof SyncCasConflictError) {
+                    console.error(`↻ sync CAS conflict on ${documentId} (attempt ${attempt}/3) — restarting from file read`);
+                    if (attempt === 3)
+                        throw e;
+                    continue;
+                }
+                throw e;
+            }
+            this.coordinator?.invalidateCoverage();
+            if (lazySync)
+                this.coordinator?.kick();
+            // Entity linking AFTER commit. Non-destructive + idempotent (INSERT OR IGNORE).
+            const linkedEntities = await this.autoLinkEntities(documentId);
+            let explicitlyLinked;
+            if (options.entityNames && options.entityNames.length > 0) {
+                const linkResult = await this.linkEntitiesToDocument(documentId, options.entityNames);
+                explicitlyLinked = linkResult.linkedEntities;
+            }
+            const result = {
+                documentId, bytes, chunks: segments.length,
+                embeddedChunks: reusedChunks + newlyEmbeddedChunks, // spec §5.3
+                linkedEntities,
+                embedding_status: lazySync ? (this.gate.isDisabled ? 'disabled' : 'queued') : 'embedded',
+                reusedChunks, newlyEmbeddedChunks,
+                queuedChunks: lazySync ? segments.length : 0,
+                deletedChunks: oldRowids.length,
+                chunkerTransitioned: snap !== undefined && snap.chunking_signature !== signature,
+                ...(explicitlyLinked !== undefined ? { explicitlyLinked } : {}),
+            };
+            if (linkedEntities === 0 && explicitlyLinked === undefined) {
+                result.warning = 'linkedEntities=0: ensure the file content contains entity-name literals (e.g. a wiki anchor line "RAG entity: ...") so term-matching can link entities.';
+            }
+            console.error(`✅ syncDocumentFromFile done: ${documentId} (${result.chunks} chunks, reused ${reusedChunks}, embedded ${newlyEmbeddedChunks})`);
+            return result;
         }
-        console.error(`✅ syncDocumentFromFile done: ${documentId} (${result.chunks} chunks, ${result.embeddedChunks} embedded, ${linkedEntities} linked)`);
-        return result;
+        throw new Error('unreachable');
     }
     async storeDocument(id, content, metadata = {}) {
         if (!this.db)
             throw new Error('Database not initialized');
         console.error(`📄 Storing document: ${id}`);
+        // Decide `replaced` from the document row, not from the chunk count: a document stored but
+        // never chunked still gets overwritten here, and reporting that as a fresh write would be a lie.
+        const existed = this.db.prepare(`SELECT 1 FROM documents WHERE id = ?`).get(id) !== undefined;
         // Clean up existing document
-        await this.cleanupDocument(id);
+        const cleaned = await this.cleanupDocument(id);
         // Store document
         this.db.prepare(`
       INSERT OR REPLACE INTO documents (id, content, metadata)
       VALUES (?, ?, ?)
     `).run(id, content, JSON.stringify(metadata));
         console.error(`✅ Document stored: ${id}`);
-        return { id, stored: true };
+        return { id, stored: true, replaced: existed, deletedChunks: cleaned.deletedChunks };
     }
     async chunkDocument(documentId, options = {}) {
         if (!this.db)
@@ -2036,12 +2153,12 @@ export class RAGKnowledgeGraphManager {
         if (!document) {
             throw new Error(`Document with ID ${documentId} not found`);
         }
-        const { maxTokens = 800, overlap = 160 } = options;
-        console.error(`🔪 Chunking document: ${documentId} (maxTokens: ${maxTokens}, overlap: ${overlap})`);
+        const { maxTokens } = this.validateChunkParams(options);
+        console.error(`🔪 Chunking document: ${documentId} (maxTokens: ${maxTokens}, chunker: c1)`);
         // Clean up existing chunks
         await this.cleanupDocument(documentId);
         // Create chunks
-        const chunks = this.chunkText(document.content, maxTokens, overlap);
+        const chunks = this.chunkStructured(document.content, maxTokens);
         const resultChunks = [];
         for (const chunk of chunks) {
             const chunkId = `${documentId}_chunk_${chunk.chunk_index}`;
@@ -2061,6 +2178,9 @@ export class RAGKnowledgeGraphManager {
             });
         }
         console.error(`✅ Document chunked: ${chunks.length} chunks created`);
+        // spec §7.1: 두 번째 chunk 생성 경로 — 스탬프를 안 박으면 §5.1 관측이 조용히 샌다.
+        this.db.prepare(`UPDATE documents SET chunking_signature = ? WHERE id = ?`)
+            .run(effectiveSignature(maxTokens), documentId);
         // Indirect missing-row producer (spec §5): freshly chunked rows have no
         // vectors yet — let the coordinator recover them without a restart.
         this.coordinator?.invalidateCoverage();
@@ -2113,6 +2233,73 @@ export class RAGKnowledgeGraphManager {
     hasCJK(text) {
         return /[\u3000-\u9fff\uac00-\ud7af\uff00-\uffef]/.test(text);
     }
+    // spec §5.4 (r7-2·r8-1·r9): primary name 의 본문 occurrence range [sCp, eCp).
+    // 의미 = buildEntityMatcher 와 동일 (CJK substring / Latin word-boundary) — 여기서
+    // 어긋나면 'Data' 가 'Database' 에 새로 링크되는 식으로 의미가 확장된다.
+    buildEntityRangeFinder(content) {
+        // 원문 UTF-16 -> codepoint 표. Latin 경로는 folded 가 아니라 **원문**에 regex 를 건다
+        // (r9-1: folded 에 걸면 fooİ -> fooi̇ 로 접힌 뒤 매치돼 현행 matcher 의미가 확장된다).
+        const origU16ToCp = [];
+        let origTotalCp = 0;
+        for (let u = 0; u < content.length;) {
+            const c = content.codePointAt(u);
+            origU16ToCp.push(origTotalCp);
+            if (c > 0xffff) {
+                origU16ToCp.push(origTotalCp);
+                u += 2;
+            }
+            else
+                u += 1;
+            origTotalCp++;
+        }
+        const origCpAt = (u16) => (u16 < origU16ToCp.length ? origU16ToCp[u16] : origTotalCp);
+        // folded 표 (CJK substring / fallback 경로 전용). unit -> 유래한 원문 cp.
+        let folded = '';
+        const u16ToCp = [];
+        let cp = 0;
+        for (const ch of content) { // for..of = codepoint 순회
+            const f = ch.toLowerCase(); // 다단위 fold 가능 (İ -> 'i̇')
+            for (let i = 0; i < f.length; i++)
+                u16ToCp.push(cp);
+            folded += f;
+            cp++;
+        }
+        // r9-1: exclusive end = "마지막으로 소비한 unit 의 원문 cp + 1".
+        // 경계 unit 을 읽으면 매치가 fold 전개 중간에서 끝날 때 1 모자란다 (漢İ/漢i 실측 [0,1)).
+        const endCp = (u16) => (u16 === 0 ? 0 : u16ToCp[Math.min(u16, u16ToCp.length) - 1] + 1);
+        return (name, isCjk) => {
+            const out = [];
+            const lower = name.toLowerCase();
+            const pushAllSubstr = () => {
+                let from = 0;
+                while (true) {
+                    const u = folded.indexOf(lower, from);
+                    if (u < 0)
+                        break;
+                    out.push({ s: u16ToCp[u], e: endCp(u + lower.length) });
+                    from = u + 1; // r9-2: 중첩 occurrence 보존
+                }
+            };
+            if (isCjk) {
+                pushAllSubstr();
+                return out;
+            }
+            try {
+                const escaped = lower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                const re = new RegExp(`\\b${escaped}\\b`, 'gi'); // buildEntityMatcher 와 동일 규칙,
+                let m; // 단 원문에 실행 (의미 확장 방지)
+                while ((m = re.exec(content)) !== null) {
+                    out.push({ s: origCpAt(m.index), e: origCpAt(m.index + m[0].length) });
+                    if (re.lastIndex === m.index)
+                        re.lastIndex++;
+                }
+            }
+            catch {
+                pushAllSubstr();
+            } // matcher 의 fallback 과 동일
+            return out;
+        };
+    }
     // Build a match pattern for an entity name — word-boundary for Latin, substring for CJK
     buildEntityMatcher(name) {
         const lower = name.toLowerCase();
@@ -2136,9 +2323,12 @@ export class RAGKnowledgeGraphManager {
             return 0;
         try {
             // Get all chunk text for this document
-            const chunks = this.db.prepare(`SELECT rowid, text FROM chunk_metadata WHERE document_id = ?`).all(documentId);
+            const chunks = this.db.prepare(`SELECT rowid, text, start_pos, end_pos FROM chunk_metadata WHERE document_id = ?`).all(documentId);
             if (chunks.length === 0)
                 return 0;
+            // spec §5.4: range 링킹용 — 문서 본문과 finder 를 1회 준비
+            const docRow = this.db.prepare(`SELECT content FROM documents WHERE id = ?`).get(documentId);
+            const findRanges = docRow ? this.buildEntityRangeFinder(docRow.content) : null;
             // Get all entities with observations for richer matching
             const entities = this.db.prepare(`SELECT id, name, entityType, observations FROM entities`).all();
             // Minimum name length: 2 for CJK (e.g. "할랄"), 4 for Latin (avoid "API", "Bug")
@@ -2184,6 +2374,19 @@ export class RAGKnowledgeGraphManager {
                         entityLinked = true;
                     }
                 }
+                // spec §5.4 (r7-2): chunk 단위 매칭은 경계에 잘린 이름을 영원히 놓친다 — c1 은
+                // overlap 이 없어 흡수도 안 된다. primary name 의 본문 occurrence range 와
+                // 교차하는 chunk 에 링크한다 (aliases 는 predicate 라 chunk 단위 유지).
+                if (findRanges) {
+                    for (const { s, e } of findRanges(entity.name, this.hasCJK(entity.name))) {
+                        for (const chunk of chunks) {
+                            if (chunk.start_pos !== null && chunk.end_pos !== null && chunk.start_pos < e && chunk.end_pos > s) {
+                                insertStmt.run(chunk.rowid, entity.id); // INSERT OR IGNORE — 중복 무해
+                                entityLinked = true;
+                            }
+                        }
+                    }
+                }
                 if (entityLinked)
                     linkedCount++;
             }
@@ -2256,9 +2459,13 @@ export class RAGKnowledgeGraphManager {
         console.error(`✅ Entities linked: ${linkedCount} entities linked to document`);
         return { documentId, linkedEntities: linkedCount };
     }
+    // Report what was destroyed. The counts were already computed here and thrown away, so a caller
+    // that replaces a document could not tell from the return value that anything was deleted
+    // (2026-08-05 field report from a deployed project: "{stored:true} came back and I did not know
+    // what I had just wiped"). Silent destruction is the defect; the numbers are free.
     async cleanupDocument(documentId) {
         if (!this.db)
-            return;
+            return { deletedChunks: 0, deletedAssociations: 0, deletedVectors: 0 };
         console.error(`🧹 Cleaning up document: ${documentId}`);
         // Get existing chunks
         const existingChunks = this.db.prepare(`
@@ -2286,6 +2493,7 @@ export class RAGKnowledgeGraphManager {
             console.error(`  ├─ Deleted ${deletedVectors} vector embeddings`);
             console.error(`  └─ Deleted ${metadata.changes} chunk metadata records`);
         }
+        return { deletedChunks: existingChunks.length, deletedAssociations, deletedVectors };
     }
     async deleteDocument(documentId) {
         if (!this.db)
@@ -3031,8 +3239,12 @@ export class RAGKnowledgeGraphManager {
                 graphBoost += Math.min(entityBoost, 0.4);
             }
             // Generate semantic summary (skip when degraded — no embeddings available).
+            // RAG_MEMORY_SEARCH_SUMMARIES=off: diagnostic escape hatch (v5) — the summary
+            // path embeds EVERY sentence of EVERY candidate (~100+ inferences per search,
+            // measured 90-120s cold). Off = preview slices + relevanceScore 0; ranking
+            // then rests on vectorSimilarity + boosts. Default unchanged.
             let summary, keyHighlight, relevanceScore;
-            if (vectorDegraded || !primaryQueryEmbedding) {
+            if (vectorDegraded || !primaryQueryEmbedding || process.env.RAG_MEMORY_SEARCH_SUMMARIES === 'off') {
                 keyHighlight = result.text.slice(0, 150);
                 summary = result.text.slice(0, 300);
                 relevanceScore = 0;
@@ -3210,6 +3422,20 @@ export class RAGKnowledgeGraphManager {
         // reads version, model/reconciliation state, and provenance coverage here.
         const gs = this.gate.status;
         const cov = this.coordinator?.coverage();
+        // v14 (spec §7.2): document 기준 chunking 전환 상태 — 상호배타, 합 = documents.
+        // regex 분류는 SQL 밖(JS)에서: current = 런타임이 인식하는 c1 형식(강한 파서),
+        // legacy = 'legacy-unknown', unknown = 그 외 전부.
+        const sigRows = this.db.prepare(`SELECT chunking_signature AS s, count(*) AS n FROM documents GROUP BY chunking_signature`)
+            .all();
+        let sigCur = 0, sigLeg = 0, sigUnk = 0;
+        for (const r of sigRows) {
+            if (r.s === LEGACY_SIGNATURE)
+                sigLeg += r.n;
+            else if (isCurrentFormatSignature(r.s))
+                sigCur += r.n;
+            else
+                sigUnk += r.n;
+        }
         return {
             entities: {
                 total: entityStats.reduce((sum, stat) => sum + stat.count, 0),
@@ -3221,6 +3447,8 @@ export class RAGKnowledgeGraphManager {
             },
             documents: documentCount.count,
             chunks: chunkCount.count,
+            chunking: { current: sigCur, legacy: sigLeg, unknown: sigUnk,
+                default_signature: effectiveSignature(DEFAULT_MAX_TOKENS) },
             server: {
                 version: PKG_VERSION,
                 node: process.versions.node,
@@ -3517,7 +3745,7 @@ export class RAGKnowledgeGraphManager {
             .filter(m => m.version > targetVersion && m.version <= currentVersion)
             .sort((a, b) => b.version - a.version);
         migrationManager.rollback(targetVersion);
-        return {
+        const result = {
             rolledBack: migrationsToRollback.length,
             currentVersion: migrationManager.getCurrentVersion(),
             rolledBackMigrations: migrationsToRollback.map(m => ({
@@ -3525,6 +3753,18 @@ export class RAGKnowledgeGraphManager {
                 description: m.description
             }))
         };
+        // v14 rollback is a compatibility rollback ONLY (spec §6.3): dropping the
+        // chunking_signature column does not restore old chunk boundaries — c1 rows
+        // read fine on v13 code. Say so in the RESPONSE, not just the tool
+        // description, so a caller who rolled back sees the limit (advisor r5-10).
+        if (result.rolledBackMigrations.some(m => m.version === 14)) {
+            return {
+                ...result,
+                semanticRollback: false,
+                warning: 'v14 rollback removes the chunking_signature column only; chunk boundaries produced by chunker c1 are NOT restored (compatibility rollback). Data restore path = pre-migration backup snapshot.'
+            };
+        }
+        return result;
     }
 }
 // Initialize the manager
@@ -3633,6 +3873,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.syncDocumentFromFile(validatedArgs.path, validatedArgs.documentId, {
                                 metadata: validatedArgs.metadata,
                                 content: validatedArgs.content,
+                                excludePattern: validatedArgs.excludePattern,
                                 entityNames: validatedArgs.entityNames,
                                 chunkParams: validatedArgs.chunkParams,
                             }), null, 2) }] };

package/dist/src/chunkerC.d.ts

@@ -0,0 +1,18 @@
+import type { Tiktoken } from 'tiktoken';
+export interface CSegment {
+    text: string;
+    start_pos: number;
+    end_pos: number;
+}
+export declare const DEFAULT_MAX_TOKENS = 800;
+export declare const LEGACY_SIGNATURE = "legacy-unknown";
+export declare function mergeMinTokens(maxTokens: number): number;
+export declare function effectiveSignature(maxTokens: number): string;
+export declare function isCurrentFormatSignature(sig: string): boolean;
+type Block = {
+    lines: string[];
+    heading: boolean;
+};
+export declare function buildBlocks(lines: string[]): Block[];
+export declare function chunkStructured(text: string, enc: Tiktoken, maxTokens?: number): CSegment[];
+export {};

package/dist/src/chunkerC.js

@@ -0,0 +1,210 @@
+export const DEFAULT_MAX_TOKENS = 800;
+export const LEGACY_SIGNATURE = 'legacy-unknown';
+// r11: heading is a preferred cut, not a mandatory one. A cut happens only when
+// the accumulated section group AND the next section are both >= this floor —
+// short adjacent sections merge, so cross-section queries still land in one
+// chunk (gate regression k05) and short log sections stop forming over-sharp
+// competitor chunks (k09). Derived, not independent: one knob (maxTokens).
+export function mergeMinTokens(maxTokens) {
+    return Math.floor(maxTokens / 2); // NOT >>1 — bitshift wraps at 2^31 (r12)
+}
+export function effectiveSignature(maxTokens) {
+    return `c1:enc=cl100k_base:max=${maxTokens}:overlap=0:fence=on:merge=${mergeMinTokens(maxTokens)}:fallback=cp-exact-${maxTokens}`;
+}
+// Strict parse (r5-15): the regex alone classified impossible signatures
+// (max=0, max/fallback mismatch) as current. Cross-check the components.
+export function isCurrentFormatSignature(sig) {
+    const m = /^c1:enc=cl100k_base:max=(\d+):overlap=0:fence=on:merge=(\d+):fallback=cp-exact-(\d+)$/.exec(sig);
+    if (!m)
+        return false;
+    const max = Number(m[1]);
+    return Number.isInteger(max) && max > 0 && m[1] === m[3] && Number(m[2]) === mergeMinTokens(max);
+}
+function splitLines(text) {
+    const lines = [];
+    let start = 0;
+    for (let i = 0; i < text.length; i++) {
+        if (text[i] === '\n') {
+            lines.push(text.slice(start, i + 1));
+            start = i + 1;
+        }
+    }
+    if (start < text.length)
+        lines.push(text.slice(start));
+    return lines;
+}
+const FENCE_RE = /^\s{0,3}(`{3,}|~{3,})/;
+const HEADING_RE = /^#{1,4}[ \t]/;
+const BULLET_RE = /^(\s*)([-*+]|\d+[.)])[ \t]/;
+const BLANK_RE = /^\s*$/;
+export function buildBlocks(lines) {
+    const blocks = [];
+    let i = 0;
+    const attachTrailingBlanks = (blk) => {
+        while (i < lines.length && BLANK_RE.test(lines[i]))
+            blk.lines.push(lines[i++]);
+    };
+    while (i < lines.length) {
+        const line = lines[i];
+        const fm = line.match(FENCE_RE);
+        if (fm) {
+            const marker = fm[1][0];
+            const openLen = fm[1].length;
+            const blk = { lines: [lines[i++]], heading: false };
+            const closeRe = new RegExp(`^\\s{0,3}\\${marker}{${openLen},}\\s*$`);
+            while (i < lines.length) {
+                const l = lines[i];
+                blk.lines.push(l);
+                i++;
+                if (closeRe.test(l.replace(/\r?\n$/, '')))
+                    break; // unclosed -> EOF
+            }
+            attachTrailingBlanks(blk);
+            blocks.push(blk);
+        }
+        else if (HEADING_RE.test(line)) {
+            const blk = { lines: [lines[i++]], heading: true };
+            attachTrailingBlanks(blk);
+            blocks.push(blk);
+        }
+        else if (BULLET_RE.test(line)) {
+            const indent = line.match(BULLET_RE)[1].length;
+            const blk = { lines: [lines[i++]], heading: false };
+            while (i < lines.length) {
+                const l = lines[i];
+                if (BLANK_RE.test(l) || HEADING_RE.test(l) || FENCE_RE.test(l))
+                    break;
+                const bm = l.match(BULLET_RE);
+                if (bm && bm[1].length <= indent)
+                    break; // sibling/outer bullet
+                const li = l.match(/^(\s*)/)[1].length;
+                if (!bm && li <= indent)
+                    break; // dedented prose
+                blk.lines.push(l);
+                i++;
+            }
+            attachTrailingBlanks(blk);
+            blocks.push(blk);
+        }
+        else if (BLANK_RE.test(line)) {
+            const blk = { lines: [], heading: false };
+            while (i < lines.length && BLANK_RE.test(lines[i]))
+                blk.lines.push(lines[i++]);
+            blocks.push(blk);
+        }
+        else {
+            const blk = { lines: [lines[i++]], heading: false };
+            while (i < lines.length && !BLANK_RE.test(lines[i]) && !HEADING_RE.test(lines[i])
+                && !FENCE_RE.test(lines[i]) && !BULLET_RE.test(lines[i])) {
+                blk.lines.push(lines[i++]);
+            }
+            attachTrailingBlanks(blk);
+            blocks.push(blk);
+        }
+    }
+    return blocks;
+}
+// cp-exact oversize split, non-monotonicity-safe (spec §4.3-6, r5-5).
+// BPE prefix token counts are NOT monotonic ('/sdkX' -> 1,1,2,1,2), so binary
+// search is invalid. Scan prefixes linearly, remember the last fitting one, and
+// keep probing LOOKAHEAD candidates past a miss to recover dips. If not even
+// one codepoint fits, throw — never emit an over-budget chunk.
+const LOOKAHEAD = 16;
+function splitOversize(block, enc, maxTokens) {
+    const cps = [...block];
+    const out = [];
+    let start = 0;
+    while (start < cps.length) {
+        let lastFit = 0;
+        let missesSinceFit = 0;
+        for (let probe = start + 1; probe <= cps.length && missesSinceFit < LOOKAHEAD; probe++) {
+            const t = enc.encode(cps.slice(start, probe).join('')).length;
+            if (t <= maxTokens) {
+                lastFit = probe - start;
+                missesSinceFit = 0;
+            }
+            else
+                missesSinceFit++;
+        }
+        if (lastFit === 0) {
+            throw new Error(`chunker c1: codepoint at offset ${start} exceeds maxTokens=${maxTokens} on its own — cannot honor the token budget`);
+        }
+        out.push(cps.slice(start, start + lastFit).join(''));
+        start += lastFit;
+    }
+    return out;
+}
+export function chunkStructured(text, enc, maxTokens = DEFAULT_MAX_TOKENS) {
+    if (text.length === 0)
+        return [];
+    const blocks = buildBlocks(splitLines(text));
+    // r11 section pass: a section = one heading block through the next heading
+    // (preamble = blocks before the first heading). Cut decisions are precomputed
+    // per section so they depend only on section sizes, not packing state.
+    const minSection = mergeMinTokens(maxTokens);
+    const sectionOfBlock = new Array(blocks.length);
+    const sectionTexts = [];
+    let sec = -1;
+    blocks.forEach((b, bi) => {
+        if (b.heading || sec === -1) {
+            sec++;
+            sectionTexts[sec] = '';
+        }
+        sectionOfBlock[bi] = sec;
+        sectionTexts[sec] += b.lines.join('');
+    });
+    const secTokens = sectionTexts.map(t => enc.encode(t).length);
+    // Cut at section s iff the group accumulated since the last cut and section s
+    // are BOTH >= minSection. Group size = sum of section token counts (token
+    // counts are not additive across joins; the sum is a deterministic threshold
+    // proxy, never used as a budget).
+    const cutAtSection = new Array(secTokens.length).fill(false);
+    {
+        let groupTokens = 0;
+        for (let s = 0; s < secTokens.length; s++) {
+            if (s > 0 && groupTokens >= minSection && secTokens[s] >= minSection) {
+                cutAtSection[s] = true;
+                groupTokens = 0;
+            }
+            groupTokens += secTokens[s];
+        }
+    }
+    const pieces = [];
+    let acc = '';
+    const flush = () => { if (acc.length > 0) {
+        pieces.push(acc);
+        acc = '';
+    } };
+    for (let bi = 0; bi < blocks.length; bi++) {
+        const b = blocks[bi];
+        const btext = b.lines.join('');
+        if (btext.length === 0)
+            continue;
+        if (b.heading) {
+            const s = sectionOfBlock[bi];
+            // Preferred cut: honor the precomputed section cut. Merged headings still
+            // flush when their whole section cannot join the open chunk — splitting
+            // at the heading beats orphaning the heading line at a budget flush.
+            if (cutAtSection[s])
+                flush();
+            else if (acc.length > 0 && enc.encode(acc + sectionTexts[s]).length > maxTokens)
+                flush();
+        }
+        if (acc.length > 0 && enc.encode(acc + btext).length > maxTokens)
+            flush();
+        if (acc.length === 0 && enc.encode(btext).length > maxTokens) {
+            pieces.push(...splitOversize(btext, enc, maxTokens)); // mega-block fallback
+            continue;
+        }
+        acc += btext;
+    }
+    flush();
+    const segments = [];
+    let cursor = 0;
+    for (const p of pieces) {
+        const len = [...p].length;
+        segments.push({ text: p, start_pos: cursor, end_pos: cursor + len });
+        cursor += len;
+    }
+    return segments;
+}

package/dist/src/migrations/migrations.js

@@ -751,5 +751,34 @@ export const migrations = [
             db.exec(`DROP TABLE IF EXISTS entity_observations`);
             db.exec(`DROP TABLE IF EXISTS observation_roots`);
         }
+    },
+    {
+        version: 14,
+        description: 'Chunking signature: documents.chunking_signature (schema-only; backfill legacy-unknown)',
+        up: (db) => {
+            // spec §6.1: schema-only. 전환은 sync 의 content 변경 시에만 (spec §5.1, r4 D3).
+            // 백필값은 'legacy-unknown', NOT 'bpe-800-160': custom chunkDocument 파라미터가
+            // 기록된 적 없어 단정하면 거짓 표기가 된다 (advisor r1).
+            const before = db.prepare(`SELECT count(*) AS n FROM documents`).get().n;
+            db.exec(`ALTER TABLE documents ADD COLUMN chunking_signature TEXT NOT NULL DEFAULT 'legacy-unknown'`);
+            // 리터럴 고정 — 마이그레이션은 동결된 역사다. 런타임 기본값 진화는 boot upsert 소관.
+            db.prepare(`INSERT INTO server_meta (key, value) VALUES ('current_default_chunker', ?)
+                  ON CONFLICT(key) DO UPDATE SET value = excluded.value`)
+                .run('c1:enc=cl100k_base:max=800:overlap=0:fence=on:fallback=cp-exact-800');
+            const after = db.prepare(`SELECT count(*) AS n FROM documents`).get().n;
+            if (after !== before)
+                throw new Error(`v14 gate: documents rows changed ${before} -> ${after}`);
+            const cols = db.prepare(`PRAGMA table_info(documents)`).all();
+            if (!cols.some(c => c.name === 'chunking_signature'))
+                throw new Error('v14 gate: column missing after ALTER');
+            const fk = db.prepare(`PRAGMA foreign_key_check`).all();
+            if (fk.length > 0)
+                throw new Error(`v14 gate: foreign_key_check reported ${fk.length} violations`);
+        },
+        down: (db) => {
+            // 호환성 rollback 뿐 (spec §6.3): chunk 경계는 복원하지 않는다. c1 행은 v13 코드가 읽는다.
+            db.exec(`ALTER TABLE documents DROP COLUMN chunking_signature`);
+            db.prepare(`DELETE FROM server_meta WHERE key = 'current_default_chunker'`).run();
+        }
     }
 ];

package/dist/src/tools/migration-tools.js

@@ -110,7 +110,7 @@ Applies all pending database migrations to bring the schema up to the latest ver
     },
     rollbackMigration: {
         capability: {
-            description: 'Rollback database to a specific migration version',
+            description: 'Rollback database to a specific migration version. v14+: rolling back the chunking_signature column does NOT restore old chunk boundaries (compatibility rollback only — the response carries semanticRollback:false and a warning).',
             parameters: {
                 type: 'object',
                 properties: {

package/dist/src/tools/rag-tools.js

@@ -75,12 +75,12 @@ const chunkDocumentCapability = {
             },
             maxTokens: {
                 type: 'number',
-                description: 'Maximum tokens per chunk (default: 200)',
+                description: 'Maximum tokens per chunk (default: 800, positive integer)',
                 optional: true
             },
             overlap: {
                 type: 'number',
-                description: 'Number of overlapping tokens between chunks (default: 20)',
+                description: 'Deprecated since v5.0.0: chunker c1 has no overlap. Omit or pass exactly 0 — any other value is rejected.',
                 optional: true
             }
         },
@@ -94,7 +94,7 @@ Create text chunks from a stored document with configurable chunking parameters.
 
 <importantNotes>
 - (!important!) **Document must be stored first** using storeDocument
-- (!important!) **Configurable chunking** - adjust maxTokens and overlap as needed
+- (!important!) **Configurable chunking** - adjust maxTokens as needed. overlap is deprecated since v5.0.0 (chunker c1 has no overlap): omit it or pass exactly 0 — any other value is rejected
 - (!important!) **Replaces existing chunks** for the document if any exist
 </importantNotes>
 
@@ -108,19 +108,19 @@ Create text chunks from a stored document with configurable chunking parameters.
 <bestPractices>
 - Smaller chunks (100-150 tokens) for precise retrieval
 - Larger chunks (300-500 tokens) for context preservation
-- Use overlap (10-30 tokens) to maintain continuity
+- Chunker c1 anchors boundaries to markdown structure and merges adjacent sections shorter than floor(maxTokens/2), so continuity needs no overlap
 - Consider document type when choosing chunk size
 </bestPractices>
 
 <examples>
 - Default chunking: {"documentId": "doc1"}
-- Custom size: {"documentId": "doc1", "maxTokens": 150, "overlap": 30}
-- Large context: {"documentId": "legal_doc", "maxTokens": 400, "overlap": 50}
+- Custom size: {"documentId": "doc1", "maxTokens": 150}
+- Large context: {"documentId": "legal_doc", "maxTokens": 400}
 </examples>`;
 const chunkDocumentSchema = {
     documentId: z.string().describe('ID of the stored document to chunk'),
-    maxTokens: z.number().default(200).optional().describe('Maximum tokens per chunk'),
-    overlap: z.number().default(20).optional().describe('Number of overlapping tokens'),
+    maxTokens: z.number().optional().default(800).describe('Maximum tokens per chunk (positive integer)'),
+    overlap: z.number().optional().default(0).describe('Deprecated: omit or 0 only (rejected otherwise since v5.0.0)'),
 };
 export const chunkDocumentTool = {
     capability: chunkDocumentCapability,
@@ -549,7 +549,7 @@ const syncDocumentFromFileCapability = {
             },
             chunkParams: {
                 type: 'object',
-                description: 'Optional chunking parameters { maxTokens, overlap }',
+                description: 'Optional chunking parameters { maxTokens }. overlap: omit or 0 only (rejected otherwise since v5.0.0 — chunker c1 has no overlap)',
                 additionalProperties: true,
                 optional: true
             }
@@ -594,8 +594,9 @@ const syncDocumentFromFileSchema = {
     documentId: z.string().describe('RAG document ID to (re)create from the file'),
     metadata: z.record(z.any()).optional().describe('Metadata merged into the stored document'),
     content: z.string().optional().describe('Optional content override; stored instead of reading the file'),
+    excludePattern: z.union([z.string(), z.array(z.string())]).optional().describe('Regular expression(s) whose matches are stripped before indexing. Applied to the file (or to `content`) first, so hash, byte count and chunking all describe what was actually indexed. Compiled with the dotAll flag, so a pattern may span lines to drop a marked block. An invalid expression fails the call rather than indexing the whole file.'),
     entityNames: z.array(z.string()).optional().describe('Optional entities to explicitly link'),
-    chunkParams: z.record(z.any()).optional().describe('Optional chunking parameters { maxTokens, overlap }'),
+    chunkParams: z.record(z.any()).optional().describe('Optional chunking parameters { maxTokens }. overlap: omit or 0 only (rejected otherwise since v5.0.0)'),
 };
 export const syncDocumentFromFileTool = {
     capability: syncDocumentFromFileCapability,

package/docs/UPDATING.md

@@ -108,6 +108,84 @@ path and holder pid (e.g. `.download-<key>.lock`). Verify the holder process
 is genuinely gone or hung (`ps -p <pid>`), then remove the lock file manually;
 the next start becomes a clean download owner.
 
+## v5.1.0 (schema v14, unchanged): destructive-replace reporting + `excludePattern`
+
+**What changes on upgrade**: nothing you have to do. No migration, no schema
+change, no re-embedding. Both changes are additive — existing calls keep their
+arguments and keep working, and the new response fields are extra keys.
+
+**`storeDocument` now says what it destroyed.** It has always deleted the
+previous document's chunks, vectors and entity links before writing, but the
+response was `{ id, stored: true }`, so a caller replacing a document could not
+tell from the return value that anything was removed. It now returns
+`{ id, stored, replaced, deletedChunks }`, matching what `syncDocumentFromFile`
+already reported. `replaced` is decided by the document row, not the chunk
+count — a document that was stored but never chunked still gets overwritten,
+and reporting that as a fresh write would be wrong.
+
+**`syncDocumentFromFile` accepts `excludePattern`** (string or array of
+strings): regions matching these regular expressions are stripped before
+indexing. Previously the only way to leave part of a file out was to read it
+yourself and pass the whole edited text through `content`, which defeats the
+point of a tool that reads server-side to keep content off the model context.
+
+Three properties worth knowing:
+
+1. **The exclusion happens first**, before hashing and chunking, so
+   `content_hash`, the reported `bytes` and the chunk boundaries all describe
+   what was actually indexed. Changing only the pattern therefore invalidates
+   the dedup gate and re-indexes; it does not silently return `unchanged`.
+2. **Patterns are compiled with the dotAll flag**, so one pattern can span
+   lines to drop a marked block (`<!-- SECRET -->[\s\S]*?<!-- /SECRET -->`).
+   JavaScript has no inline `(?s)`, so without this every such pattern would
+   quietly match nothing.
+3. **An invalid expression fails the call.** Degrading to "no exclusion" would
+   index the whole file while the caller believes it was filtered, and an index
+   is a disclosure path — a failed sync is the safer error.
+
+## v5.0.0 (schema v14): chunker c1 + vector reuse
+
+**Breaking**: `chunkParams.overlap` is rejected on BOTH public paths
+(`syncDocumentFromFile.chunkParams` and `chunkDocument`) unless omitted or
+exactly 0 — chunker c1 has no overlap. `maxTokens` must be a positive integer.
+Validation runs before the dedup gate, so invalid params fail even on
+unchanged content.
+
+**What changes on upgrade**: nothing, immediately. v14 is schema-only —
+`documents.chunking_signature` is added with DEFAULT `legacy-unknown` and no
+data row changes, so there is no coverage cliff and no re-embedding storm.
+A document transitions to c1 only when its CONTENT changes at sync time
+(signature mismatch alone is an observed state, not a trigger). The first
+sync of an edited document pays a cold transition (old BPE chunk texts rarely
+match c1 boundaries); every later sync reuses vectors for unchanged text.
+
+**Observability**: `getKnowledgeGraphStats().chunking = { current, legacy,
+unknown, default_signature }` — mutually exclusive, sums to `documents`. The
+framework's /start Step 5a reads this response.
+
+**Rollback caveat (v14)**: `rollbackMigration` drops the column and returns
+`semanticRollback: false` plus a warning — chunk boundaries produced by c1
+are NOT restored (they read fine on v13 code). Data restore path = the
+pre-migration backup snapshot.
+
+**Manual links**: full replacement re-derives `chunk_entities`; a link made
+via `linkEntitiesToDocument` that is not reproducible from body literals or
+`entityNames` is not preserved (true before v5 too). New in v5: primary
+entity names are also linked by document-level occurrence ranges, so a name
+cut across a chunk boundary still links to the intersecting chunks
+(overlap used to absorb this; c1 has none).
+
+**Fleet prerequisite before releasing/upgrading**: audit every deployment's
+`schema_migrations` for occupied slots `>= 14` — pending migrations are
+selected by MAX(version) arithmetic, so an experimental slot silently skips
+the real v14 (this is exactly how code-v8 never ran in production).
+
+**Diagnostic env (v5)**: `RAG_MEMORY_SEARCH_SUMMARIES=off` disables the per-result
+sentence-similarity summaries in `hybridSearch` (which embed every sentence of every
+candidate — 100+ inferences, 90-120s cold per search, measured). Off = preview-slice
+summaries, `relevance_score` 0, ranking rests on vector similarity + boosts. Default
+unchanged. The 3-arm release harness sets this uniformly across all arms.
+
 ## v3.6 breaking response changes
 
 1. `hybridSearch` returns an envelope: `{results, search_mode, model_state,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "4.0.0",
+  "version": "5.1.0",
   "engines": {
     "node": ">=24"
   },
@@ -45,7 +45,7 @@
     "prepare": "npm run build",
     "watch": "tsc --watch",
     "verify:invariants": "node test/chunk-invariants.test.mjs",
-    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs",
+    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/document-return-contracts.test.mjs",
     "test": "npm run build && npm run verify:invariants && npm run verify:engine",
     "prepublishOnly": "npm run build && npm run verify:invariants && npm run verify:engine"
   },