rag-memory-epf-mcp 5.3.0 → 6.0.0

package/README.md

@@ -153,6 +153,30 @@ storeDocument(id, content, metadata)
 
 ## Changelog
 
+### v6.0.0
+
+- **Breaking — observation-derived alias links are now gated.** `autoLinkEntities` used to take every
+  `stem.ext` token appearing anywhere in an entity's observations and link that entity to every chunk
+  containing the token as a substring. Measured on a live 2,891-chunk / 604-entity corpus: 65,388 of
+  66,841 `chunk_entities` rows (97.8%) existed only because of such a hit, and one token — `agents.md`,
+  held by 88 entities — accounted for 39,248 of the 80,096 distinct (chunk, entity) pairs any alias
+  could reach (49.0%). The regex also accepted things that are not filenames at all (`v3.3`,
+  `gpt-5.6`, `1.7mb`, `github.com`, `os.path`).
+  A token now has to (a) look like a filename — extension whitelist, stem ≥ 3 chars, non-numeric
+  stem — and (b) be held by at most 3 entities, and it must match on token boundaries so `foo.py` no
+  longer matches inside `notfoo.pyc`. Effect on the same corpus: alias links 100% → 4.0%. Filtering by
+  extension alone leaves 92.3%, so the owner cap is what does the work. The cap is a judgement, not a
+  discovered boundary — the sweep is smooth (`owners<=1` 1.6% … `<=10` 19.9%) — and it is a cap, not a
+  ban: a filename named by one to three records still links, which is what the alias path was for.
+- **What this means for existing databases.** Nothing is rewritten on upgrade: rows already in
+  `chunk_entities` stay, and new ingests simply link far less. Links have always been a function of
+  when a document was last processed (nothing re-links older documents when entities are added), and
+  `chunk_entities` has no provenance column, so old alias rows cannot be told apart from name matches
+  after the fact. If you want the old noise gone you have to clean it offline, before or after
+  upgrading. Callers that assumed dense `chunk_entities` coverage will see sparser graphs.
+- Regression: `test/alias-link-gate.test.mjs` (registered in `verify:engine`), verified to fail
+  without the gate.
+
 ### v5.3.0
 - (Published first as `5.3.0-rc.1` on the `next` dist-tag; promoted to `latest` after a canary run of the published artifact against a real project database: default call carries no `graph_boost` and equals explicit `useGraph:false`, the known-item probe from the 2026-08-17 measurement returns the correct gotcha at rank 1, opt-in `true` still exposes `graph_boost`, schema/MCP defaults read `false`.)
 - **Behavior change — `hybridSearch` graph re-ranking is now opt-in** (`useGraph` default `true` → `false`; tool schema, MCP exposure and the manager signature agree). Omitting the argument now means "no graph re-ranking" — a behavior change for callers that relied on the old default, hence a release-candidate first (`next` dist-tag, fleet canary) before stable. Measured 2026-08-17 on three real corpora (self-retrieval, usable samples 120/117/120, summaries off): with the additive graph boost on, the known-item chunk got worse in 46/49/52 samples and better in 3/2/0 (sign test p < 7e-11 per corpus), 106 targets left the top-10 entirely; reproduced on the summaries-on product path (HAL, 20 paired samples: hit@1 10→7, hit@5 18→13). Mechanism: only query-matched/connected entities score, but the per-entity boost saturates the cap quickly, so heavily-linked chunks can outrank the exact chunk even at `vector_similarity` 0. This is a harm-reduced default, not a validated graph improvement: the boost path is unchanged for `useGraph: true` (legacy/experimental re-ranker for back-compat and evaluation; the graph does not generate candidates — for relationship exploration use `openNodes` → `getNeighbors`). Regression lock: `test/search-graph-default.test.mjs`.

package/dist/index.d.ts

@@ -279,6 +279,8 @@ export declare class RAGKnowledgeGraphManager {
         errors?: string[];
     }>;
     private hasCJK;
+    private static readonly ALIAS_FILE_EXT;
+    private looksLikeFilename;
     private buildEntityRangeFinder;
     private buildEntityMatcher;
     private autoLinkEntities;
@@ -372,6 +374,36 @@ export declare class RAGKnowledgeGraphManager {
             to: number;
         }>;
     }>;
+    /**
+     * Diagnostic seam for the graph re-ranker (evaluation change graph-role-evaluation, R2).
+     * Returns the seed entities (query-vector matched, similarity > 0.4, top-10 per variant) and the
+     * 1-hop connected entities exactly as hybridSearch(useGraph:true) computes them, plus edge detail
+     * that hybridSearch itself does not use (edge id, type, direction, confidence). It never generates
+     * candidates and never changes ranking; hybridSearch consumes only the name sets.
+     * `opts.chunkVectorDegraded` lets a caller hand over a decision it has already made; omit it and
+     * the seam derives eligibility itself.
+     */
+    explainGraphContext(query: string, queryVariants?: string[], opts?: {
+        chunkVectorDegraded?: boolean;
+    }): Promise<{
+        status: 'vector' | 'entity-text-fallback' | 'chunk-vector-disabled' | 'error';
+        query_variants: string[];
+        seeds: Array<{
+            entity_id: string;
+            name: string;
+            similarity: number;
+        }>;
+        connected: Array<{
+            entity_id: string;
+            name: string;
+            via_seed_id: string;
+            via_seed_name: string;
+            edge_id: string;
+            relation_type: string;
+            direction: 'out' | 'in';
+            confidence: number | null;
+        }>;
+    }>;
     hybridSearch(query: string, limit?: number, useGraph?: boolean): Promise<{
         results: EnhancedSearchResult[];
         search_mode: 'hybrid' | 'hybrid-partial' | 'fts-only';

package/dist/index.js

@@ -2241,6 +2241,32 @@ export class RAGKnowledgeGraphManager {
     hasCJK(text) {
         return /[\u3000-\u9fff\uac00-\ud7af\uff00-\uffef]/.test(text);
     }
+    // The alias regex in autoLinkEntities matches anything shaped like "stem.ext", which
+    // includes version strings ("v3.3", "gpt-5.6"), measurements ("1.7mb", "0.465") and
+    // domains/module paths ("github.com", "os.path"). Those are not filenames and linking
+    // on them is pure noise. Whitelist the extensions we actually ship and store.
+    static ALIAS_FILE_EXT = new Set([
+        'md', 'py', 'js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'json',
+        'sh', 'bash', 'zsh', 'toml', 'yaml', 'yml', 'ini', 'cfg', 'conf',
+        'txt', 'log', 'csv', 'tsv', 'sql', 'db', 'zip', 'gz', 'tar',
+        'css', 'scss', 'html', 'htm', 'svg', 'png', 'jpg', 'jpeg', 'gif',
+        'pdf', 'docx', 'pptx', 'xlsx', 'hwp', 'hwpx', 'lock', 'bak',
+    ]);
+    // Deliberately absent: any 5+ character extension (jsonl, ipynb, scss is 4 so it stays).
+    // The extractor regex is `\w{1,4}`, so a longer extension never reaches this set — listing
+    // one would be a dead entry that reads as support. Extend the regex first if that changes.
+    looksLikeFilename(token) {
+        const i = token.lastIndexOf('.');
+        if (i < 1)
+            return false;
+        const stem = token.slice(0, i);
+        const ext = token.slice(i + 1);
+        if (stem.length < 3)
+            return false; // "d.ts" is a fragment, not a file
+        if (/^\d+$/.test(stem))
+            return false; // "2026.md" style numeric stems
+        return RAGKnowledgeGraphManager.ALIAS_FILE_EXT.has(ext);
+    }
     // spec §5.4 (r7-2·r8-1·r9): primary name 의 본문 occurrence range [sCp, eCp).
     // 의미 = buildEntityMatcher 와 동일 (CJK substring / Latin word-boundary) — 여기서
     // 어긋나면 'Data' 가 'Database' 에 새로 링크되는 식으로 의미가 확장된다.
@@ -2342,6 +2368,51 @@ export class RAGKnowledgeGraphManager {
             // Minimum name length: 2 for CJK (e.g. "할랄"), 4 for Latin (avoid "API", "Bug")
             const MIN_LEN_CJK = 2;
             const MIN_LEN_LATIN = 4;
+            // Observation-derived aliases are only useful when the token identifies few entities.
+            // Measured on a 2,891-chunk / 604-entity corpus (2026-08-22). Denominators matter here,
+            // so both are stated: 66,841 rows in chunk_entities, of which 65,388 (97.8%) exist only
+            // because of an alias hit; counting distinct (chunk, entity) pairs reachable by any alias
+            // gives 80,096. Against that 80,096, "agents.md" alone accounts for 39,248 (49.0%), and
+            // 35,099 (43.8%) have no other alias reason at all. It is held by 88 entities. A filename
+            // that dozens of entities mention is a stopword, not an identifier.
+            //
+            // The owner cap is what does the work: the extension whitelist alone still leaves 92.3%
+            // of alias links. But the cap value is a judgement call, not a discovered boundary — the
+            // sweep is smooth, with no natural knee (share of the 80,096 that survives):
+            //   owners<=1 1.6% · <=2 3.7% · <=3 5.2% · <=4 7.0% · <=5 8.3% · <=8 14.7% · <=10 19.9%
+            // 3 was chosen to keep the intended behaviour (a file named by one or two records, plus
+            // some slack) while cutting the stopword tail. Raising it is cheap and reversible.
+            //
+            // A chunk-frequency cap was measured (a further 1.4pp) and rejected on COST: it needs a
+            // full-corpus scan on every ingest. Note the honest caveat — it was first rejected for
+            // depending on ingest order, but this owner cap has that same property, and so does the
+            // engine as a whole: autoLinkEntities only ever sees the entities that exist at ingest
+            // time, and nothing re-links older documents when entities are added (see
+            // createEntities / addObservations — neither calls this). Links are a function of when a
+            // document was last processed. That predates this gate; it is not introduced by it.
+            const MAX_ALIAS_OWNERS = 3;
+            const aliasOwners = new Map();
+            for (const e of entities) {
+                if (!e.observations)
+                    continue;
+                let obs;
+                try {
+                    obs = JSON.parse(e.observations);
+                }
+                catch {
+                    continue;
+                }
+                const seen = new Set();
+                for (const ob of obs) {
+                    const pm = String(ob).match(/[\w\-]+\.\w{1,4}\b/g);
+                    if (pm)
+                        for (const p of pm)
+                            if (p.length >= 4)
+                                seen.add(p.toLowerCase());
+                }
+                for (const t of seen)
+                    aliasOwners.set(t, (aliasOwners.get(t) ?? 0) + 1);
+            }
             const insertStmt = this.db.prepare(`
         INSERT OR IGNORE INTO chunk_entities (chunk_rowid, entity_id) VALUES (?, ?)
       `);
@@ -2351,7 +2422,9 @@ export class RAGKnowledgeGraphManager {
                 if (entity.name.length < minLen)
                     continue;
                 const nameMatcher = this.buildEntityMatcher(entity.name);
-                // Also collect observation-derived aliases (short keywords from observations)
+                // Also collect observation-derived aliases (short keywords from observations).
+                // Gated: the token must look like a filename AND identify at most MAX_ALIAS_OWNERS
+                // entities. Ungated, "agents.md" linked 88 entities to every chunk that mentioned it.
                 const aliases = [];
                 if (entity.observations) {
                     let obs;
@@ -2366,9 +2439,30 @@ export class RAGKnowledgeGraphManager {
                         const pathMatch = ob.match(/[\w\-]+\.\w{1,4}\b/g);
                         if (pathMatch) {
                             for (const p of pathMatch) {
-                                if (p.length >= 4) {
-                                    aliases.push((text) => text.toLowerCase().includes(p.toLowerCase()));
-                                }
+                                if (p.length < 4)
+                                    continue;
+                                const tok = p.toLowerCase();
+                                if (!this.looksLikeFilename(tok))
+                                    continue; // "v3.3", "gpt-5.6", "0.465"
+                                if ((aliasOwners.get(tok) ?? 0) > MAX_ALIAS_OWNERS)
+                                    continue; // shared = identifies nothing
+                                // Bare substring matching links "foo.py" to a chunk saying "notfoo.pyc".
+                                // Require the token to stand alone: no filename character on either side.
+                                const boundary = /[\w\-.]/;
+                                aliases.push((text) => {
+                                    const hay = text.toLowerCase();
+                                    let from = 0;
+                                    for (;;) {
+                                        const i = hay.indexOf(tok, from);
+                                        if (i < 0)
+                                            return false;
+                                        const before = i === 0 ? '' : hay[i - 1];
+                                        const after = hay[i + tok.length] ?? '';
+                                        if (!boundary.test(before) && !boundary.test(after))
+                                            return true;
+                                        from = i + 1;
+                                    }
+                                });
                             }
                         }
                     }
@@ -2923,6 +3017,97 @@ export class RAGKnowledgeGraphManager {
         this.coordinator?.kick();
         return { imported, skipped, observation_order_remap: remapReport };
     }
+    /**
+     * Diagnostic seam for the graph re-ranker (evaluation change graph-role-evaluation, R2).
+     * Returns the seed entities (query-vector matched, similarity > 0.4, top-10 per variant) and the
+     * 1-hop connected entities exactly as hybridSearch(useGraph:true) computes them, plus edge detail
+     * that hybridSearch itself does not use (edge id, type, direction, confidence). It never generates
+     * candidates and never changes ranking; hybridSearch consumes only the name sets.
+     * `opts.chunkVectorDegraded` lets a caller hand over a decision it has already made; omit it and
+     * the seam derives eligibility itself.
+     */
+    async explainGraphContext(query, queryVariants, opts) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        const variants = queryVariants ?? this.buildCrossLingualVariants(query);
+        const empty = { query_variants: variants, seeds: [], connected: [] };
+        // The caller's latched decision wins (review finding I2). hybridSearch decides chunk-vector
+        // degradation once, before the chunk-embedding awaits, and passes that value down; re-deriving
+        // it here would let an eligibility flip during those awaits give the seam a different answer
+        // than the ranking path already acted on — the pre-extraction code read it once, so this keeps
+        // behaviour identical. A standalone caller passes nothing and gets the live derivation.
+        const chunkVectorDegraded = opts?.chunkVectorDegraded ?? !(this.coordinator?.eligible ?? false);
+        if (chunkVectorDegraded)
+            return { status: 'chunk-vector-disabled', ...empty };
+        try {
+            const searchEntities = (embedding) => this.db.prepare(`
+            SELECT em.entity_id, e.name, ee.distance
+            FROM entity_embeddings ee
+            JOIN entity_embedding_metadata em ON ee.rowid = em.rowid
+            JOIN entities e ON e.id = em.entity_id
+            WHERE ee.embedding MATCH ? AND k = 10
+            ORDER BY ee.distance
+          `).all(Buffer.from(embedding.buffer));
+            const entityMap = new Map();
+            for (const variant of variants) {
+                const embedding = await this.generateEmbedding(variant, 1024, true);
+                for (const e of searchEntities(embedding)) {
+                    const existing = entityMap.get(e.entity_id);
+                    if (!existing || e.distance < existing.distance)
+                        entityMap.set(e.entity_id, e);
+                }
+            }
+            const similar = Array.from(entityMap.values()).sort((a, b) => a.distance - b.distance || (a.entity_id < b.entity_id ? -1 : 1));
+            const seeds = [];
+            const connected = [];
+            const edgeStmt = this.db.prepare(`
+            SELECT r.id AS edge_id, r.relationType AS relation_type, r.confidence,
+                   CASE WHEN r.source_entity = ? THEN e2.id ELSE e1.id END AS entity_id,
+                   CASE WHEN r.source_entity = ? THEN e2.name ELSE e1.name END AS name,
+                   CASE WHEN r.source_entity = ? THEN 'out' ELSE 'in' END AS direction
+            FROM relationships r
+            JOIN entities e1 ON e1.id = r.source_entity
+            JOIN entities e2 ON e2.id = r.target_entity
+            WHERE r.source_entity = ? OR r.target_entity = ?
+            ORDER BY r.id`);
+            for (const entity of similar) {
+                const similarity = Math.max(0, 1 - entity.distance / 2);
+                if (similarity > 0.4) {
+                    seeds.push({ entity_id: entity.entity_id, name: entity.name, similarity });
+                    for (const row of edgeStmt.all(entity.entity_id, entity.entity_id, entity.entity_id, entity.entity_id, entity.entity_id)) {
+                        connected.push({ entity_id: row.entity_id, name: row.name, via_seed_id: entity.entity_id, via_seed_name: entity.name,
+                            edge_id: row.edge_id, relation_type: row.relation_type, direction: row.direction,
+                            confidence: row.confidence === null || row.confidence === undefined ? null : Number(row.confidence) });
+                    }
+                }
+            }
+            return { status: 'vector', query_variants: variants, seeds, connected };
+        }
+        catch (error) {
+            console.error('⚠️ Entity vector search for graph enhancement failed:', error);
+            // Fallback: text-based matching (original behavior) — same SQL as before extraction.
+            const connected = [];
+            const queryEntities = this.extractTermsFromText(query);
+            for (const entity of queryEntities) {
+                const rows = this.db.prepare(`
+            SELECT DISTINCT
+              CASE WHEN r.source_entity = e1.id THEN e2.name ELSE e1.name END as connected_name,
+              CASE WHEN r.source_entity = e1.id THEN e2.id ELSE e1.id END as connected_id,
+              r.id AS edge_id, r.relationType AS relation_type, r.confidence,
+              CASE WHEN r.source_entity = e1.id THEN 'out' ELSE 'in' END AS direction
+            FROM entities e1
+            JOIN relationships r ON (r.source_entity = e1.id OR r.target_entity = e1.id)
+            JOIN entities e2 ON (e2.id = r.source_entity OR e2.id = r.target_entity)
+            WHERE e1.name = ? AND e2.name != ?
+            ORDER BY r.id`).all(entity, entity);
+                for (const row of rows)
+                    connected.push({ entity_id: row.connected_id, name: row.connected_name, via_seed_id: '', via_seed_name: entity,
+                        edge_id: row.edge_id, relation_type: row.relation_type, direction: row.direction,
+                        confidence: row.confidence === null || row.confidence === undefined ? null : Number(row.confidence) });
+            }
+            return { status: 'entity-text-fallback', query_variants: variants, seeds: [], connected };
+        }
+    }
     // v5.3.0: the graph re-ranker is OPT-IN (harm-reduced default, not a validated improvement).
     // Measured 2026-08-17 on three real corpora (self-retrieval, usable samples 120/117/120,
     // summaries off): with the additive graph boost on, the known-item chunk got WORSE in
@@ -3102,77 +3287,16 @@ export class RAGKnowledgeGraphManager {
                 ...(ftsUnsearchable ? { warning: 'query has no searchable terms for FTS' } : {}),
             };
         }
-        // Get entity information for graph enhancement via vector similarity
+        // Get entity information for graph enhancement — via the diagnostic seam (evaluation change
+        // graph-role-evaluation R2). Same SQL, same threshold, same fallback; hybridSearch consumes only names.
         let connectedEntities = new Set();
         let queryMatchedEntities = new Set();
         if (useGraph && !vectorDegraded) {
-            // Vector search: find entities semantically similar to the query (dual search)
-            try {
-                const searchEntities = (embedding) => {
-                    return this.db.prepare(`
-            SELECT
-              em.entity_id,
-              e.name,
-              ee.distance
-            FROM entity_embeddings ee
-            JOIN entity_embedding_metadata em ON ee.rowid = em.rowid
-            JOIN entities e ON e.id = em.entity_id
-            WHERE ee.embedding MATCH ?
-              AND k = 10
-            ORDER BY ee.distance
-          `).all(Buffer.from(embedding.buffer));
-                };
-                // Merge all query variant entity results
-                const entityMap = new Map();
-                for (const variant of queryVariants) {
-                    const embedding = await this.generateEmbedding(variant, 1024, true);
-                    for (const e of searchEntities(embedding)) {
-                        const existing = entityMap.get(e.entity_id);
-                        if (!existing || e.distance < existing.distance) {
-                            entityMap.set(e.entity_id, e);
-                        }
-                    }
-                }
-                const similarEntities = Array.from(entityMap.values()).sort((a, b) => a.distance - b.distance);
-                for (const entity of similarEntities) {
-                    const similarity = Math.max(0, 1 - entity.distance / 2);
-                    if (similarity > 0.4) {
-                        queryMatchedEntities.add(entity.name);
-                        // Get connected entities via relationships
-                        const connected = this.db.prepare(`
-              SELECT DISTINCT
-                CASE
-                  WHEN r.source_entity = ? THEN e2.name
-                  ELSE e1.name
-                END as connected_name
-              FROM relationships r
-              JOIN entities e1 ON e1.id = r.source_entity
-              JOIN entities e2 ON e2.id = r.target_entity
-              WHERE r.source_entity = ? OR r.target_entity = ?
-            `).all(entity.entity_id, entity.entity_id, entity.entity_id);
-                        connected.forEach((row) => connectedEntities.add(row.connected_name));
-                    }
-                }
-            }
-            catch (error) {
-                console.error('⚠️ Entity vector search for graph enhancement failed:', error);
-                // Fallback: text-based matching (original behavior)
-                const queryEntities = this.extractTermsFromText(query);
-                for (const entity of queryEntities) {
-                    const connected = this.db.prepare(`
-            SELECT DISTINCT
-              CASE
-                WHEN r.source_entity = e1.id THEN e2.name
-                ELSE e1.name
-              END as connected_name
-            FROM entities e1
-            JOIN relationships r ON (r.source_entity = e1.id OR r.target_entity = e1.id)
-            JOIN entities e2 ON (e2.id = r.source_entity OR e2.id = r.target_entity)
-            WHERE e1.name = ? AND e2.name != ?
-          `).all(entity, entity);
-                    connected.forEach((row) => connectedEntities.add(row.connected_name));
-                }
-            }
+            const ctx = await this.explainGraphContext(query, queryVariants, { chunkVectorDegraded: vectorDegraded });
+            for (const s of ctx.seeds)
+                queryMatchedEntities.add(s.name);
+            for (const c of ctx.connected)
+                connectedEntities.add(c.name);
         }
         // Process results with semantic summaries
         const enhancedResults = [];

package/dist/src/backup/preflight.js

@@ -1,4 +1,4 @@
-import { existsSync, openSync, readSync, closeSync, linkSync, unlinkSync } from 'node:fs';
+import { existsSync, openSync, readSync, closeSync, copyFileSync, unlinkSync, constants } from 'node:fs';
 import { createHash } from 'node:crypto';
 import Database from 'better-sqlite3';
 // 이 파일이 지켜야 하는 것은 두 줄이다:
@@ -101,21 +101,51 @@ function verifyRecoveryPoint(path) {
         v.close();
     }
 }
-// 슬롯 게시. `link()` 는 목적지가 있으면 EEXIST 로 실패하므로 **원자적 no-clobber** 다
-// (rename 은 조용히 덮어쓴다). 그래서 경쟁하는 프로세스가 있어도 복구점을 잃지 않는다.
+// 슬롯 게시. **`COPYFILE_EXCL` 복사**로 발행한다 — 목적지가 있으면 EEXIST 로 실패하므로
+// no-clobber 의미는 `link()` 와 같고(rename 은 조용히 덮어쓴다), 경쟁 프로세스가 있어도
+// 복구점을 잃지 않는다.
+//
+// **왜 `link()` 가 아닌가** (2026-08-22, 필드 보고): Google Drive File Stream(Windows `G:\`)은
+// 하드링크를 지원하지 않는다. 실측 = 그 FS 에서 `ln` 이 "Invalid request code" 로 실패하고
+// node 의 `linkSync` 는 같은 조건에서 `EISDIR`(errno -4068)로 표면화한다. `EEXIST` 가 아니므로
+// 위 루프가 그대로 throw 했고, 이 함수는 `server.connect()` **전에** 도는 fail-closed 경로라
+// 사용자에게는 원인 없는 "MCP 연결 실패"로만 보였다. 대기 마이그레이션이 없는 동안에는
+// 이 코드가 아예 안 돌기 때문에 **배포 시점이 아니라 스키마 범프 시점에** 터진다.
+// 🔴 이 파일 위쪽 주석이 *"이 프로젝트군은 non-git 환경(Google Drive 폴더)에 배포된다"* 고
+// 적어 놓고 그 FS 가 없는 syscall 로 발행하고 있었다.
+//
+// **FS 별 분기를 두지 않는다.** 하드링크가 되는 곳에서만 link 를 쓰는 폴백 구조는 드문 경로가
+// 영영 안 밟혀서, 정작 필요한 날 처음 실행된다. 모든 FS 가 같은 경로를 타게 한다.
+// 비용은 복사 한 번 추가인데 백업 자체가 이미 `db.backup()` 전체 복사다.
+//
+// ⚠ **복사는 원자적이지 않다** — link 는 O(1) 이라 사실상 원자적이었지만 복사는 바이트를 다시
+// 쓴다. 중간에 죽으면 잘린 파일이 슬롯을 차지하고, `pickRecoverySlot` 은 기존 파일을 **의도적으로**
+// 검증하지 않으므로(아래 주석) 그 파일은 영영 복구점 행세를 한다. 그래서 **목적지 기준으로
+// 다시 검증**하고, 실패하면 슬롯을 비운다. tmp 는 이미 검증했지만 그 뒤에 바이트가 다시 쓰였다.
 function publishNoClobber(tmp, base) {
     for (let attempt = 0; attempt < MAX_RECOVERY_POINTS; attempt++) {
         const slot = pickRecoverySlot(base);
         try {
-            linkSync(tmp, slot);
-            unlinkSync(tmp);
-            return slot;
+            copyFileSync(tmp, slot, constants.COPYFILE_EXCL);
+        }
+        catch (e) {
+            if (e.code === 'EEXIST')
+                continue; // 슬롯 경쟁 — 다음 빈 슬롯으로
+            throw e;
+        }
+        try {
+            verifyRecoveryPoint(slot);
         }
         catch (e) {
-            if (e.code !== 'EEXIST')
-                throw e;
-            // 그 슬롯을 누가 먼저 가져갔다 — 다음 빈 슬롯으로.
+            // 잘린/손상된 사본이 슬롯을 점유한 채 복구점 행세를 하지 못하게 한다.
+            try {
+                unlinkSync(slot);
+            }
+            catch { /* 정리 실패는 원인을 가리지 않는다 */ }
+            throw e;
         }
+        unlinkSync(tmp);
+        return slot;
     }
     throw slotsFullError(base);
 }

package/dist/src/chunkText.d.ts

@@ -0,0 +1,10 @@
+import type { Tiktoken } from 'tiktoken';
+export interface ChunkSegment {
+    text: string;
+    start_pos: number | null;
+    end_pos: number | null;
+    start_token: number;
+    end_token: number;
+}
+export declare function trimIncompleteUtf8(bytes: Uint8Array, trimHead: boolean, trimTail: boolean): Uint8Array;
+export declare function chunkText(text: string, encoding: Tiktoken, maxTokens?: number, overlap?: number): ChunkSegment[];

package/dist/src/chunkText.js

@@ -0,0 +1,104 @@
+// Tokenize and chunk text using a BPE encoder while reporting both token-space
+// and char-space (Unicode codepoint) offsets back into the original string.
+//
+// BPE tokenizers (cl100k_base) split multi-byte UTF-8 sequences across tokens.
+// Slicing token arrays at arbitrary boundaries can leave incomplete UTF-8
+// prefix/suffix bytes, which TextDecoder replaces with U+FFFD (�). We trim the
+// incomplete sequences at chunk boundaries; overlap covers the removed bytes.
+//
+// Each chunk records both token-space offsets (start_token/end_token from the
+// BPE encoder loop) and char-space offsets (start_pos/end_pos into the original
+// text). Char offsets are Unicode codepoint counts — language-neutral, so SQL
+// substr, Python str slicing, and JS [...str] iteration all line up. JS's
+// native UTF-16 indexing differs for supplementary characters (emoji, rare CJK),
+// so the function maintains parallel UTF-16 and codepoint cursors and reports
+// codepoint offsets. On a coincidental indexOf miss the char offsets are NULL.
+//
+// Extracted to a standalone module so publish-time invariant tests can exercise
+// the algorithm directly without booting the full RAG-Memory stack.
+// trimIncompleteUtf8: strip incomplete UTF-8 sequences from the head/tail of a
+// byte buffer produced by decoding an arbitrary token slice. A multi-byte
+// codepoint that begins or ends on the cut edge belongs to an adjacent chunk
+// and must be removed so TextDecoder does not emit U+FFFD. Pass
+// trimHead/trimTail=false to preserve head/tail bytes (first/last chunks).
+export function trimIncompleteUtf8(bytes, trimHead, trimTail) {
+    let start = 0;
+    let end = bytes.length;
+    if (trimHead) {
+        while (start < end && (bytes[start] & 0xC0) === 0x80)
+            start++;
+    }
+    if (trimTail) {
+        let i = end - 1;
+        while (i >= start && (bytes[i] & 0xC0) === 0x80)
+            i--;
+        if (i >= start) {
+            const lead = bytes[i];
+            let needed = 1;
+            if ((lead & 0x80) === 0)
+                needed = 1;
+            else if ((lead & 0xE0) === 0xC0)
+                needed = 2;
+            else if ((lead & 0xF0) === 0xE0)
+                needed = 3;
+            else if ((lead & 0xF8) === 0xF0)
+                needed = 4;
+            if (end - i < needed)
+                end = i;
+        }
+    }
+    return bytes.subarray(start, end);
+}
+export function chunkText(text, encoding, maxTokens = 800, overlap = 160) {
+    const tokens = encoding.encode(text);
+    const segments = [];
+    let utf16Cursor = 0;
+    let cpCursor = 0;
+    for (let i = 0; i < tokens.length; i += maxTokens - overlap) {
+        const chunkTokens = tokens.slice(i, i + maxTokens);
+        const decodedBytes = encoding.decode(chunkTokens);
+        const isFirst = i === 0;
+        const isLast = i + chunkTokens.length >= tokens.length;
+        const safeBytes = trimIncompleteUtf8(decodedBytes, !isFirst, !isLast);
+        const chunkTextStr = new TextDecoder('utf-8').decode(safeBytes);
+        let startPos;
+        let endPos;
+        if (isFirst) {
+            startPos = 0;
+            endPos = [...chunkTextStr].length;
+            utf16Cursor = 0;
+            cpCursor = 0;
+        }
+        else if (chunkTextStr.length === 0) {
+            startPos = null;
+            endPos = null;
+        }
+        else {
+            const utfIdx = text.indexOf(chunkTextStr, utf16Cursor);
+            if (utfIdx >= 0) {
+                // Advance cpCursor by codepoints between the previous cursor and the
+                // new chunk's start (handles overlap by anchoring at the previous
+                // chunk's start, not its end).
+                if (utfIdx > utf16Cursor) {
+                    cpCursor += [...text.slice(utf16Cursor, utfIdx)].length;
+                    utf16Cursor = utfIdx;
+                }
+                const cpLen = [...chunkTextStr].length;
+                startPos = cpCursor;
+                endPos = cpCursor + cpLen;
+            }
+            else {
+                startPos = null;
+                endPos = null;
+            }
+        }
+        segments.push({
+            text: chunkTextStr,
+            start_pos: startPos,
+            end_pos: endPos,
+            start_token: i,
+            end_token: i + chunkTokens.length
+        });
+    }
+    return segments;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "5.3.0",
+  "version": "6.0.0",
   "engines": {
     "node": ">=24"
   },
@@ -45,7 +45,7 @@
     "prepare": "npm run build",
     "watch": "tsc --watch",
     "verify:invariants": "node test/chunk-invariants.test.mjs",
-    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/search-graph-default.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/document-return-contracts.test.mjs && node test/observation-date-prefix.test.mjs && node test/delete-entities-cascade.test.mjs",
+    "verify:engine": "node test/engine-smoke.test.mjs && node test/launch-smoke.test.mjs && node test/sync-atomicity.test.mjs && node test/dedup.test.mjs && node test/search-degradation.test.mjs && node test/entity-embed-cap.test.mjs && node test/migration12.test.mjs && node test/model-cache.test.mjs && node test/embedding-gate.test.mjs && node test/lazy-boot.test.mjs && node test/reconciliation.test.mjs && node test/backfill.test.mjs && node test/fts-query.test.mjs && node test/search-contracts.test.mjs && node test/tool-contracts.test.mjs && node test/bounded-exit.test.mjs && node test/observation-schema.test.mjs && node test/search-graph-default.test.mjs && node test/observation-migration.test.mjs && node test/observation-lifecycle.test.mjs && node test/observation-contracts.test.mjs && node test/observation-search.test.mjs && node test/observation-cascade.test.mjs && node test/observation-realdata.test.mjs && node test/chunker-c.test.mjs && node test/migration14.test.mjs && node test/chunk-params-validation.test.mjs && node test/vector-reuse.test.mjs && node test/entity-range-linking.test.mjs && node test/alias-link-gate.test.mjs && node test/stats-chunking.test.mjs && node test/migration14-realdata.test.mjs && node test/migration14-realdata-sync.test.mjs && node test/search-summaries-off.test.mjs && node test/document-return-contracts.test.mjs && node test/observation-date-prefix.test.mjs && node test/delete-entities-cascade.test.mjs && node test/backup-publish-portable.test.mjs && node test/graph-context-explain.test.mjs && node test/eval-graph-role-libs.test.mjs && node test/eval-graph-role-t5b.test.mjs && node --test test/eval-graph-role-t8-fix.test.mjs && node --test test/eval-graph-role-t7-upstream.test.mjs && node --test test/eval-graph-role-t11-decision.test.mjs && node --test test/eval-graph-role-prereq-fix.test.mjs",
     "test": "npm run build && npm run verify:invariants && npm run verify:engine",
     "prepublishOnly": "npm run build && npm run verify:invariants && npm run verify:engine"
   },