rag-memory-epf-mcp 6.4.0 → 6.4.2

package/dist/src/chunkerC.d.ts

@@ -14,5 +14,6 @@ type Block = {
     heading: boolean;
 };
 export declare function buildBlocks(lines: string[]): Block[];
+export declare function splitOversize(block: string, enc: Tiktoken, maxTokens: number): string[];
 export declare function chunkStructured(text: string, enc: Tiktoken, maxTokens?: number): CSegment[];
 export {};

package/dist/src/chunkerC.js

@@ -8,13 +8,18 @@ export const LEGACY_SIGNATURE = 'legacy-unknown';
 export function mergeMinTokens(maxTokens) {
     return Math.floor(maxTokens / 2); // NOT >>1 — bitshift wraps at 2^31 (r12)
 }
+// kg-relation-evidence D3: fallback=cp-window-N names the windowed oversize split below. Boundaries
+// differ from cp-exact only for blocks that reach the fallback, so the signature says so.
 export function effectiveSignature(maxTokens) {
-    return `c1:enc=cl100k_base:max=${maxTokens}:overlap=0:fence=on:merge=${mergeMinTokens(maxTokens)}:fallback=cp-exact-${maxTokens}`;
+    return `c1:enc=cl100k_base:max=${maxTokens}:overlap=0:fence=on:merge=${mergeMinTokens(maxTokens)}:fallback=cp-window-${maxTokens}`;
 }
 // Strict parse (r5-15): the regex alone classified impossible signatures
 // (max=0, max/fallback mismatch) as current. Cross-check the components.
+// Both c1 fallbacks are the current FORMAT (stats "current" vs legacy/unknown, index.ts:4826-4831):
+// a document stamped cp-exact by 6.4.0 is not re-chunked by an unchanged-content sync and must not be
+// reported as an unknown format.
 export function isCurrentFormatSignature(sig) {
-    const m = /^c1:enc=cl100k_base:max=(\d+):overlap=0:fence=on:merge=(\d+):fallback=cp-exact-(\d+)$/.exec(sig);
+    const m = /^c1:enc=cl100k_base:max=(\d+):overlap=0:fence=on:merge=(\d+):fallback=cp-(?:exact|window)-(\d+)$/.exec(sig);
     if (!m)
         return false;
     const max = Number(m[1]);
@@ -104,33 +109,75 @@ export function buildBlocks(lines) {
     }
     return blocks;
 }
-// cp-exact oversize split, non-monotonicity-safe (spec §4.3-6, r5-5).
-// BPE prefix token counts are NOT monotonic ('/sdkX' -> 1,1,2,1,2), so binary
-// search is invalid. Scan prefixes linearly, remember the last fitting one, and
-// keep probing LOOKAHEAD candidates past a miss to recover dips. If not even
-// one codepoint fits, throw — never emit an over-budget chunk.
+// Oversize split for a block with no structural cut (spec §4.3-6; kg-relation-evidence D3).
+// 6.4.0 scanned every codepoint prefix and re-encoded it (one encode per codepoint, quadratic per
+// chunk; 20k chars 4.6 s, 575k 162 s measured). Now: encode a window once, map the first maxTokens
+// window tokens back to a codepoint count, then VERIFY that candidate by encoding the real substring —
+// the window tokenisation is only an estimate, because BPE re-tokenises a truncated final piece
+// differently (prefix counts are not monotonic: '/sdkX' -> 1,1,2,1,2). Shrink the token count
+// (1, 2, 4, ... fewer) until a candidate verifies; if none does, fall back to the 6.4.0 linear scan for
+// this chunk only, which also keeps its throw. Contract (chunk-invariants, chunker-c): lossless,
+// contiguous codepoint offsets, every chunk's ACTUAL encode length <= maxTokens, deterministic, always
+// advances. Not a longest-fit guarantee — neither was LOOKAHEAD.
 const LOOKAHEAD = 16;
-function splitOversize(block, enc, maxTokens) {
+const WINDOW_MIN_CP = 64;
+const WINDOW_CP_PER_TOKEN = 8;
+function utf8Bytes(cp) {
+    const c = cp.codePointAt(0);
+    return c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; // a lone surrogate encodes as U+FFFD = 3
+}
+function linearFit(cps, start, enc, maxTokens) {
+    let lastFit = 0;
+    let missesSinceFit = 0;
+    for (let probe = start + 1; probe <= cps.length && missesSinceFit < LOOKAHEAD; probe++) {
+        const t = enc.encode(cps.slice(start, probe).join('')).length;
+        if (t <= maxTokens) {
+            lastFit = probe - start;
+            missesSinceFit = 0;
+        }
+        else
+            missesSinceFit++;
+    }
+    return lastFit;
+}
+function windowFit(cps, start, enc, maxTokens) {
+    const rest = cps.length - start;
+    let w = Math.min(rest, Math.max(WINDOW_MIN_CP, maxTokens * WINDOW_CP_PER_TOKEN));
+    let toks = enc.encode(cps.slice(start, start + w).join(''));
+    while (toks.length <= maxTokens && w < rest) {
+        w = Math.min(rest, w * 2);
+        toks = enc.encode(cps.slice(start, start + w).join(''));
+    }
+    if (toks.length <= maxTokens)
+        return rest; // the whole remainder fits
+    for (let keep = maxTokens, step = 1; keep >= 1; keep -= step, step *= 2) {
+        const bytes = enc.decode(toks.slice(0, keep)).length;
+        let used = 0;
+        let c = 0;
+        while (c < w && used + utf8Bytes(cps[start + c]) <= bytes) {
+            used += utf8Bytes(cps[start + c]);
+            c++;
+        }
+        if (c === 0)
+            continue;
+        if (enc.encode(cps.slice(start, start + c).join('')).length <= maxTokens)
+            return c;
+    }
+    return 0;
+}
+export function splitOversize(block, enc, maxTokens) {
     const cps = [...block];
     const out = [];
     let start = 0;
     while (start < cps.length) {
-        let lastFit = 0;
-        let missesSinceFit = 0;
-        for (let probe = start + 1; probe <= cps.length && missesSinceFit < LOOKAHEAD; probe++) {
-            const t = enc.encode(cps.slice(start, probe).join('')).length;
-            if (t <= maxTokens) {
-                lastFit = probe - start;
-                missesSinceFit = 0;
-            }
-            else
-                missesSinceFit++;
-        }
-        if (lastFit === 0) {
+        let take = windowFit(cps, start, enc, maxTokens);
+        if (take === 0)
+            take = linearFit(cps, start, enc, maxTokens);
+        if (take === 0) {
             throw new Error(`chunker c1: codepoint at offset ${start} exceeds maxTokens=${maxTokens} on its own — cannot honor the token budget`);
         }
-        out.push(cps.slice(start, start + lastFit).join(''));
-        start += lastFit;
+        out.push(cps.slice(start, start + take).join(''));
+        start += take;
     }
     return out;
 }

package/dist/src/kg/relation-evidence.d.ts

@@ -0,0 +1,156 @@
+import type Database from 'better-sqlite3';
+export declare const EVIDENCE_FORMAT: "evidence-v1";
+export declare const EVIDENCE_CAPS: Readonly<{
+    maxEntries: 8;
+    maxQuoteCodepoints: 4000;
+    maxBodyCodepoints: 8000;
+}>;
+export declare const FINGERPRINT_TAG = "kg-evidence-fp/1";
+export declare const LEGACY_CMP_TAG = "kg-legacy-cmp/1";
+export declare const EVIDENCE_SEMANTICS = "caller-selected evidence; the engine verified the references exist verbatim; not an independent proof of the relation";
+export type WhyStatus = 'reason-stated' | 'relation-only' | 'unjudged';
+export interface EvidenceSupportInput {
+    why_status?: WhyStatus;
+    valid_time?: string;
+    scope_note?: string;
+}
+export type EvidenceEntryInput = {
+    kind: 'document';
+    chunk_id: string;
+    start: number;
+    end: number;
+    quote_sha256: string;
+} | {
+    kind: 'observation';
+    observation_id: string;
+    revision_no: number;
+    content_sha256: string;
+};
+export interface EvidenceItemInput {
+    relationship_id: string;
+    evidence: EvidenceEntryInput[];
+    support?: EvidenceSupportInput;
+}
+export interface SourceTuple {
+    source_kind: string;
+    source_ref: string;
+    source_hash: string | null;
+}
+export interface DocumentRefMeta {
+    kind: 'document';
+    document_id: string;
+    chunk_id: string;
+    start: number;
+    end: number;
+    chunk_text_sha256: string;
+    quote_sha256: string;
+}
+export interface ObservationRefMeta {
+    kind: 'observation';
+    observation_id: string;
+    entity_id: string;
+    revision_no: number;
+    content_sha256: string;
+    status_at_generation: 'active';
+    sources: SourceTuple[];
+    source_summary: 'recorded' | 'unrecorded' | 'import-only';
+}
+export type EvidenceRefMeta = DocumentRefMeta | ObservationRefMeta;
+export interface EvidenceChunkMetadataV1 {
+    kg_format: typeof EVIDENCE_FORMAT;
+    relationship_id: string;
+    source_entity_id: string;
+    target_entity_id: string;
+    source_entity: string;
+    target_entity: string;
+    relation_type: string;
+    direction: 'source->target';
+    confidence: number;
+    evidence_refs: EvidenceRefMeta[];
+    support: {
+        why_status: WhyStatus;
+        valid_time: string | null;
+        scope_note: string | null;
+    };
+    generated_ref_status: 'refs-resolved';
+    input_fingerprint: string;
+    fingerprint_profile_id: number;
+    body_codepoints: number;
+    embedded_truncation: 'unchecked';
+    evidence_semantics: string;
+}
+export interface FingerprintInput {
+    relationship_id: string;
+    source_entity_id: string;
+    target_entity_id: string;
+    relation_type: string;
+    source_name: string;
+    target_name: string;
+    kg_format: typeof EVIDENCE_FORMAT;
+    refs: EvidenceRefMeta[];
+    profile_id: number;
+}
+export interface PreparedEvidenceChunk {
+    chunk_id: string;
+    relationship_id: string;
+    text: string;
+    body_codepoints: number;
+    metadata: EvidenceChunkMetadataV1;
+}
+export interface KgChunkRow {
+    rowid: number;
+    chunk_id: string;
+    chunk_type: string | null;
+    entity_id: string | null;
+    relationship_id: string | null;
+    text: string | null;
+    metadata: string | null;
+}
+export interface KgEmbedCapture {
+    rowid: number;
+    chunk_id: string;
+    text_sha256: string;
+    cmp: string | null;
+    profile_id: number;
+}
+export declare function cpLength(s: string): number;
+export declare function cpSlice(s: string, start: number, end: number): string;
+export declare function sha256Utf8(s: string): string;
+export declare function compareSourceTuples(a: SourceTuple, b: SourceTuple): number;
+export declare function fingerprintSerialization(f: FingerprintInput): string;
+export declare function computeInputFingerprint(f: FingerprintInput): string;
+export declare function fingerprintFromMetadata(m: EvidenceChunkMetadataV1, profileId: number): string;
+export declare function buildEvidenceBody(sourceName: string, relationType: string, targetName: string, quotes: string[]): string;
+export declare function evidenceChunkId(relationshipId: string): string;
+export declare function parseEvidenceMetadata(raw: string | null): EvidenceChunkMetadataV1 | null;
+export declare function kgComparisonValue(row: KgChunkRow): string | null;
+export declare function captureKgEmbedInput(row: KgChunkRow, profileId: number): KgEmbedCapture;
+export declare class EvidenceInputError extends Error {
+    readonly code: string;
+    readonly itemIndex: number;
+    constructor(code: string, itemIndex: number, detail: string);
+}
+export declare class EvidenceCommitConflictError extends Error {
+    readonly itemIndex: number;
+    readonly reason: string;
+    constructor(itemIndex: number, reason: string);
+}
+export declare function readSourceTuples(db: Database.Database, observationId: string): SourceTuple[];
+export declare function resolveEvidenceItem(db: Database.Database, item: EvidenceItemInput, itemIndex: number, profileId: number): PreparedEvidenceChunk;
+export type EvidenceStatus = 'claim_only' | 'refs_available' | 'refs_unavailable';
+export interface EvidenceRefStatus {
+    index: number;
+    kind: string;
+    status: 'available' | 'missing' | 'mismatch';
+    ref: unknown;
+    text: string | null;
+}
+export declare function resolveEvidenceStatus(db: Database.Database, meta: Record<string, unknown> | null): {
+    status: EvidenceStatus;
+    refs: EvidenceRefStatus[];
+};
+export declare function isSourceTuple(v: unknown): v is SourceTuple;
+export declare function isWellFormedRef(v: unknown): boolean;
+export declare function collectEvidenceRowidsCitingObservations(db: Database.Database, observationIds: string[]): number[];
+export declare function collectEvidenceRowidsCitingDocument(db: Database.Database, documentId: string): number[];
+export declare function collectEvidenceRowidsCitingOwnerEntity(db: Database.Database, entityId: string): number[];