ralph-hero-knowledge-index 0.1.20 → 0.1.21

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "ralph-knowledge",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Knowledge graph for ralph-hero: semantic search, relationship traversal, and document indexing across thoughts/ documents. Optional companion to ralph-hero.",
   "author": {
     "name": "Chad Dubiel",

package/.mcp.json

@@ -2,7 +2,7 @@
   "mcpServers": {
     "ralph-knowledge": {
       "command": "npx",
-      "args": ["-y", "ralph-hero-knowledge-index@0.1.20"]
+      "args": ["-y", "ralph-hero-knowledge-index@0.1.21"]
     }
   }
 }

package/dist/chunker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * RecursiveCharacterTextSplitter-style chunker.
+ *
+ * Splits long text into overlapping chunks while preserving the original
+ * character offsets (charStart, charEnd) so downstream code can reconstruct
+ * positions. Mirrors the semantics of LangChain's RecursiveCharacterTextSplitter:
+ * tries each separator in order, snapping chunk boundaries to the highest-priority
+ * separator that keeps pieces under chunkSize.
+ *
+ * Defaults correspond to 512-token chunks with 64-token overlap
+ * (approx: 1 token ~= 4 chars for English text).
+ */
+export interface Chunk {
+    index: number;
+    content: string;
+    charStart: number;
+    charEnd: number;
+}
+export interface ChunkerOptions {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    separators?: string[];
+}
+/**
+ * Split `text` into overlapping chunks.
+ *
+ * Semantics:
+ * - Empty input -> empty array.
+ * - Short input (<= chunkSize) -> single chunk covering the whole text.
+ * - For each chunk, `text.slice(charStart, charEnd) === content`.
+ * - `charStart` is monotonically non-decreasing across chunks.
+ * - Consecutive chunks overlap by ~`chunkOverlap` chars (snapped to atom
+ *   boundaries; may differ by up to the largest atom size).
+ * - Each chunk's content length is bounded by chunkSize + a small slack for
+ *   the separator that snapped the boundary.
+ */
+export declare function chunkText(text: string, opts?: ChunkerOptions): Chunk[];

package/dist/chunker.js

@@ -0,0 +1,219 @@
+/**
+ * RecursiveCharacterTextSplitter-style chunker.
+ *
+ * Splits long text into overlapping chunks while preserving the original
+ * character offsets (charStart, charEnd) so downstream code can reconstruct
+ * positions. Mirrors the semantics of LangChain's RecursiveCharacterTextSplitter:
+ * tries each separator in order, snapping chunk boundaries to the highest-priority
+ * separator that keeps pieces under chunkSize.
+ *
+ * Defaults correspond to 512-token chunks with 64-token overlap
+ * (approx: 1 token ~= 4 chars for English text).
+ */
+const DEFAULT_CHUNK_SIZE = 2048;
+const DEFAULT_CHUNK_OVERLAP = 256;
+const DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""];
+/**
+ * Pick the first separator from `separators` that occurs in `piece.text`.
+ * Falls back to the last separator (typically `""`) if none match — this is
+ * the sentinel that lets us character-split oversized pieces with no natural
+ * boundary.
+ */
+function pickSeparator(pieceText, separators) {
+    for (let i = 0; i < separators.length; i++) {
+        const sep = separators[i];
+        if (sep === "") {
+            return { separator: sep, remaining: separators.slice(i + 1) };
+        }
+        if (pieceText.includes(sep)) {
+            return { separator: sep, remaining: separators.slice(i + 1) };
+        }
+    }
+    // Should be unreachable when DEFAULT_SEPARATORS ends with "".
+    return { separator: "", remaining: [] };
+}
+/**
+ * Split a piece on `separator` while retaining absolute char offsets.
+ * When separator is empty, split into single-character pieces.
+ */
+function splitOnSeparator(piece, separator) {
+    if (separator === "") {
+        const out = [];
+        for (let i = 0; i < piece.text.length; i++) {
+            out.push({ text: piece.text[i], start: piece.start + i });
+        }
+        return out;
+    }
+    const out = [];
+    let cursor = 0;
+    let idx = piece.text.indexOf(separator, cursor);
+    while (idx !== -1) {
+        // Keep the separator attached to the preceding piece so reconstruction
+        // via text.slice(charStart, charEnd) works bit-for-bit.
+        const sliceEnd = idx + separator.length;
+        out.push({
+            text: piece.text.slice(cursor, sliceEnd),
+            start: piece.start + cursor,
+        });
+        cursor = sliceEnd;
+        idx = piece.text.indexOf(separator, cursor);
+    }
+    if (cursor < piece.text.length) {
+        out.push({
+            text: piece.text.slice(cursor),
+            start: piece.start + cursor,
+        });
+    }
+    return out;
+}
+/**
+ * Recursively flatten a piece into "atoms" — pieces small enough to merge
+ * greedily into chunks. Pieces larger than chunkSize are split with the next
+ * separator in line; pieces that fit are returned as-is.
+ */
+function flattenToAtoms(piece, separators, chunkSize) {
+    if (piece.text.length <= chunkSize) {
+        return [piece];
+    }
+    const { separator, remaining } = pickSeparator(piece.text, separators);
+    const splits = splitOnSeparator(piece, separator);
+    // If the separator didn't actually reduce the piece (e.g., no occurrence),
+    // fall through to the next separator with the original piece.
+    if (splits.length <= 1) {
+        if (remaining.length === 0) {
+            // No more separators — return whatever we have, even if oversized.
+            return [piece];
+        }
+        return flattenToAtoms(piece, remaining, chunkSize);
+    }
+    const out = [];
+    for (const sub of splits) {
+        if (sub.text.length <= chunkSize) {
+            out.push(sub);
+        }
+        else if (remaining.length > 0) {
+            for (const leaf of flattenToAtoms(sub, remaining, chunkSize)) {
+                out.push(leaf);
+            }
+        }
+        else {
+            // Last-resort: character-split oversized atom so we never return a
+            // single atom larger than chunkSize.
+            out.push(...splitOnSeparator(sub, ""));
+        }
+    }
+    return out;
+}
+/**
+ * Build a chunk object from a contiguous run of atoms.
+ * `charStart` is taken from the first atom, `charEnd` from the last atom's
+ * end boundary, and `content` is `text.slice(start, end)` — this guarantees
+ * `text.slice(charStart, charEnd) === content`.
+ */
+function buildChunk(originalText, atoms, index) {
+    const first = atoms[0];
+    const last = atoms[atoms.length - 1];
+    const charStart = first.start;
+    const charEnd = last.start + last.text.length;
+    return {
+        index,
+        content: originalText.slice(charStart, charEnd),
+        charStart,
+        charEnd,
+    };
+}
+/**
+ * Compute the start position for the next chunk's atoms given the previous
+ * chunk ended at `prevEnd`. We walk backward through the atom list to find
+ * the atom whose start >= prevEnd - chunkOverlap; that atom begins the
+ * overlap region.
+ */
+function findOverlapStartIndex(atoms, lastEndAtomIndex, prevEnd, chunkOverlap) {
+    if (chunkOverlap <= 0) {
+        return lastEndAtomIndex + 1;
+    }
+    const targetStart = prevEnd - chunkOverlap;
+    // Find the earliest atom in [0..lastEndAtomIndex] whose start >= targetStart.
+    let overlapAtomIdx = lastEndAtomIndex + 1;
+    for (let i = lastEndAtomIndex; i >= 0; i--) {
+        if (atoms[i].start >= targetStart) {
+            overlapAtomIdx = i;
+        }
+        else {
+            break;
+        }
+    }
+    // If overlap produced no progress (no atoms found), step forward to avoid
+    // an infinite loop.
+    if (overlapAtomIdx > lastEndAtomIndex) {
+        overlapAtomIdx = lastEndAtomIndex + 1;
+    }
+    return overlapAtomIdx;
+}
+/**
+ * Split `text` into overlapping chunks.
+ *
+ * Semantics:
+ * - Empty input -> empty array.
+ * - Short input (<= chunkSize) -> single chunk covering the whole text.
+ * - For each chunk, `text.slice(charStart, charEnd) === content`.
+ * - `charStart` is monotonically non-decreasing across chunks.
+ * - Consecutive chunks overlap by ~`chunkOverlap` chars (snapped to atom
+ *   boundaries; may differ by up to the largest atom size).
+ * - Each chunk's content length is bounded by chunkSize + a small slack for
+ *   the separator that snapped the boundary.
+ */
+export function chunkText(text, opts = {}) {
+    const chunkSize = opts.chunkSize ?? DEFAULT_CHUNK_SIZE;
+    const chunkOverlap = opts.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
+    const separators = opts.separators ?? DEFAULT_SEPARATORS;
+    if (text.length === 0) {
+        return [];
+    }
+    if (chunkOverlap >= chunkSize) {
+        throw new Error(`chunker: chunkOverlap (${chunkOverlap}) must be smaller than chunkSize (${chunkSize})`);
+    }
+    // Fast path for short docs — no need to walk separators.
+    if (text.length <= chunkSize) {
+        return [
+            {
+                index: 0,
+                content: text,
+                charStart: 0,
+                charEnd: text.length,
+            },
+        ];
+    }
+    const atoms = flattenToAtoms({ text, start: 0 }, separators, chunkSize);
+    const chunks = [];
+    let chunkIdx = 0;
+    let i = 0;
+    while (i < atoms.length) {
+        // Greedily pack atoms into this chunk until adding one more would push
+        // the chunk content past chunkSize.
+        let runLen = 0;
+        let j = i;
+        while (j < atoms.length) {
+            const atomLen = atoms[j].text.length;
+            // Always include at least one atom per chunk to ensure progress.
+            if (j > i && runLen + atomLen > chunkSize) {
+                break;
+            }
+            runLen += atomLen;
+            j++;
+        }
+        const runAtoms = atoms.slice(i, j);
+        const chunk = buildChunk(text, runAtoms, chunkIdx);
+        chunks.push(chunk);
+        chunkIdx++;
+        if (j >= atoms.length) {
+            break;
+        }
+        // Compute overlap start for the next chunk.
+        const lastEndAtomIdx = j - 1;
+        const nextStart = findOverlapStartIndex(atoms, lastEndAtomIdx, chunk.charEnd, chunkOverlap);
+        i = nextStart;
+    }
+    return chunks;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAeH,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,kBAAkB,GAAa,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAWnE;;;;;GAKG;AACH,SAAS,aAAa,CAAC,SAAiB,EAAE,UAAoB;IAI5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;QAC3B,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACf,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,CAAC;QACD,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,CAAC;IACH,CAAC;IACD,8DAA8D;IAC9D,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,KAAY,EAAE,SAAiB;IACvD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;QACrB,MAAM,GAAG,GAAY,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,OAAO,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,uEAAuE;QACvE,wDAAwD;QACxD,MAAM,QAAQ,GAAG,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QACxC,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC;YACxC,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM;SAC5B,CAAC,CAAC;QACH,MAAM,GAAG,QAAQ,CAAC;QAClB,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CACrB,KAAY,EACZ,UAAoB,EACpB,SAAiB;IAEjB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IACD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAElD,2EAA2E;IAC3E,8DAA8D;IAC9D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACvB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,mEAAmE;YACnE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC7D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,mEAAmE;YACnE,qCAAqC;YACrC,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,SAAS,UAAU,CACjB,YAAoB,EACpB,KAAc,EACd,KAAa;IAEb,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IACtC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC9C,OAAO;QACL,KAAK;QACL,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC;QAC/C,SAAS;QACT,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,qBAAqB,CAC5B,KAAc,EACd,gBAAwB,EACxB,OAAe,EACf,YAAoB;IAEpB,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,gBAAgB,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,GAAG,YAAY,CAAC;IAC3C,8EAA8E;IAC9E,IAAI,cAAc,GAAG,gBAAgB,GAAG,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,gBAAgB,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,KAAK,IAAI,WAAW,EAAE,CAAC;YACnC,cAAc,GAAG,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,MAAM;QACR,CAAC;IACH,CAAC;IACD,0EAA0E;IAC1E,oBAAoB;IACpB,IAAI,cAAc,GAAG,gBAAgB,EAAE,CAAC;QACtC,cAAc,GAAG,gBAAgB,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,qBAAqB,CAAC;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAEzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,0BAA0B,YAAY,qCAAqC,SAAS,GAAG,CACxF,CAAC;IACJ,CAAC;IAED,yDAAyD;IACzD,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC7B,OAAO;YACL;gBACE,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,IAAI,CAAC,MAAM;aACrB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAExE,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,uEAAuE;QACvE,oCAAoC;QACpC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACtC,iEAAiE;YACjE,IAAI,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG,OAAO,GAAG,SAAS,EAAE,CAAC;gBAC1C,MAAM;YACR,CAAC;YACD,MAAM,IAAI,OAAO,CAAC;YAClB,CAAC,EAAE,CAAC;QACN,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,QAAQ,EAAE,CAAC;QAEX,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM;QACR,CAAC;QAED,4CAA4C;QAC5C,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,qBAAqB,CACrC,KAAK,EACL,cAAc,EACd,KAAK,CAAC,OAAO,EACb,YAAY,CACb,CAAC;QACF,CAAC,GAAG,SAAS,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ralph-hero-knowledge-index",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "type": "module",
   "main": "dist/index.js",
   "bin": {

package/src/__tests__/chunker.test.ts

@@ -0,0 +1,246 @@
+import { describe, it, expect } from "vitest";
+import { chunkText, type Chunk } from "../chunker.js";
+
+/**
+ * Test suite for the recursive character text splitter.
+ *
+ * Invariants asserted (all must hold for every chunk returned):
+ *   - text.slice(charStart, charEnd) === content
+ *   - charStart is monotonically non-decreasing
+ *   - content.length <= chunkSize + (longest separator length)
+ */
+
+function assertCoreInvariants(
+  text: string,
+  chunks: Chunk[],
+  chunkSize: number,
+  separatorSlack = 2,
+): void {
+  let prevStart = -1;
+  for (let i = 0; i < chunks.length; i++) {
+    const c = chunks[i]!;
+    expect(c.index).toBe(i);
+    expect(text.slice(c.charStart, c.charEnd)).toBe(c.content);
+    expect(c.charStart).toBeGreaterThanOrEqual(prevStart);
+    expect(c.content.length).toBeLessThanOrEqual(chunkSize + separatorSlack);
+    prevStart = c.charStart;
+  }
+}
+
+describe("chunkText — empty and short inputs", () => {
+  it("returns [] for empty string", () => {
+    expect(chunkText("")).toEqual([]);
+  });
+
+  it("returns a single chunk for a short doc", () => {
+    const text = "short doc";
+    const chunks = chunkText(text);
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]!.charStart).toBe(0);
+    expect(chunks[0]!.charEnd).toBe(text.length);
+    expect(chunks[0]!.content).toBe(text);
+    expect(chunks[0]!.index).toBe(0);
+  });
+
+  it("returns a single chunk at exactly chunkSize", () => {
+    const text = "x".repeat(2048);
+    const chunks = chunkText(text, { chunkSize: 2048 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]!.content).toBe(text);
+  });
+
+  it("returns a single chunk when text.length < chunkSize even with unusual separators", () => {
+    const chunks = chunkText("hello world", {
+      chunkSize: 100,
+      chunkOverlap: 10,
+      separators: ["##", "\n"],
+    });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]!.content).toBe("hello world");
+  });
+});
+
+describe("chunkText — long documents", () => {
+  it("produces >= 4 chunks for an 8K-char paragraph-rich doc with chunkSize=2048", () => {
+    const paragraph = "The quick brown fox jumps over the lazy dog. ".repeat(20);
+    const text = Array.from({ length: 10 }, () => paragraph).join("\n\n");
+    expect(text.length).toBeGreaterThan(8000);
+
+    const chunks = chunkText(text, { chunkSize: 2048, chunkOverlap: 256 });
+    expect(chunks.length).toBeGreaterThanOrEqual(4);
+    assertCoreInvariants(text, chunks, 2048);
+  });
+
+  it("bounds content.length at chunkSize + separator slack", () => {
+    const text = Array.from({ length: 400 }, (_, i) => `Sentence ${i}.`).join(" ");
+    const chunks = chunkText(text, { chunkSize: 512, chunkOverlap: 64 });
+    assertCoreInvariants(text, chunks, 512);
+  });
+
+  it("keeps charStart monotonically non-decreasing", () => {
+    const text = "x".repeat(10_000);
+    const chunks = chunkText(text, { chunkSize: 256, chunkOverlap: 32 });
+    for (let i = 1; i < chunks.length; i++) {
+      expect(chunks[i]!.charStart).toBeGreaterThanOrEqual(chunks[i - 1]!.charStart);
+    }
+  });
+
+  it("reconstructs content bit-for-bit from offsets for every chunk", () => {
+    const text = Array.from({ length: 500 }, (_, i) => `Para ${i}.\n\n`).join("");
+    const chunks = chunkText(text, { chunkSize: 1024, chunkOverlap: 128 });
+    for (const c of chunks) {
+      expect(text.slice(c.charStart, c.charEnd)).toBe(c.content);
+    }
+  });
+});
+
+describe("chunkText — overlap behavior", () => {
+  it("produces overlap of approximately chunkOverlap between consecutive chunks", () => {
+    const text = "abcdefghij ".repeat(500); // ~5500 chars
+    const chunkOverlap = 256;
+    const chunks = chunkText(text, { chunkSize: 1024, chunkOverlap });
+
+    expect(chunks.length).toBeGreaterThan(1);
+
+    for (let i = 1; i < chunks.length; i++) {
+      const prev = chunks[i - 1]!;
+      const curr = chunks[i]!;
+      // Overlap in characters = prev.charEnd - curr.charStart.
+      const overlap = prev.charEnd - curr.charStart;
+      expect(overlap).toBeGreaterThan(0);
+      // Tolerance: +/- 16 chars (allows for snap to atom boundary).
+      expect(Math.abs(overlap - chunkOverlap)).toBeLessThanOrEqual(16);
+    }
+  });
+
+  it("makes forward progress even when chunkOverlap is zero", () => {
+    const text = "abcdefghij ".repeat(300);
+    const chunks = chunkText(text, { chunkSize: 512, chunkOverlap: 0 });
+    for (let i = 1; i < chunks.length; i++) {
+      expect(chunks[i]!.charStart).toBeGreaterThanOrEqual(chunks[i - 1]!.charEnd);
+    }
+  });
+
+  it("rejects chunkOverlap >= chunkSize", () => {
+    // Only need enough text to trigger the chunking path past the fast return.
+    const text = "x".repeat(2048);
+    expect(() =>
+      chunkText(text, { chunkSize: 100, chunkOverlap: 100 }),
+    ).toThrow(/chunkOverlap/);
+    expect(() =>
+      chunkText(text, { chunkSize: 100, chunkOverlap: 200 }),
+    ).toThrow(/chunkOverlap/);
+  });
+});
+
+describe("chunkText — unicode correctness", () => {
+  it("preserves character boundaries with emoji + CJK", () => {
+    // Mix emoji (surrogate pairs), CJK, and ASCII; repeat enough to exceed chunkSize.
+    const segment =
+      "Hello. こんにちは。 你好。 안녕하세요。 Emojis: 🎉🚀✨. ";
+    const text = segment.repeat(100); // plenty of chunks
+    const chunks = chunkText(text, { chunkSize: 512, chunkOverlap: 64 });
+
+    assertCoreInvariants(text, chunks, 512);
+
+    // Slicing must produce the same string as content — if indices split a
+    // surrogate pair the JS string would still compare equal char-by-char,
+    // but we additionally verify no lone surrogate prefix/suffix on boundaries.
+    for (const c of chunks) {
+      expect(text.slice(c.charStart, c.charEnd)).toBe(c.content);
+    }
+  });
+
+  it("handles pure CJK doc (no ASCII separators beyond ideographic space)", () => {
+    const text = "这是一个很长的中文文档。".repeat(300);
+    const chunks = chunkText(text, { chunkSize: 256, chunkOverlap: 32 });
+    assertCoreInvariants(text, chunks, 256);
+    // Must cover the whole doc (last chunk ends at text.length).
+    expect(chunks[chunks.length - 1]!.charEnd).toBe(text.length);
+  });
+});
+
+describe("chunkText — separator priority", () => {
+  it("prefers \\n\\n over other separators so code fences stay intact", () => {
+    // A large markdown doc with a code fence that would exceed chunkSize if
+    // we naively split on every newline; ensure it's kept whole by preferring
+    // \n\n as the outer boundary.
+    const prefix = "# Heading\n\nSome text before the fence.\n\n";
+    const fence =
+      "```typescript\n" +
+      "const x = 1;\n".repeat(60) +
+      "```";
+    const suffix = "\n\nText after the fence.";
+    const text = prefix + fence + suffix;
+
+    const chunkSize = fence.length + 50; // Large enough to keep fence whole.
+    const chunks = chunkText(text, {
+      chunkSize,
+      chunkOverlap: 64,
+      separators: ["\n\n", "\n", ". ", " ", ""],
+    });
+
+    // Fence should appear intact in at least one chunk.
+    const anyChunkContainsWholeFence = chunks.some(c => c.content.includes(fence));
+    expect(anyChunkContainsWholeFence).toBe(true);
+
+    // And no chunk should contain just a fragment of the opening backticks
+    // separated from its closing ones — i.e., if a chunk starts with ``` it
+    // must also contain the matching closing ```.
+    for (const c of chunks) {
+      const opens = (c.content.match(/```/g) ?? []).length;
+      // Either zero fences (text-only chunk) or an even number (complete fences).
+      expect(opens % 2 === 0 || c.content.includes(fence)).toBe(true);
+    }
+  });
+
+  it("honors custom separators — '##' heading splitter", () => {
+    const text =
+      "Intro paragraph here.\n\n" +
+      "## Section A\nContent for A.\n\n" +
+      "## Section B\nContent for B with a bit more text.\n\n" +
+      "## Section C\nFinal content goes here.";
+    const chunks = chunkText(text, {
+      chunkSize: 40, // small -> forces splitting
+      chunkOverlap: 8,
+      separators: ["##", "\n", " ", ""],
+    });
+
+    // Custom separator must be honored: the chunker must split on '##'.
+    // Because the separator stays attached to the preceding piece (to keep
+    // text.slice(start,end) === content), chunk boundaries occur *just after*
+    // each '##' occurrence — verify that at least one chunk's charEnd aligns
+    // with a '##' position + 2.
+    expect(chunks.length).toBeGreaterThan(1);
+    const sectionStarts = [
+      text.indexOf("## Section A"),
+      text.indexOf("## Section B"),
+      text.indexOf("## Section C"),
+    ];
+    const endPositions = chunks.map(c => c.charEnd);
+    const boundariesHit = sectionStarts.filter(pos =>
+      endPositions.includes(pos + 2),
+    );
+    expect(boundariesHit.length).toBeGreaterThan(0);
+  });
+
+  it("character-splits as a last resort when no separator applies", () => {
+    // A single token longer than chunkSize with no separators; must still
+    // produce chunks (via the "" sentinel).
+    const text = "a".repeat(300);
+    const chunks = chunkText(text, {
+      chunkSize: 100,
+      chunkOverlap: 10,
+      separators: ["\n\n", "\n", ""],
+    });
+    expect(chunks.length).toBeGreaterThan(1);
+    for (const c of chunks) {
+      expect(c.content.length).toBeLessThanOrEqual(100);
+    }
+    // Overall coverage: concatenating (with overlap removed) must reconstruct.
+    const firstChar = chunks[0]!.charStart;
+    const lastEnd = chunks[chunks.length - 1]!.charEnd;
+    expect(firstChar).toBe(0);
+    expect(lastEnd).toBe(text.length);
+  });
+});

package/src/chunker.ts

@@ -0,0 +1,279 @@
+/**
+ * RecursiveCharacterTextSplitter-style chunker.
+ *
+ * Splits long text into overlapping chunks while preserving the original
+ * character offsets (charStart, charEnd) so downstream code can reconstruct
+ * positions. Mirrors the semantics of LangChain's RecursiveCharacterTextSplitter:
+ * tries each separator in order, snapping chunk boundaries to the highest-priority
+ * separator that keeps pieces under chunkSize.
+ *
+ * Defaults correspond to 512-token chunks with 64-token overlap
+ * (approx: 1 token ~= 4 chars for English text).
+ */
+
+export interface Chunk {
+  index: number;
+  content: string;
+  charStart: number;
+  charEnd: number;
+}
+
+export interface ChunkerOptions {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  separators?: string[];
+}
+
+const DEFAULT_CHUNK_SIZE = 2048;
+const DEFAULT_CHUNK_OVERLAP = 256;
+const DEFAULT_SEPARATORS: string[] = ["\n\n", "\n", ". ", " ", ""];
+
+/**
+ * A piece of the original text with an absolute offset back to the source.
+ * Used as the internal working type while recursing through separators.
+ */
+interface Piece {
+  text: string;
+  start: number;
+}
+
+/**
+ * Pick the first separator from `separators` that occurs in `piece.text`.
+ * Falls back to the last separator (typically `""`) if none match — this is
+ * the sentinel that lets us character-split oversized pieces with no natural
+ * boundary.
+ */
+function pickSeparator(pieceText: string, separators: string[]): {
+  separator: string;
+  remaining: string[];
+} {
+  for (let i = 0; i < separators.length; i++) {
+    const sep = separators[i]!;
+    if (sep === "") {
+      return { separator: sep, remaining: separators.slice(i + 1) };
+    }
+    if (pieceText.includes(sep)) {
+      return { separator: sep, remaining: separators.slice(i + 1) };
+    }
+  }
+  // Should be unreachable when DEFAULT_SEPARATORS ends with "".
+  return { separator: "", remaining: [] };
+}
+
+/**
+ * Split a piece on `separator` while retaining absolute char offsets.
+ * When separator is empty, split into single-character pieces.
+ */
+function splitOnSeparator(piece: Piece, separator: string): Piece[] {
+  if (separator === "") {
+    const out: Piece[] = [];
+    for (let i = 0; i < piece.text.length; i++) {
+      out.push({ text: piece.text[i]!, start: piece.start + i });
+    }
+    return out;
+  }
+
+  const out: Piece[] = [];
+  let cursor = 0;
+  let idx = piece.text.indexOf(separator, cursor);
+  while (idx !== -1) {
+    // Keep the separator attached to the preceding piece so reconstruction
+    // via text.slice(charStart, charEnd) works bit-for-bit.
+    const sliceEnd = idx + separator.length;
+    out.push({
+      text: piece.text.slice(cursor, sliceEnd),
+      start: piece.start + cursor,
+    });
+    cursor = sliceEnd;
+    idx = piece.text.indexOf(separator, cursor);
+  }
+  if (cursor < piece.text.length) {
+    out.push({
+      text: piece.text.slice(cursor),
+      start: piece.start + cursor,
+    });
+  }
+  return out;
+}
+
+/**
+ * Recursively flatten a piece into "atoms" — pieces small enough to merge
+ * greedily into chunks. Pieces larger than chunkSize are split with the next
+ * separator in line; pieces that fit are returned as-is.
+ */
+function flattenToAtoms(
+  piece: Piece,
+  separators: string[],
+  chunkSize: number,
+): Piece[] {
+  if (piece.text.length <= chunkSize) {
+    return [piece];
+  }
+  const { separator, remaining } = pickSeparator(piece.text, separators);
+  const splits = splitOnSeparator(piece, separator);
+
+  // If the separator didn't actually reduce the piece (e.g., no occurrence),
+  // fall through to the next separator with the original piece.
+  if (splits.length <= 1) {
+    if (remaining.length === 0) {
+      // No more separators — return whatever we have, even if oversized.
+      return [piece];
+    }
+    return flattenToAtoms(piece, remaining, chunkSize);
+  }
+
+  const out: Piece[] = [];
+  for (const sub of splits) {
+    if (sub.text.length <= chunkSize) {
+      out.push(sub);
+    } else if (remaining.length > 0) {
+      for (const leaf of flattenToAtoms(sub, remaining, chunkSize)) {
+        out.push(leaf);
+      }
+    } else {
+      // Last-resort: character-split oversized atom so we never return a
+      // single atom larger than chunkSize.
+      out.push(...splitOnSeparator(sub, ""));
+    }
+  }
+  return out;
+}
+
+/**
+ * Build a chunk object from a contiguous run of atoms.
+ * `charStart` is taken from the first atom, `charEnd` from the last atom's
+ * end boundary, and `content` is `text.slice(start, end)` — this guarantees
+ * `text.slice(charStart, charEnd) === content`.
+ */
+function buildChunk(
+  originalText: string,
+  atoms: Piece[],
+  index: number,
+): Chunk {
+  const first = atoms[0]!;
+  const last = atoms[atoms.length - 1]!;
+  const charStart = first.start;
+  const charEnd = last.start + last.text.length;
+  return {
+    index,
+    content: originalText.slice(charStart, charEnd),
+    charStart,
+    charEnd,
+  };
+}
+
+/**
+ * Compute the start position for the next chunk's atoms given the previous
+ * chunk ended at `prevEnd`. We walk backward through the atom list to find
+ * the atom whose start >= prevEnd - chunkOverlap; that atom begins the
+ * overlap region.
+ */
+function findOverlapStartIndex(
+  atoms: Piece[],
+  lastEndAtomIndex: number,
+  prevEnd: number,
+  chunkOverlap: number,
+): number {
+  if (chunkOverlap <= 0) {
+    return lastEndAtomIndex + 1;
+  }
+  const targetStart = prevEnd - chunkOverlap;
+  // Find the earliest atom in [0..lastEndAtomIndex] whose start >= targetStart.
+  let overlapAtomIdx = lastEndAtomIndex + 1;
+  for (let i = lastEndAtomIndex; i >= 0; i--) {
+    if (atoms[i]!.start >= targetStart) {
+      overlapAtomIdx = i;
+    } else {
+      break;
+    }
+  }
+  // If overlap produced no progress (no atoms found), step forward to avoid
+  // an infinite loop.
+  if (overlapAtomIdx > lastEndAtomIndex) {
+    overlapAtomIdx = lastEndAtomIndex + 1;
+  }
+  return overlapAtomIdx;
+}
+
+/**
+ * Split `text` into overlapping chunks.
+ *
+ * Semantics:
+ * - Empty input -> empty array.
+ * - Short input (<= chunkSize) -> single chunk covering the whole text.
+ * - For each chunk, `text.slice(charStart, charEnd) === content`.
+ * - `charStart` is monotonically non-decreasing across chunks.
+ * - Consecutive chunks overlap by ~`chunkOverlap` chars (snapped to atom
+ *   boundaries; may differ by up to the largest atom size).
+ * - Each chunk's content length is bounded by chunkSize + a small slack for
+ *   the separator that snapped the boundary.
+ */
+export function chunkText(text: string, opts: ChunkerOptions = {}): Chunk[] {
+  const chunkSize = opts.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const chunkOverlap = opts.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
+  const separators = opts.separators ?? DEFAULT_SEPARATORS;
+
+  if (text.length === 0) {
+    return [];
+  }
+
+  if (chunkOverlap >= chunkSize) {
+    throw new Error(
+      `chunker: chunkOverlap (${chunkOverlap}) must be smaller than chunkSize (${chunkSize})`,
+    );
+  }
+
+  // Fast path for short docs — no need to walk separators.
+  if (text.length <= chunkSize) {
+    return [
+      {
+        index: 0,
+        content: text,
+        charStart: 0,
+        charEnd: text.length,
+      },
+    ];
+  }
+
+  const atoms = flattenToAtoms({ text, start: 0 }, separators, chunkSize);
+
+  const chunks: Chunk[] = [];
+  let chunkIdx = 0;
+  let i = 0;
+
+  while (i < atoms.length) {
+    // Greedily pack atoms into this chunk until adding one more would push
+    // the chunk content past chunkSize.
+    let runLen = 0;
+    let j = i;
+    while (j < atoms.length) {
+      const atomLen = atoms[j]!.text.length;
+      // Always include at least one atom per chunk to ensure progress.
+      if (j > i && runLen + atomLen > chunkSize) {
+        break;
+      }
+      runLen += atomLen;
+      j++;
+    }
+    const runAtoms = atoms.slice(i, j);
+    const chunk = buildChunk(text, runAtoms, chunkIdx);
+    chunks.push(chunk);
+    chunkIdx++;
+
+    if (j >= atoms.length) {
+      break;
+    }
+
+    // Compute overlap start for the next chunk.
+    const lastEndAtomIdx = j - 1;
+    const nextStart = findOverlapStartIndex(
+      atoms,
+      lastEndAtomIdx,
+      chunk.charEnd,
+      chunkOverlap,
+    );
+    i = nextStart;
+  }
+
+  return chunks;
+}