react-native-pageindex 0.1.0

package/dist/reverseIndex.js

@@ -0,0 +1,248 @@
+"use strict";
+/**
+ * Reverse (inverted) index — maps terms → tree nodes that contain them.
+ *
+ * Two modes:
+ *   'keyword' — fast, no LLM. Extracts stopword-filtered terms with TF scoring.
+ *   'llm'     — slower, semantic. Uses LLM to extract concept terms per node.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildReverseIndex = buildReverseIndex;
+exports.searchReverseIndex = searchReverseIndex;
+const tree_1 = require("./utils/tree");
+// ─── Stopwords ────────────────────────────────────────────────────────────────
+const STOPWORDS = new Set([
+    'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+    'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'be',
+    'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will',
+    'would', 'could', 'should', 'may', 'might', 'shall', 'can', 'not',
+    'this', 'that', 'these', 'those', 'it', 'its', 'we', 'our', 'you',
+    'your', 'he', 'she', 'they', 'their', 'my', 'his', 'her', 'which',
+    'who', 'what', 'when', 'where', 'how', 'if', 'then', 'so', 'than',
+    'more', 'also', 'about', 'into', 'up', 'out', 'no', 'all', 'each',
+    'any', 'some', 'other', 'new', 'one', 'two', 'such', 'only', 'over',
+    'after', 'before', 'between', 'through', 'during', 'including', 'without',
+    'within', 'along', 'following', 'across', 'behind', 'beyond', 'plus',
+    'except', 'however', 'therefore', 'thus', 'hence', 'while', 'although',
+    'because', 'since', 'unless', 'until', 'whether', 'both', 'either',
+    'neither', 'per', 'via', 'etc', 'ie', 'eg',
+]);
+// ─── Keyword Extraction ───────────────────────────────────────────────────────
+function tokenise(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s-]/g, ' ')
+        .split(/\s+/)
+        .map((t) => t.replace(/^-+|-+$/g, '').trim())
+        .filter(Boolean);
+}
+function extractKeywords(text, minLength) {
+    const tokens = tokenise(text);
+    const tf = new Map();
+    for (const token of tokens) {
+        if (token.length < minLength)
+            continue;
+        if (STOPWORDS.has(token))
+            continue;
+        if (/^\d+$/.test(token))
+            continue; // pure numbers
+        tf.set(token, (tf.get(token) ?? 0) + 1);
+    }
+    return tf;
+}
+// Normalise TF into 0-1 score (log-normalised)
+function normaliseTf(count, maxCount) {
+    if (maxCount === 0)
+        return 0;
+    return Math.log1p(count) / Math.log1p(maxCount);
+}
+// ─── LLM Prompt ───────────────────────────────────────────────────────────────
+function buildLlmPrompt(nodeTitle, nodeSummary, maxTerms) {
+    return `Extract up to ${maxTerms} key concepts, named entities, or important terms from the section below.
+Return a JSON array of strings only — short terms or phrases (1–4 words each), no duplicates, no stopwords.
+
+Section title: ${nodeTitle}
+Section summary: ${nodeSummary}
+
+Respond with ONLY a JSON array, e.g.: ["machine learning", "gradient descent", "neural network"]`;
+}
+async function extractLlmTerms(node, llm, maxTerms) {
+    const title = node.title ?? '';
+    const summary = node.summary ?? node.prefix_summary ?? node.text ?? '';
+    if (!title && !summary)
+        return [];
+    const prompt = buildLlmPrompt(title, summary, maxTerms);
+    try {
+        const result = await llm(prompt);
+        const match = result.content.match(/\[[\s\S]*\]/);
+        if (!match)
+            return [];
+        const parsed = JSON.parse(match[0]);
+        if (!Array.isArray(parsed))
+            return [];
+        return parsed
+            .filter((t) => typeof t === 'string')
+            .map((t) => t.toLowerCase().trim())
+            .filter((t) => t.length >= 2)
+            .slice(0, maxTerms);
+    }
+    catch {
+        return [];
+    }
+}
+// ─── Node Text for Keyword Mode ───────────────────────────────────────────────
+function nodeTextForKeywords(node) {
+    const parts = [];
+    if (node.title)
+        parts.push(node.title, node.title); // weight title 2x
+    if (node.summary)
+        parts.push(node.summary);
+    if (node.prefix_summary)
+        parts.push(node.prefix_summary);
+    if (node.text)
+        parts.push(node.text);
+    return parts.join(' ');
+}
+// ─── Build Reverse Index ──────────────────────────────────────────────────────
+/**
+ * Builds an inverted index from a `PageIndexResult`.
+ *
+ * In **keyword** mode (default), terms are extracted via stopword-filtered TF
+ * scoring — fast, no LLM calls needed.
+ *
+ * In **llm** mode, the LLM extracts semantic concept terms from each node's
+ * title + summary — slower but catches synonyms/concepts.
+ *
+ * @param result   The forward-index output from `pageIndex()` / `pageIndexMd()`
+ * @param pages    Original page data (optional; used for extra keyword signal)
+ * @param llm      LLM provider (required for mode 'llm')
+ * @param options  Index options
+ */
+async function buildReverseIndex(input) {
+    const { result, llm, options = {} } = input;
+    const { mode = 'keyword', minTermLength = 3, maxTermsPerNode = 10, onProgress, } = options;
+    if (mode === 'llm' && !llm) {
+        throw new Error('[PageIndex] LLM provider is required when mode is "llm"');
+    }
+    // Flatten tree into leaf + branch nodes (all nodes)
+    const nodes = (0, tree_1.getNodes)(result.structure);
+    const terms = {};
+    const total = nodes.length;
+    for (let i = 0; i < total; i++) {
+        const node = nodes[i];
+        onProgress?.({
+            step: mode === 'llm' ? 'Extracting concepts via LLM' : 'Extracting keywords',
+            percent: Math.round((i / total) * 90),
+            detail: `Node ${i + 1} / ${total}: ${node.title ?? ''}`,
+        });
+        const entry = {
+            nodeId: node.node_id,
+            nodeTitle: node.title ?? '',
+            startIndex: node.start_index,
+            endIndex: node.end_index,
+            score: 0, // filled in below
+        };
+        if (mode === 'keyword') {
+            const text = nodeTextForKeywords(node);
+            const tf = extractKeywords(text, minTermLength);
+            if (tf.size === 0)
+                continue;
+            const maxCount = Math.max(...tf.values());
+            // Keep top-N by count
+            const sorted = [...tf.entries()]
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, maxTermsPerNode);
+            for (const [term, count] of sorted) {
+                const score = normaliseTf(count, maxCount);
+                const entryWithScore = { ...entry, score };
+                if (!terms[term])
+                    terms[term] = [];
+                terms[term].push(entryWithScore);
+            }
+        }
+        else {
+            // LLM mode
+            const termList = await extractLlmTerms(node, llm, maxTermsPerNode);
+            for (let rank = 0; rank < termList.length; rank++) {
+                const term = termList[rank];
+                // Score decays with position (first term most important)
+                const score = 1 - rank / termList.length;
+                const entryWithScore = { ...entry, score };
+                if (!terms[term])
+                    terms[term] = [];
+                terms[term].push(entryWithScore);
+            }
+        }
+    }
+    // Sort each term's entries by score descending
+    for (const term of Object.keys(terms)) {
+        terms[term].sort((a, b) => b.score - a.score);
+    }
+    onProgress?.({
+        step: 'Reverse index complete',
+        percent: 100,
+        detail: `${Object.keys(terms).length} terms across ${total} nodes`,
+    });
+    return {
+        docName: result.doc_name,
+        terms,
+        stats: {
+            totalTerms: Object.keys(terms).length,
+            totalNodes: total,
+            indexMode: mode,
+            indexedAt: new Date().toISOString(),
+        },
+    };
+}
+// ─── Search ───────────────────────────────────────────────────────────────────
+/**
+ * Queries the reverse index for one or more terms.
+ * Multi-word queries are split and each term is looked up separately;
+ * nodes matching multiple terms get a combined score boost.
+ *
+ * @param index  The reverse index (from `buildReverseIndex`)
+ * @param query  Free-text query string
+ * @param topK   Max results to return (default: 10)
+ */
+function searchReverseIndex(index, query, topK = 10) {
+    const queryTerms = tokenise(query).filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+    if (queryTerms.length === 0)
+        return [];
+    // nodeId (or nodeTitle as fallback) → combined result
+    const combined = new Map();
+    for (const qTerm of queryTerms) {
+        // Exact match
+        const exactHits = index.terms[qTerm] ?? [];
+        // Prefix / partial match
+        const partialHits = [];
+        for (const [term, entries] of Object.entries(index.terms)) {
+            if (term !== qTerm && term.includes(qTerm)) {
+                partialHits.push(...entries.map((e) => ({ ...e, score: e.score * 0.6 })));
+            }
+        }
+        const allHits = [...exactHits, ...partialHits];
+        for (const hit of allHits) {
+            const key = hit.nodeId ?? hit.nodeTitle;
+            const existing = combined.get(key);
+            if (existing) {
+                existing.totalScore += hit.score;
+                // keep the best individual term score
+                if (hit.score > existing.score) {
+                    existing.score = hit.score;
+                    existing.matchedTerm = qTerm;
+                }
+            }
+            else {
+                combined.set(key, {
+                    ...hit,
+                    matchedTerm: qTerm,
+                    totalScore: hit.score,
+                });
+            }
+        }
+    }
+    return [...combined.values()]
+        .sort((a, b) => b.totalScore - a.totalScore)
+        .slice(0, topK);
+}
+//# sourceMappingURL=reverseIndex.js.map

package/dist/reverseIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reverseIndex.js","sourceRoot":"","sources":["../src/reverseIndex.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;AAoIH,8CAiGC;AAaD,gDAoDC;AA1RD,uCAAwC;AAExC,iFAAiF;AAEjF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACnE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI;IAClE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAClE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IACjE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IACjE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO;IACjE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;IACjE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM;IACjE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACnE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS;IACzE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM;IACpE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU;IACtE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ;IAClE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;CAC3C,CAAC,CAAC;AAEH,iFAAiF;AAEjF,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,eAAe,CACtB,IAAY,EACZ,SAAiB;IAEjB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS;YAAE,SAAS;QACvC,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QACnC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,SAAS,CAAC,eAAe;QAClD,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,+CAA+C;AAC/C,SAAS,WAAW,CAAC,KAAa,EAAE,QAAgB;IAClD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AAClD,CAAC;AAED,iFAAiF;AAEjF,SAAS,cAAc,CAAC,SAAiB,EAAE,WAAmB,EAAE,QAAgB;IAC9E,OAAO,iBAAiB,QAAQ;;;iBAGjB,SAAS;mBACP,WAAW;;iGAEmE,CAAC;AAClG,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,IAAc,EACd,GAAgB,EAChB,QAAgB;IAEhB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IACvE,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QACtC,OAAQ,MAAoB;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;aACpC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAY,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;aAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;aAC5B,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,SAAS,mBAAmB,CAAC,IAAc;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB;IACtE,IAAI,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,IAAI,CAAC,cAAc;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACzD,IAAI,IAAI,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;;;;;;GAaG;AACI,KAAK,UAAU,iBAAiB,CAAC,KAKvC;IACC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC;IAC5C,MAAM,EACJ,IAAI,GAAG,SAAS,EAChB,aAAa,GAAG,CAAC,EACjB,eAAe,GAAG,EAAE,EACpB,UAAU,GACX,GAAG,OAAO,CAAC;IAEZ,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,oDAAoD;IACpD,MAAM,KAAK,GAAG,IAAA,eAAQ,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAEzC,MAAM,KAAK,GAAwC,EAAE,CAAC;IAEtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEtB,UAAU,EAAE,CAAC;YACX,IAAI,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,qBAAqB;YAC5E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACrC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE;SACxD,CAAC,CAAC;QAEH,MAAM,KAAK,GAAsB;YAC/B,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,SAAS,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;YAC3B,UAAU,EAAE,IAAI,CAAC,WAAW;YAC5B,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,KAAK,EAAE,CAAC,EAAE,kBAAkB;SAC7B,CAAC;QAEF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;YAEhD,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YAE5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAE1C,sBAAsB;YACtB,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;iBAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC3B,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;YAE7B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;gBACnC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC3C,MAAM,cAAc,GAAsB,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC9D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,WAAW;YACX,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,GAAI,EAAE,eAAe,CAAC,CAAC;YAEpE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC5B,yDAAyD;gBACzD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,MAAM,cAAc,GAAsB,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC9D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC;IAED,UAAU,EAAE,CAAC;QACX,IAAI,EAAE,wBAAwB;QAC9B,OAAO,EAAE,GAAG;QACZ,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,iBAAiB,KAAK,QAAQ;KACnE,CAAC,CAAC;IAEH,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,QAAQ;QACxB,KAAK;QACL,KAAK,EAAE;YACL,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM;YACrC,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;KACF,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;GAQG;AACH,SAAgB,kBAAkB,CAChC,KAAmB,EACnB,KAAa,EACb,IAAI,GAAG,EAAE;IAET,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAC1C,CAAC;IAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,sDAAsD;IACtD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEjD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,cAAc;QACd,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC3C,yBAAyB;QACzB,MAAM,WAAW,GAAwB,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1D,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3C,WAAW,CAAC,IAAI,CACd,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,CAAC,CACxD,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,WAAW,CAAC,CAAC;QAE/C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC;YACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACjC,sCAAsC;gBACtC,IAAI,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAC/B,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAC3B,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;oBAChB,GAAG,GAAG;oBACN,WAAW,EAAE,KAAK;oBAClB,UAAU,EAAE,GAAG,CAAC,KAAK;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;SAC3C,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACpB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,190 @@
+/** A single message in a chat conversation */
+export interface LLMMessage {
+    role: 'user' | 'assistant';
+    content: string;
+}
+/** The finish reason returned by the LLM */
+export type LLMFinishReason = 'stop' | 'length' | string;
+/** The result of an LLM call */
+export interface LLMResult {
+    content: string;
+    finishReason: LLMFinishReason;
+}
+/**
+ * Provider-agnostic LLM callback.
+ * Wire up OpenAI, Anthropic, Ollama, or any other provider here.
+ *
+ * @example
+ * const llm: LLMProvider = async (prompt, opts) => {
+ *   const response = await openai.chat.completions.create({
+ *     model: 'gpt-4o',
+ *     messages: [
+ *       ...(opts?.chatHistory ?? []),
+ *       { role: 'user', content: prompt },
+ *     ],
+ *   });
+ *   return {
+ *     content: response.choices[0].message.content ?? '',
+ *     finishReason: response.choices[0].finish_reason ?? 'stop',
+ *   };
+ * };
+ */
+export type LLMProvider = (prompt: string, options?: {
+    chatHistory?: LLMMessage[];
+}) => Promise<LLMResult>;
+/** Emitted at every major milestone during processing */
+export interface ProgressInfo {
+    /** Human-readable description of the current step */
+    step: string;
+    /** Overall progress 0–100 */
+    percent: number;
+    /** Optional extra detail, e.g. "Page 3 / 45" or "Node 2 / 8" */
+    detail?: string;
+}
+export type ProgressCallback = (info: ProgressInfo) => void;
+/**
+ * Returns the approximate number of tokens in `text`.
+ * Default implementation: Math.ceil(text.length / 4)
+ * Plug in `js-tiktoken` or similar for exact counts.
+ */
+export type TokenCounter = (text: string) => number;
+/** Text and token count for a single document page */
+export interface PageData {
+    text: string;
+    tokenCount: number;
+}
+/** A node in the hierarchical tree index */
+export interface TreeNode {
+    title: string;
+    node_id?: string;
+    start_index?: number;
+    end_index?: number;
+    summary?: string;
+    prefix_summary?: string;
+    text?: string;
+    nodes?: TreeNode[];
+    structure?: string;
+    physical_index?: number | null;
+    appear_start?: string;
+    list_index?: number;
+    page?: number | null;
+}
+/** The final output of pageIndex() or pageIndexMd() */
+export interface PageIndexResult {
+    doc_name: string;
+    doc_description?: string;
+    structure: TreeNode[];
+}
+/** File formats supported by pageIndexDocument() */
+export type DocumentFileType = 'pdf' | 'docx' | 'csv' | 'xlsx' | 'md';
+/** Options for CSV parsing */
+export interface CsvParseOptions {
+    /** Column delimiter (default: auto-detect from ',', ';', '\t') */
+    delimiter?: string;
+    /** Rows per page-chunk (default: 100) */
+    rowsPerPage?: number;
+    /** Treat first row as header (default: true) */
+    hasHeader?: boolean;
+}
+/** Options for XLSX parsing */
+export interface XlsxParseOptions {
+    /** Sheet names to include (default: all sheets) */
+    sheets?: string[];
+    /** Max rows per sheet-chunk (default: 200) */
+    rowsPerChunk?: number;
+}
+/** A single reverse-index hit — points back to the tree node that contains the term */
+export interface ReverseIndexEntry {
+    nodeId?: string;
+    nodeTitle: string;
+    /** Start page of the node (1-based) */
+    startIndex?: number;
+    /** End page of the node (1-based) */
+    endIndex?: number;
+    /**
+     * Relevance score 0–1.
+     * Keyword mode: TF-based score.
+     * LLM mode: LLM-assigned importance.
+     */
+    score: number;
+}
+/** The complete reverse (inverted) index for one document */
+export interface ReverseIndex {
+    docName: string;
+    /**
+     * Maps every indexed term → list of nodes that contain it, sorted by score desc.
+     * Keys are lowercase, normalised terms.
+     */
+    terms: Record<string, ReverseIndexEntry[]>;
+    stats: {
+        totalTerms: number;
+        totalNodes: number;
+        indexMode: 'keyword' | 'llm';
+        indexedAt: string;
+    };
+}
+/** A ranked search result returned by searchReverseIndex() */
+export interface SearchResult extends ReverseIndexEntry {
+    /** The term that matched the query */
+    matchedTerm: string;
+    /** Combined score when multiple terms match */
+    totalScore: number;
+}
+/** Options for building the reverse index */
+export interface ReverseIndexOptions {
+    /**
+     * 'keyword' — fast, no LLM calls, extracts terms via stopword-filtered TF
+     * 'llm'     — slower, semantic; uses LLM to extract concepts per node
+     * default: 'keyword'
+     */
+    mode?: 'keyword' | 'llm';
+    /** Minimum term length to index (default: 3) */
+    minTermLength?: number;
+    /** Max number of terms to extract per node in LLM mode (default: 10) */
+    maxTermsPerNode?: number;
+    /** Called at each major milestone */
+    onProgress?: ProgressCallback;
+}
+/** Options for the PDF pipeline */
+export interface PageIndexOptions {
+    /** Number of pages to scan for an existing Table of Contents (default: 20) */
+    tocCheckPageNum?: number;
+    /** Max pages a single tree node may span before sub-indexing (default: 10) */
+    maxPageNumEachNode?: number;
+    /** Max tokens a single tree node may contain before sub-indexing (default: 20000) */
+    maxTokenNumEachNode?: number;
+    /** Whether to add a sequential node_id to each node (default: true) */
+    ifAddNodeId?: boolean;
+    /** Whether to generate an LLM summary for each node (default: true) */
+    ifAddNodeSummary?: boolean;
+    /** Whether to generate a one-sentence document description (default: false) */
+    ifAddDocDescription?: boolean;
+    /** Whether to include raw page text in each node (default: false) */
+    ifAddNodeText?: boolean;
+    /** Custom token counting function (default: ~4 chars/token approximation) */
+    tokenCounter?: TokenCounter;
+    /** Called at each major processing milestone with step name and 0–100 percent */
+    onProgress?: ProgressCallback;
+}
+/** Options for the Markdown pipeline */
+export interface MdPageIndexOptions {
+    /** Whether to merge small nodes together (default: false) */
+    ifThinning?: boolean;
+    /** Minimum token threshold for thinning (default: 5000) */
+    minTokenThreshold?: number;
+    /** Whether to generate an LLM summary for each node (default: true) */
+    ifAddNodeSummary?: boolean;
+    /** Token threshold below which raw text is used instead of generating a summary (default: 200) */
+    summaryTokenThreshold?: number;
+    /** Whether to generate a one-sentence document description (default: false) */
+    ifAddDocDescription?: boolean;
+    /** Whether to include raw text in each node (default: false) */
+    ifAddNodeText?: boolean;
+    /** Whether to add a sequential node_id to each node (default: true) */
+    ifAddNodeId?: boolean;
+    /** Custom token counting function */
+    tokenCounter?: TokenCounter;
+    /** Called at each major processing milestone with step name and 0–100 percent */
+    onProgress?: ProgressCallback;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,8CAA8C;AAC9C,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,4CAA4C;AAC5C,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEzD,gCAAgC;AAChC,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,eAAe,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,MAAM,WAAW,GAAG,CACxB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,WAAW,CAAC,EAAE,UAAU,EAAE,CAAA;CAAE,KACrC,OAAO,CAAC,SAAS,CAAC,CAAC;AAIxB,yDAAyD;AACzD,MAAM,WAAW,YAAY;IAC3B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,CAAC;AAI5D;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAIpD,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,4CAA4C;AAC5C,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB;AAID,oDAAoD;AACpD,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAEtE,8BAA8B;AAC9B,MAAM,WAAW,eAAe;IAC9B,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gDAAgD;IAChD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,+BAA+B;AAC/B,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,8CAA8C;IAC9C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAID,uFAAuF;AACvF,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC3C,KAAK,EAAE;QACL,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,SAAS,GAAG,KAAK,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,8DAA8D;AAC9D,MAAM,WAAW,YAAa,SAAQ,iBAAiB;IACrD,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,6CAA6C;AAC7C,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,IAAI,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC;IACzB,gDAAgD;IAChD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wEAAwE;IACxE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qCAAqC;IACrC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAID,mCAAmC;AACnC,MAAM,WAAW,gBAAgB;IAC/B,8EAA8E;IAC9E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,8EAA8E;IAC9E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qFAAqF;IACrF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uEAAuE;IACvE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,qEAAqE;IACrE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,iFAAiF;IACjF,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IACjC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,2DAA2D;IAC3D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kGAAkG;IAClG,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,gEAAgE;IAChE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,uEAAuE;IACvE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,qCAAqC;IACrC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,iFAAiF;IACjF,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B"}

package/dist/types.js

@@ -0,0 +1,4 @@
+"use strict";
+// ─── LLM Provider ────────────────────────────────────────────────────────────
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,gFAAgF"}

package/dist/utils/json.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Extracts the text between ```json ... ``` fences, or returns the raw string.
+ * Port of `get_json_content()` from pageindex/utils.py
+ */
+export declare function getJsonContent(response: string): string;
+/**
+ * Parses a JSON string from an LLM response, handling common quirks
+ * (fenced code blocks, Python `None` → JSON `null`, trailing commas).
+ * Returns an empty object `{}` on failure.
+ * Port of `extract_json()` from pageindex/utils.py
+ */
+export declare function extractJson(content: string): Record<string, unknown> | unknown[];
+//# sourceMappingURL=json.d.ts.map

package/dist/utils/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/utils/json.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAQvD;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,EAAE,CA6ChF"}

package/dist/utils/json.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getJsonContent = getJsonContent;
+exports.extractJson = extractJson;
+/**
+ * Extracts the text between ```json ... ``` fences, or returns the raw string.
+ * Port of `get_json_content()` from pageindex/utils.py
+ */
+function getJsonContent(response) {
+    const start = response.indexOf('```json');
+    if (start !== -1) {
+        const after = response.slice(start + 7);
+        const end = after.lastIndexOf('```');
+        return (end !== -1 ? after.slice(0, end) : after).trim();
+    }
+    return response.trim();
+}
+/**
+ * Parses a JSON string from an LLM response, handling common quirks
+ * (fenced code blocks, Python `None` → JSON `null`, trailing commas).
+ * Returns an empty object `{}` on failure.
+ * Port of `extract_json()` from pageindex/utils.py
+ */
+function extractJson(content) {
+    try {
+        // Strip ```json ... ``` fences if present
+        let jsonContent;
+        const fenceStart = content.indexOf('```json');
+        if (fenceStart !== -1) {
+            const afterFence = content.slice(fenceStart + 7);
+            const fenceEnd = afterFence.lastIndexOf('```');
+            jsonContent = (fenceEnd !== -1 ? afterFence.slice(0, fenceEnd) : afterFence).trim();
+        }
+        else {
+            jsonContent = content.trim();
+        }
+        // Common cleanup
+        jsonContent = jsonContent
+            .replace(/\bNone\b/g, 'null') // Python None → JSON null
+            .replace(/\bTrue\b/g, 'true') // Python True → JSON true
+            .replace(/\bFalse\b/g, 'false') // Python False → JSON false
+            .replace(/\r?\n/g, ' ') // Remove newlines
+            .replace(/\s+/g, ' '); // Normalize whitespace
+        return JSON.parse(jsonContent);
+    }
+    catch {
+        // Second attempt: remove trailing commas before ] or }
+        try {
+            let jsonContent = content.trim()
+                .replace(/\bNone\b/g, 'null')
+                .replace(/\bTrue\b/g, 'true')
+                .replace(/\bFalse\b/g, 'false')
+                .replace(/,\s*]/g, ']')
+                .replace(/,\s*}/g, '}');
+            const fenceStart = jsonContent.indexOf('```json');
+            if (fenceStart !== -1) {
+                const afterFence = jsonContent.slice(fenceStart + 7);
+                const fenceEnd = afterFence.lastIndexOf('```');
+                jsonContent = (fenceEnd !== -1 ? afterFence.slice(0, fenceEnd) : afterFence).trim();
+            }
+            return JSON.parse(jsonContent);
+        }
+        catch {
+            console.warn('[PageIndex] Failed to parse JSON from LLM response:', content.slice(0, 200));
+            return {};
+        }
+    }
+}
+//# sourceMappingURL=json.js.map

package/dist/utils/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/utils/json.ts"],"names":[],"mappings":";;AAIA,wCAQC;AAQD,kCA6CC;AAjED;;;GAGG;AACH,SAAgB,cAAc,CAAC,QAAgB;IAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,OAAe;IACzC,IAAI,CAAC;QACH,0CAA0C;QAC1C,IAAI,WAAmB,CAAC;QACxB,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC/C,WAAW,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;QACtF,CAAC;aAAM,CAAC;YACN,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,iBAAiB;QACjB,WAAW,GAAG,WAAW;aACtB,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAG,0BAA0B;aACzD,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAG,0BAA0B;aACzD,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,4BAA4B;aAC3D,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAS,kBAAkB;aACjD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAU,uBAAuB;QAEzD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAwC,CAAC;IACxE,CAAC;IAAC,MAAM,CAAC;QACP,uDAAuD;QACvD,IAAI,CAAC;YACH,IAAI,WAAW,GAAG,OAAO,CAAC,IAAI,EAAE;iBAC7B,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;iBAC5B,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;iBAC5B,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;iBAC9B,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;iBACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAE1B,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAClD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtB,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC/C,WAAW,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;YACtF,CAAC;YAED,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAwC,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,qDAAqD,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3F,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/utils/pdf.d.ts

@@ -0,0 +1,20 @@
+import type { PageData, TokenCounter } from '../types';
+/**
+ * Extracts per-page text from a PDF buffer using `pdfjs-dist`.
+ *
+ * This is an **optional helper** — install `pdfjs-dist` (>=4.0.0) to use it.
+ * If you already have page text (e.g., from `react-native-pdf` or a backend),
+ * you can pass `PageData[]` directly to `pageIndex()` without calling this.
+ *
+ * @param data    Raw PDF bytes (ArrayBuffer or Uint8Array)
+ * @param counter Token counter function (defaults to ~4 chars/token)
+ * @returns       Array of `{ text, tokenCount }` — one entry per page
+ *
+ * @example
+ * import RNFS from 'react-native-fs';
+ * const base64 = await RNFS.readFile(filePath, 'base64');
+ * const bytes = Uint8Array.from(atob(base64), c => c.charCodeAt(0));
+ * const pages = await extractPdfPages(bytes.buffer as ArrayBuffer);
+ */
+export declare function extractPdfPages(data: ArrayBuffer | Uint8Array, counter?: TokenCounter): Promise<PageData[]>;
+//# sourceMappingURL=pdf.d.ts.map

package/dist/utils/pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../src/utils/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGvD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,WAAW,GAAG,UAAU,EAC9B,OAAO,GAAE,YAAkC,GAC1C,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6CrB"}