treedex 0.1.0

package/dist/index.cjs

@@ -0,0 +1,1213 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/pdf-parser.ts
+var pdf_parser_exports = {};
+__export(pdf_parser_exports, {
+  countTokens: () => countTokens,
+  extractPages: () => extractPages,
+  groupPages: () => groupPages,
+  pagesToTaggedText: () => pagesToTaggedText
+});
+function countTokens(text) {
+  return (0, import_gpt_tokenizer.encode)(text).length;
+}
+async function extractPages(pdfPath) {
+  const fs = await import("fs/promises");
+  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
+  const data = await fs.readFile(pdfPath);
+  const doc = await pdfjs.getDocument({ data }).promise;
+  const pages = [];
+  for (let i = 0; i < doc.numPages; i++) {
+    const page = await doc.getPage(i + 1);
+    const content = await page.getTextContent();
+    const text = content.items.map((item) => {
+      const obj = item;
+      return typeof obj.str === "string" ? obj.str : "";
+    }).join(" ");
+    pages.push({
+      page_num: i,
+      text,
+      token_count: countTokens(text)
+    });
+  }
+  return pages;
+}
+function pagesToTaggedText(pages, start, end) {
+  const parts = [];
+  for (const page of pages.slice(start, end + 1)) {
+    const n = page.page_num;
+    parts.push(`<physical_index_${n}>${page.text}</physical_index_${n}>`);
+  }
+  return parts.join("\n");
+}
+function groupPages(pages, maxTokens = 2e4, overlap = 1) {
+  const totalTokens = pages.reduce((sum, p) => sum + p.token_count, 0);
+  if (totalTokens <= maxTokens) {
+    return [pagesToTaggedText(pages, 0, pages.length - 1)];
+  }
+  const groups = [];
+  let groupStart = 0;
+  while (groupStart < pages.length) {
+    let running = 0;
+    let groupEnd = groupStart;
+    while (groupEnd < pages.length) {
+      const pageTokens = pages[groupEnd].token_count;
+      if (running + pageTokens > maxTokens && groupEnd > groupStart) {
+        groupEnd -= 1;
+        break;
+      }
+      running += pageTokens;
+      groupEnd += 1;
+    }
+    if (groupEnd >= pages.length) {
+      groupEnd = pages.length - 1;
+    }
+    groupEnd = Math.min(groupEnd, pages.length - 1);
+    groups.push(pagesToTaggedText(pages, groupStart, groupEnd));
+    if (groupEnd >= pages.length - 1) {
+      break;
+    }
+    const nextStart = groupEnd + 1 - overlap;
+    groupStart = Math.max(nextStart, groupStart + 1);
+  }
+  return groups;
+}
+var import_gpt_tokenizer;
+var init_pdf_parser = __esm({
+  "src/pdf-parser.ts"() {
+    "use strict";
+    import_gpt_tokenizer = require("gpt-tokenizer");
+  }
+});
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  BaseLLM: () => BaseLLM,
+  CerebrasLLM: () => CerebrasLLM,
+  ClaudeLLM: () => ClaudeLLM,
+  CohereLLM: () => CohereLLM,
+  DOCXLoader: () => DOCXLoader,
+  DeepSeekLLM: () => DeepSeekLLM,
+  FireworksLLM: () => FireworksLLM,
+  FunctionLLM: () => FunctionLLM,
+  GeminiLLM: () => GeminiLLM,
+  GroqLLM: () => GroqLLM,
+  HTMLLoader: () => HTMLLoader,
+  HuggingFaceLLM: () => HuggingFaceLLM,
+  MistralLLM: () => MistralLLM,
+  OllamaLLM: () => OllamaLLM,
+  OpenAICompatibleLLM: () => OpenAICompatibleLLM,
+  OpenAILLM: () => OpenAILLM,
+  OpenRouterLLM: () => OpenRouterLLM,
+  PDFLoader: () => PDFLoader,
+  QueryResult: () => QueryResult,
+  SambanovaLLM: () => SambanovaLLM,
+  TextLoader: () => TextLoader,
+  TogetherLLM: () => TogetherLLM,
+  TreeDex: () => TreeDex,
+  assignNodeIds: () => assignNodeIds,
+  assignPageRanges: () => assignPageRanges,
+  autoLoader: () => autoLoader,
+  collectNodeTexts: () => collectNodeTexts,
+  countNodes: () => countNodes,
+  countTokens: () => countTokens,
+  createNodeMapping: () => createNodeMapping,
+  embedTextInTree: () => embedTextInTree,
+  extractJson: () => extractJson,
+  extractPages: () => extractPages,
+  findLargeNodes: () => findLargeNodes,
+  getLeafNodes: () => getLeafNodes,
+  groupPages: () => groupPages,
+  listToTree: () => listToTree,
+  pagesToTaggedText: () => pagesToTaggedText,
+  printTree: () => printTree,
+  retrievalPrompt: () => retrievalPrompt,
+  stripTextFromTree: () => stripTextFromTree,
+  structureContinuePrompt: () => structureContinuePrompt,
+  structureExtractionPrompt: () => structureExtractionPrompt,
+  textToPages: () => textToPages,
+  treeToFlatList: () => treeToFlatList
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/loaders.ts
+init_pdf_parser();
+function textToPages(text, charsPerPage = 3e3) {
+  const pages = [];
+  for (let i = 0; i < text.length; i += charsPerPage) {
+    const chunk = text.slice(i, i + charsPerPage);
+    pages.push({
+      page_num: pages.length,
+      text: chunk,
+      token_count: countTokens(chunk)
+    });
+  }
+  return pages;
+}
+var PDFLoader = class {
+  async load(path) {
+    const { extractPages: extractPages2 } = await Promise.resolve().then(() => (init_pdf_parser(), pdf_parser_exports));
+    return extractPages2(path);
+  }
+};
+var TextLoader = class {
+  charsPerPage;
+  constructor(charsPerPage = 3e3) {
+    this.charsPerPage = charsPerPage;
+  }
+  async load(path) {
+    const fs = await import("fs/promises");
+    const text = await fs.readFile(path, "utf-8");
+    return textToPages(text, this.charsPerPage);
+  }
+};
+var HTMLLoader = class {
+  charsPerPage;
+  constructor(charsPerPage = 3e3) {
+    this.charsPerPage = charsPerPage;
+  }
+  async load(path) {
+    const fs = await import("fs/promises");
+    const html = await fs.readFile(path, "utf-8");
+    const text = await this.stripHtml(html);
+    return textToPages(text, this.charsPerPage);
+  }
+  async stripHtml(html) {
+    try {
+      const { Parser } = await import("htmlparser2");
+      return new Promise((resolve) => {
+        const parts = [];
+        let skip = false;
+        const parser = new Parser({
+          onopentag(name) {
+            if (name === "script" || name === "style") skip = true;
+          },
+          onclosetag(name) {
+            if (name === "script" || name === "style") skip = false;
+            if (["p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr"].includes(name)) {
+              parts.push("\n");
+            }
+          },
+          ontext(data) {
+            if (!skip) parts.push(data);
+          },
+          onend() {
+            const raw = parts.join("");
+            resolve(raw.replace(/\n{3,}/g, "\n\n").trim());
+          }
+        });
+        parser.write(html);
+        parser.end();
+      });
+    } catch {
+      return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+    }
+  }
+};
+var DOCXLoader = class {
+  charsPerPage;
+  constructor(charsPerPage = 3e3) {
+    this.charsPerPage = charsPerPage;
+  }
+  async load(path) {
+    const fs = await import("fs/promises");
+    const mammoth = await import("mammoth");
+    const buffer = await fs.readFile(path);
+    const result = await mammoth.extractRawText({ buffer });
+    return textToPages(result.value, this.charsPerPage);
+  }
+};
+var EXTENSION_MAP = {
+  ".pdf": PDFLoader,
+  ".txt": TextLoader,
+  ".md": TextLoader,
+  ".html": HTMLLoader,
+  ".htm": HTMLLoader,
+  ".docx": DOCXLoader
+};
+async function autoLoader(filePath) {
+  const { extname } = await import("path");
+  const ext = extname(filePath).toLowerCase();
+  const LoaderClass = EXTENSION_MAP[ext];
+  if (!LoaderClass) {
+    const supported = Object.keys(EXTENSION_MAP).join(", ");
+    throw new Error(
+      `Unsupported file extension '${ext}'. Supported: ${supported}`
+    );
+  }
+  const loader = new LoaderClass();
+  return loader.load(filePath);
+}
+
+// src/core.ts
+init_pdf_parser();
+
+// src/tree-builder.ts
+function listToTree(flatList) {
+  const nodesByStructure = {};
+  const roots = [];
+  for (const item of flatList) {
+    const node = { ...item, nodes: [] };
+    const structure = node.structure;
+    nodesByStructure[structure] = node;
+    const parts = structure.split(".");
+    if (parts.length === 1) {
+      roots.push(node);
+    } else {
+      const parentStructure = parts.slice(0, -1).join(".");
+      const parent = nodesByStructure[parentStructure];
+      if (parent !== void 0) {
+        parent.nodes.push(node);
+      } else {
+        roots.push(node);
+      }
+    }
+  }
+  return roots;
+}
+function assignRanges(nodes, boundaryEnd) {
+  for (let i = 0; i < nodes.length; i++) {
+    const node = nodes[i];
+    node.start_index = node.physical_index ?? 0;
+    if (i + 1 < nodes.length) {
+      node.end_index = (nodes[i + 1].physical_index ?? 0) - 1;
+    } else {
+      node.end_index = boundaryEnd;
+    }
+    if (node.nodes.length > 0) {
+      assignRanges(node.nodes, node.end_index);
+    }
+  }
+}
+function assignPageRanges(tree, totalPages) {
+  assignRanges(tree, totalPages - 1);
+  return tree;
+}
+function assignNodeIds(tree) {
+  let counter = 0;
+  function walk(nodes) {
+    for (const node of nodes) {
+      counter++;
+      node.node_id = String(counter).padStart(4, "0");
+      walk(node.nodes);
+    }
+  }
+  walk(tree);
+  return tree;
+}
+function findLargeNodes(tree, options = {}) {
+  const { maxPages = 10, maxTokens = 2e4, pages = null } = options;
+  const large = [];
+  function walk(nodes) {
+    for (const node of nodes) {
+      const start = node.start_index ?? 0;
+      const end = node.end_index ?? 0;
+      const pageCount = end - start + 1;
+      let isLarge = pageCount > maxPages;
+      if (!isLarge && pages !== null) {
+        const tokenSum = pages.filter((p) => p.page_num >= start && p.page_num <= end).reduce((sum, p) => sum + p.token_count, 0);
+        isLarge = tokenSum > maxTokens;
+      }
+      if (isLarge) {
+        large.push(node);
+      }
+      walk(node.nodes);
+    }
+  }
+  walk(tree);
+  return large;
+}
+function embedTextInTree(tree, pages) {
+  function walk(nodes) {
+    for (const node of nodes) {
+      const start = node.start_index ?? 0;
+      const end = node.end_index ?? 0;
+      node.text = pages.filter((p) => p.page_num >= start && p.page_num <= end).map((p) => p.text).join("\n");
+      walk(node.nodes);
+    }
+  }
+  walk(tree);
+  return tree;
+}
+
+// src/tree-utils.ts
+function createNodeMapping(tree) {
+  const mapping = {};
+  function walk(nodes) {
+    for (const node of nodes) {
+      if (node.node_id !== void 0) {
+        mapping[node.node_id] = node;
+      }
+      walk(node.nodes);
+    }
+  }
+  walk(tree);
+  return mapping;
+}
+function stripTextFromTree(tree) {
+  const stripped = JSON.parse(JSON.stringify(tree));
+  function strip(nodes) {
+    for (const node of nodes) {
+      delete node.text;
+      strip(node.nodes);
+    }
+  }
+  strip(stripped);
+  return stripped;
+}
+function collectNodeTexts(nodeIds, nodeMap) {
+  const parts = [];
+  for (const nid of nodeIds) {
+    const node = nodeMap[nid];
+    if (node === void 0) continue;
+    const title = node.title ?? "Untitled";
+    const structure = node.structure ?? "";
+    const text = node.text ?? "";
+    const header = structure ? `[${structure}: ${title}]` : `[${title}]`;
+    parts.push(`${header}
+${text}`);
+  }
+  return parts.join("\n\n");
+}
+function countNodes(tree) {
+  let total = 0;
+  for (const node of tree) {
+    total += 1;
+    total += countNodes(node.nodes);
+  }
+  return total;
+}
+function getLeafNodes(tree) {
+  const leaves = [];
+  function walk(nodes) {
+    for (const node of nodes) {
+      if (node.nodes.length === 0) {
+        leaves.push(node);
+      } else {
+        walk(node.nodes);
+      }
+    }
+  }
+  walk(tree);
+  return leaves;
+}
+function treeToFlatList(tree) {
+  const result = [];
+  function walk(nodes) {
+    for (const node of nodes) {
+      const { nodes: children, ...flat } = node;
+      result.push(flat);
+      walk(children);
+    }
+  }
+  walk(tree);
+  return result;
+}
+function extractJson(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+  }
+  const blockMatch = text.match(/```(?:json)?\s*\n?(.*?)```/s);
+  if (blockMatch) {
+    const block = blockMatch[1].trim();
+    try {
+      return JSON.parse(block);
+    } catch {
+      const cleaned = block.replace(/,\s*([}\]])/g, "$1");
+      try {
+        return JSON.parse(cleaned);
+      } catch {
+      }
+    }
+  }
+  for (const [startChar, endChar] of [
+    ["{", "}"],
+    ["[", "]"]
+  ]) {
+    const start = text.indexOf(startChar);
+    if (start === -1) continue;
+    let depth = 0;
+    for (let i = start; i < text.length; i++) {
+      if (text[i] === startChar) depth++;
+      else if (text[i] === endChar) {
+        depth--;
+        if (depth === 0) {
+          const candidate = text.slice(start, i + 1);
+          try {
+            return JSON.parse(candidate);
+          } catch {
+            const cleaned = candidate.replace(/,\s*([}\]])/g, "$1");
+            try {
+              return JSON.parse(cleaned);
+            } catch {
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
+  throw new Error(
+    `Could not extract JSON from text: ${text.slice(0, 200)}...`
+  );
+}
+function printTree(tree, indent = 0) {
+  const prefix = "  ".repeat(indent);
+  for (const node of tree) {
+    const nodeId = node.node_id ?? "????";
+    const structure = node.structure ?? "";
+    const title = node.title ?? "Untitled";
+    const start = node.start_index ?? "?";
+    const end = node.end_index ?? "?";
+    console.log(
+      `${prefix}[${nodeId}] ${structure}: ${title} (pages ${start}-${end})`
+    );
+    printTree(node.nodes, indent + 1);
+  }
+}
+
+// src/prompts.ts
+function structureExtractionPrompt(text) {
+  return `You are a document structure analyzer. Given the following document text with physical page index tags, extract the hierarchical structure (table of contents).
+
+Return a JSON list of objects, each with:
+- "structure": hierarchical numbering like "1", "1.1", "1.2.3"
+- "title": the section/chapter title
+- "physical_index": the page number (from the <physical_index_N> tag) where this section starts
+
+Rules:
+- Use the physical_index tags to determine page numbers
+- Create a logical hierarchy: chapters -> sections -> subsections
+- Every section must have a unique structure ID
+- Return ONLY valid JSON \u2014 no extra text
+
+Document text:
+${text}
+
+JSON output:
+`;
+}
+function structureContinuePrompt(previousStructure, text) {
+  return `You are continuing to extract the hierarchical structure of a document.
+
+Here is the structure extracted so far:
+${previousStructure}
+
+Now extract the structure from the next portion of the document. Continue the numbering from where the previous structure left off. If a section from the previous portion continues into this portion, do NOT duplicate it.
+
+Return a JSON list of NEW sections only (same format as before).
+
+Document text:
+${text}
+
+JSON output:
+`;
+}
+function retrievalPrompt(treeStructure, query) {
+  return `You are a document retrieval system. Given a document's tree structure and a user query, select the most relevant sections that would contain the answer.
+
+Document structure:
+${treeStructure}
+
+User query: ${query}
+
+Return a JSON object with:
+- "node_ids": list of node IDs (strings like "0001", "0005") that are most relevant to the query
+- "reasoning": brief explanation of why these sections were selected
+
+Select the smallest set of sections that fully covers the answer. Prefer leaf nodes over parent nodes when the leaf contains the specific content. Return ONLY valid JSON.
+
+JSON output:
+`;
+}
+
+// src/core.ts
+var QueryResult = class {
+  context;
+  nodeIds;
+  pageRanges;
+  reasoning;
+  constructor(context, nodeIds, pageRanges, reasoning) {
+    this.context = context;
+    this.nodeIds = nodeIds;
+    this.pageRanges = pageRanges;
+    this.reasoning = reasoning;
+  }
+  /** Human-readable page ranges like 'pages 5-8, 12-15'. */
+  get pagesStr() {
+    if (this.pageRanges.length === 0) return "no pages";
+    const parts = [];
+    for (const [start, end] of this.pageRanges) {
+      if (start === end) {
+        parts.push(String(start + 1));
+      } else {
+        parts.push(`${start + 1}-${end + 1}`);
+      }
+    }
+    return "pages " + parts.join(", ");
+  }
+  toString() {
+    return `QueryResult(nodes=${JSON.stringify(this.nodeIds)}, ${this.pagesStr}, context_len=${this.context.length})`;
+  }
+};
+var TreeDex = class _TreeDex {
+  tree;
+  pages;
+  llm;
+  _nodeMap;
+  constructor(tree, pages, llm = null) {
+    this.tree = tree;
+    this.pages = pages;
+    this.llm = llm;
+    this._nodeMap = createNodeMapping(tree);
+  }
+  /**
+   * Build a TreeDex index from a file.
+   *
+   * @param path - Path to document (PDF, TXT, HTML, DOCX)
+   * @param llm - LLM backend with .generate(prompt) method
+   * @param options - Optional configuration
+   */
+  static async fromFile(path, llm, options) {
+    const {
+      loader,
+      maxTokens = 2e4,
+      overlap = 1,
+      verbose = true
+    } = options ?? {};
+    if (verbose) {
+      const { basename } = await import("path");
+      console.log(`Loading: ${basename(path)}`);
+    }
+    let pages;
+    if (loader) {
+      pages = await loader.load(path);
+    } else {
+      pages = await autoLoader(path);
+    }
+    if (verbose) {
+      const totalTokens = pages.reduce((s, p) => s + p.token_count, 0);
+      console.log(`  ${pages.length} pages, ${totalTokens.toLocaleString()} tokens`);
+    }
+    return _TreeDex.fromPages(pages, llm, { maxTokens, overlap, verbose });
+  }
+  /** Build a TreeDex index from pre-extracted pages. */
+  static async fromPages(pages, llm, options) {
+    const { maxTokens = 2e4, overlap = 1, verbose = true } = options ?? {};
+    const groups = groupPages(pages, maxTokens, overlap);
+    if (verbose) {
+      console.log(`  ${groups.length} page group(s) for structure extraction`);
+    }
+    const allSections = [];
+    for (let i = 0; i < groups.length; i++) {
+      if (verbose) {
+        console.log(
+          `  Extracting structure from group ${i + 1}/${groups.length}...`
+        );
+      }
+      let prompt;
+      if (i === 0) {
+        prompt = structureExtractionPrompt(groups[i]);
+      } else {
+        const prevJson = JSON.stringify(allSections, null, 2);
+        prompt = structureContinuePrompt(prevJson, groups[i]);
+      }
+      const response = await llm.generate(prompt);
+      const sections = extractJson(response);
+      if (Array.isArray(sections)) {
+        allSections.push(
+          ...sections
+        );
+      } else if (sections !== null && typeof sections === "object" && "sections" in sections) {
+        allSections.push(
+          ...sections.sections
+        );
+      }
+    }
+    if (verbose) {
+      console.log(`  Extracted ${allSections.length} sections`);
+    }
+    const tree = listToTree(allSections);
+    assignPageRanges(tree, pages.length);
+    assignNodeIds(tree);
+    embedTextInTree(tree, pages);
+    if (verbose) {
+      console.log(`  Tree: ${countNodes(tree)} nodes`);
+    }
+    return new _TreeDex(tree, pages, llm);
+  }
+  /** Create a TreeDex from an existing tree and pages. */
+  static fromTree(tree, pages, llm = null) {
+    return new _TreeDex(tree, pages, llm);
+  }
+  /**
+   * Query the index and return relevant context.
+   *
+   * @param question - The user's question
+   * @param llm - Optional LLM override. Uses this.llm if not provided.
+   */
+  async query(question, llm) {
+    const activeLlm = llm ?? this.llm;
+    if (!activeLlm) {
+      throw new Error(
+        "No LLM provided. Pass llm to query() or TreeDex constructor."
+      );
+    }
+    const stripped = stripTextFromTree(this.tree);
+    const treeJson = JSON.stringify(stripped, null, 2);
+    const prompt = retrievalPrompt(treeJson, question);
+    const response = await activeLlm.generate(prompt);
+    const result = extractJson(response);
+    const nodeIds = result.node_ids ?? [];
+    const reasoning = result.reasoning ?? "";
+    const context = collectNodeTexts(nodeIds, this._nodeMap);
+    const pageRanges = [];
+    for (const nid of nodeIds) {
+      const node = this._nodeMap[nid];
+      if (node) {
+        const start = node.start_index ?? 0;
+        const end = node.end_index ?? 0;
+        pageRanges.push([start, end]);
+      }
+    }
+    return new QueryResult(context, nodeIds, pageRanges, reasoning);
+  }
+  /** Save the index to a JSON file. */
+  async save(path) {
+    const fs = await import("fs/promises");
+    const stripped = stripTextFromTree(this.tree);
+    const data = {
+      version: "1.0",
+      framework: "TreeDex",
+      tree: stripped,
+      pages: this.pages
+    };
+    await fs.writeFile(path, JSON.stringify(data, null, 2), "utf-8");
+    return path;
+  }
+  /** Load a TreeDex index from a JSON file. */
+  static async load(path, llm) {
+    const fs = await import("fs/promises");
+    const raw = await fs.readFile(path, "utf-8");
+    const data = JSON.parse(raw);
+    const tree = data.tree;
+    const pages = data.pages;
+    assignPageRanges(tree, pages.length);
+    embedTextInTree(tree, pages);
+    return new _TreeDex(tree, pages, llm ?? null);
+  }
+  /** Pretty-print the tree structure. */
+  showTree() {
+    printTree(this.tree);
+  }
+  /** Return index statistics. */
+  stats() {
+    const totalTokens = this.pages.reduce((s, p) => s + p.token_count, 0);
+    const leaves = getLeafNodes(this.tree);
+    return {
+      total_pages: this.pages.length,
+      total_tokens: totalTokens,
+      total_nodes: countNodes(this.tree),
+      leaf_nodes: leaves.length,
+      root_sections: this.tree.length
+    };
+  }
+  /** Find sections that exceed size thresholds. */
+  findLargeSections(options) {
+    return findLargeNodes(this.tree, {
+      maxPages: options?.maxPages ?? 10,
+      maxTokens: options?.maxTokens ?? 2e4,
+      pages: this.pages
+    });
+  }
+};
+
+// src/llm-backends.ts
+var BaseLLM = class {
+  toString() {
+    return `${this.constructor.name}()`;
+  }
+};
+var GeminiLLM = class extends BaseLLM {
+  apiKey;
+  modelName;
+  _client = null;
+  constructor(apiKey, model = "gemini-2.0-flash") {
+    super();
+    this.apiKey = apiKey;
+    this.modelName = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { GoogleGenerativeAI } = await import("@google/generative-ai");
+      const genai = new GoogleGenerativeAI(this.apiKey);
+      this._client = genai.getGenerativeModel({ model: this.modelName });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const model = await this.getClient();
+    const response = await model.generateContent(prompt);
+    return response.response.text();
+  }
+  toString() {
+    return `GeminiLLM(model=${JSON.stringify(this.modelName)})`;
+  }
+};
+var OpenAILLM = class extends BaseLLM {
+  apiKey;
+  modelName;
+  _client = null;
+  constructor(apiKey, model = "gpt-4o") {
+    super();
+    this.apiKey = apiKey;
+    this.modelName = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { default: OpenAI } = await import("openai");
+      this._client = new OpenAI({ apiKey: this.apiKey });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const client = await this.getClient();
+    const response = await client.chat.completions.create({
+      model: this.modelName,
+      messages: [{ role: "user", content: prompt }]
+    });
+    return response.choices[0].message.content;
+  }
+  toString() {
+    return `OpenAILLM(model=${JSON.stringify(this.modelName)})`;
+  }
+};
+var ClaudeLLM = class extends BaseLLM {
+  apiKey;
+  modelName;
+  _client = null;
+  constructor(apiKey, model = "claude-sonnet-4-20250514") {
+    super();
+    this.apiKey = apiKey;
+    this.modelName = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { default: Anthropic } = await import("@anthropic-ai/sdk");
+      this._client = new Anthropic({ apiKey: this.apiKey });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const client = await this.getClient();
+    const response = await client.messages.create({
+      model: this.modelName,
+      max_tokens: 4096,
+      messages: [{ role: "user", content: prompt }]
+    });
+    return response.content[0].text;
+  }
+  toString() {
+    return `ClaudeLLM(model=${JSON.stringify(this.modelName)})`;
+  }
+};
+var MistralLLM = class extends BaseLLM {
+  apiKey;
+  modelName;
+  _client = null;
+  constructor(apiKey, model = "mistral-large-latest") {
+    super();
+    this.apiKey = apiKey;
+    this.modelName = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { Mistral } = await import("@mistralai/mistralai");
+      this._client = new Mistral({ apiKey: this.apiKey });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const client = await this.getClient();
+    const response = await client.chat.complete({
+      model: this.modelName,
+      messages: [{ role: "user", content: prompt }]
+    });
+    return response.choices[0].message.content;
+  }
+  toString() {
+    return `MistralLLM(model=${JSON.stringify(this.modelName)})`;
+  }
+};
+var CohereLLM = class extends BaseLLM {
+  apiKey;
+  modelName;
+  _client = null;
+  constructor(apiKey, model = "command-r-plus") {
+    super();
+    this.apiKey = apiKey;
+    this.modelName = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { CohereClientV2 } = await import("cohere-ai");
+      this._client = new CohereClientV2({ token: this.apiKey });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const client = await this.getClient();
+    const response = await client.chat({
+      model: this.modelName,
+      messages: [{ role: "user", content: prompt }]
+    });
+    return response.message.content[0].text;
+  }
+  toString() {
+    return `CohereLLM(model=${JSON.stringify(this.modelName)})`;
+  }
+};
+var OpenAICompatibleLLM = class extends BaseLLM {
+  baseUrl;
+  model;
+  apiKey;
+  maxTokens;
+  temperature;
+  extraHeaders;
+  constructor(options) {
+    super();
+    this.baseUrl = options.baseUrl.replace(/\/+$/, "");
+    this.model = options.model;
+    this.apiKey = options.apiKey ?? null;
+    this.maxTokens = options.maxTokens ?? 4096;
+    this.temperature = options.temperature ?? 0;
+    this.extraHeaders = options.extraHeaders ?? {};
+  }
+  buildHeaders() {
+    const headers = {
+      "Content-Type": "application/json",
+      "User-Agent": "TreeDex/0.1"
+    };
+    if (this.apiKey) {
+      headers["Authorization"] = `Bearer ${this.apiKey}`;
+    }
+    Object.assign(headers, this.extraHeaders);
+    return headers;
+  }
+  async generate(prompt) {
+    const url = `${this.baseUrl}/chat/completions`;
+    const payload = {
+      model: this.model,
+      messages: [{ role: "user", content: prompt }],
+      max_tokens: this.maxTokens,
+      temperature: this.temperature
+    };
+    const resp = await fetch(url, {
+      method: "POST",
+      headers: this.buildHeaders(),
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(12e4)
+    });
+    if (!resp.ok) {
+      const errorBody = await resp.text();
+      throw new Error(
+        `API request failed (${resp.status}): ${errorBody}`
+      );
+    }
+    const body = await resp.json();
+    return body.choices[0].message.content;
+  }
+  toString() {
+    return `OpenAICompatibleLLM(baseUrl=${JSON.stringify(this.baseUrl)}, model=${JSON.stringify(this.model)})`;
+  }
+};
+var GroqLLM = class extends BaseLLM {
+  apiKey;
+  model;
+  _client = null;
+  constructor(apiKey, model = "llama-3.3-70b-versatile") {
+    super();
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+  async getClient() {
+    if (this._client === null) {
+      const { default: Groq } = await import("groq-sdk");
+      this._client = new Groq({ apiKey: this.apiKey });
+    }
+    return this._client;
+  }
+  async generate(prompt) {
+    const client = await this.getClient();
+    const response = await client.chat.completions.create({
+      model: this.model,
+      messages: [{ role: "user", content: prompt }]
+    });
+    return response.choices[0].message.content;
+  }
+  toString() {
+    return `GroqLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var TogetherLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "meta-llama/Llama-3-70b-chat-hf", options) {
+    super({
+      baseUrl: "https://api.together.xyz/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `TogetherLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var FireworksLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "accounts/fireworks/models/llama-v3p1-70b-instruct", options) {
+    super({
+      baseUrl: "https://api.fireworks.ai/inference/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `FireworksLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var OpenRouterLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "anthropic/claude-sonnet-4", options) {
+    super({
+      baseUrl: "https://openrouter.ai/api/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `OpenRouterLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var DeepSeekLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "deepseek-chat", options) {
+    super({
+      baseUrl: "https://api.deepseek.com/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `DeepSeekLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var CerebrasLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "llama-3.3-70b", options) {
+    super({
+      baseUrl: "https://api.cerebras.ai/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `CerebrasLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var SambanovaLLM = class extends OpenAICompatibleLLM {
+  constructor(apiKey, model = "Meta-Llama-3.1-70B-Instruct", options) {
+    super({
+      baseUrl: "https://api.sambanova.ai/v1",
+      model,
+      apiKey,
+      ...options
+    });
+  }
+  toString() {
+    return `SambanovaLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var HuggingFaceLLM = class extends BaseLLM {
+  apiKey;
+  model;
+  maxTokens;
+  constructor(apiKey, model = "mistralai/Mistral-7B-Instruct-v0.3", maxTokens = 4096) {
+    super();
+    this.apiKey = apiKey;
+    this.model = model;
+    this.maxTokens = maxTokens;
+  }
+  async generate(prompt) {
+    const url = `https://api-inference.huggingface.co/models/${this.model}/v1/chat/completions`;
+    const payload = {
+      model: this.model,
+      messages: [{ role: "user", content: prompt }],
+      max_tokens: this.maxTokens
+    };
+    const resp = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": "TreeDex/0.1",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(12e4)
+    });
+    if (!resp.ok) {
+      const errorBody = await resp.text();
+      throw new Error(
+        `HuggingFace request failed (${resp.status}): ${errorBody}`
+      );
+    }
+    const body = await resp.json();
+    return body.choices[0].message.content;
+  }
+  toString() {
+    return `HuggingFaceLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var OllamaLLM = class extends BaseLLM {
+  model;
+  baseUrl;
+  constructor(model = "llama3", baseUrl = "http://localhost:11434") {
+    super();
+    this.model = model;
+    this.baseUrl = baseUrl.replace(/\/+$/, "");
+  }
+  async generate(prompt) {
+    const url = `${this.baseUrl}/api/generate`;
+    const payload = {
+      model: this.model,
+      prompt,
+      stream: false
+    };
+    const resp = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": "TreeDex/0.1"
+      },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(12e4)
+    });
+    if (!resp.ok) {
+      const errorBody = await resp.text();
+      throw new Error(
+        `Ollama request failed (${resp.status}): ${errorBody}`
+      );
+    }
+    const body = await resp.json();
+    return body.response;
+  }
+  toString() {
+    return `OllamaLLM(model=${JSON.stringify(this.model)})`;
+  }
+};
+var FunctionLLM = class extends BaseLLM {
+  _fn;
+  constructor(fn) {
+    super();
+    if (typeof fn !== "function") {
+      throw new TypeError(`Expected a function, got ${typeof fn}`);
+    }
+    this._fn = fn;
+  }
+  async generate(prompt) {
+    const result = await this._fn(prompt);
+    if (typeof result !== "string") {
+      throw new TypeError(
+        `LLM function must return string, got ${typeof result}`
+      );
+    }
+    return result;
+  }
+  toString() {
+    const name = this._fn.name || "anonymous";
+    return `FunctionLLM(fn=${name})`;
+  }
+};
+
+// src/index.ts
+init_pdf_parser();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BaseLLM,
+  CerebrasLLM,
+  ClaudeLLM,
+  CohereLLM,
+  DOCXLoader,
+  DeepSeekLLM,
+  FireworksLLM,
+  FunctionLLM,
+  GeminiLLM,
+  GroqLLM,
+  HTMLLoader,
+  HuggingFaceLLM,
+  MistralLLM,
+  OllamaLLM,
+  OpenAICompatibleLLM,
+  OpenAILLM,
+  OpenRouterLLM,
+  PDFLoader,
+  QueryResult,
+  SambanovaLLM,
+  TextLoader,
+  TogetherLLM,
+  TreeDex,
+  assignNodeIds,
+  assignPageRanges,
+  autoLoader,
+  collectNodeTexts,
+  countNodes,
+  countTokens,
+  createNodeMapping,
+  embedTextInTree,
+  extractJson,
+  extractPages,
+  findLargeNodes,
+  getLeafNodes,
+  groupPages,
+  listToTree,
+  pagesToTaggedText,
+  printTree,
+  retrievalPrompt,
+  stripTextFromTree,
+  structureContinuePrompt,
+  structureExtractionPrompt,
+  textToPages,
+  treeToFlatList
+});
+//# sourceMappingURL=index.cjs.map