chunksmith-pageindex 0.3.0__tar.gz

chunksmith_pageindex-0.3.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: chunksmith-pageindex
+Version: 0.3.0
+Summary: ChunkSmith PDF page-index outline pipeline (no Unstructured partition).
+Author-email: AnshulParate2004 <anshulnparate@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/AnshulParate2004/chunksmith-lib
+Project-URL: Repository, https://github.com/AnshulParate2004/chunksmith-lib
+Project-URL: Changelog, https://github.com/AnshulParate2004/chunksmith-lib/blob/main/CHANGELOG.md
+Requires-Python: >=3.10
+Requires-Dist: chunksmith-core>=0.3.0
+Provides-Extra: pdf
+Requires-Dist: pymupdf>=1.24.0; extra == "pdf"
+Requires-Dist: PyPDF2>=3.0.0; extra == "pdf"
+Provides-Extra: llm
+Requires-Dist: httpx>=0.27.0; extra == "llm"
+Requires-Dist: tiktoken>=0.7.0; extra == "llm"
+Requires-Dist: langchain-core>=0.3.28; extra == "llm"
+Requires-Dist: langchain-litellm>=0.2.0; extra == "llm"

chunksmith_pageindex-0.3.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chunksmith-pageindex"
+version = "0.3.0"
+description = "ChunkSmith PDF page-index outline pipeline (no Unstructured partition)."
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "AnshulParate2004", email = "anshulnparate@gmail.com" }]
+dependencies = [
+    "chunksmith-core>=0.3.0",
+]
+
+[project.optional-dependencies]
+pdf = ["pymupdf>=1.24.0", "PyPDF2>=3.0.0"]
+llm = [
+    "httpx>=0.27.0",
+    "tiktoken>=0.7.0",
+    "langchain-core>=0.3.28",
+    "langchain-litellm>=0.2.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/AnshulParate2004/chunksmith-lib"
+Repository = "https://github.com/AnshulParate2004/chunksmith-lib"
+Changelog = "https://github.com/AnshulParate2004/chunksmith-lib/blob/main/CHANGELOG.md"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["chunksmith_pageindex", "chunksmith_pageindex.*"]

chunksmith_pageindex-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/__init__.py

@@ -0,0 +1,6 @@
+"""ChunkSmith PageIndexer (PDF parser + LLM outline)."""
+
+from chunksmith_pageindex.config import RuntimeSettings, load_settings
+from chunksmith_pageindex.tree.builder import build_outline_from_pdf
+
+__all__ = ["RuntimeSettings", "load_settings", "build_outline_from_pdf"]

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/config.py

@@ -0,0 +1,75 @@
+"""Runtime settings: env vars + Python defaults (no secrets in defaults module)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+from dotenv import load_dotenv
+
+from chunksmith_pageindex.config_defaults import DEFAULTS
+from chunksmith_pageindex.llm_config import _clean_env, resolve_litellm_config
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    v = os.getenv(name)
+    if v is None or v == "":
+        return default
+    return v.strip().lower() in ("1", "true", "yes", "on")
+
+
+@dataclass(frozen=True)
+class RuntimeSettings:
+    llm_model: str
+    openai_api_key: str | None
+    pageindex_model: str
+    pdf_parser: str
+    max_tokens_per_chunk: int
+    overlap_pages: int
+    generate_doc_summary: bool
+    litellm_kwargs: dict[str, Any] = field(default_factory=dict)
+
+
+def _default_values(overrides: dict[str, Any] | None = None) -> dict[str, Any]:
+    merged = dict(DEFAULTS)
+    if overrides:
+        merged.update(overrides)
+    return merged
+
+
+def load_settings(*, defaults: dict[str, Any] | None = None) -> RuntimeSettings:
+    load_dotenv()
+    y = _default_values(defaults)
+
+    openai_key = _clean_env("OPENAI_API_KEY") or _clean_env("CHATGPT_API_KEY")
+    model = (
+        _clean_env("CHUNKSMITH_LLM_MODEL")
+        or os.getenv("PAGEINDEX_MODEL")
+        or os.getenv("LLM_MODEL")
+        or y.get("model")
+        or "gpt-4o-2024-11-20"
+    )
+    pdf_parser = os.getenv("CHUNKSMITH_PDF_PARSER") or y.get("pdf_parser") or "PyPDF2"
+    max_tokens = int(os.getenv("CHUNKSMITH_MAX_TOKENS_PER_CHUNK") or y.get("max_tokens_per_chunk") or 20000)
+    overlap = int(os.getenv("CHUNKSMITH_OVERLAP_PAGES") or y.get("overlap_pages") or 1)
+    gen_doc = _env_bool("CHUNKSMITH_GENERATE_DOC_SUMMARY", bool(y.get("generate_doc_summary", False)))
+
+    litellm = resolve_litellm_config(pageindex_model=str(model).strip())
+
+    if litellm.model.startswith("azure/"):
+        if not (_clean_env("AZURE_API_KEY") and _clean_env("AZURE_API_BASE")):
+            raise ValueError(f"Azure model {litellm.model!r} requires AZURE_API_KEY and AZURE_API_BASE in .env")
+    elif not openai_key:
+        raise ValueError("Missing OPENAI_API_KEY in .env")
+
+    return RuntimeSettings(
+        llm_model=litellm.model,
+        openai_api_key=openai_key,
+        pageindex_model=str(model).strip(),
+        pdf_parser=pdf_parser.strip(),
+        max_tokens_per_chunk=max_tokens,
+        overlap_pages=max(0, overlap),
+        generate_doc_summary=gen_doc,
+        litellm_kwargs=dict(litellm.kwargs),
+    )

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/config_defaults.py

@@ -0,0 +1,11 @@
+"""Non-secret defaults (override with env vars — see repo .env.example)."""
+
+from __future__ import annotations
+
+DEFAULTS: dict[str, object] = {
+    "model": "gpt-4o-2024-11-20",
+    "pdf_parser": "PyPDF2",
+    "max_tokens_per_chunk": 20_000,
+    "overlap_pages": 1,
+    "generate_doc_summary": False,
+}

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/exceptions.py

@@ -0,0 +1,13 @@
+"""Domain-specific errors for the page-index outline pipeline."""
+
+
+class ChunksmithError(Exception):
+    """Base error for this package."""
+
+
+class PdfLoadError(ChunksmithError):
+    """Failed to open or read a PDF."""
+
+
+class OutlineExtractionError(ChunksmithError):
+    """LLM returned unusable output or finish_reason was not ``finished``."""

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/indexer/__init__.py

@@ -0,0 +1,5 @@
+"""LLM client and outline extractors."""
+
+from chunksmith_pageindex.indexer import client, doc_description, extractor, prompts
+
+__all__ = ["client", "doc_description", "extractor", "prompts"]

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/indexer/client.py

@@ -0,0 +1,170 @@
+"""ChatLiteLLM client, token counting, and JSON extraction."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any, List, Optional, Tuple
+
+import tiktoken
+from langchain_core.messages import AIMessage, HumanMessage
+from langchain_litellm import ChatLiteLLM
+
+from chunksmith_pageindex.config import RuntimeSettings
+
+logger = logging.getLogger(__name__)
+
+_USAGE_TOTALS: dict[str, int] = {
+    "calls": 0,
+    "prompt_tokens": 0,
+    "completion_tokens": 0,
+    "total_tokens": 0,
+}
+
+
+def reset_usage_totals() -> None:
+    for k in _USAGE_TOTALS:
+        _USAGE_TOTALS[k] = 0
+
+
+def get_usage_totals() -> dict[str, int]:
+    return dict(_USAGE_TOTALS)
+
+
+def _record_usage(response: Any) -> None:
+    meta = getattr(response, "response_metadata", None) or {}
+    usage = meta.get("token_usage") or meta.get("usage") or {}
+    if not usage:
+        return
+    _USAGE_TOTALS["calls"] += 1
+    _USAGE_TOTALS["prompt_tokens"] += int(usage.get("prompt_tokens", 0) or 0)
+    _USAGE_TOTALS["completion_tokens"] += int(usage.get("completion_tokens", 0) or 0)
+    _USAGE_TOTALS["total_tokens"] += int(usage.get("total_tokens", 0) or 0)
+
+
+def _tiktoken_encoding_for(model: str | None) -> tiktoken.Encoding:
+    probe = (model or "gpt-4o").split("/")[-1]
+    try:
+        return tiktoken.encoding_for_model(probe)
+    except KeyError:
+        return tiktoken.get_encoding("cl100k_base")
+
+
+def count_tokens(text: str | None, model: str | None = None) -> int:
+    if not text:
+        return 0
+    enc = _tiktoken_encoding_for(model)
+    return len(enc.encode(text))
+
+
+def build_chat_model(settings: RuntimeSettings, model: str | None = None) -> ChatLiteLLM:
+    return ChatLiteLLM(
+        model=model or settings.llm_model,
+        temperature=0,
+        **settings.litellm_kwargs,
+    )
+
+
+def _to_langchain_messages(
+    prompt: str,
+    chat_history: Optional[List[dict]] = None,
+) -> list[HumanMessage | AIMessage]:
+    if not chat_history:
+        return [HumanMessage(content=prompt)]
+    messages: list[HumanMessage | AIMessage] = []
+    for row in chat_history:
+        role = str(row.get("role") or "").strip().lower()
+        content = str(row.get("content") or "")
+        if role == "assistant":
+            messages.append(AIMessage(content=content))
+        else:
+            messages.append(HumanMessage(content=content))
+    messages.append(HumanMessage(content=prompt))
+    return messages
+
+
+def _finish_reason_from_response(response: Any) -> str:
+    meta = getattr(response, "response_metadata", None) or {}
+    reason = meta.get("finish_reason") or meta.get("stop_reason")
+    if isinstance(reason, str) and reason.strip():
+        if reason in ("length", "max_tokens"):
+            return "max_output_reached"
+        return "finished"
+    return "finished"
+
+
+def ChatGPT_API_with_finish_reason(
+    settings: RuntimeSettings,
+    model: str,
+    prompt: str,
+    chat_history: Optional[List[dict]] = None,
+) -> Tuple[str, str]:
+    max_retries = 10
+    llm = build_chat_model(settings, model=model or None)
+    messages = _to_langchain_messages(prompt, chat_history)
+    for i in range(max_retries):
+        try:
+            response = llm.invoke(messages)
+            _record_usage(response)
+            if not (getattr(response, "response_metadata", None) or {}).get("token_usage"):
+                _USAGE_TOTALS["calls"] += 1
+            text = response.content if isinstance(response.content, str) else str(response.content or "")
+            return text, _finish_reason_from_response(response)
+        except Exception as e:
+            logger.warning("Chat completion retry %s: %s", i + 1, e)
+            if i < max_retries - 1:
+                time.sleep(1)
+            else:
+                logger.error("Max retries reached for chat completion")
+                return "Error", "error"
+    return "Error", "error"
+
+
+def ChatGPT_API(
+    settings: RuntimeSettings,
+    model: str,
+    prompt: str,
+    chat_history: Optional[List[dict]] = None,
+) -> str:
+    text, _ = ChatGPT_API_with_finish_reason(settings, model, prompt, chat_history)
+    return text
+
+
+def extract_json(content: str) -> Any:
+    try:
+        start_idx = content.find("```json")
+        if start_idx != -1:
+            start_idx += 7
+            end_idx = content.rfind("```")
+            json_content = content[start_idx:end_idx].strip()
+        else:
+            json_content = content.strip()
+
+        json_content = json_content.replace("None", "null")
+        json_content = " ".join(json_content.replace("\n", " ").replace("\r", " ").split())
+
+        return json.loads(json_content)
+    except json.JSONDecodeError:
+        try:
+            json_content = json_content.replace(",]", "]").replace(",}", "}")
+            return json.loads(json_content)
+        except Exception:
+            logger.exception("Failed to parse JSON from model output")
+            return {}
+    except Exception:
+        logger.exception("Unexpected error while extracting JSON")
+        return {}
+
+
+def llm_completion(
+    settings: RuntimeSettings,
+    model: str | None,
+    prompt: str,
+    chat_history: Optional[List[dict]] = None,
+    return_finish_reason: bool = False,
+):
+    m = model or settings.llm_model
+    if return_finish_reason:
+        return ChatGPT_API_with_finish_reason(settings, m, prompt, chat_history=chat_history)
+    return ChatGPT_API(settings, m, prompt, chat_history=chat_history)

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/indexer/doc_description.py

@@ -0,0 +1,51 @@
+"""One-shot document description after the outline tree is built (separate LLM call from TOC extraction)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from chunksmith_pageindex.config import RuntimeSettings
+from chunksmith_pageindex.indexer import prompts
+from chunksmith_pageindex.indexer.client import llm_completion
+
+logger = logging.getLogger(__name__)
+
+
+def create_clean_structure_for_description(structure: Any) -> Any:
+    """
+    Strip heavy fields (e.g. full ``text``) before sending the tree to the description model.
+    Keeps ``title``, ``node_id``, ``summary``, and nested ``nodes``.
+    """
+    if isinstance(structure, dict):
+        clean_node: dict[str, Any] = {}
+        for key in ("title", "node_id", "summary"):
+            if key in structure:
+                clean_node[key] = structure[key]
+        children = structure.get("nodes")
+        if isinstance(children, list) and children:
+            clean_node["nodes"] = create_clean_structure_for_description(children)
+        return clean_node
+    if isinstance(structure, list):
+        return [create_clean_structure_for_description(item) for item in structure]
+    return structure
+
+
+def generate_doc_description(
+    settings: RuntimeSettings,
+    structure: Any,
+    *,
+    model: str | None = None,
+) -> str:
+    """
+    Second-phase LLM: one plain completion, no chat history, unrelated to TOC init/continue prompts.
+    """
+    clean = create_clean_structure_for_description(structure)
+    structure_json = json.dumps(clean, ensure_ascii=False, indent=2)
+    prompt = prompts.build_doc_description_prompt(structure_json)
+    raw = llm_completion(settings, model, prompt, chat_history=None)
+    text = (raw or "").strip()
+    if not text or text == "Error":
+        logger.warning("Document description LLM returned empty or error placeholder")
+    return text

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/indexer/extractor.py

@@ -0,0 +1,96 @@
+"""LLM calls for init/continue outline extraction."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, List
+
+from chunksmith_pageindex.config import RuntimeSettings
+from chunksmith_pageindex.exceptions import OutlineExtractionError
+from chunksmith_pageindex.indexer import client
+from chunksmith_pageindex.indexer.prompts import (
+    ANCHOR_INSTRUCTION,
+    SUMMARY_INSTRUCTION,
+    TOC_CONTINUE_SYSTEM,
+    TOC_INIT_SYSTEM,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _outline_extra_instructions(*, include_summary: bool, include_word_range: bool) -> str:
+    parts: List[str] = []
+    if include_summary:
+        parts.append(SUMMARY_INSTRUCTION.strip())
+    if include_word_range:
+        parts.append(ANCHOR_INSTRUCTION.strip())
+    if not parts:
+        return ""
+    return "\n" + "\n".join(parts)
+
+
+def format_given_text_block(part: str) -> str:
+    """Prefix the document excerpt for the outline prompt (verbatim anchor must match this text)."""
+    return f"\nGiven text\n:{part}"
+
+
+def _rows_from_llm_response(response: str) -> List[dict[str, Any]]:
+    """Parse assistant text: prefer ``{"sections": [...]}``, else bare array."""
+    parsed = client.extract_json(response)
+    if isinstance(parsed, dict) and "sections" in parsed:
+        inner = parsed["sections"]
+        if isinstance(inner, list):
+            return inner
+    if isinstance(parsed, list):
+        return parsed
+    return []
+
+
+def generate_toc_init(
+    settings: RuntimeSettings,
+    part: str,
+    model: str | None = None,
+    *,
+    include_summary: bool = False,
+    include_word_range: bool = False,
+) -> List[dict[str, Any]]:
+    m = model or settings.pageindex_model
+    head = TOC_INIT_SYSTEM.strip() + _outline_extra_instructions(
+        include_summary=include_summary,
+        include_word_range=include_word_range,
+    )
+    prompt = head + format_given_text_block(part)
+
+    response, finish_reason = client.llm_completion(settings, m, prompt, return_finish_reason=True)
+    if finish_reason != "finished":
+        raise OutlineExtractionError(f"generate_toc_init finish_reason={finish_reason!r}")
+    rows = _rows_from_llm_response(response)
+    if not rows:
+        raise OutlineExtractionError("generate_toc_init: no sections in model output")
+    return rows
+
+
+def generate_toc_continue(
+    settings: RuntimeSettings,
+    toc_content: List[dict[str, Any]],
+    part: str,
+    model: str | None = None,
+    *,
+    include_summary: bool = False,
+    include_word_range: bool = False,
+) -> List[dict[str, Any]]:
+    m = model or settings.pageindex_model
+    head = TOC_CONTINUE_SYSTEM.strip() + _outline_extra_instructions(
+        include_summary=include_summary,
+        include_word_range=include_word_range,
+    )
+    prompt = head + format_given_text_block(part) + "\nPrevious outline (JSON)\n:" + json.dumps(toc_content, indent=2)
+
+    response, finish_reason = client.llm_completion(settings, m, prompt, return_finish_reason=True)
+    if finish_reason != "finished":
+        raise OutlineExtractionError(f"generate_toc_continue finish_reason={finish_reason!r}")
+    rows = _rows_from_llm_response(response)
+    if not rows:
+        raise OutlineExtractionError("generate_toc_continue: no sections in model output")
+    return rows

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/indexer/prompts.py

@@ -0,0 +1,56 @@
+"""Prompt templates for flat outline extraction (no-TOC path)."""
+
+# Appended to init/continue instructions when ``add_summary`` is enabled (same completion as outline rows).
+SUMMARY_INSTRUCTION = """
+Each row must also include "summary" (string): one or two concise sentences describing what that section covers,
+using only information visible in the given excerpt (no outside knowledge)."""
+
+# Appended when ``add_word_range=True``: one verbatim alignment string per row (no word indices).
+ANCHOR_INSTRUCTION = """
+When section anchors are requested in the **same** API response, every row must include the usual outline keys
+(structure, title, physical_index, and "summary" only if that was also requested) **plus**:
+
+  • ``split_document_anchor`` (string) — a **short verbatim substring** copied from the **Given text** excerpt below
+    (the text after ``Given text`` + newline + ``:``). Use text at or immediately after where that section begins
+    in this excerpt (e.g. heading like ``1 Introduction``, ``Abstract``, ``3.2 Attention``). Used to locate and
+    verify the section start in the excerpt; it must appear exactly as in the Given text.
+
+Apply this on **every** row for this message. For **continue** calls, anchors refer only to the **current** excerpt
+(the new Given text in this message), not to prior parts."""
+
+TOC_INIT_SYSTEM = """
+You are an expert in extracting hierarchical tree structure. Generate the tree structure of the document.
+structure: numeric index, e.g. "1", "1.1", "1.2", "1.2.1", "1.2.2", "1.2.3", "1.2.4", "1.2.5", "1.2.6", "1.2.7", "1.2.8", "1.2.9", "1.2.10".
+Tags <physical_index_X> in the given text mark where PDF page X begins. Long PDFs are split into multiple excerpts; this message is the first excerpt only.
+Several sections may start on the same page—that is normal; set each row's physical_index from the tag nearest that section's start.
+**Important:** ``physical_index`` is always the **printed PDF page** from those tags only—never infer it from section numbers in headings (e.g. a section titled ``3 Model Architecture`` may start on page 2 if the ``<physical_index_2>`` block contains that heading).
+Each section row must include: "structure" (string), "title" (string), and "physical_index"
+(either an integer start page or a string tag like "<physical_index_5>").
+If the prompt also includes appended instructions (e.g. per-section summaries or ``split_document_anchor`` on the excerpt),
+include those fields on every row as specified there.
+Return JSON only: a markdown code block ```json containing either a JSON array of rows in reading order,
+or an object {"sections": [ ...rows... ]}."""
+
+TOC_CONTINUE_SYSTEM = """
+You are an expert in extracting hierarchical tree structure.
+You are given the previous outline as JSON and the text of the current part of the document.
+Continue the outline: add new rows only for sections that appear in the current part.
+structure: numeric index, e.g. "1", "1.1", "1.2", "1.2.1", "1.2.2", "1.2.3", "1.2.4", "1.2.5", "1.2.6", "1.2.7", "1.2.8", "1.2.9", "1.2.10".
+Tags <physical_index_X> in the given text mark where PDF page X begins. The start of this excerpt often overlaps the end of the prior excerpt (the same pages can appear again)—use the tags to assign physical_index and do **not** duplicate sections already present in the previous outline JSON.
+Several sections may start on the same page; that is normal.
+Each row: "structure" (string), "title" (string), "physical_index" (int or "<physical_index_N>" string).
+``physical_index`` must be the PDF page from ``<physical_index_N>`` tags only, not from numeric prefixes in section titles.
+When ``split_document_anchor`` is requested, it must be copied from **this** message's Given text excerpt only.
+If the prompt also includes appended instructions (e.g. summaries or anchors), include those fields on each new row.
+Return JSON only: ```json with either an array of only the additional rows, or {"sections": [ ... ]}."""
+
+
+def build_doc_description_prompt(structure_json: str) -> str:
+    """User message for a standalone completion after the TOC tree exists (not mixed with outline extraction)."""
+    return f"""You are an expert in generating descriptions for a document.
+You are given a structure of a document. Your task is to generate a one-sentence description for the document, which makes it easy to distinguish the document from other documents.
+
+Document Structure:
+{structure_json}
+
+Directly return the description, do not include any other text."""

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/llm_config.py

@@ -0,0 +1,43 @@
+"""LiteLLM / ChatLiteLLM env resolution for PageIndexer."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+
+def _clean_env(name: str) -> str | None:
+    raw = os.getenv(name)
+    if raw is None:
+        return None
+    value = str(raw).split("#", 1)[0].strip()
+    return value or None
+
+
+@dataclass(frozen=True)
+class LiteLLMConfig:
+    model: str
+    kwargs: dict[str, Any]
+
+
+def resolve_litellm_config(*, pageindex_model: str) -> LiteLLMConfig:
+    model = (_clean_env("CHUNKSMITH_LLM_MODEL") or pageindex_model).strip()
+    kwargs: dict[str, Any] = {}
+
+    openai_key = _clean_env("OPENAI_API_KEY") or _clean_env("CHATGPT_API_KEY")
+    azure_key = _clean_env("AZURE_API_KEY")
+    azure_base = _clean_env("AZURE_API_BASE")
+    azure_version = _clean_env("AZURE_API_VERSION")
+
+    if model.startswith("azure/"):
+        if azure_key:
+            kwargs["api_key"] = azure_key
+        if azure_base:
+            kwargs["api_base"] = azure_base.rstrip("/")
+        if azure_version:
+            kwargs["api_version"] = azure_version
+    elif openai_key:
+        kwargs["api_key"] = openai_key
+
+    return LiteLLMConfig(model=model, kwargs=kwargs)

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/parser/__init__.py

@@ -0,0 +1,5 @@
+"""PDF loading and page tagging."""
+
+from chunksmith_pageindex.parser import chunker, document
+
+__all__ = ["chunker", "document"]

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/parser/chunker.py

@@ -0,0 +1,65 @@
+"""Tag pages with physical_index markers and split into token-budget groups."""
+
+from __future__ import annotations
+
+import math
+from typing import List, Tuple
+
+from chunksmith_pageindex.indexer.client import count_tokens
+
+
+def build_tagged_page_strings(
+    page_list: List[Tuple[str, int]],
+    *,
+    start_index: int = 1,
+    model: str | None = None,
+) -> Tuple[List[str], List[int]]:
+    """
+    Wrap each page's text in ``<physical_index_N>`` tags.
+
+    Token counts are computed on the **tagged** strings (same as original PageIndex).
+    """
+    page_contents: List[str] = []
+    token_lengths: List[int] = []
+    for i, (page_text, _tok_unused) in enumerate(page_list):
+        page_num = start_index + i
+        wrapped = f"<physical_index_{page_num}>\n{page_text}\n<physical_index_{page_num}>\n\n"
+        page_contents.append(wrapped)
+        token_lengths.append(count_tokens(wrapped, model))
+    return page_contents, token_lengths
+
+
+def page_list_to_group_text(
+    page_contents: List[str],
+    token_lengths: List[int],
+    *,
+    max_tokens: int = 20000,
+    overlap_page: int = 1,
+) -> List[str]:
+    """
+    Split tagged page strings into groups that each stay near ``max_tokens`` total.
+
+    Mirrors the original PageIndex grouping logic (with overlap between groups).
+    """
+    num_tokens = sum(token_lengths)
+    if num_tokens <= max_tokens:
+        return ["".join(page_contents)]
+
+    subsets: List[str] = []
+    current_subset: List[str] = []
+    current_token_count = 0
+    expected_parts_num = math.ceil(num_tokens / max_tokens)
+    average_tokens_per_part = math.ceil(((num_tokens / expected_parts_num) + max_tokens) / 2)
+
+    for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)):
+        if current_token_count + page_tokens > average_tokens_per_part:
+            subsets.append("".join(current_subset))
+            overlap_start = max(i - overlap_page, 0)
+            current_subset = list(page_contents[overlap_start:i])
+            current_token_count = sum(token_lengths[overlap_start:i])
+        current_subset.append(page_content)
+        current_token_count += page_tokens
+
+    if current_subset:
+        subsets.append("".join(current_subset))
+    return subsets

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/parser/document.py

@@ -0,0 +1,66 @@
+"""Load PDF pages as (text, token_length) pairs."""
+
+from __future__ import annotations
+
+import os
+from io import BytesIO
+from pathlib import Path
+from typing import List, Tuple, Union
+
+import PyPDF2
+import pymupdf
+
+from chunksmith_pageindex.config import RuntimeSettings
+from chunksmith_pageindex.exceptions import PdfLoadError
+from chunksmith_pageindex.indexer.client import count_tokens
+
+PdfSource = Union[str, Path, BytesIO]
+
+
+def load_pdf_pages(source: PdfSource, settings: RuntimeSettings) -> List[Tuple[str, int]]:
+    """
+    Extract one string per PDF page and count tokens (for chunking).
+
+    ``settings.pdf_parser`` must be ``PyPDF2`` or ``PyMuPDF``.
+    """
+    parser = settings.pdf_parser
+    model = settings.pageindex_model
+
+    if parser == "PyPDF2":
+        return _load_pypdf2(source, model)
+    if parser == "PyMuPDF":
+        return _load_pymupdf(source, model)
+    raise PdfLoadError(f"Unsupported pdf_parser: {parser!r} (use PyPDF2 or PyMuPDF)")
+
+
+def _load_pypdf2(source: PdfSource, model: str) -> List[Tuple[str, int]]:
+    if isinstance(source, (str, Path)):
+        path = str(source)
+        if not os.path.isfile(path):
+            raise PdfLoadError(f"Not a file: {path}")
+        reader = PyPDF2.PdfReader(path)
+    else:
+        reader = PyPDF2.PdfReader(source)
+    out: List[Tuple[str, int]] = []
+    for page in reader.pages:
+        text = page.extract_text() or ""
+        out.append((text, count_tokens(text, model)))
+    return out
+
+
+def _load_pymupdf(source: PdfSource, model: str) -> List[Tuple[str, int]]:
+    if isinstance(source, (str, Path)):
+        path = str(source)
+        if not os.path.isfile(path):
+            raise PdfLoadError(f"Not a file: {path}")
+        doc = pymupdf.open(path)
+    else:
+        doc = pymupdf.open(stream=source, filetype="pdf")
+    try:
+        out: List[Tuple[str, int]] = []
+        for page in doc:
+            text = page.get_text() or ""
+            out.append((text, count_tokens(text, model)))
+        return out
+    finally:
+        doc.close()

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/__init__.py

@@ -0,0 +1,5 @@
+"""Flat row validation and nested tree assembly."""
+
+from chunksmith_pageindex.tree import builder, schema
+
+__all__ = ["builder", "schema"]

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/builder.py

@@ -0,0 +1,127 @@
+"""Merge flat TOC rows, convert indices, build nested tree, assign node IDs."""
+
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from chunksmith_pageindex.config import RuntimeSettings, load_settings
+from chunksmith_pageindex.exceptions import OutlineExtractionError
+from chunksmith_pageindex.indexer import doc_description, extractor
+from chunksmith_pageindex.parser import chunker, document
+from chunksmith_pageindex.tree import schema
+from chunksmith_pageindex.tree.node_text import add_node_text
+from chunksmith_pageindex.tree.physical_index import (
+    convert_physical_index_to_int,
+    refine_physical_index_from_excerpt_tags,
+    refine_physical_index_from_pdf_page_text,
+)
+from chunksmith_pageindex.tree.tree_transform import (
+    add_preface_if_needed,
+    post_processing,
+    write_node_id,
+)
+
+
+def build_outline_from_pdf(
+    pdf_path: Union[str, Path],
+    settings: Optional[RuntimeSettings] = None,
+    *,
+    add_text: bool = False,
+    assign_node_ids: bool = False,
+    add_summary: bool = False,
+    add_word_range: bool = False,
+    generate_doc_summary: Optional[bool] = None,
+) -> dict[str, Any]:
+    """
+    End-to-end no-TOC outline: PDF → tagged chunks → LLM flat rows → tree.
+
+    If ``add_word_range`` is True, each row also has ``split_document_anchor`` (verbatim excerpt substring at the
+    section start) and ``chunk_excerpt_index`` (which LLM chunk produced that row).
+    """
+    settings = settings or load_settings()
+    path = Path(pdf_path).resolve()
+    page_list = document.load_pdf_pages(path, settings)
+    if not page_list:
+        raise OutlineExtractionError("No pages extracted from PDF")
+
+    tagged, lengths = chunker.build_tagged_page_strings(page_list, start_index=1, model=settings.pageindex_model)
+    groups = chunker.page_list_to_group_text(
+        tagged,
+        lengths,
+        max_tokens=settings.max_tokens_per_chunk,
+        overlap_page=settings.overlap_pages,
+    )
+
+    init_rows = extractor.generate_toc_init(
+        settings,
+        groups[0],
+        include_summary=add_summary,
+        include_word_range=add_word_range,
+    )
+    chunk_of_row: list[int] = [0] * len(init_rows)
+    flat = list(init_rows)
+    for chunk_idx, part in enumerate(groups[1:], start=1):
+        cont_rows = extractor.generate_toc_continue(
+            settings,
+            flat,
+            part,
+            include_summary=add_summary,
+            include_word_range=add_word_range,
+        )
+        chunk_of_row.extend([chunk_idx] * len(cont_rows))
+        flat.extend(cont_rows)
+
+    convert_physical_index_to_int(flat)
+    before_preface = len(chunk_of_row)
+    flat = add_preface_if_needed(flat, include_summary=add_summary)
+    if len(flat) == before_preface + 1:
+        chunk_of_row.insert(0, 0)
+    if add_word_range and flat and str(flat[0].get("title", "")).strip() == "Preface":
+        flat[0].setdefault("split_document_anchor", "Preface")
+
+    paired = [(c, x) for c, x in zip(chunk_of_row, flat) if x.get("physical_index") is not None]
+    chunk_of_row = [p[0] for p in paired]
+    flat = [p[1] for p in paired]
+
+    refine_physical_index_from_excerpt_tags(flat, chunk_of_row, groups)
+    refine_physical_index_from_pdf_page_text(flat, page_list)
+
+    try:
+        validated = schema.validate_toc_rows(flat)
+        use_rows: list[dict[str, Any]] = []
+        for i, r in enumerate(validated):
+            d = r.model_dump()
+            if add_word_range:
+                d["chunk_excerpt_index"] = chunk_of_row[i]
+            use_rows.append(d)
+    except Exception as e:
+        raise OutlineExtractionError(f"TOC row validation failed: {e}") from e
+    if not add_summary:
+        for d in use_rows:
+            d.pop("summary", None)
+    if not add_word_range:
+        for d in use_rows:
+            d.pop("split_document_anchor", None)
+            d.pop("chunk_excerpt_index", None)
+
+    end_page = len(page_list)
+    tree = post_processing(copy.deepcopy(use_rows), end_page)
+
+    if assign_node_ids:
+        write_node_id(tree)
+
+    if add_text:
+        add_node_text(tree, page_list)
+
+    do_doc_summary = settings.generate_doc_summary if generate_doc_summary is None else generate_doc_summary
+    out: dict[str, Any] = {
+        "doc_name": path.name,
+        "structure": tree,
+    }
+    if do_doc_summary:
+        out["doc_description"] = doc_description.generate_doc_description(
+            settings, tree, model=settings.pageindex_model
+        )
+    return out

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/node_text.py

@@ -0,0 +1,84 @@
+"""Fill outline node ``text`` from PDF pages using ``start_index`` / ``end_index``."""
+
+from __future__ import annotations
+
+from typing import Any, List
+
+
+def _needle_in_blob(node: dict[str, Any], blob: str) -> str:
+    """Prefer ``split_document_anchor``, then ``title``, first substring that appears in ``blob``."""
+    for key in ("split_document_anchor", "title"):
+        v = str(node.get(key) or "").strip()
+        if v and v in blob:
+            return v
+    return ""
+
+
+def page_span_plain_text(
+    start_1b: int,
+    end_1b_inclusive: int,
+    pdf_pages: List[tuple[str, int]],
+) -> str:
+    """Concatenate extracted text for PDF pages ``start_1b`` … ``end_1b_inclusive`` (1-based, inclusive)."""
+    s = int(start_1b)
+    e = int(end_1b_inclusive)
+    if e < s:
+        s, e = e, s
+    parts: List[str] = []
+    for pi in range(s - 1, e):
+        if 0 <= pi < len(pdf_pages):
+            parts.append(pdf_pages[pi][0])
+    return "".join(parts)
+
+
+def iter_outline_nodes_preorder(tree: Any) -> List[dict[str, Any]]:
+    """Depth-first pre-order over nested ``nodes`` (same order as typical reading / JSON walk)."""
+    out: List[dict[str, Any]] = []
+
+    def walk(x: Any) -> None:
+        if isinstance(x, dict):
+            out.append(x)
+            ch = x.get("nodes")
+            if isinstance(ch, list):
+                for c in ch:
+                    walk(c)
+        elif isinstance(x, list):
+            for item in x:
+                walk(item)
+
+    walk(tree)
+    return out
+
+
+def add_node_text(tree: Any, pdf_pages: List[tuple[str, int]]) -> None:
+    """
+    Set each node's ``text`` to the slice of its page span that belongs to that section.
+
+    Builds plain text from ``start_index`` … ``end_index`` (inclusive), then cuts from this node's
+    heading needle to the next outline node's needle in pre-order (so siblings on the same page do
+    not duplicate each other's bodies). If no needle matches, uses the full page span.
+    """
+    nodes = iter_outline_nodes_preorder(tree)
+    for i, node in enumerate(nodes):
+        start_p = node.get("start_index")
+        end_p = node.get("end_index")
+        if start_p is None or end_p is None:
+            continue
+        blob = page_span_plain_text(int(start_p), int(end_p), pdf_pages).strip()
+        if not blob:
+            node["text"] = ""
+            continue
+        start_needle = _needle_in_blob(node, blob)
+        next_needle = ""
+        if i + 1 < len(nodes):
+            next_needle = _needle_in_blob(nodes[i + 1], blob)
+        if start_needle:
+            start_char = blob.find(start_needle)
+            end_char = len(blob)
+            if next_needle:
+                nx = blob.find(next_needle, start_char + max(1, len(start_needle)))
+                if nx >= 0:
+                    end_char = nx
+            node["text"] = blob[start_char:end_char].strip()
+        else:
+            node["text"] = blob

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/physical_index.py

@@ -0,0 +1,114 @@
+"""Physical index refinement from excerpt tags and PDF page text."""
+
+from __future__ import annotations
+
+from typing import Any, List
+
+
+def convert_physical_index_to_int(data: List[dict[str, Any]]) -> None:
+    """Normalize ``physical_index`` strings like ``<physical_index_5>`` to ints in-place."""
+    for item in data:
+        if not isinstance(item, dict) or "physical_index" not in item:
+            continue
+        pi = item["physical_index"]
+        if isinstance(pi, str):
+            if pi.startswith("<physical_index_"):
+                item["physical_index"] = int(pi.split("_")[-1].rstrip(">").strip())
+            elif pi.startswith("physical_index_"):
+                item["physical_index"] = int(pi.split("_")[-1].strip())
+
+
+def physical_page_at_char(tagged_excerpt: str, char_index: int) -> int:
+    i = 0
+    current = 1
+    limit = max(0, min(len(tagged_excerpt), char_index))
+    prefix = "<physical_index_"
+    plen = len(prefix)
+    while i < limit:
+        if tagged_excerpt.startswith(prefix, i):
+            j = tagged_excerpt.find(">", i)
+            if j == -1:
+                break
+            current = int(tagged_excerpt[i + plen : j])
+            i = j + 1
+        else:
+            i += 1
+    return current
+
+
+def find_row_start_in_excerpt(excerpt: str, row: dict[str, Any], search_from: int) -> int:
+    anchor = str(row.get("split_document_anchor") or "").strip()
+    if anchor:
+        p = excerpt.find(anchor, search_from)
+        if p >= 0:
+            return p
+    title = str(row.get("title") or "").strip()
+    if title:
+        p = excerpt.find(title, search_from)
+        if p >= 0:
+            return p
+    return -1
+
+
+def refine_physical_index_from_excerpt_tags(
+    rows: List[dict[str, Any]],
+    chunk_idx_per_row: List[int],
+    groups: List[str],
+) -> None:
+    if not rows or len(chunk_idx_per_row) != len(rows):
+        return
+    prev_chunk = -1
+    scan_from = 0
+    for row, chi in zip(rows, chunk_idx_per_row):
+        if not isinstance(row, dict) or row.get("physical_index") is None:
+            continue
+        if not isinstance(chi, int) or chi < 0 or chi >= len(groups):
+            continue
+        excerpt = groups[chi]
+        if chi != prev_chunk:
+            scan_from = 0
+            prev_chunk = chi
+        pos = find_row_start_in_excerpt(excerpt, row, scan_from)
+        if pos < 0:
+            continue
+        try:
+            row["physical_index"] = physical_page_at_char(excerpt, pos)
+        except ValueError:
+            continue
+        scan_from = pos + 1
+
+
+def refine_physical_index_from_pdf_page_text(
+    rows: List[dict[str, Any]],
+    page_list: List[tuple[str, int]],
+) -> None:
+    if not rows or not page_list:
+        return
+    texts = [p[0] for p in page_list]
+    n = len(texts)
+    prev_page = 1
+    for row in rows:
+        if not isinstance(row, dict) or row.get("physical_index") is None:
+            continue
+        anchor = str(row.get("split_document_anchor") or "").strip()
+        title = str(row.get("title") or "").strip()
+        needles: List[str] = []
+        if anchor:
+            needles.append(anchor)
+        if title and title not in needles:
+            needles.append(title)
+        if not needles:
+            continue
+        found: int | None = None
+        lo = max(0, prev_page - 1)
+        for pi in range(lo, n):
+            body = texts[pi]
+            for needle in needles:
+                if needle in body:
+                    found = pi + 1
+                    break
+            if found is not None:
+                break
+        if found is not None:
+            row["physical_index"] = found
+            prev_page = found

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/schema.py

@@ -0,0 +1,50 @@
+"""Pydantic models for LLM flat rows and optional nested outline nodes."""
+
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, TypeAdapter
+
+
+class TocFlatRow(BaseModel):
+    """One outline row (LLM output)."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    structure: str
+    title: str = ""
+    physical_index: str | int | None = None
+    # Filled when ``add_summary=True`` on ``build_outline_from_pdf`` (same outline API calls).
+    summary: str = ""
+    # When ``add_word_range=True``: verbatim substring from the excerpt at the section start.
+    split_document_anchor: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "split_document_anchor",
+            "document_anchor_word",
+            "split_from_document_word",
+        ),
+    )
+
+
+def validate_toc_rows(raw: Any) -> List[TocFlatRow]:
+    if not isinstance(raw, list):
+        return []
+    adapter = TypeAdapter(List[TocFlatRow])
+    return adapter.validate_python(raw)
+
+
+class OutlineNode(BaseModel):
+    """Nested node after ``post_processing`` / ``list_to_tree`` (optional validation)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    title: str = ""
+    start_index: Optional[int] = None
+    end_index: Optional[int] = None
+    node_id: Optional[str] = None
+    summary: Optional[str] = None
+    split_document_anchor: Optional[str] = None
+    text: Optional[str] = None
+    nodes: Optional[List[OutlineNode]] = None

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex/tree/tree_transform.py

@@ -0,0 +1,98 @@
+"""Flat TOC row → nested tree transforms."""
+
+from __future__ import annotations
+
+from typing import Any, List
+
+
+def add_preface_if_needed(data: List[dict[str, Any]], *, include_summary: bool = False) -> List[dict[str, Any]]:
+    if not data:
+        return data
+    if data[0].get("physical_index") is not None and data[0]["physical_index"] > 1:
+        row: dict[str, Any] = {"structure": "0", "title": "Preface", "physical_index": 1}
+        if include_summary:
+            row["summary"] = ""
+        data.insert(0, row)
+    return data
+
+
+def list_to_tree(data: List[dict[str, Any]]) -> List[dict[str, Any]]:
+    def get_parent_structure(structure: str | None) -> str | None:
+        if not structure:
+            return None
+        parts = str(structure).split(".")
+        return ".".join(parts[:-1]) if len(parts) > 1 else None
+
+    nodes: dict[str, dict[str, Any]] = {}
+    root_nodes: List[dict[str, Any]] = []
+
+    for item in data:
+        structure = item.get("structure")
+        sk = str(structure) if structure is not None else ""
+        node = {
+            "title": item.get("title"),
+            "start_index": item.get("start_index"),
+            "end_index": item.get("end_index"),
+            "nodes": [],
+        }
+        if "summary" in item:
+            node["summary"] = str(item.get("summary") or "").strip()
+        anchor = str(item.get("split_document_anchor") or "").strip()
+        if anchor:
+            node["split_document_anchor"] = anchor
+        cxi = item.get("chunk_excerpt_index")
+        if cxi is not None:
+            node["chunk_excerpt_index"] = int(cxi)
+        nodes[sk] = node
+        parent_structure = get_parent_structure(sk if sk else None)
+
+        if parent_structure:
+            if parent_structure in nodes:
+                nodes[parent_structure]["nodes"].append(node)
+            else:
+                root_nodes.append(node)
+        else:
+            root_nodes.append(node)
+
+    def clean_node(node: dict[str, Any]) -> dict[str, Any]:
+        if not node.get("nodes"):
+            node.pop("nodes", None)
+        else:
+            for child in node["nodes"]:
+                clean_node(child)
+        return node
+
+    return [clean_node(n) for n in root_nodes]
+
+
+def post_processing(structure: List[dict[str, Any]], end_physical_index: int) -> List[dict[str, Any]]:
+    for i, item in enumerate(structure):
+        item["start_index"] = item.get("physical_index")
+        if i < len(structure) - 1:
+            if structure[i + 1].get("appear_start") == "yes":
+                item["end_index"] = structure[i + 1]["physical_index"] - 1
+            else:
+                item["end_index"] = structure[i + 1]["physical_index"]
+        else:
+            item["end_index"] = end_physical_index
+    tree = list_to_tree(structure)
+    if tree:
+        return tree
+    for node in structure:
+        node.pop("appear_start", None)
+        node.pop("physical_index", None)
+    return structure
+
+
+def write_node_id(data: Any, node_id: int = 0) -> int:
+    if isinstance(data, dict):
+        data["node_id"] = str(node_id).zfill(4)
+        node_id += 1
+        children = data.get("nodes")
+        if isinstance(children, list):
+            for item in children:
+                node_id = write_node_id(item, node_id)
+    elif isinstance(data, list):
+        for item in data:
+            node_id = write_node_id(item, node_id)
+    return node_id

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: chunksmith-pageindex
+Version: 0.3.0
+Summary: ChunkSmith PDF page-index outline pipeline (no Unstructured partition).
+Author-email: AnshulParate2004 <anshulnparate@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/AnshulParate2004/chunksmith-lib
+Project-URL: Repository, https://github.com/AnshulParate2004/chunksmith-lib
+Project-URL: Changelog, https://github.com/AnshulParate2004/chunksmith-lib/blob/main/CHANGELOG.md
+Requires-Python: >=3.10
+Requires-Dist: chunksmith-core>=0.3.0
+Provides-Extra: pdf
+Requires-Dist: pymupdf>=1.24.0; extra == "pdf"
+Requires-Dist: PyPDF2>=3.0.0; extra == "pdf"
+Provides-Extra: llm
+Requires-Dist: httpx>=0.27.0; extra == "llm"
+Requires-Dist: tiktoken>=0.7.0; extra == "llm"
+Requires-Dist: langchain-core>=0.3.28; extra == "llm"
+Requires-Dist: langchain-litellm>=0.2.0; extra == "llm"

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex.egg-info/SOURCES.txt

@@ -0,0 +1,25 @@
+pyproject.toml
+src/chunksmith_pageindex/__init__.py
+src/chunksmith_pageindex/config.py
+src/chunksmith_pageindex/config_defaults.py
+src/chunksmith_pageindex/exceptions.py
+src/chunksmith_pageindex/llm_config.py
+src/chunksmith_pageindex.egg-info/PKG-INFO
+src/chunksmith_pageindex.egg-info/SOURCES.txt
+src/chunksmith_pageindex.egg-info/dependency_links.txt
+src/chunksmith_pageindex.egg-info/requires.txt
+src/chunksmith_pageindex.egg-info/top_level.txt
+src/chunksmith_pageindex/indexer/__init__.py
+src/chunksmith_pageindex/indexer/client.py
+src/chunksmith_pageindex/indexer/doc_description.py
+src/chunksmith_pageindex/indexer/extractor.py
+src/chunksmith_pageindex/indexer/prompts.py
+src/chunksmith_pageindex/parser/__init__.py
+src/chunksmith_pageindex/parser/chunker.py
+src/chunksmith_pageindex/parser/document.py
+src/chunksmith_pageindex/tree/__init__.py
+src/chunksmith_pageindex/tree/builder.py
+src/chunksmith_pageindex/tree/node_text.py
+src/chunksmith_pageindex/tree/physical_index.py
+src/chunksmith_pageindex/tree/schema.py
+src/chunksmith_pageindex/tree/tree_transform.py

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex.egg-info/requires.txt

@@ -0,0 +1,11 @@
+chunksmith-core>=0.3.0
+
+[llm]
+httpx>=0.27.0
+tiktoken>=0.7.0
+langchain-core>=0.3.28
+langchain-litellm>=0.2.0
+
+[pdf]
+pymupdf>=1.24.0
+PyPDF2>=3.0.0

chunksmith_pageindex-0.3.0/src/chunksmith_pageindex.egg-info/top_level.txt

@@ -0,0 +1 @@
+chunksmith_pageindex