pageindex 0.3.0.dev2__tar.gz → 0.3.0.dev3__tar.gz

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pageindex
-Version: 0.3.0.dev2
+Version: 0.3.0.dev3
 Summary: Python SDK for PageIndex
 License: MIT
 License-File: LICENSE
@@ -275,10 +275,13 @@ You can customize the processing with additional optional arguments:
 --toc-check-pages       Pages to check for table of contents (default: 20)
 --max-pages-per-node    Max pages per node (default: 10)
 --max-tokens-per-node   Max tokens per node (default: 20000)
---if-add-node-id        Add node ID (yes/no, default: yes)
---if-add-node-summary   Add node summary (yes/no, default: yes)
---if-add-doc-description Add doc description (yes/no, default: yes)
+--if-add-node-id        Add node IDs (on by default; disable with: --if-add-node-id no)
+--if-add-node-summary   Add node summaries (on by default; disable with: --if-add-node-summary no)
+--if-add-doc-description Add a document description (on by default; disable with: --if-add-doc-description no)
+--if-add-node-text      Add raw text to nodes (off by default; enable with: --if-add-node-text)
 ```
+These flags take no value by default (a bare `--if-add-node-id` turns it on); the
+legacy `--if-add-node-id no` form still works for turning an option off.
 </details>
 
 <details>

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/README.md

@@ -238,10 +238,13 @@ You can customize the processing with additional optional arguments:
 --toc-check-pages       Pages to check for table of contents (default: 20)
 --max-pages-per-node    Max pages per node (default: 10)
 --max-tokens-per-node   Max tokens per node (default: 20000)
---if-add-node-id        Add node ID (yes/no, default: yes)
---if-add-node-summary   Add node summary (yes/no, default: yes)
---if-add-doc-description Add doc description (yes/no, default: yes)
+--if-add-node-id        Add node IDs (on by default; disable with: --if-add-node-id no)
+--if-add-node-summary   Add node summaries (on by default; disable with: --if-add-node-summary no)
+--if-add-doc-description Add a document description (on by default; disable with: --if-add-doc-description no)
+--if-add-node-text      Add raw text to nodes (off by default; enable with: --if-add-node-text)
 ```
+These flags take no value by default (a bare `--if-add-node-id` turns it on); the
+legacy `--if-add-node-id no` form still works for turning an option off.
 </details>
 
 <details>

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/__init__.py

@@ -61,4 +61,11 @@ __all__ = [
     "IndexingError",
     "CloudAPIError",
     "FileTypeError",
+    # Legacy top-level exports (pre-SDK API), kept so `from pageindex import *`
+    # still binds them.
+    "page_index",
+    "md_to_tree",
+    "get_document",
+    "get_document_structure",
+    "get_page_content",
 ]

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/agent.py

@@ -20,7 +20,7 @@ OPEN_SYSTEM_PROMPT = """
 You are PageIndex, a document QA assistant.
 TOOL USE:
 - Call list_documents() to see available documents; use doc_name and doc_description to pick which doc(s) are relevant.
-- Call get_document(doc_id) to confirm status and page/line count.
+- Call get_document(doc_id) to confirm the document's name and type.
 - Call get_document_structure(doc_id) to identify relevant page ranges.
 - Call get_page_content(doc_id, pages="5-7") with tight ranges; never fetch the whole document.
 - Before each tool call, output one short sentence explaining the reason.
@@ -33,7 +33,7 @@ Answer based only on tool output. Be concise.
 SCOPED_SYSTEM_PROMPT = """
 You are PageIndex, a document QA assistant.
 TOOL USE:
-- Call get_document(doc_id) to confirm status and page/line count.
+- Call get_document(doc_id) to confirm the document's name and type.
 - Call get_document_structure(doc_id) to identify relevant page ranges.
 - Call get_page_content(doc_id, pages="5-7") with tight ranges; never fetch the whole document.
 - Before each tool call, output one short sentence explaining the reason.
@@ -46,20 +46,29 @@ Answer based only on tool output. Be concise.
 """
 
 
+def _defang_delimiters(text: str) -> str:
+    """Strip '<'/'>' so untrusted text can never form a literal <docs>/</docs>
+    (or any other tag-shaped string) that would prematurely close the
+    wrap_with_doc_context() delimiter and escape the untrusted-data boundary."""
+    return text.replace("<", "").replace(">", "")
+
+
 def wrap_with_doc_context(docs: list[dict], question: str) -> str:
     """Prepend a doc-context block to the user question for scoped queries.
 
     Document fields (especially doc_description, which is LLM-generated at
     index time) are untrusted text that may contain adversarial instructions.
     We wrap them in a <docs>...</docs> delimiter and tell the agent in the
-    system prompt to treat the block as data only.
+    system prompt to treat the block as data only. '<'/'>' are stripped from
+    the untrusted fields first so embedded content can never form a literal
+    </docs> (or any other tag) that closes the delimiter early.
     """
     lines = []
     for d in docs:
-        line = f"- {d['doc_id']}: {d.get('doc_name', '')}"
+        line = f"- {d['doc_id']}: {_defang_delimiters(d.get('doc_name', ''))}"
         desc = d.get("doc_description") or ""
         if desc:
-            line += f" — {desc}"
+            line += f" — {_defang_delimiters(desc)}"
         lines.append(line)
     label = "document" if len(docs) == 1 else "documents"
     return (

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/backend/cloud.py

@@ -13,13 +13,12 @@ import urllib.parse
 import requests
 from typing import AsyncIterator
 
+from ..cloud_api import API_BASE  # single source of truth for the cloud base URL
 from ..errors import CloudAPIError, DocumentNotFoundError, PageIndexError
 from ..events import QueryEvent
 
 logger = logging.getLogger(__name__)
 
-API_BASE = "https://api.pageindex.ai"
-
 _INTERNAL_TOOLS = frozenset({"ToolSearch", "Read", "Grep", "Glob", "Bash", "Edit", "Write"})
 
 
@@ -89,7 +88,9 @@ class CloudBackend:
 
     @staticmethod
     def _validate_collection_name(name: str) -> None:
-        if not re.match(r'^[a-zA-Z0-9_-]{1,128}$', name):
+        # .fullmatch() (not .match()): a $-anchored .match() would accept a
+        # trailing newline ("papers\n") because $ matches just before a final \n.
+        if not re.fullmatch(r'[a-zA-Z0-9_-]{1,128}', name):
             raise PageIndexError(
                 f"Invalid collection name: {name!r}. "
                 "Must be 1-128 chars of [a-zA-Z0-9_-]."
@@ -162,6 +163,12 @@ class CloudBackend:
         folder_id = self._get_folder_id(name)
         if folder_id:
             self._request("DELETE", f"/folder/{self._enc(folder_id)}/")
+            # Drop the cached id so a later same-name op re-resolves instead of
+            # reusing the now-deleted folder_id. Only when it was a REAL id —
+            # if folder_id was falsy, the cache holds the "folders unavailable
+            # on this plan" None sentinel, which must survive so we don't
+            # re-issue a doomed GET /folders/ on the next call.
+            self._folder_id_cache.pop(name, None)
 
     # ── Document management ───────────────────────────────────────────────
 
@@ -239,7 +246,12 @@ class CloudBackend:
         if isinstance(all_pages, list):
             return [
                 {"page": p.get("page", p.get("page_index")),
-                 "content": p.get("content", p.get("markdown", ""))}
+                 "content": p.get("content", p.get("markdown", "")),
+                 # Cloud OCR pages carry an `images` list (empty on text-only
+                 # pages). Preserve it — omitting when empty, mirroring the local
+                 # backend — so cloud callers get the same PageContent shape and
+                 # the SDK-prompted UI can render figures.
+                 **({"images": p["images"]} if p.get("images") else {})}
                 for p in all_pages
                 if p.get("page", p.get("page_index")) in page_nums
             ]
@@ -307,6 +319,8 @@ class CloudBackend:
                 "doc_ids cannot be empty; pass None to query the whole collection"
             )
         doc_id = doc_ids if doc_ids else self._get_all_doc_ids(collection)
+        if not doc_id:
+            raise ValueError("collection has no documents to query")
         # A non-streaming completion returns nothing until generation
         # finishes, so it needs far more than the default 30s. retries=1:
         # retrying this non-idempotent call would redo the full server-side
@@ -340,6 +354,8 @@ class CloudBackend:
                 "doc_ids cannot be empty; pass None to query the whole collection"
             )
         doc_id = doc_ids if doc_ids else self._get_all_doc_ids(collection)
+        if not doc_id:
+            raise ValueError("collection has no documents to query")
         headers = self._headers
         # Queue carries QueryEvent, an Exception to re-raise, or None (end).
         queue: asyncio.Queue[QueryEvent | Exception | None] = asyncio.Queue()
@@ -380,6 +396,13 @@ class CloudBackend:
                     timeout=120,
                 )
                 resp_holder["resp"] = resp
+                # The consumer may have abandoned the stream while we were still
+                # blocked in requests.post() (its connect phase, before resp
+                # existed to close). Now that resp exists, bail immediately
+                # rather than reading/draining a stream nobody is listening to;
+                # the finally block closes resp and pushes the sentinel.
+                if stop.is_set():
+                    return
                 if resp.status_code != 200:
                     body = resp.text[:500] if resp.text else ""
                     raise CloudAPIError(

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/backend/local.py

@@ -12,12 +12,14 @@ from ..parser.pdf import PdfParser
 from ..parser.markdown import MarkdownParser
 from ..storage.protocol import StorageEngine
 from ..index.pipeline import build_index
-from ..index.utils import parse_pages, get_pdf_page_content, get_md_page_content, remove_fields
+from ..index.utils import parse_pages, get_pdf_page_content, remove_fields
 from ..backend.protocol import AgentTools
 from ..errors import (FileTypeError, DocumentNotFoundError, CollectionNotFoundError,
                       IndexingError, PageIndexError)
 
-_COLLECTION_NAME_RE = re.compile(r'^[a-zA-Z0-9_-]{1,128}$')
+# Matched with .fullmatch() (not .match()): a $-anchored .match() would accept a
+# trailing newline ("papers\n") because $ matches just before a final \n.
+_COLLECTION_NAME_RE = re.compile(r'[a-zA-Z0-9_-]{1,128}')
 
 
 class LocalBackend:
@@ -45,7 +47,7 @@ class LocalBackend:
 
     # Collection management
     def _validate_collection_name(self, name: str) -> None:
-        if not _COLLECTION_NAME_RE.match(name):
+        if not _COLLECTION_NAME_RE.fullmatch(name):
             raise PageIndexError(f"Invalid collection name: {name!r}. Must be 1-128 chars of [a-zA-Z0-9_-].")
 
     def create_collection(self, name: str) -> None:
@@ -116,26 +118,23 @@ class LocalBackend:
             parsed = parser.parse(file_path, model=self._model, images_dir=images_dir)
             result = build_index(parsed, model=self._model, opt=self._index_config)
 
-            # Cache page text for fast retrieval (avoids re-reading files)
+            # Cache page text for fast retrieval (avoids re-reading files) and to
+            # reconstruct node text on demand (get_document(include_text=True),
+            # get_page_content fallback) independent of whether IndexConfig kept
+            # text in the stored structure. build_index() already applies
+            # if_add_node_text to result["structure"] for every strategy, so no
+            # extra stripping is needed here.
             pages = [{"page": n.index, "content": n.content,
                       **({"images": n.images} if n.images else {})}
                      for n in parsed.nodes if n.content]
 
-            # Strip text from structure to save storage space (PDF only;
-            # markdown needs text in structure for fallback retrieval)
-            doc_type = ext.lstrip(".")
-            if doc_type == "pdf":
-                clean_structure = remove_fields(result["structure"], fields=["text"])
-            else:
-                clean_structure = result["structure"]
-
             self._storage.save_document(collection, doc_id, {
                 "doc_name": parsed.doc_name,
                 "doc_description": result.get("doc_description", ""),
                 "file_path": str(managed_path),
                 "file_hash": file_hash,
-                "doc_type": doc_type,
-                "structure": clean_structure,
+                "doc_type": ext.lstrip("."),
+                "structure": result["structure"],
                 "pages": pages,
             })
         except sqlite3.IntegrityError:
@@ -158,6 +157,19 @@ class LocalBackend:
 
         return doc_id
 
+    def _require_document(self, collection: str, doc_id: str) -> dict:
+        """Return the document's storage row, or raise DocumentNotFoundError.
+
+        Single source of truth for "does this doc exist" — every public method
+        and agent tool below goes through this, so a missing doc always
+        surfaces the same way instead of each caller re-implementing its own
+        (and potentially inconsistent) existence check.
+        """
+        doc = self._storage.get_document(collection, doc_id)
+        if not doc:
+            raise DocumentNotFoundError(f"Document {doc_id} not found")
+        return doc
+
     def get_document(self, collection: str, doc_id: str, include_text: bool = False) -> dict:
         """Get document metadata with structure.
 
@@ -166,9 +178,7 @@ class LocalBackend:
                 from cached page content. WARNING: may be very large — do NOT
                 use in agent/LLM contexts as it can exhaust the context window.
         """
-        doc = self._storage.get_document(collection, doc_id)
-        if not doc:
-            raise DocumentNotFoundError(f"Document {doc_id} not found")
+        doc = self._require_document(collection, doc_id)
         doc["structure"] = self._storage.get_document_structure(collection, doc_id)
         if include_text:
             pages = self._storage.get_pages(collection, doc_id) or []
@@ -178,7 +188,13 @@ class LocalBackend:
 
     @staticmethod
     def _fill_node_text(nodes: list, page_map: dict) -> None:
-        """Recursively fill 'text' on structure nodes from cached page content."""
+        """Recursively fill 'text' on structure nodes from cached page content.
+
+        Two node conventions, one per indexing strategy: content_based (PDF)
+        nodes span a start_index..end_index page range; level_based (Markdown)
+        nodes map 1:1 to a single page keyed by line_num. Handling only the
+        first would silently leave Markdown nodes with no text.
+        """
         for node in nodes:
             start = node.get("start_index")
             end = node.get("end_index")
@@ -186,16 +202,17 @@ class LocalBackend:
                 node["text"] = "\n".join(
                     page_map.get(p, "") for p in range(start, end + 1)
                 )
+            elif "line_num" in node:
+                node["text"] = page_map.get(node["line_num"], "")
             if "nodes" in node:
                 LocalBackend._fill_node_text(node["nodes"], page_map)
 
     def get_document_structure(self, collection: str, doc_id: str) -> list:
+        self._require_document(collection, doc_id)
         return self._storage.get_document_structure(collection, doc_id)
 
     def get_page_content(self, collection: str, doc_id: str, pages: str) -> list:
-        doc = self._storage.get_document(collection, doc_id)
-        if not doc:
-            raise DocumentNotFoundError(f"Document {doc_id} not found")
+        doc = self._require_document(collection, doc_id)
         page_nums = parse_pages(pages)
 
         # Try cached pages first (fast, no file I/O)
@@ -203,22 +220,24 @@ class LocalBackend:
         if cached_pages:
             return [p for p in cached_pages if p["page"] in page_nums]
 
-        # Fallback to reading from file
+        # Fallback: re-derive from the source file, same as the PDF path below
+        # — never from the stored structure, whose 'text' field may have been
+        # stripped (if_add_node_text=False, the default). Reachable only for a
+        # custom StorageEngine that doesn't cache pages (the built-in
+        # SQLiteStorage always does).
         if doc["doc_type"] == "pdf":
             return get_pdf_page_content(doc["file_path"], page_nums)
         else:
-            structure = self._storage.get_document_structure(collection, doc_id)
-            return get_md_page_content(structure, page_nums)
+            parser = self._resolve_parser(doc["file_path"])
+            parsed = parser.parse(doc["file_path"], model=self._model)
+            page_map = {n.index: n.content for n in parsed.nodes}
+            return [{"page": p, "content": page_map[p]} for p in page_nums if p in page_map]
 
     def list_documents(self, collection: str) -> list[dict]:
         return self._storage.list_documents(collection)
 
     def delete_document(self, collection: str, doc_id: str) -> None:
-        doc = self._storage.get_document(collection, doc_id)
-        if not doc:
-            # Parity with the cloud backend, which surfaces HTTP 404 as
-            # DocumentNotFoundError — a typo'd doc_id should not pass silently.
-            raise DocumentNotFoundError(f"Document {doc_id} not found")
+        doc = self._require_document(collection, doc_id)
         if doc.get("file_path"):
             Path(doc["file_path"]).unlink(missing_ok=True)
         # Clean up images directory: files/{collection}/{doc_id}/
@@ -233,13 +252,17 @@ class LocalBackend:
         - doc_ids=None (open mode): includes ``list_documents``; agent picks docs itself.
         - doc_ids=[...] (scoped mode): no ``list_documents``; the other tools
           hard-enforce the whitelist and reject out-of-scope doc_ids.
+
+        Note ``is not None``: an empty list is a scope of *nothing* (reject every
+        doc), NOT open mode. Using truthiness would let ``doc_ids=[]`` collapse to
+        ``None`` and silently grant access to the whole collection.
         """
         from agents import function_tool
         import json
         storage = self._storage
         col_name = collection
         backend = self
-        scope = set(doc_ids) if doc_ids else None
+        scope = set(doc_ids) if doc_ids is not None else None
 
         def _reject(doc_id: str) -> str | None:
             if scope is not None and doc_id not in scope:
@@ -255,7 +278,14 @@ class LocalBackend:
             rejection = _reject(doc_id)
             if rejection:
                 return rejection
-            return json.dumps(storage.get_document(col_name, doc_id))
+            try:
+                # _require_document (not backend.get_document) deliberately:
+                # the metadata-only row, no 'structure' — keeps this tool's
+                # output small for the agent's context window.
+                doc = backend._require_document(col_name, doc_id)
+            except DocumentNotFoundError:
+                return json.dumps({"error": f"doc_id '{doc_id}' not found."})
+            return json.dumps(doc)
 
         @function_tool
         def get_document_structure(doc_id: str) -> str:
@@ -263,6 +293,10 @@ class LocalBackend:
             rejection = _reject(doc_id)
             if rejection:
                 return rejection
+            try:
+                backend._require_document(col_name, doc_id)
+            except DocumentNotFoundError:
+                return json.dumps({"error": f"doc_id '{doc_id}' not found."})
             structure = storage.get_document_structure(col_name, doc_id)
             return json.dumps(remove_fields(structure, fields=["text"]), ensure_ascii=False)
 
@@ -272,7 +306,18 @@ class LocalBackend:
             rejection = _reject(doc_id)
             if rejection:
                 return rejection
-            result = backend.get_page_content(col_name, doc_id, pages)
+            try:
+                result = backend.get_page_content(col_name, doc_id, pages)
+            except DocumentNotFoundError:
+                return json.dumps({"error": f"doc_id '{doc_id}' not found."})
+            except (ValueError, AttributeError) as e:
+                # A malformed page spec ("all", "5-") is a recoverable bad tool
+                # argument: hand the model an actionable error it can correct
+                # (mirroring the legacy retrieval tool) rather than letting the
+                # ValueError surface as the agent SDK's generic tool-failure text.
+                return json.dumps({
+                    "error": f"Invalid pages format: {pages!r}. Use '5-7', '3,8', or '12'. Error: {e}"
+                })
             return json.dumps(result, ensure_ascii=False)
 
         tools = [get_document, get_document_structure, get_page_content]

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/client.py

@@ -5,6 +5,7 @@ from typing import Any, Iterator
 
 from typing_extensions import deprecated
 
+from .cloud_api import API_BASE
 from .collection import Collection
 from .config import IndexConfig
 from .errors import PageIndexAPIError
@@ -50,7 +51,7 @@ class PageIndexClient:
         # Or use LocalClient / CloudClient for explicit mode selection
     """
 
-    BASE_URL = "https://api.pageindex.ai"
+    BASE_URL = API_BASE  # single source of truth lives in cloud_api
 
     def __init__(self, api_key: str | None = None, model: str = None,
                  retrieve_model: str = None, storage_path: str = None,
@@ -113,14 +114,30 @@ class PageIndexClient:
 
     @staticmethod
     def _validate_llm_provider(model: str) -> None:
-        """Validate model and check API key via litellm. Warns if key seems missing."""
+        """Validate the model string and require an API key for providers that
+        need one. Local / keyless providers (ollama, lm_studio, …) are skipped so
+        a keyless LiteLLM model isn't rejected at construction time."""
         try:
             import litellm
-            litellm.model_cost_map_url = ""
             _, provider, _, _ = litellm.get_llm_provider(model=model)
         except Exception:
             return
 
+        # LiteLLM providers that run locally / self-hosted and need no API key
+        # by default (litellm itself falls back to a placeholder key for these
+        # rather than erroring — see e.g. hosted_vllm's transformation.py).
+        # This list is necessarily a manual allowlist (litellm.validate_environment
+        # isn't reliable enough to derive it from); extend it as litellm adds
+        # more local-inference providers.
+        keyless = {
+            "ollama", "ollama_chat", "lm_studio", "hosted_vllm", "vllm",
+            "xinference", "llamafile", "triton", "oobabooga",
+            "openai_like", "custom_openai", "custom", "docker_model_runner",
+            "petals",
+        }
+        if provider in keyless:
+            return
+
         key = litellm.get_api_key(llm_provider=provider, dynamic_api_key=None)
         if not key:
             import os

{pageindex-0.3.0.dev2 → pageindex-0.3.0.dev3}/pageindex/cloud_api.py

@@ -8,11 +8,16 @@ import requests
 
 from .errors import PageIndexAPIError
 
+# Single source of truth for the cloud API base URL — imported by the modern
+# CloudBackend (as API_BASE) and PageIndexClient so a staging/migration change
+# only has to happen here.
+API_BASE = "https://api.pageindex.ai"
+
 
 class LegacyCloudAPI:
     """Compatibility layer for the pageindex 0.2.x cloud SDK API."""
 
-    BASE_URL = "https://api.pageindex.ai"
+    BASE_URL = API_BASE
 
     def __init__(self, api_key: str, base_url: str | None = None):
         self.api_key = api_key
@@ -85,7 +90,11 @@ class LegacyCloudAPI:
     def get_tree(self, doc_id: str, node_summary: bool = False) -> dict[str, Any]:
         response = self._request(
             "GET",
-            f"/doc/{self._enc(doc_id)}/?type=tree&summary={node_summary}",
+            # Lowercase the bool: a Python f-string renders True/False with a
+            # capital letter, but the API expects summary=true/false (the modern
+            # CloudBackend sends lowercase). A case-sensitive server would
+            # otherwise silently drop node summaries.
+            f"/doc/{self._enc(doc_id)}/?type=tree&summary={'true' if node_summary else 'false'}",
             "Failed to get tree result",
         )
         return response.json()
@@ -147,6 +156,12 @@ class LegacyCloudAPI:
             payload["temperature"] = temperature
         if enable_citations:
             payload["enable_citations"] = enable_citations
+        # Forward stream_metadata so the wire request matches the caller's intent
+        # (and stays correct if the server ever gates metadata chunks behind it),
+        # mirroring the modern CloudBackend which always sends it. It only affects
+        # streaming responses, where it selects the raw dict-chunk parser below.
+        if stream_metadata:
+            payload["stream_metadata"] = stream_metadata
 
         response = self._request(
             "POST",
@@ -224,7 +239,11 @@ class LegacyCloudAPI:
             f"/doc/{self._enc(doc_id)}/",
             "Failed to delete document",
         )
-        return response.json()
+        # A successful DELETE may come back with an empty body (the documented
+        # examples don't consume one, and REST APIs commonly return no content
+        # for deletes). Don't let json() raise JSONDecodeError on success —
+        # the document is already gone; return an empty dict.
+        return response.json() if response.content else {}
 
     def list_documents(
         self,