langroid 0.37.0__tar.gz → 0.37.1__tar.gz

{langroid-0.37.0 → langroid-0.37.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.37.0
+Version: 0.37.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -102,6 +102,7 @@ Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'doc-chat'
 Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'doc-chat'
 Provides-Extra: docling
 Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'docling'
+Requires-Dist: pypdf>=5.1.0; extra == 'docling'
 Provides-Extra: docx
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'docx'
 Provides-Extra: fastembed

{langroid-0.37.0 → langroid-0.37.1}/langroid/parsing/document_parser.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import itertools
 import logging
 import re
-import tempfile
 from enum import Enum
 from io import BytesIO
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
+from langroid.parsing.pdf_utils import pdf_split_pages
 from langroid.utils.object_registry import ObjectRegistry
 
 try:
@@ -515,29 +515,23 @@ class DoclingParser(DocumentParser):
             raise LangroidImportError(
                 "docling", ["docling", "pdf-parsers", "all", "doc-chat"]
             )
-        from docling.datamodel.document import TextItem  # type: ignore
+
         from docling.document_converter import (  # type: ignore
             ConversionResult,
             DocumentConverter,
         )
+        from docling_core.types.doc import ImageRefMode  # type: ignore
 
+        page_files, tmp_dir = pdf_split_pages(self.doc_bytes)
         converter = DocumentConverter()
-        file_path = self.source
-        if file_path == "bytes":
-            with tempfile.NamedTemporaryFile(delete=False) as tmp:
-                tmp.write(self.doc_bytes.getvalue())
-                file_path = tmp.name
-        result: ConversionResult = converter.convert(file_path)
-        doc = result.document
-        n_pages = doc.num_pages()  # type: ignore
-        for i in range(n_pages):
-            texts = [
-                item[0].text
-                for item in doc.iterate_items(page_no=i + 1)
-                if isinstance(item[0], TextItem)
-            ]
-            text = "\n".join(texts)
-            yield i, text
+        for i, page_file in enumerate(page_files):
+            result: ConversionResult = converter.convert(page_file)
+            md_text = result.document.export_to_markdown(
+                image_mode=ImageRefMode.REFERENCED
+            )
+            yield i, md_text
+
+        tmp_dir.cleanup()
 
     def get_document_from_page(self, page: str) -> Document:
         """

langroid-0.37.1/langroid/parsing/pdf_utils.py

@@ -0,0 +1,55 @@
+import tempfile
+from io import BytesIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, BinaryIO, List, Tuple, Union
+
+try:
+    import pypdf
+except ImportError:
+    if not TYPE_CHECKING:
+        pypdf = None
+
+from langroid.exceptions import LangroidImportError
+
+if pypdf is None:
+    raise LangroidImportError(
+        "pypdf", ["pypdf", "docling", "all", "pdf-parsers", "doc-chat"]
+    )
+from pypdf import PdfReader, PdfWriter
+
+
+def pdf_split_pages(
+    input_pdf: Union[str, Path, BytesIO, BinaryIO],
+) -> Tuple[List[Path], TemporaryDirectory[Any]]:
+    """Splits a PDF into individual pages in a temporary directory.
+
+    Args:
+        input_pdf: Input PDF file path or file-like object
+        max_workers: Maximum number of concurrent workers for parallel processing
+
+    Returns:
+        Tuple containing:
+            - List of paths to individual PDF pages
+            - Temporary directory object (caller must call cleanup())
+
+    Example:
+        paths, tmp_dir = split_pdf_temp("input.pdf")
+        # Use paths...
+        tmp_dir.cleanup()  # Clean up temp files when done
+    """
+    tmp_dir = tempfile.TemporaryDirectory()
+    reader = PdfReader(input_pdf)
+    paths = []
+
+    for i in range(len(reader.pages)):
+        writer = PdfWriter()
+        writer.add_page(reader.pages[i])
+        writer.add_metadata(reader.metadata or {})
+
+        output = Path(tmp_dir.name) / f"page_{i+1}.pdf"
+        with open(output, "wb") as f:
+            writer.write(f)
+        paths.append(output)
+
+    return paths, tmp_dir  # Return dir object so caller can control cleanup

{langroid-0.37.0 → langroid-0.37.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.37.0"
+version = "0.37.1"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]
@@ -128,6 +128,7 @@ lancedb = [
 
 docling = [
     "docling<3.0.0,>=2.16.0",
+    "pypdf>=5.1.0", # needed to split pdf into pages, then use docling
 ]
 
 pymupdf4llm = [