langroid 0.37.0__py3-none-any.whl → 0.37.1__py3-none-any.whl

langroid/parsing/document_parser.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import itertools
 import logging
 import re
-import tempfile
 from enum import Enum
 from io import BytesIO
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
+from langroid.parsing.pdf_utils import pdf_split_pages
 from langroid.utils.object_registry import ObjectRegistry
 
 try:
@@ -515,29 +515,23 @@ class DoclingParser(DocumentParser):
             raise LangroidImportError(
                 "docling", ["docling", "pdf-parsers", "all", "doc-chat"]
             )
-        from docling.datamodel.document import TextItem  # type: ignore
+
         from docling.document_converter import (  # type: ignore
             ConversionResult,
             DocumentConverter,
         )
+        from docling_core.types.doc import ImageRefMode  # type: ignore
 
+        page_files, tmp_dir = pdf_split_pages(self.doc_bytes)
         converter = DocumentConverter()
-        file_path = self.source
-        if file_path == "bytes":
-            with tempfile.NamedTemporaryFile(delete=False) as tmp:
-                tmp.write(self.doc_bytes.getvalue())
-                file_path = tmp.name
-        result: ConversionResult = converter.convert(file_path)
-        doc = result.document
-        n_pages = doc.num_pages()  # type: ignore
-        for i in range(n_pages):
-            texts = [
-                item[0].text
-                for item in doc.iterate_items(page_no=i + 1)
-                if isinstance(item[0], TextItem)
-            ]
-            text = "\n".join(texts)
-            yield i, text
+        for i, page_file in enumerate(page_files):
+            result: ConversionResult = converter.convert(page_file)
+            md_text = result.document.export_to_markdown(
+                image_mode=ImageRefMode.REFERENCED
+            )
+            yield i, md_text
+
+        tmp_dir.cleanup()
 
     def get_document_from_page(self, page: str) -> Document:
         """

langroid/parsing/pdf_utils.py

@@ -0,0 +1,55 @@
+import tempfile
+from io import BytesIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, BinaryIO, List, Tuple, Union
+
+try:
+    import pypdf
+except ImportError:
+    if not TYPE_CHECKING:
+        pypdf = None
+
+from langroid.exceptions import LangroidImportError
+
+if pypdf is None:
+    raise LangroidImportError(
+        "pypdf", ["pypdf", "docling", "all", "pdf-parsers", "doc-chat"]
+    )
+from pypdf import PdfReader, PdfWriter
+
+
+def pdf_split_pages(
+    input_pdf: Union[str, Path, BytesIO, BinaryIO],
+) -> Tuple[List[Path], TemporaryDirectory[Any]]:
+    """Splits a PDF into individual pages in a temporary directory.
+
+    Args:
+        input_pdf: Input PDF file path or file-like object
+        max_workers: Maximum number of concurrent workers for parallel processing
+
+    Returns:
+        Tuple containing:
+            - List of paths to individual PDF pages
+            - Temporary directory object (caller must call cleanup())
+
+    Example:
+        paths, tmp_dir = split_pdf_temp("input.pdf")
+        # Use paths...
+        tmp_dir.cleanup()  # Clean up temp files when done
+    """
+    tmp_dir = tempfile.TemporaryDirectory()
+    reader = PdfReader(input_pdf)
+    paths = []
+
+    for i in range(len(reader.pages)):
+        writer = PdfWriter()
+        writer.add_page(reader.pages[i])
+        writer.add_metadata(reader.metadata or {})
+
+        output = Path(tmp_dir.name) / f"page_{i+1}.pdf"
+        with open(output, "wb") as f:
+            writer.write(f)
+        paths.append(output)
+
+    return paths, tmp_dir  # Return dir object so caller can control cleanup

{langroid-0.37.0.dist-info → langroid-0.37.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.37.0
+Version: 0.37.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -102,6 +102,7 @@ Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'doc-chat'
 Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'doc-chat'
 Provides-Extra: docling
 Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'docling'
+Requires-Dist: pypdf>=5.1.0; extra == 'docling'
 Provides-Extra: docx
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'docx'
 Provides-Extra: fastembed

{langroid-0.37.0.dist-info → langroid-0.37.1.dist-info}/RECORD

@@ -78,10 +78,11 @@ langroid/language_models/prompt_formatter/llama2_formatter.py,sha256=YdcO88qyBeu
 langroid/parsing/__init__.py,sha256=ZgSAfgTC6VsTLFlRSWT-TwYco7SQeRMeZG-49MnKYGY,936
 langroid/parsing/agent_chats.py,sha256=sbZRV9ujdM5QXvvuHVjIi2ysYSYlap-uqfMMUKulrW0,1068
 langroid/parsing/code_parser.py,sha256=5ze0MBytrGGkU69pA_bJDjRm6QZz_QYfPcIwkagUa7U,3796
-langroid/parsing/document_parser.py,sha256=1DjkoiieuPxlPtX-3FGzr3frDSKOjfKM4PhaKbVNQ1c,28570
+langroid/parsing/document_parser.py,sha256=Xcf_yA4admhx75N123_ouWcgnYXHztxX0S3TxqlWKNU,28334
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
 langroid/parsing/parse_json.py,sha256=aADo38bAHQhC8on4aWZZzVzSDy-dK35vRLZsFI2ewh8,4756
 langroid/parsing/parser.py,sha256=WDv4QnNtAcLSiPe6cPhHOa-aMhrt3OV-kKnVXdgwtmI,12276
+langroid/parsing/pdf_utils.py,sha256=IFs2GH9_ZOYJ159YF5MomQ8RKRj1YPBIxkv0gx4Xz7o,1629
 langroid/parsing/repo_loader.py,sha256=3GjvPJS6Vf5L6gV2zOU8s-Tf1oq_fZm-IB_RL_7CTsY,29373
 langroid/parsing/routing.py,sha256=-FcnlqldzL4ZoxuDwXjQPNHgBe9F9-F4R6q7b_z9CvI,1232
 langroid/parsing/search.py,sha256=0i_r0ESb5HEQfagA2g7_uMQyxYPADWVbdcN9ixZhS4E,8992
@@ -122,7 +123,7 @@ langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3Hmh
 langroid/vector_store/momento.py,sha256=UNHGT6jXuQtqY9f6MdqGU14bVnS0zHgIJUa30ULpUJo,10474
 langroid/vector_store/qdrantdb.py,sha256=Cen6f-y6witiR53UQ-5a605Reo0gTj3ygXpE_ehYoZo,18116
 langroid/vector_store/weaviatedb.py,sha256=C6jd1Twl5_jux3JYyrcTfQb63Lk9HuiUzVF4NahXuGo,10642
-langroid-0.37.0.dist-info/METADATA,sha256=hlweiAhkVzVb_sVOPF-adwqwDPpAUUsgE1wJFRYNnKg,60524
-langroid-0.37.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.37.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.37.0.dist-info/RECORD,,
+langroid-0.37.1.dist-info/METADATA,sha256=XL8VnB7r3uUJ6-BkwZkUPeSQO4pfvo8YfH3GvbX_gFg,60572
+langroid-0.37.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.37.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.37.1.dist-info/RECORD,,