langroid 0.36.1__py3-none-any.whl → 0.37.1__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -100,7 +100,7 @@ hf_embed_config = SentenceTransformerEmbeddingsConfig(
 
 oai_embed_config = OpenAIEmbeddingsConfig(
     model_type="openai",
-    model_name="text-embedding-ada-002",
+    model_name="text-embedding-3-small",
     dims=1536,
 )
 
@@ -189,8 +189,8 @@ class DocChatAgentConfig(ChatAgentConfig):
             # NOTE: PDF parsing is extremely challenging, and each library
             # has its own strengths and weaknesses.
             # Try one that works for your use case.
-            # or "unstructured", "pdfplumber", "fitz", "pypdf"
-            library="pdfplumber",
+            # or "unstructured", "fitz", "pymupdf4llm", "pypdf"
+            library="pymupdf4llm",
         ),
     )
 

langroid/embedding_models/models.py

@@ -18,7 +18,7 @@ AzureADTokenProvider = Callable[[], str]
 
 class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "openai"
-    model_name: str = "text-embedding-ada-002"
+    model_name: str = "text-embedding-3-large"
     api_key: str = ""
     api_base: Optional[str] = None
     organization: str = ""
@@ -28,7 +28,7 @@ class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
 
 class AzureOpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "azure-openai"
-    model_name: str = "text-embedding-ada-002"
+    model_name: str = "text-embedding-3-large"
     api_key: str = ""
     api_base: str = ""
     deployment_name: Optional[str] = None

langroid/exceptions.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 
 class XMLException(Exception):
@@ -15,7 +15,7 @@ class LangroidImportError(ImportError):
     def __init__(
         self,
         package: Optional[str] = None,
-        extra: Optional[str] = None,
+        extra: Optional[str | List[str]] = None,
         error: str = "",
         *args: object,
     ) -> None:
@@ -33,9 +33,21 @@ class LangroidImportError(ImportError):
             error = f"{package} is not installed by default with Langroid.\n"
 
         if extra:
+            if isinstance(extra, list):
+                help_preamble = f"""
+                If you want to use it, please install langroid with one of these 
+                extras: {', '.join(extra)}. The examples below use the first one, 
+                i.e. {extra[0]}.
+                """
+                extra = extra[0]
+            else:
+                help_preamble = f"""
+                If you want to use it, please install langroid with the
+                `{extra}` extra.
+                """
+
             install_help = f"""
-                If you want to use it, please install langroid 
-                with the `{extra}` extra, for example:
+                {help_preamble}
                 
                 If you are using pip:
                 pip install "langroid[{extra}]"

langroid/parsing/code_parser.py

@@ -65,7 +65,7 @@ class CodeParsingConfig(BaseSettings):
         "bash",
     ]
     chunk_size: int = 500  # tokens
-    token_encoding_model: str = "text-embedding-ada-002"
+    token_encoding_model: str = "text-embedding-3-small"
     n_similar_docs: int = 4
 
 

langroid/parsing/document_parser.py

@@ -5,9 +5,10 @@ import logging
 import re
 from enum import Enum
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, Generator, List, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
+from langroid.parsing.pdf_utils import pdf_split_pages
 from langroid.utils.object_registry import ObjectRegistry
 
 try:
@@ -15,18 +16,24 @@ try:
 except ImportError:
     if not TYPE_CHECKING:
         fitz = None
+try:
+    import pymupdf4llm
+except ImportError:
+    if not TYPE_CHECKING:
+        pymupdf4llm = None
 
 try:
-    import pypdf
+    import docling
 except ImportError:
     if not TYPE_CHECKING:
-        pypdf = None
+        docling = None
 
 try:
-    import pdfplumber
+    import pypdf
 except ImportError:
     if not TYPE_CHECKING:
-        pdfplumber = None
+        pypdf = None
+
 
 import requests
 from bs4 import BeautifulSoup
@@ -41,6 +48,7 @@ logger = logging.getLogger(__name__)
 
 
 class DocumentType(str, Enum):
+    # TODO add `md` (Markdown) and `html`
     PDF = "pdf"
     DOCX = "docx"
     DOC = "doc"
@@ -139,10 +147,12 @@ class DocumentParser(Parser):
         if inferred_doc_type == DocumentType.PDF:
             if config.pdf.library == "fitz":
                 return FitzPDFParser(source, config)
+            elif config.pdf.library == "pymupdf4llm":
+                return PyMuPDF4LLMParser(source, config)
+            elif config.pdf.library == "docling":
+                return DoclingParser(source, config)
             elif config.pdf.library == "pypdf":
                 return PyPDFParser(source, config)
-            elif config.pdf.library == "pdfplumber":
-                return PDFPlumberParser(source, config)
             elif config.pdf.library == "unstructured":
                 return UnstructuredPDFParser(source, config)
             elif config.pdf.library == "pdf2image":
@@ -307,8 +317,11 @@ class DocumentParser(Parser):
         """Yield each page in the PDF."""
         raise NotImplementedError
 
-    def extract_text_from_page(self, page: Any) -> str:
-        """Extract text from a given page."""
+    def get_document_from_page(self, page: Any) -> Document:
+        """
+        Get Langroid Document object (with possible metadata)
+        corresponding to a given page.
+        """
         raise NotImplementedError
 
     def fix_text(self, text: str) -> str:
@@ -335,7 +348,10 @@ class DocumentParser(Parser):
         """
 
         text = "".join(
-            [self.extract_text_from_page(page) for _, page in self.iterate_pages()]
+            [
+                self.get_document_from_page(page).content
+                for _, page in self.iterate_pages()
+            ]
         )
         return Document(content=text, metadata=DocMetaData(source=self.source))
 
@@ -359,7 +375,10 @@ class DocumentParser(Parser):
         common_id = ObjectRegistry.new_id()
         n_chunks = 0  # how many chunk so far
         for i, page in self.iterate_pages():
-            page_text = self.extract_text_from_page(page)
+            # not used but could be useful, esp to blend the
+            # metadata from the pages into the chunks
+            page_doc = self.get_document_from_page(page)
+            page_text = page_doc.content
             split += self.tokenizer.encode(page_text)
             pages.append(str(i + 1))
             # split could be so long it needs to be split
@@ -422,81 +441,146 @@ class FitzPDFParser(DocumentParser):
             yield i, page
         doc.close()
 
-    def extract_text_from_page(self, page: "fitz.Page") -> str:
+    def get_document_from_page(self, page: "fitz.Page") -> Document:
         """
-        Extract text from a given `fitz` page.
+        Get Document object from a given `fitz` page.
 
         Args:
             page (fitz.Page): The `fitz` page object.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.get_text())
+        return Document(
+            content=self.fix_text(page.get_text()),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
-class PyPDFParser(DocumentParser):
+class PyMuPDF4LLMParser(DocumentParser):
     """
-    Parser for processing PDFs using the `pypdf` library.
+    Parser for processing PDFs using the `pymupdf4llm` library.
     """
 
-    def iterate_pages(self) -> Generator[Tuple[int, pypdf.PageObject], None, None]:
+    def iterate_pages(self) -> Generator[Tuple[int, "fitz.Page"], None, None]:
         """
-        Yield each page in the PDF using `pypdf`.
+        Yield each page in the PDF using `fitz`.
 
         Returns:
-            Generator[pypdf.pdf.PageObject]: Generator yielding each page.
+            Generator[fitz.Page]: Generator yielding each page.
         """
-        if pypdf is None:
-            raise LangroidImportError("pypdf", "pdf-parsers")
-        reader = pypdf.PdfReader(self.doc_bytes)
-        for i, page in enumerate(reader.pages):
+        if fitz is None:
+            raise LangroidImportError(
+                "pymupdf4llm", ["pymupdf4llm", "all", "pdf-parsers", "doc-chat"]
+            )
+        doc: fitz.Document = fitz.open(stream=self.doc_bytes, filetype="pdf")
+        pages: List[Dict[str, Any]] = pymupdf4llm.to_markdown(doc, page_chunks=True)
+        for i, page in enumerate(pages):
             yield i, page
+        doc.close()
 
-    def extract_text_from_page(self, page: pypdf.PageObject) -> str:
+    def get_document_from_page(self, page: Dict[str, Any]) -> Document:
         """
-        Extract text from a given `pypdf` page.
+        Get Document object corresponding to a given "page-chunk"
+        dictionary, see:
+         https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/api.html
+
 
         Args:
-            page (pypdf.pdf.PageObject): The `pypdf` page object.
+            page (Dict[str,Any]): The "page-chunk" dictionary.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.extract_text())
+        return Document(
+            content=self.fix_text(page.get("text", "")),
+            # TODO could possible use other metadata from page, see above link.
+            metadata=DocMetaData(source=self.source),
+        )
 
 
-class PDFPlumberParser(DocumentParser):
+class DoclingParser(DocumentParser):
     """
-    Parser for processing PDFs using the `pdfplumber` library.
+    Parser for processing PDFs using the `docling` library.
     """
 
-    def iterate_pages(
-        self,
-    ) -> (Generator)[Tuple[int, pdfplumber.pdf.Page], None, None]:  # type: ignore
+    def iterate_pages(self) -> Generator[Tuple[int, Any], None, None]:
+        """
+        Yield each page in the PDF using `docling`.
+
+        Returns:
+            Generator[docling.Page]: Generator yielding each page.
         """
-        Yield each page in the PDF using `pdfplumber`.
+        if docling is None:
+            raise LangroidImportError(
+                "docling", ["docling", "pdf-parsers", "all", "doc-chat"]
+            )
+
+        from docling.document_converter import (  # type: ignore
+            ConversionResult,
+            DocumentConverter,
+        )
+        from docling_core.types.doc import ImageRefMode  # type: ignore
+
+        page_files, tmp_dir = pdf_split_pages(self.doc_bytes)
+        converter = DocumentConverter()
+        for i, page_file in enumerate(page_files):
+            result: ConversionResult = converter.convert(page_file)
+            md_text = result.document.export_to_markdown(
+                image_mode=ImageRefMode.REFERENCED
+            )
+            yield i, md_text
+
+        tmp_dir.cleanup()
+
+    def get_document_from_page(self, page: str) -> Document:
+        """
+        Get Document object from a given `docling` "page" (actually a chunk).
+
+        Args:
+            page (docling.chunking.DocChunk): The `docling` chunk
 
         Returns:
-            Generator[pdfplumber.Page]: Generator yielding each page.
+            Document: Document object, with content and possible metadata.
         """
-        if pdfplumber is None:
-            raise LangroidImportError("pdfplumber", "pdf-parsers")
-        with pdfplumber.open(self.doc_bytes) as pdf:
-            for i, page in enumerate(pdf.pages):
-                yield i, page
+        return Document(
+            content=self.fix_text(page),
+            metadata=DocMetaData(source=self.source),
+        )
+
 
-    def extract_text_from_page(self, page: pdfplumber.pdf.Page) -> str:  # type: ignore
+class PyPDFParser(DocumentParser):
+    """
+    Parser for processing PDFs using the `pypdf` library.
+    """
+
+    def iterate_pages(self) -> Generator[Tuple[int, pypdf.PageObject], None, None]:
         """
-        Extract text from a given `pdfplumber` page.
+        Yield each page in the PDF using `pypdf`.
+
+        Returns:
+            Generator[pypdf.pdf.PageObject]: Generator yielding each page.
+        """
+        if pypdf is None:
+            raise LangroidImportError("pypdf", "pdf-parsers")
+        reader = pypdf.PdfReader(self.doc_bytes)
+        for i, page in enumerate(reader.pages):
+            yield i, page
+
+    def get_document_from_page(self, page: pypdf.PageObject) -> Document:
+        """
+        Get Document object from a given `pypdf` page.
 
         Args:
-            page (pdfplumber.Page): The `pdfplumber` page object.
+            page (pypdf.pdf.PageObject): The `pypdf` page object.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.extract_text())
+        return Document(
+            content=self.fix_text(page.extract_text()),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class ImagePdfParser(DocumentParser):
@@ -516,15 +600,15 @@ class ImagePdfParser(DocumentParser):
         for i, image in enumerate(images):
             yield i, image
 
-    def extract_text_from_page(self, page: "Image") -> str:  # type: ignore
+    def get_document_from_page(self, page: "Image") -> Document:  # type: ignore
         """
-        Extract text from a given `pdf2image` page.
+        Get Document object corresponding to a given `pdf2image` page.
 
         Args:
             page (Image): The PIL Image object.
 
         Returns:
-            str: Extracted text from the image.
+            Document: Document object, with content and possible metadata.
         """
         try:
             import pytesseract
@@ -532,7 +616,10 @@ class ImagePdfParser(DocumentParser):
             raise LangroidImportError("pytesseract", "pdf-parsers")
 
         text = pytesseract.image_to_string(page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredPDFParser(DocumentParser):
@@ -564,8 +651,8 @@ class UnstructuredPDFParser(DocumentParser):
                 The `unstructured` library failed to parse the pdf.
                 Please try a different library by setting the `library` field
                 in the `pdf` section of the `parsing` field in the config file.
-                Supported libraries are:
-                fitz, pypdf, pdfplumber, unstructured
+                Other supported libraries are:
+                fitz, pymupdf4llm, pypdf
                 """
             )
 
@@ -584,18 +671,21 @@ class UnstructuredPDFParser(DocumentParser):
         if page_elements:
             yield page_number, page_elements
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given `unstructured` element.
+        Get Document object from a given `unstructured` element.
 
         Args:
             page (unstructured element): The `unstructured` element object.
 
         Returns:
-            str: Extracted text from the element.
+            Document: Document object, with content and possible metadata.
         """
         text = " ".join(el.text for el in page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredDocxParser(DocumentParser):
@@ -632,9 +722,9 @@ class UnstructuredDocxParser(DocumentParser):
         if page_elements:
             yield page_number, page_elements
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given `unstructured` element.
+        Get Document object from a given `unstructured` element.
 
         Note:
             The concept of "pages" doesn't actually exist in the .docx file format in
@@ -647,10 +737,13 @@ class UnstructuredDocxParser(DocumentParser):
             page (unstructured element): The `unstructured` element object.
 
         Returns:
-            str: Extracted text from the element.
+            Document object, with content and possible metadata.
         """
         text = " ".join(el.text for el in page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredDocParser(UnstructuredDocxParser):
@@ -704,15 +797,19 @@ class PythonDocxParser(DocumentParser):
         for i, para in enumerate(doc.paragraphs, start=1):
             yield i, [para]
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given 'page', which in this case is a single paragraph.
+        Get Document object from a given 'page', which in this case is a single
+        paragraph.
 
         Args:
             page (list): A list containing a single Paragraph object.
 
         Returns:
-            str: Extracted text from the paragraph.
+            Document: Document object, with content and possible metadata.
         """
         paragraph = page[0]
-        return self.fix_text(paragraph.text)
+        return Document(
+            content=self.fix_text(paragraph.text),
+            metadata=DocMetaData(source=self.source),
+        )

langroid/parsing/parser.py

@@ -23,11 +23,12 @@ class Splitter(str, Enum):
 class PdfParsingConfig(BaseSettings):
     library: Literal[
         "fitz",
-        "pdfplumber",
+        "pymupdf4llm",
+        "docling",
         "pypdf",
         "unstructured",
         "pdf2image",
-    ] = "pdfplumber"
+    ] = "pymupdf4llm"
 
 
 class DocxParsingConfig(BaseSettings):
@@ -40,6 +41,7 @@ class DocParsingConfig(BaseSettings):
 
 class ParsingConfig(BaseSettings):
     splitter: str = Splitter.TOKENS
+    chunk_by_page: bool = False  # split by page?
     chunk_size: int = 200  # aim for this many tokens per chunk
     overlap: int = 50  # overlap between chunks
     max_chunks: int = 10_000
@@ -49,7 +51,7 @@ class ParsingConfig(BaseSettings):
     n_similar_docs: int = 4
     n_neighbor_ids: int = 5  # window size to store around each chunk
     separators: List[str] = ["\n\n", "\n", " ", ""]
-    token_encoding_model: str = "text-embedding-ada-002"
+    token_encoding_model: str = "text-embedding-3-large"
     pdf: PdfParsingConfig = PdfParsingConfig()
     docx: DocxParsingConfig = DocxParsingConfig()
     doc: DocParsingConfig = DocParsingConfig()
@@ -61,7 +63,7 @@ class Parser:
         try:
             self.tokenizer = tiktoken.encoding_for_model(config.token_encoding_model)
         except Exception:
-            self.tokenizer = tiktoken.encoding_for_model("text-embedding-ada-002")
+            self.tokenizer = tiktoken.encoding_for_model("text-embedding-3-small")
 
     def num_tokens(self, text: str) -> int:
         tokens = self.tokenizer.encode(text)

langroid/parsing/pdf_utils.py

@@ -0,0 +1,55 @@
+import tempfile
+from io import BytesIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, BinaryIO, List, Tuple, Union
+
+try:
+    import pypdf
+except ImportError:
+    if not TYPE_CHECKING:
+        pypdf = None
+
+from langroid.exceptions import LangroidImportError
+
+if pypdf is None:
+    raise LangroidImportError(
+        "pypdf", ["pypdf", "docling", "all", "pdf-parsers", "doc-chat"]
+    )
+from pypdf import PdfReader, PdfWriter
+
+
+def pdf_split_pages(
+    input_pdf: Union[str, Path, BytesIO, BinaryIO],
+) -> Tuple[List[Path], TemporaryDirectory[Any]]:
+    """Splits a PDF into individual pages in a temporary directory.
+
+    Args:
+        input_pdf: Input PDF file path or file-like object
+        max_workers: Maximum number of concurrent workers for parallel processing
+
+    Returns:
+        Tuple containing:
+            - List of paths to individual PDF pages
+            - Temporary directory object (caller must call cleanup())
+
+    Example:
+        paths, tmp_dir = split_pdf_temp("input.pdf")
+        # Use paths...
+        tmp_dir.cleanup()  # Clean up temp files when done
+    """
+    tmp_dir = tempfile.TemporaryDirectory()
+    reader = PdfReader(input_pdf)
+    paths = []
+
+    for i in range(len(reader.pages)):
+        writer = PdfWriter()
+        writer.add_page(reader.pages[i])
+        writer.add_metadata(reader.metadata or {})
+
+        output = Path(tmp_dir.name) / f"page_{i+1}.pdf"
+        with open(output, "wb") as f:
+            writer.write(f)
+        paths.append(output)
+
+    return paths, tmp_dir  # Return dir object so caller can control cleanup

langroid/vector_store/chromadb.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
 
 from langroid.embedding_models.base import (
     EmbeddingModelsConfig,
@@ -18,6 +18,10 @@ logger = logging.getLogger(__name__)
 class ChromaDBConfig(VectorStoreConfig):
     collection_name: str = "temp"
     storage_path: str = ".chroma/data"
+    distance: Literal["cosine", "l2", "ip"] = "cosine"
+    construction_ef: int = 100
+    search_ef: int = 100
+    max_neighbors: int = 16
     embedding: EmbeddingModelsConfig = OpenAIEmbeddingsConfig()
     host: str = "127.0.0.1"
     port: int = 6333
@@ -109,6 +113,13 @@ class ChromaDB(VectorStore):
             name=self.config.collection_name,
             embedding_function=self.embedding_fn,
             get_or_create=not replace,
+            metadata={
+                "hnsw:space": self.config.distance,
+                "hnsw:construction_ef": self.config.construction_ef,
+                "hnsw:search_ef": self.config.search_ef,
+                # we could expose other configs, see:
+                # https://docs.trychroma.com/docs/collections/configure
+            },
         )
 
     def add_documents(self, documents: Sequence[Document]) -> None:

langroid/vector_store/qdrantdb.py

@@ -78,7 +78,7 @@ class QdrantDB(VectorStore):
         super().__init__(config)
         self.config: QdrantDBConfig = config
         self.embedding_fn: EmbeddingFunction = self.embedding_model.embedding_fn()
-        self.embedding_dim = self.embedding_model.embedding_dims
+        self.embedding_dim = len(self.embedding_fn(["test"])[0])
         if self.config.use_sparse_embeddings:
             try:
                 from transformers import AutoModelForMaskedLM, AutoTokenizer

langroid/vector_store/weaviatedb.py

@@ -43,8 +43,8 @@ class WeaviateDB(VectorStore):
         load_dotenv()
         key = os.getenv("WEAVIATE_API_KEY")
         url = os.getenv("WEAVIATE_API_URL")
-        if None in [key, url]:
-            logger.warning(
+        if url is None or key is None:
+            raise ValueError(
                 """WEAVIATE_API_KEY, WEAVIATE_API_URL env variable must be set to use
                 WeaviateDB in cloud mode. Please set these values
                 in your .env file.
@@ -130,9 +130,9 @@ class WeaviateDB(VectorStore):
         vector_index_config = Configure.VectorIndex.hnsw(
             distance_metric=VectorDistances.COSINE,
         )
-        if self.config.embedding == OpenAIEmbeddingsConfig:
+        if isinstance(self.config.embedding, OpenAIEmbeddingsConfig):
             vectorizer_config = Configure.Vectorizer.text2vec_openai(
-                model=self.embedding_model
+                model=self.config.embedding.model_name,
             )
         else:
             vectorizer_config = None
@@ -212,7 +212,7 @@ class WeaviateDB(VectorStore):
             return_metadata=MetadataQuery(distance=True),
         )
         return [
-            (self.weaviate_obj_to_doc(item), 1 - item.metadata.distance)
+            (self.weaviate_obj_to_doc(item), 1 - (item.metadata.distance or 1))
             for item in response.objects
         ]
 

{langroid-0.36.1.dist-info → langroid-0.37.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.36.1
+Version: 0.37.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -12,6 +12,7 @@ Requires-Dist: async-generator<2.0,>=1.10
 Requires-Dist: bs4<1.0.0,>=0.0.1
 Requires-Dist: cerebras-cloud-sdk<2.0.0,>=1.1.0
 Requires-Dist: colorlog<7.0.0,>=6.7.0
+Requires-Dist: docling<3.0.0,>=2.16.0
 Requires-Dist: docstring-parser<1.0,>=0.16
 Requires-Dist: duckduckgo-search<7.0.0,>=6.0.0
 Requires-Dist: faker<19.0.0,>=18.9.0
@@ -32,9 +33,10 @@ Requires-Dist: onnxruntime<2.0.0,>=1.16.1
 Requires-Dist: openai<2.0.0,>=1.45.0
 Requires-Dist: pandas<3.0.0,>=2.0.3
 Requires-Dist: prettytable<4.0.0,>=3.8.0
-Requires-Dist: pydantic<2.10.2,>=1
+Requires-Dist: pydantic<3.0.0,>=1
 Requires-Dist: pygithub<2.0.0,>=1.58.1
 Requires-Dist: pygments<3.0.0,>=2.15.1
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17
 Requires-Dist: pyparsing<4.0.0,>=3.0.9
 Requires-Dist: pytest-rerunfailures<16.0,>=15.0
 Requires-Dist: python-dotenv<2.0.0,>=1.0.0
@@ -55,14 +57,15 @@ Provides-Extra: all
 Requires-Dist: arango-datasets<2.0.0,>=1.2.2; extra == 'all'
 Requires-Dist: chainlit<3.0.0,>=2.0.1; extra == 'all'
 Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'all'
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'all'
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'all'
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'all'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'all'
 Requires-Dist: litellm<2.0.0,>=1.30.1; extra == 'all'
 Requires-Dist: metaphor-python<0.2.0,>=0.1.23; extra == 'all'
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'all'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'all'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'all'
 Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'all'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'all'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'all'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'all'
 Requires-Dist: pypdf>=5.1.0; extra == 'all'
@@ -74,7 +77,7 @@ Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'all'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'all'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'all'
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'all'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'all'
 Requires-Dist: weaviate-client>=4.9.6; extra == 'all'
 Provides-Extra: arango
 Requires-Dist: arango-datasets<2.0.0,>=1.2.2; extra == 'arango'
@@ -89,13 +92,17 @@ Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'db'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'db'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'db'
 Provides-Extra: doc-chat
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'doc-chat'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'doc-chat'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'doc-chat'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'doc-chat'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'doc-chat'
 Requires-Dist: pypdf>=5.1.0; extra == 'doc-chat'
 Requires-Dist: pytesseract<0.4.0,>=0.3.10; extra == 'doc-chat'
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'doc-chat'
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'doc-chat'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'doc-chat'
+Provides-Extra: docling
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'docling'
+Requires-Dist: pypdf>=5.1.0; extra == 'docling'
 Provides-Extra: docx
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'docx'
 Provides-Extra: fastembed
@@ -104,7 +111,7 @@ Provides-Extra: hf-embeddings
 Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'hf-embeddings'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'hf-embeddings'
 Provides-Extra: hf-transformers
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'hf-transformers'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'hf-transformers'
 Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'hf-transformers'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'hf-transformers'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'hf-transformers'
@@ -125,13 +132,16 @@ Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'mysql'
 Provides-Extra: neo4j
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'neo4j'
 Provides-Extra: pdf-parsers
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'pdf-parsers'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'pdf-parsers'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'pdf-parsers'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'pdf-parsers'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'pdf-parsers'
 Requires-Dist: pypdf>=5.1.0; extra == 'pdf-parsers'
 Requires-Dist: pytesseract<0.4.0,>=0.3.10; extra == 'pdf-parsers'
 Provides-Extra: postgres
 Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'postgres'
+Provides-Extra: pymupdf4llm
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'pymupdf4llm'
 Provides-Extra: scrapy
 Requires-Dist: scrapy<3.0.0,>=2.11.0; extra == 'scrapy'
 Provides-Extra: sql
@@ -139,11 +149,11 @@ Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'sql'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'sql'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'sql'
 Provides-Extra: transformers
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'transformers'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'transformers'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'transformers'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'transformers'
 Provides-Extra: unstructured
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'unstructured'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'unstructured'
 Provides-Extra: vecdbs
 Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'vecdbs'
 Requires-Dist: lancedb<0.9.0,>=0.8.2; extra == 'vecdbs'

{langroid-0.36.1.dist-info → langroid-0.37.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 langroid/__init__.py,sha256=z_fCOLQJPOw3LLRPBlFB5-2HyCjpPgQa4m4iY5Fvb8Y,1800
-langroid/exceptions.py,sha256=gp6ku4ZLdXXCUQIwUNVFojJNGTzGnkevi2PLvG7HOhc,2555
+langroid/exceptions.py,sha256=OPjece_8cwg94DLPcOGA1ddzy5bGh65pxzcHMnssTz8,2995
 langroid/mytypes.py,sha256=h1eMq1ZwTLVezObPfCseWNWbEOzP7mAKu2XoS63W1cM,2647
 langroid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
@@ -14,7 +14,7 @@ langroid/agent/xml_tool_message.py,sha256=6SshYZJKIfi4mkE-gIoSwjkEYekQ8GwcSiCv7a
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=RH8qUXaZE5o2WQz3WJQ1SdFtASGlxWCA6_HYz_3meDQ,20822
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=PCpdaVocIWt6ftO5OfmI1l20abVbKUSZWdMcu9hJDD0,64816
+langroid/agent/special/doc_chat_agent.py,sha256=qoXp6PKI7oAQs8rgj934NzZaEEKsPICcgYl_iQY0bac,64818
 langroid/agent/special/lance_doc_chat_agent.py,sha256=s8xoRs0gGaFtDYFUSIRchsgDVbS5Q3C2b2mr3V1Fd-Q,10419
 langroid/agent/special/lance_tools.py,sha256=qS8x4wi8mrqfbYV2ztFzrcxyhHQ0ZWOc-zkYiH7awj0,2105
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -57,7 +57,7 @@ langroid/cachedb/momento_cachedb.py,sha256=YEOJ62hEcV6iIeMr5aGgRYgWQqFYaej9gEDEc
 langroid/cachedb/redis_cachedb.py,sha256=7kgnbf4b5CKsCrlL97mHWKvdvlLt8zgn7lc528jEpiE,5141
 langroid/embedding_models/__init__.py,sha256=XhVIMQJbQRpImcnhA9sJR7h6r7QgPo1SKDCvwEUD9j4,851
 langroid/embedding_models/base.py,sha256=DUhvzALoW2UMbtmLxP4eJTfPii99WjUNX7bwFpj_K-0,2395
-langroid/embedding_models/models.py,sha256=sW6baTvFSeZBZ5w-Kd9Vgo93gokesJ3aHP4x9htoF2E,16776
+langroid/embedding_models/models.py,sha256=YppD52U1lbeygt8_SuPNi6piOV_FgBltZWH5e3l7iso,16776
 langroid/embedding_models/remote_embeds.py,sha256=6_kjXByVbqhY9cGwl9R83ZcYC2km-nGieNNAo1McHaY,5151
 langroid/embedding_models/protoc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/embedding_models/protoc/embeddings.proto,sha256=_O-SgFpTaylQeOTgSpxhEJ7CUw7PeCQQJLaPqpPYKJg,321
@@ -77,11 +77,12 @@ langroid/language_models/prompt_formatter/hf_formatter.py,sha256=PVJppmjRvD-2DF-
 langroid/language_models/prompt_formatter/llama2_formatter.py,sha256=YdcO88qyBeuMENVIVvVqSYuEpvYSTndUe_jd6hVTko4,2899
 langroid/parsing/__init__.py,sha256=ZgSAfgTC6VsTLFlRSWT-TwYco7SQeRMeZG-49MnKYGY,936
 langroid/parsing/agent_chats.py,sha256=sbZRV9ujdM5QXvvuHVjIi2ysYSYlap-uqfMMUKulrW0,1068
-langroid/parsing/code_parser.py,sha256=AOxb3xbYpTBPP3goOm5dKfJdh5hS_2BhLVCEkifWZN8,3796
-langroid/parsing/document_parser.py,sha256=9xUOyrVNBAS9cpCvCptr2XK4Kq47W574i8zzGEoXc3c,24933
+langroid/parsing/code_parser.py,sha256=5ze0MBytrGGkU69pA_bJDjRm6QZz_QYfPcIwkagUa7U,3796
+langroid/parsing/document_parser.py,sha256=Xcf_yA4admhx75N123_ouWcgnYXHztxX0S3TxqlWKNU,28334
 langroid/parsing/para_sentence_split.py,sha256=AJBzZojP3zpB-_IMiiHismhqcvkrVBQ3ZINoQyx_bE4,2000
 langroid/parsing/parse_json.py,sha256=aADo38bAHQhC8on4aWZZzVzSDy-dK35vRLZsFI2ewh8,4756
-langroid/parsing/parser.py,sha256=Wq204V1wqLdXS8kZ5J81dU2jE8fjoRY9zzNFbzLFDIs,12205
+langroid/parsing/parser.py,sha256=WDv4QnNtAcLSiPe6cPhHOa-aMhrt3OV-kKnVXdgwtmI,12276
+langroid/parsing/pdf_utils.py,sha256=IFs2GH9_ZOYJ159YF5MomQ8RKRj1YPBIxkv0gx4Xz7o,1629
 langroid/parsing/repo_loader.py,sha256=3GjvPJS6Vf5L6gV2zOU8s-Tf1oq_fZm-IB_RL_7CTsY,29373
 langroid/parsing/routing.py,sha256=-FcnlqldzL4ZoxuDwXjQPNHgBe9F9-F4R6q7b_z9CvI,1232
 langroid/parsing/search.py,sha256=0i_r0ESb5HEQfagA2g7_uMQyxYPADWVbdcN9ixZhS4E,8992
@@ -116,13 +117,13 @@ langroid/utils/output/printing.py,sha256=yzPJZN-8_jyOJmI9N_oLwEDfjMwVgk3IDiwnZ4e
 langroid/utils/output/status.py,sha256=rzbE7mDJcgNNvdtylCseQcPGCGghtJvVq3lB-OPJ49E,1049
 langroid/vector_store/__init__.py,sha256=BcoOm1tG3y0EqjkIGmMOHkY9iTUhDHgyruknWDKgqIg,1214
 langroid/vector_store/base.py,sha256=suBanIt0iKEgnMnGdQOyWS58guG20Jyy-GK4DMMuYL0,14208
-langroid/vector_store/chromadb.py,sha256=9WXW9IoSnhOmGEtMruVhEtVWL_VO6NXnPIz-nzh0gIQ,8235
+langroid/vector_store/chromadb.py,sha256=XkpW7pnSf6Lk7Nf1BEIw-zjYGYchoWHgrhnJX7YmxD8,8725
 langroid/vector_store/lancedb.py,sha256=b3_vWkTjG8mweZ7ZNlUD-NjmQP_rLBZfyKWcxt2vosA,14855
 langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3HmhHQICXLs,11663
 langroid/vector_store/momento.py,sha256=UNHGT6jXuQtqY9f6MdqGU14bVnS0zHgIJUa30ULpUJo,10474
-langroid/vector_store/qdrantdb.py,sha256=HRLCt-FG8y4718omwpFaQZnWeYxPj0XCwS4tjokI1sU,18116
-langroid/vector_store/weaviatedb.py,sha256=Jxe-cp2PyZdQ9NQVNZJ-CnsYsNxgUBdfAOoLZQEN650,10602
-langroid-0.36.1.dist-info/METADATA,sha256=OzErGoPlFwxWia7jrFUx4M9FolTjexpJbgpTfhwT9Nk,60103
-langroid-0.36.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.36.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.36.1.dist-info/RECORD,,
+langroid/vector_store/qdrantdb.py,sha256=Cen6f-y6witiR53UQ-5a605Reo0gTj3ygXpE_ehYoZo,18116
+langroid/vector_store/weaviatedb.py,sha256=C6jd1Twl5_jux3JYyrcTfQb63Lk9HuiUzVF4NahXuGo,10642
+langroid-0.37.1.dist-info/METADATA,sha256=XL8VnB7r3uUJ6-BkwZkUPeSQO4pfvo8YfH3GvbX_gFg,60572
+langroid-0.37.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.37.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.37.1.dist-info/RECORD,,