langroid 0.36.1__tar.gz → 0.37.0__tar.gz

{langroid-0.36.1 → langroid-0.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.36.1
+Version: 0.37.0
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -12,6 +12,7 @@ Requires-Dist: async-generator<2.0,>=1.10
 Requires-Dist: bs4<1.0.0,>=0.0.1
 Requires-Dist: cerebras-cloud-sdk<2.0.0,>=1.1.0
 Requires-Dist: colorlog<7.0.0,>=6.7.0
+Requires-Dist: docling<3.0.0,>=2.16.0
 Requires-Dist: docstring-parser<1.0,>=0.16
 Requires-Dist: duckduckgo-search<7.0.0,>=6.0.0
 Requires-Dist: faker<19.0.0,>=18.9.0
@@ -32,9 +33,10 @@ Requires-Dist: onnxruntime<2.0.0,>=1.16.1
 Requires-Dist: openai<2.0.0,>=1.45.0
 Requires-Dist: pandas<3.0.0,>=2.0.3
 Requires-Dist: prettytable<4.0.0,>=3.8.0
-Requires-Dist: pydantic<2.10.2,>=1
+Requires-Dist: pydantic<3.0.0,>=1
 Requires-Dist: pygithub<2.0.0,>=1.58.1
 Requires-Dist: pygments<3.0.0,>=2.15.1
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17
 Requires-Dist: pyparsing<4.0.0,>=3.0.9
 Requires-Dist: pytest-rerunfailures<16.0,>=15.0
 Requires-Dist: python-dotenv<2.0.0,>=1.0.0
@@ -55,14 +57,15 @@ Provides-Extra: all
 Requires-Dist: arango-datasets<2.0.0,>=1.2.2; extra == 'all'
 Requires-Dist: chainlit<3.0.0,>=2.0.1; extra == 'all'
 Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'all'
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'all'
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'all'
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'all'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'all'
 Requires-Dist: litellm<2.0.0,>=1.30.1; extra == 'all'
 Requires-Dist: metaphor-python<0.2.0,>=0.1.23; extra == 'all'
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'all'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'all'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'all'
 Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'all'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'all'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'all'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'all'
 Requires-Dist: pypdf>=5.1.0; extra == 'all'
@@ -74,7 +77,7 @@ Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'all'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'all'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'all'
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'all'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'all'
 Requires-Dist: weaviate-client>=4.9.6; extra == 'all'
 Provides-Extra: arango
 Requires-Dist: arango-datasets<2.0.0,>=1.2.2; extra == 'arango'
@@ -89,13 +92,16 @@ Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'db'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'db'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'db'
 Provides-Extra: doc-chat
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'doc-chat'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'doc-chat'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'doc-chat'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'doc-chat'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'doc-chat'
 Requires-Dist: pypdf>=5.1.0; extra == 'doc-chat'
 Requires-Dist: pytesseract<0.4.0,>=0.3.10; extra == 'doc-chat'
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'doc-chat'
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'doc-chat'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'doc-chat'
+Provides-Extra: docling
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'docling'
 Provides-Extra: docx
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'docx'
 Provides-Extra: fastembed
@@ -104,7 +110,7 @@ Provides-Extra: hf-embeddings
 Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'hf-embeddings'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'hf-embeddings'
 Provides-Extra: hf-transformers
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'hf-transformers'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'hf-transformers'
 Requires-Dist: sentence-transformers<3.0.0,>=2.2.2; extra == 'hf-transformers'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'hf-transformers'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'hf-transformers'
@@ -125,13 +131,16 @@ Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'mysql'
 Provides-Extra: neo4j
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'neo4j'
 Provides-Extra: pdf-parsers
+Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'pdf-parsers'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'pdf-parsers'
-Requires-Dist: pdfplumber<0.11.0,>=0.10.2; extra == 'pdf-parsers'
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'pdf-parsers'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'pdf-parsers'
 Requires-Dist: pypdf>=5.1.0; extra == 'pdf-parsers'
 Requires-Dist: pytesseract<0.4.0,>=0.3.10; extra == 'pdf-parsers'
 Provides-Extra: postgres
 Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'postgres'
+Provides-Extra: pymupdf4llm
+Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'pymupdf4llm'
 Provides-Extra: scrapy
 Requires-Dist: scrapy<3.0.0,>=2.11.0; extra == 'scrapy'
 Provides-Extra: sql
@@ -139,11 +148,11 @@ Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'sql'
 Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'sql'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'sql'
 Provides-Extra: transformers
-Requires-Dist: huggingface-hub<0.22.0,>=0.21.2; extra == 'transformers'
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'transformers'
 Requires-Dist: torch<3.0.0,>=2.0.0; extra == 'transformers'
 Requires-Dist: transformers<5.0.0,>=4.40.1; extra == 'transformers'
 Provides-Extra: unstructured
-Requires-Dist: unstructured[docx,pdf,pptx]<0.10.18,>=0.10.16; extra == 'unstructured'
+Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'unstructured'
 Provides-Extra: vecdbs
 Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'vecdbs'
 Requires-Dist: lancedb<0.9.0,>=0.8.2; extra == 'vecdbs'

{langroid-0.36.1 → langroid-0.37.0}/langroid/agent/special/doc_chat_agent.py

@@ -100,7 +100,7 @@ hf_embed_config = SentenceTransformerEmbeddingsConfig(
 
 oai_embed_config = OpenAIEmbeddingsConfig(
     model_type="openai",
-    model_name="text-embedding-ada-002",
+    model_name="text-embedding-3-small",
     dims=1536,
 )
 
@@ -189,8 +189,8 @@ class DocChatAgentConfig(ChatAgentConfig):
             # NOTE: PDF parsing is extremely challenging, and each library
             # has its own strengths and weaknesses.
             # Try one that works for your use case.
-            # or "unstructured", "pdfplumber", "fitz", "pypdf"
-            library="pdfplumber",
+            # or "unstructured", "fitz", "pymupdf4llm", "pypdf"
+            library="pymupdf4llm",
         ),
     )
 

{langroid-0.36.1 → langroid-0.37.0}/langroid/embedding_models/models.py

@@ -18,7 +18,7 @@ AzureADTokenProvider = Callable[[], str]
 
 class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "openai"
-    model_name: str = "text-embedding-ada-002"
+    model_name: str = "text-embedding-3-large"
     api_key: str = ""
     api_base: Optional[str] = None
     organization: str = ""
@@ -28,7 +28,7 @@ class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
 
 class AzureOpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "azure-openai"
-    model_name: str = "text-embedding-ada-002"
+    model_name: str = "text-embedding-3-large"
     api_key: str = ""
     api_base: str = ""
     deployment_name: Optional[str] = None

{langroid-0.36.1 → langroid-0.37.0}/langroid/exceptions.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 
 class XMLException(Exception):
@@ -15,7 +15,7 @@ class LangroidImportError(ImportError):
     def __init__(
         self,
         package: Optional[str] = None,
-        extra: Optional[str] = None,
+        extra: Optional[str | List[str]] = None,
         error: str = "",
         *args: object,
     ) -> None:
@@ -33,9 +33,21 @@ class LangroidImportError(ImportError):
             error = f"{package} is not installed by default with Langroid.\n"
 
         if extra:
+            if isinstance(extra, list):
+                help_preamble = f"""
+                If you want to use it, please install langroid with one of these 
+                extras: {', '.join(extra)}. The examples below use the first one, 
+                i.e. {extra[0]}.
+                """
+                extra = extra[0]
+            else:
+                help_preamble = f"""
+                If you want to use it, please install langroid with the
+                `{extra}` extra.
+                """
+
             install_help = f"""
-                If you want to use it, please install langroid 
-                with the `{extra}` extra, for example:
+                {help_preamble}
                 
                 If you are using pip:
                 pip install "langroid[{extra}]"

{langroid-0.36.1 → langroid-0.37.0}/langroid/parsing/code_parser.py

@@ -65,7 +65,7 @@ class CodeParsingConfig(BaseSettings):
         "bash",
     ]
     chunk_size: int = 500  # tokens
-    token_encoding_model: str = "text-embedding-ada-002"
+    token_encoding_model: str = "text-embedding-3-small"
     n_similar_docs: int = 4
 
 

{langroid-0.36.1 → langroid-0.37.0}/langroid/parsing/document_parser.py

@@ -3,9 +3,10 @@ from __future__ import annotations
 import itertools
 import logging
 import re
+import tempfile
 from enum import Enum
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, Generator, List, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
 from langroid.utils.object_registry import ObjectRegistry
@@ -15,18 +16,24 @@ try:
 except ImportError:
     if not TYPE_CHECKING:
         fitz = None
+try:
+    import pymupdf4llm
+except ImportError:
+    if not TYPE_CHECKING:
+        pymupdf4llm = None
 
 try:
-    import pypdf
+    import docling
 except ImportError:
     if not TYPE_CHECKING:
-        pypdf = None
+        docling = None
 
 try:
-    import pdfplumber
+    import pypdf
 except ImportError:
     if not TYPE_CHECKING:
-        pdfplumber = None
+        pypdf = None
+
 
 import requests
 from bs4 import BeautifulSoup
@@ -41,6 +48,7 @@ logger = logging.getLogger(__name__)
 
 
 class DocumentType(str, Enum):
+    # TODO add `md` (Markdown) and `html`
     PDF = "pdf"
     DOCX = "docx"
     DOC = "doc"
@@ -139,10 +147,12 @@ class DocumentParser(Parser):
         if inferred_doc_type == DocumentType.PDF:
             if config.pdf.library == "fitz":
                 return FitzPDFParser(source, config)
+            elif config.pdf.library == "pymupdf4llm":
+                return PyMuPDF4LLMParser(source, config)
+            elif config.pdf.library == "docling":
+                return DoclingParser(source, config)
             elif config.pdf.library == "pypdf":
                 return PyPDFParser(source, config)
-            elif config.pdf.library == "pdfplumber":
-                return PDFPlumberParser(source, config)
             elif config.pdf.library == "unstructured":
                 return UnstructuredPDFParser(source, config)
             elif config.pdf.library == "pdf2image":
@@ -307,8 +317,11 @@ class DocumentParser(Parser):
         """Yield each page in the PDF."""
         raise NotImplementedError
 
-    def extract_text_from_page(self, page: Any) -> str:
-        """Extract text from a given page."""
+    def get_document_from_page(self, page: Any) -> Document:
+        """
+        Get Langroid Document object (with possible metadata)
+        corresponding to a given page.
+        """
         raise NotImplementedError
 
     def fix_text(self, text: str) -> str:
@@ -335,7 +348,10 @@ class DocumentParser(Parser):
         """
 
         text = "".join(
-            [self.extract_text_from_page(page) for _, page in self.iterate_pages()]
+            [
+                self.get_document_from_page(page).content
+                for _, page in self.iterate_pages()
+            ]
         )
         return Document(content=text, metadata=DocMetaData(source=self.source))
 
@@ -359,7 +375,10 @@ class DocumentParser(Parser):
         common_id = ObjectRegistry.new_id()
         n_chunks = 0  # how many chunk so far
         for i, page in self.iterate_pages():
-            page_text = self.extract_text_from_page(page)
+            # not used but could be useful, esp to blend the
+            # metadata from the pages into the chunks
+            page_doc = self.get_document_from_page(page)
+            page_text = page_doc.content
             split += self.tokenizer.encode(page_text)
             pages.append(str(i + 1))
             # split could be so long it needs to be split
@@ -422,81 +441,152 @@ class FitzPDFParser(DocumentParser):
             yield i, page
         doc.close()
 
-    def extract_text_from_page(self, page: "fitz.Page") -> str:
+    def get_document_from_page(self, page: "fitz.Page") -> Document:
         """
-        Extract text from a given `fitz` page.
+        Get Document object from a given `fitz` page.
 
         Args:
             page (fitz.Page): The `fitz` page object.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.get_text())
+        return Document(
+            content=self.fix_text(page.get_text()),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
-class PyPDFParser(DocumentParser):
+class PyMuPDF4LLMParser(DocumentParser):
     """
-    Parser for processing PDFs using the `pypdf` library.
+    Parser for processing PDFs using the `pymupdf4llm` library.
     """
 
-    def iterate_pages(self) -> Generator[Tuple[int, pypdf.PageObject], None, None]:
+    def iterate_pages(self) -> Generator[Tuple[int, "fitz.Page"], None, None]:
         """
-        Yield each page in the PDF using `pypdf`.
+        Yield each page in the PDF using `fitz`.
 
         Returns:
-            Generator[pypdf.pdf.PageObject]: Generator yielding each page.
+            Generator[fitz.Page]: Generator yielding each page.
         """
-        if pypdf is None:
-            raise LangroidImportError("pypdf", "pdf-parsers")
-        reader = pypdf.PdfReader(self.doc_bytes)
-        for i, page in enumerate(reader.pages):
+        if fitz is None:
+            raise LangroidImportError(
+                "pymupdf4llm", ["pymupdf4llm", "all", "pdf-parsers", "doc-chat"]
+            )
+        doc: fitz.Document = fitz.open(stream=self.doc_bytes, filetype="pdf")
+        pages: List[Dict[str, Any]] = pymupdf4llm.to_markdown(doc, page_chunks=True)
+        for i, page in enumerate(pages):
             yield i, page
+        doc.close()
 
-    def extract_text_from_page(self, page: pypdf.PageObject) -> str:
+    def get_document_from_page(self, page: Dict[str, Any]) -> Document:
         """
-        Extract text from a given `pypdf` page.
+        Get Document object corresponding to a given "page-chunk"
+        dictionary, see:
+         https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/api.html
+
 
         Args:
-            page (pypdf.pdf.PageObject): The `pypdf` page object.
+            page (Dict[str,Any]): The "page-chunk" dictionary.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.extract_text())
+        return Document(
+            content=self.fix_text(page.get("text", "")),
+            # TODO could possible use other metadata from page, see above link.
+            metadata=DocMetaData(source=self.source),
+        )
 
 
-class PDFPlumberParser(DocumentParser):
+class DoclingParser(DocumentParser):
     """
-    Parser for processing PDFs using the `pdfplumber` library.
+    Parser for processing PDFs using the `docling` library.
     """
 
-    def iterate_pages(
-        self,
-    ) -> (Generator)[Tuple[int, pdfplumber.pdf.Page], None, None]:  # type: ignore
+    def iterate_pages(self) -> Generator[Tuple[int, Any], None, None]:
+        """
+        Yield each page in the PDF using `docling`.
+
+        Returns:
+            Generator[docling.Page]: Generator yielding each page.
         """
-        Yield each page in the PDF using `pdfplumber`.
+        if docling is None:
+            raise LangroidImportError(
+                "docling", ["docling", "pdf-parsers", "all", "doc-chat"]
+            )
+        from docling.datamodel.document import TextItem  # type: ignore
+        from docling.document_converter import (  # type: ignore
+            ConversionResult,
+            DocumentConverter,
+        )
+
+        converter = DocumentConverter()
+        file_path = self.source
+        if file_path == "bytes":
+            with tempfile.NamedTemporaryFile(delete=False) as tmp:
+                tmp.write(self.doc_bytes.getvalue())
+                file_path = tmp.name
+        result: ConversionResult = converter.convert(file_path)
+        doc = result.document
+        n_pages = doc.num_pages()  # type: ignore
+        for i in range(n_pages):
+            texts = [
+                item[0].text
+                for item in doc.iterate_items(page_no=i + 1)
+                if isinstance(item[0], TextItem)
+            ]
+            text = "\n".join(texts)
+            yield i, text
+
+    def get_document_from_page(self, page: str) -> Document:
+        """
+        Get Document object from a given `docling` "page" (actually a chunk).
+
+        Args:
+            page (docling.chunking.DocChunk): The `docling` chunk
 
         Returns:
-            Generator[pdfplumber.Page]: Generator yielding each page.
+            Document: Document object, with content and possible metadata.
         """
-        if pdfplumber is None:
-            raise LangroidImportError("pdfplumber", "pdf-parsers")
-        with pdfplumber.open(self.doc_bytes) as pdf:
-            for i, page in enumerate(pdf.pages):
-                yield i, page
+        return Document(
+            content=self.fix_text(page),
+            metadata=DocMetaData(source=self.source),
+        )
+
 
-    def extract_text_from_page(self, page: pdfplumber.pdf.Page) -> str:  # type: ignore
+class PyPDFParser(DocumentParser):
+    """
+    Parser for processing PDFs using the `pypdf` library.
+    """
+
+    def iterate_pages(self) -> Generator[Tuple[int, pypdf.PageObject], None, None]:
         """
-        Extract text from a given `pdfplumber` page.
+        Yield each page in the PDF using `pypdf`.
+
+        Returns:
+            Generator[pypdf.pdf.PageObject]: Generator yielding each page.
+        """
+        if pypdf is None:
+            raise LangroidImportError("pypdf", "pdf-parsers")
+        reader = pypdf.PdfReader(self.doc_bytes)
+        for i, page in enumerate(reader.pages):
+            yield i, page
+
+    def get_document_from_page(self, page: pypdf.PageObject) -> Document:
+        """
+        Get Document object from a given `pypdf` page.
 
         Args:
-            page (pdfplumber.Page): The `pdfplumber` page object.
+            page (pypdf.pdf.PageObject): The `pypdf` page object.
 
         Returns:
-            str: Extracted text from the page.
+            Document: Document object, with content and possible metadata.
         """
-        return self.fix_text(page.extract_text())
+        return Document(
+            content=self.fix_text(page.extract_text()),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class ImagePdfParser(DocumentParser):
@@ -516,15 +606,15 @@ class ImagePdfParser(DocumentParser):
         for i, image in enumerate(images):
             yield i, image
 
-    def extract_text_from_page(self, page: "Image") -> str:  # type: ignore
+    def get_document_from_page(self, page: "Image") -> Document:  # type: ignore
         """
-        Extract text from a given `pdf2image` page.
+        Get Document object corresponding to a given `pdf2image` page.
 
         Args:
             page (Image): The PIL Image object.
 
         Returns:
-            str: Extracted text from the image.
+            Document: Document object, with content and possible metadata.
         """
         try:
             import pytesseract
@@ -532,7 +622,10 @@ class ImagePdfParser(DocumentParser):
             raise LangroidImportError("pytesseract", "pdf-parsers")
 
         text = pytesseract.image_to_string(page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredPDFParser(DocumentParser):
@@ -564,8 +657,8 @@ class UnstructuredPDFParser(DocumentParser):
                 The `unstructured` library failed to parse the pdf.
                 Please try a different library by setting the `library` field
                 in the `pdf` section of the `parsing` field in the config file.
-                Supported libraries are:
-                fitz, pypdf, pdfplumber, unstructured
+                Other supported libraries are:
+                fitz, pymupdf4llm, pypdf
                 """
             )
 
@@ -584,18 +677,21 @@ class UnstructuredPDFParser(DocumentParser):
         if page_elements:
             yield page_number, page_elements
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given `unstructured` element.
+        Get Document object from a given `unstructured` element.
 
         Args:
             page (unstructured element): The `unstructured` element object.
 
         Returns:
-            str: Extracted text from the element.
+            Document: Document object, with content and possible metadata.
         """
         text = " ".join(el.text for el in page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredDocxParser(DocumentParser):
@@ -632,9 +728,9 @@ class UnstructuredDocxParser(DocumentParser):
         if page_elements:
             yield page_number, page_elements
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given `unstructured` element.
+        Get Document object from a given `unstructured` element.
 
         Note:
             The concept of "pages" doesn't actually exist in the .docx file format in
@@ -647,10 +743,13 @@ class UnstructuredDocxParser(DocumentParser):
             page (unstructured element): The `unstructured` element object.
 
         Returns:
-            str: Extracted text from the element.
+            Document object, with content and possible metadata.
         """
         text = " ".join(el.text for el in page)
-        return self.fix_text(text)
+        return Document(
+            content=self.fix_text(text),
+            metadata=DocMetaData(source=self.source),
+        )
 
 
 class UnstructuredDocParser(UnstructuredDocxParser):
@@ -704,15 +803,19 @@ class PythonDocxParser(DocumentParser):
         for i, para in enumerate(doc.paragraphs, start=1):
             yield i, [para]
 
-    def extract_text_from_page(self, page: Any) -> str:
+    def get_document_from_page(self, page: Any) -> Document:
         """
-        Extract text from a given 'page', which in this case is a single paragraph.
+        Get Document object from a given 'page', which in this case is a single
+        paragraph.
 
         Args:
             page (list): A list containing a single Paragraph object.
 
         Returns:
-            str: Extracted text from the paragraph.
+            Document: Document object, with content and possible metadata.
         """
         paragraph = page[0]
-        return self.fix_text(paragraph.text)
+        return Document(
+            content=self.fix_text(paragraph.text),
+            metadata=DocMetaData(source=self.source),
+        )

{langroid-0.36.1 → langroid-0.37.0}/langroid/parsing/parser.py

@@ -23,11 +23,12 @@ class Splitter(str, Enum):
 class PdfParsingConfig(BaseSettings):
     library: Literal[
         "fitz",
-        "pdfplumber",
+        "pymupdf4llm",
+        "docling",
         "pypdf",
         "unstructured",
         "pdf2image",
-    ] = "pdfplumber"
+    ] = "pymupdf4llm"
 
 
 class DocxParsingConfig(BaseSettings):
@@ -40,6 +41,7 @@ class DocParsingConfig(BaseSettings):
 
 class ParsingConfig(BaseSettings):
     splitter: str = Splitter.TOKENS
+    chunk_by_page: bool = False  # split by page?
     chunk_size: int = 200  # aim for this many tokens per chunk
     overlap: int = 50  # overlap between chunks
     max_chunks: int = 10_000
@@ -49,7 +51,7 @@ class ParsingConfig(BaseSettings):
     n_similar_docs: int = 4
     n_neighbor_ids: int = 5  # window size to store around each chunk
     separators: List[str] = ["\n\n", "\n", " ", ""]
-    token_encoding_model: str = "text-embedding-ada-002"
+    token_encoding_model: str = "text-embedding-3-large"
     pdf: PdfParsingConfig = PdfParsingConfig()
     docx: DocxParsingConfig = DocxParsingConfig()
     doc: DocParsingConfig = DocParsingConfig()
@@ -61,7 +63,7 @@ class Parser:
         try:
             self.tokenizer = tiktoken.encoding_for_model(config.token_encoding_model)
         except Exception:
-            self.tokenizer = tiktoken.encoding_for_model("text-embedding-ada-002")
+            self.tokenizer = tiktoken.encoding_for_model("text-embedding-3-small")
 
     def num_tokens(self, text: str) -> int:
         tokens = self.tokenizer.encode(text)