langroid 0.37.1__tar.gz → 0.37.3__tar.gz

{langroid-0.37.1 → langroid-0.37.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.37.1
+Version: 0.37.3
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -102,7 +102,6 @@ Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'doc-chat'
 Requires-Dist: unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15; extra == 'doc-chat'
 Provides-Extra: docling
 Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'docling'
-Requires-Dist: pypdf>=5.1.0; extra == 'docling'
 Provides-Extra: docx
 Requires-Dist: python-docx<2.0.0,>=1.1.0; extra == 'docx'
 Provides-Extra: fastembed

{langroid-0.37.1 → langroid-0.37.3}/langroid/agent/special/doc_chat_agent.py

@@ -15,7 +15,6 @@ pip install "langroid[hf-embeddings]"
 """
 
 import logging
-import textwrap
 from collections import OrderedDict
 from functools import cache
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, no_type_check
@@ -82,7 +81,7 @@ You will be given various passages from these documents, and asked to answer que
 about them, or summarize them into coherent answers.
 """
 
-CHUNK_ENRICHMENT_DELIMITER = "\n<##-##-##>"
+CHUNK_ENRICHMENT_DELIMITER = "\n<##-##-##>\n"
 
 has_sentence_transformers = False
 try:
@@ -805,9 +804,9 @@ class DocChatAgent(ChatAgent):
         Returns:
             str: string representation
         """
-        contents = [f"Extract: {d.content}" for d in docs]
+        contents = [d.content for d in docs]
         sources = [d.metadata.source for d in docs]
-        sources = [f"Source: {s}" if s is not None else "" for s in sources]
+        sources = [f"SOURCE: {s}" if s is not None else "" for s in sources]
         return "\n".join(
             [
                 f"""
@@ -952,12 +951,8 @@ class DocChatAgent(ChatAgent):
                     continue
 
                 # Combine original content with questions in a structured way
-                combined_content = textwrap.dedent(
-                    f"""\
-                {doc.content}                
-                {enrichment_config.delimiter}
-                {enrichment}
-                """
+                combined_content = (
+                    f"{doc.content}{enrichment_config.delimiter}{enrichment}"
                 )
 
                 new_doc = doc.copy(

{langroid-0.37.1 → langroid-0.37.3}/langroid/embedding_models/models.py

@@ -18,7 +18,7 @@ AzureADTokenProvider = Callable[[], str]
 
 class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "openai"
-    model_name: str = "text-embedding-3-large"
+    model_name: str = "text-embedding-3-small"
     api_key: str = ""
     api_base: Optional[str] = None
     organization: str = ""
@@ -28,7 +28,7 @@ class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
 
 class AzureOpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     model_type: str = "azure-openai"
-    model_name: str = "text-embedding-3-large"
+    model_name: str = "text-embedding-3-small"
     api_key: str = ""
     api_base: str = ""
     deployment_name: Optional[str] = None

{langroid-0.37.1 → langroid-0.37.3}/langroid/parsing/document_parser.py

@@ -2,13 +2,16 @@ from __future__ import annotations
 
 import itertools
 import logging
+import os
 import re
+import tempfile
 from enum import Enum
 from io import BytesIO
+from itertools import accumulate
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
-from langroid.parsing.pdf_utils import pdf_split_pages
 from langroid.utils.object_registry import ObjectRegistry
 
 try:
@@ -507,6 +510,8 @@ class DoclingParser(DocumentParser):
     def iterate_pages(self) -> Generator[Tuple[int, Any], None, None]:
         """
         Yield each page in the PDF using `docling`.
+        Code largely from this example:
+        https://github.com/DS4SD/docling/blob/4d41db3f7abb86c8c65386bf94e7eb0bf22bb82b/docs/examples/export_figures.py
 
         Returns:
             Generator[docling.Page]: Generator yielding each page.
@@ -516,35 +521,74 @@ class DoclingParser(DocumentParser):
                 "docling", ["docling", "pdf-parsers", "all", "doc-chat"]
             )
 
+        from docling.datamodel.base_models import InputFormat  # type: ignore
+        from docling.datamodel.pipeline_options import PdfPipelineOptions
         from docling.document_converter import (  # type: ignore
             ConversionResult,
             DocumentConverter,
+            PdfFormatOption,
         )
         from docling_core.types.doc import ImageRefMode  # type: ignore
 
-        page_files, tmp_dir = pdf_split_pages(self.doc_bytes)
-        converter = DocumentConverter()
-        for i, page_file in enumerate(page_files):
-            result: ConversionResult = converter.convert(page_file)
-            md_text = result.document.export_to_markdown(
-                image_mode=ImageRefMode.REFERENCED
-            )
-            yield i, md_text
+        IMAGE_RESOLUTION_SCALE = 2.0
+        pipeline_options = PdfPipelineOptions()
+        pipeline_options.images_scale = IMAGE_RESOLUTION_SCALE
+        pipeline_options.generate_page_images = True
+        pipeline_options.generate_picture_images = True
 
-        tmp_dir.cleanup()
+        converter = DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
+            }
+        )
+        doc_path = self.source
+        if doc_path == "bytes":
+            # write to tmp file, then use that path
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
+                temp_file.write(self.doc_bytes.getvalue())
+                doc_path = temp_file.name
+
+        output_dir = Path(str(Path(doc_path).with_suffix("")) + "-pages")
+        os.makedirs(output_dir, exist_ok=True)
+
+        result: ConversionResult = converter.convert(doc_path)
+
+        def n_page_elements(page) -> int:  # type: ignore
+            if page.assembled is None:
+                return 0
+            return 1 + len(page.assembled.elements)
+
+        page_element_count = [n_page_elements(i) for i in result.pages]
+        element_page_cutoff = list(accumulate([1] + page_element_count))
+        for i, page in enumerate(result.pages):
+            page_start = element_page_cutoff[i]
+            page_end = element_page_cutoff[i + 1]
+            md_file = output_dir / f"page_{i}.md"
+            # we could have just directly exported to a markdown string,
+            # but we need to save to a file to force generation of image-files.
+            result.document.save_as_markdown(
+                md_file,
+                image_mode=ImageRefMode.REFERENCED,
+                from_element=page_start,
+                to_element=page_end,
+            )
+            yield i, md_file
 
-    def get_document_from_page(self, page: str) -> Document:
+    def get_document_from_page(self, md_file: str) -> Document:
         """
-        Get Document object from a given `docling` "page" (actually a chunk).
+        Get Document object from a given 1-page markdown file,
+        possibly containing image refs.
 
         Args:
-            page (docling.chunking.DocChunk): The `docling` chunk
+            md_file (str): The markdown file path for the page.
 
         Returns:
             Document: Document object, with content and possible metadata.
         """
+        with open(md_file, "r") as f:
+            text = f.read()
         return Document(
-            content=self.fix_text(page),
+            content=self.fix_text(text),
             metadata=DocMetaData(source=self.source),
         )
 

{langroid-0.37.1 → langroid-0.37.3}/langroid/parsing/parser.py

@@ -51,7 +51,7 @@ class ParsingConfig(BaseSettings):
     n_similar_docs: int = 4
     n_neighbor_ids: int = 5  # window size to store around each chunk
     separators: List[str] = ["\n\n", "\n", " ", ""]
-    token_encoding_model: str = "text-embedding-3-large"
+    token_encoding_model: str = "text-embedding-3-small"
     pdf: PdfParsingConfig = PdfParsingConfig()
     docx: DocxParsingConfig = DocxParsingConfig()
     doc: DocParsingConfig = DocParsingConfig()

{langroid-0.37.1 → langroid-0.37.3}/langroid/parsing/pdf_utils.py

@@ -5,27 +5,24 @@ from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, BinaryIO, List, Tuple, Union
 
 try:
-    import pypdf
+    import fitz
 except ImportError:
     if not TYPE_CHECKING:
-        pypdf = None
+        fitz = None
 
 from langroid.exceptions import LangroidImportError
 
-if pypdf is None:
-    raise LangroidImportError(
-        "pypdf", ["pypdf", "docling", "all", "pdf-parsers", "doc-chat"]
-    )
-from pypdf import PdfReader, PdfWriter
+if fitz is None:
+    raise LangroidImportError("fitz", ["pymupdf", "all", "pdf-parsers", "doc-chat"])
 
 
 def pdf_split_pages(
-    input_pdf: Union[str, Path, BytesIO, BinaryIO],
+    input_pdf: Union[BytesIO, BinaryIO],
 ) -> Tuple[List[Path], TemporaryDirectory[Any]]:
     """Splits a PDF into individual pages in a temporary directory.
 
     Args:
-        input_pdf: Input PDF file path or file-like object
+        input_pdf: Input PDF file in bytes or binary mode
         max_workers: Maximum number of concurrent workers for parallel processing
 
     Returns:
@@ -39,17 +36,16 @@ def pdf_split_pages(
         tmp_dir.cleanup()  # Clean up temp files when done
     """
     tmp_dir = tempfile.TemporaryDirectory()
-    reader = PdfReader(input_pdf)
+    doc = fitz.open(stream=input_pdf, filetype="pdf")
     paths = []
 
-    for i in range(len(reader.pages)):
-        writer = PdfWriter()
-        writer.add_page(reader.pages[i])
-        writer.add_metadata(reader.metadata or {})
-
-        output = Path(tmp_dir.name) / f"page_{i+1}.pdf"
-        with open(output, "wb") as f:
-            writer.write(f)
+    for page_num in range(len(doc)):
+        new_doc = fitz.open()
+        new_doc.insert_pdf(doc, from_page=page_num, to_page=page_num)
+        output = Path(tmp_dir.name) / f"page_{page_num + 1}.pdf"
+        new_doc.save(str(output))
+        new_doc.close()
         paths.append(output)
 
-    return paths, tmp_dir  # Return dir object so caller can control cleanup
+    doc.close()
+    return paths, tmp_dir

{langroid-0.37.1 → langroid-0.37.3}/langroid/parsing/search.py

@@ -7,6 +7,7 @@ See tests for examples: tests/main/test_string_search.py
 """
 
 import difflib
+import re
 from typing import List, Tuple
 
 from nltk.corpus import stopwords
@@ -195,8 +196,10 @@ def get_context(
 
     Returns:
     str: A string containing b words before, the match, and a words after
-        the best approximate match position of the query in the text. If no
-        match is found, returns empty string.
+        the best approximate match position of the query in the text.
+        The text is extracted from the original `text`, preserving formatting,
+        whitespace, etc, so it does not disturb any downstream processing.
+        If no match is found, returns empty string.
     int: The start position of the match in the text.
     int: The end position of the match in the text.
 
@@ -204,6 +207,8 @@ def get_context(
     >>> get_context("apple", "The quick brown fox jumps over the apple.", 3, 2)
     # 'fox jumps over the apple.'
     """
+
+    # If no word limits specified, return full text
     if words_after is None and words_before is None:
         # return entire text since we're not asked to return a bounded context
         return text, 0, 0
@@ -212,23 +217,35 @@ def get_context(
     if fuzz.partial_ratio(query, text) < 40:
         return "", 0, 0
 
+    # Find best matching position of query in text
     sequence_matcher = difflib.SequenceMatcher(None, text, query)
     match = sequence_matcher.find_longest_match(0, len(text), 0, len(query))
 
     if match.size == 0:
         return "", 0, 0
 
+    # Count words before match point
     segments = text.split()
     n_segs = len(segments)
-
     start_segment_pos = len(text[: match.a].split())
 
+    # Calculate word window boundaries
     words_before = words_before or n_segs
     words_after = words_after or n_segs
     start_pos = max(0, start_segment_pos - words_before)
     end_pos = min(len(segments), start_segment_pos + words_after + len(query.split()))
 
-    return " ".join(segments[start_pos:end_pos]), start_pos, end_pos
+    # Find character positions where words start
+    word_positions = [m.start() for m in re.finditer(r"\S+", text)]
+
+    # Convert word positions to character positions
+    start_char = word_positions[start_pos] if start_pos < len(word_positions) else 0
+    end_char = word_positions[min(end_pos, len(word_positions) - 1)] + len(
+        text.split()[min(end_pos - 1, len(word_positions) - 1)]
+    )
+
+    # return exact substring with original formatting
+    return text[start_char:end_char], start_pos, end_pos
 
 
 def eliminate_near_duplicates(passages: List[str], threshold: float = 0.8) -> List[str]:

{langroid-0.37.1 → langroid-0.37.3}/langroid/vector_store/weaviatedb.py

@@ -211,10 +211,10 @@ class WeaviateDB(VectorStore):
             return_properties=True,
             return_metadata=MetadataQuery(distance=True),
         )
-        return [
-            (self.weaviate_obj_to_doc(item), 1 - (item.metadata.distance or 1))
-            for item in response.objects
-        ]
+        maybe_distances = [item.metadata.distance for item in response.objects]
+        similarities = [0 if d is None else 1 - d for d in maybe_distances]
+        docs = [self.weaviate_obj_to_doc(item) for item in response.objects]
+        return list(zip(docs, similarities))
 
     def _create_valid_uuid_id(self, id: str) -> Any:
         try:

{langroid-0.37.1 → langroid-0.37.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.37.1"
+version = "0.37.3"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]
@@ -128,7 +128,6 @@ lancedb = [
 
 docling = [
     "docling<3.0.0,>=2.16.0",
-    "pypdf>=5.1.0", # needed to split pdf into pages, then use docling
 ]
 
 pymupdf4llm = [