langroid 0.1.211__tar.gz → 0.1.213__tar.gz

{langroid-0.1.211 → langroid-0.1.213}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.211
+Version: 0.1.213
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -63,6 +63,7 @@ Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: onnxruntime (==1.16.1)
 Requires-Dist: openai (>=1.2.3,<2.0.0)
 Requires-Dist: pandas (>=2.0.3,<3.0.0)
+Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pdfplumber (>=0.10.2,<0.11.0)
 Requires-Dist: pre-commit (>=3.3.2,<4.0.0)
 Requires-Dist: prettytable (>=3.8.0,<4.0.0)
@@ -75,6 +76,7 @@ Requires-Dist: pymupdf (>=1.23.3,<2.0.0)
 Requires-Dist: pymysql (>=1.1.0,<2.0.0) ; extra == "mysql"
 Requires-Dist: pyparsing (>=3.0.9,<4.0.0)
 Requires-Dist: pypdf (>=3.12.2,<4.0.0)
+Requires-Dist: pytesseract (>=0.3.10,<0.4.0)
 Requires-Dist: pytest-asyncio (>=0.21.1,<0.22.0)
 Requires-Dist: pytest-mysql (>=2.4.2,<3.0.0) ; extra == "mysql"
 Requires-Dist: pytest-postgresql (>=5.0.0,<6.0.0) ; extra == "postgres"
@@ -228,6 +230,22 @@ teacher_task.run()
 <details>
 <summary> <b>Click to expand</b></summary>
 
+- **Mar 2024:**
+  - **0.1.212:** ImagePdfParser: support for extracting text from image-based PDFs.
+    (this means `DocChatAgent` will now work with image-pdfs).
+  - **0.1.194 - 0.1.211:** Misc fixes, improvements, and features:
+    - Big enhancement in RAG performance (mainly, recall) due to a [fix in Relevance 
+      Extractor](https://github.com/langroid/langroid/releases/tag/0.1.209)
+    - `DocChatAgent` [context-window fixes](https://github.com/langroid/langroid/releases/tag/0.1.208)
+    - Anthropic/Claude3 support via Litellm
+    - `URLLoader`: detect file time from header when URL doesn't end with a 
+      recognizable suffix like `.pdf`, `.docx`, etc.
+    - Misc lancedb integration fixes
+    - Auto-select embedding config based on whether `sentence_transformer` module is available.
+    - Slim down dependencies, make some heavy ones optional, e.g. `unstructured`, 
+      `haystack`, `chromadb`, `mkdocs`, `huggingface-hub`, `sentence-transformers`.
+    - Easier top-level imports from `import langroid as lr`
+    - Improve JSON detection, esp from weak LLMs
 - **Feb 2024:** 
   - **0.1.193:** Support local LLMs using Ollama's new OpenAI-Compatible server: 
      simply specify `chat_model="ollama/mistral"`. See [release notes](https://github.com/langroid/langroid/releases/tag/0.1.193).

{langroid-0.1.211 → langroid-0.1.213}/README.md

@@ -122,6 +122,22 @@ teacher_task.run()
 <details>
 <summary> <b>Click to expand</b></summary>
 
+- **Mar 2024:**
+  - **0.1.212:** ImagePdfParser: support for extracting text from image-based PDFs.
+    (this means `DocChatAgent` will now work with image-pdfs).
+  - **0.1.194 - 0.1.211:** Misc fixes, improvements, and features:
+    - Big enhancement in RAG performance (mainly, recall) due to a [fix in Relevance 
+      Extractor](https://github.com/langroid/langroid/releases/tag/0.1.209)
+    - `DocChatAgent` [context-window fixes](https://github.com/langroid/langroid/releases/tag/0.1.208)
+    - Anthropic/Claude3 support via Litellm
+    - `URLLoader`: detect file time from header when URL doesn't end with a 
+      recognizable suffix like `.pdf`, `.docx`, etc.
+    - Misc lancedb integration fixes
+    - Auto-select embedding config based on whether `sentence_transformer` module is available.
+    - Slim down dependencies, make some heavy ones optional, e.g. `unstructured`, 
+      `haystack`, `chromadb`, `mkdocs`, `huggingface-hub`, `sentence-transformers`.
+    - Easier top-level imports from `import langroid as lr`
+    - Improve JSON detection, esp from weak LLMs
 - **Feb 2024:** 
   - **0.1.193:** Support local LLMs using Ollama's new OpenAI-Compatible server: 
      simply specify `chat_model="ollama/mistral"`. See [release notes](https://github.com/langroid/langroid/releases/tag/0.1.193).

{langroid-0.1.211 → langroid-0.1.213}/langroid/agent/base.py

@@ -37,7 +37,7 @@ from langroid.language_models.base import (
 )
 from langroid.language_models.openai_gpt import OpenAIGPTConfig
 from langroid.mytypes import Entity
-from langroid.parsing.json import extract_top_level_json
+from langroid.parsing.parse_json import extract_top_level_json
 from langroid.parsing.parser import Parser, ParsingConfig
 from langroid.prompts.prompts_config import PromptsConfig
 from langroid.utils.configuration import settings

{langroid-0.1.211 → langroid-0.1.213}/langroid/agent/batch.py

@@ -26,6 +26,7 @@ def run_batch_tasks(
     input_map: Callable[[Any], str | ChatDocument] = lambda x: str(x),
     output_map: Callable[[ChatDocument | None], Any] = lambda x: x,
     sequential: bool = True,
+    turns: int = -1,
 ) -> List[Any]:
     """
     Run copies of `task` async/concurrently one per item in `items` list.
@@ -40,6 +41,7 @@ def run_batch_tasks(
             to final result
         sequential (bool): whether to run sequentially
             (e.g. some APIs such as ooba don't support concurrent requests)
+        turns (int): number of turns to run, -1 for infinite
 
     Returns:
         List[Any]: list of final results
@@ -53,7 +55,7 @@ def run_batch_tasks(
             task_i.agent.llm.set_stream(False)
         task_i.agent.config.show_stats = False
 
-        result = await task_i.run_async(input)
+        result = await task_i.run_async(input, turns=turns)
         return output_map(result)
 
     async def _do_all() -> List[Any]:

{langroid-0.1.211 → langroid-0.1.213}/langroid/agent/chat_document.py

@@ -13,7 +13,7 @@ from langroid.language_models.base import (
 )
 from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.agent_chats import parse_message
-from langroid.parsing.json import extract_top_level_json, top_level_json_field
+from langroid.parsing.parse_json import extract_top_level_json, top_level_json_field
 from langroid.utils.output.printing import shorten_text
 
 

{langroid-0.1.211 → langroid-0.1.213}/langroid/agent/openai_assistant.py

@@ -8,7 +8,7 @@ from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Type, cast, no_type_check
 
 from openai.types.beta import Assistant, Thread
-from openai.types.beta.threads import Run, ThreadMessage
+from openai.types.beta.threads import Message, Run
 from openai.types.beta.threads.runs import RunStep
 from pydantic import BaseModel
 from rich import print
@@ -41,6 +41,8 @@ class AssistantTool(BaseModel):
     def dct(self) -> Dict[str, Any]:
         d = super().dict()
         d["type"] = d["type"].value
+        if self.type != ToolType.FUNCTION:
+            d.pop("function")
         return d
 
 
@@ -257,22 +259,22 @@ class OpenAIAssistant(ChatAgent):
         self.llm.cache.store(assistant_key, self.assistant.id)
 
     @staticmethod
-    def thread_msg_to_llm_msg(msg: ThreadMessage) -> LLMMessage:
+    def thread_msg_to_llm_msg(msg: Message) -> LLMMessage:
         """
-        Convert a ThreadMessage to an LLMMessage
+        Convert a Message to an LLMMessage
         """
         return LLMMessage(
             content=msg.content[0].text.value,  # type: ignore
             role=msg.role,
         )
 
-    def _update_messages_hash(self, msg: ThreadMessage | LLMMessage) -> None:
+    def _update_messages_hash(self, msg: Message | LLMMessage) -> None:
         """
         Update the hash-state in the thread with the given message.
         """
         if self.thread is None:
             raise ValueError("Thread is None")
-        if isinstance(msg, ThreadMessage):
+        if isinstance(msg, Message):
             llm_msg = self.thread_msg_to_llm_msg(msg)
         else:
             llm_msg = msg
@@ -491,7 +493,7 @@ class OpenAIAssistant(ChatAgent):
             LLMMessage(
                 # TODO: could be image, deal with it later
                 content=m.content[0].text.value,  # type: ignore
-                role=m.role,
+                role=Role(m.role),
             )
             for m in thread_msgs
         ]
@@ -646,7 +648,7 @@ class OpenAIAssistant(ChatAgent):
             tool_outputs=tool_outputs,  # type: ignore
         )
 
-    def process_citations(self, thread_msg: ThreadMessage) -> None:
+    def process_citations(self, thread_msg: Message) -> None:
         """
         Process citations in the thread message.
         Modifies the thread message in-place.

{langroid-0.1.211 → langroid-0.1.213}/langroid/agent/task.py

@@ -29,7 +29,7 @@ from langroid.agent.chat_document import (
     ChatDocument,
 )
 from langroid.mytypes import Entity
-from langroid.parsing.json import extract_top_level_json
+from langroid.parsing.parse_json import extract_top_level_json
 from langroid.utils.configuration import settings
 from langroid.utils.constants import DONE, NO_ANSWER, PASS, PASS_TO, SEND_TO, USER_QUIT
 from langroid.utils.logging import RichFileLogger, setup_file_logger

{langroid-0.1.211 → langroid-0.1.213}/langroid/language_models/base.py

@@ -14,7 +14,7 @@ from langroid.cachedb.momento_cachedb import MomentoCacheConfig
 from langroid.cachedb.redis_cachedb import RedisCacheConfig
 from langroid.mytypes import Document
 from langroid.parsing.agent_chats import parse_message
-from langroid.parsing.json import top_level_json_field
+from langroid.parsing.parse_json import top_level_json_field
 from langroid.prompts.dialog import collate_chat_history
 from langroid.prompts.templates import (
     EXTRACTION_PROMPT_GPT4,

{langroid-0.1.211 → langroid-0.1.213}/langroid/parsing/__init__.py

@@ -2,7 +2,7 @@ from . import parser
 from . import agent_chats
 from . import code_parser
 from . import document_parser
-from . import json
+from . import parse_json
 from . import para_sentence_split
 from . import repo_loader
 from . import url_loader
@@ -13,12 +13,14 @@ from . import search
 from . import web_search
 from . import spider
 
+parse_json  # appease mypy
+
 __all__ = [
     "parser",
     "agent_chats",
     "code_parser",
     "document_parser",
-    "json",
+    "parse_json.py",
     "para_sentence_split",
     "repo_loader",
     "url_loader",

{langroid-0.1.211 → langroid-0.1.213}/langroid/parsing/document_parser.py

@@ -8,6 +8,7 @@ import fitz
 import pdfplumber
 import pypdf
 import requests
+from PIL import Image
 
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.parser import Parser, ParsingConfig
@@ -53,6 +54,8 @@ class DocumentParser(Parser):
                 return PDFPlumberParser(source, config)
             elif config.pdf.library == "unstructured":
                 return UnstructuredPDFParser(source, config)
+            elif config.pdf.library == "pdf2image":
+                return ImagePdfParser(source, config)
             else:
                 raise ValueError(
                     f"Unsupported PDF library specified: {config.pdf.library}"
@@ -298,6 +301,36 @@ class PDFPlumberParser(DocumentParser):
         return self.fix_text(page.extract_text())
 
 
+class ImagePdfParser(DocumentParser):
+    """
+    Parser for processing PDFs that are images, i.e. not "true" PDFs.
+    """
+
+    def iterate_pages(
+        self,
+    ) -> Generator[Tuple[int, Image], None, None]:
+        from pdf2image import convert_from_bytes
+
+        images = convert_from_bytes(self.doc_bytes.getvalue())
+        for i, image in enumerate(images):
+            yield i, image
+
+    def extract_text_from_page(self, page: Image) -> str:
+        """
+        Extract text from a given `pdf2image` page.
+
+        Args:
+            page (Image): The PIL Image object.
+
+        Returns:
+            str: Extracted text from the image.
+        """
+        import pytesseract
+
+        text = pytesseract.image_to_string(page)
+        return self.fix_text(text)
+
+
 class UnstructuredPDFParser(DocumentParser):
     """
     Parser for processing PDF files using the `unstructured` library.

langroid-0.1.213/langroid/parsing/image_text.py

@@ -0,0 +1,32 @@
+from typing import Union
+
+import pytesseract
+from pdf2image import convert_from_bytes, convert_from_path
+
+
+def pdf_image_to_text(input_data: Union[str, bytes]) -> str:
+    """
+    Converts a PDF that contains images to text using OCR.
+
+    Args:
+        input_data (Union[str, bytes]): The file path to the PDF or a bytes-like object
+            of the PDF content.
+
+    Returns:
+        str: The extracted text from the PDF.
+    """
+
+    # Check if the input is a file path (str) or bytes, and
+    # convert PDF to images accordingly
+    if isinstance(input_data, str):
+        images = convert_from_path(input_data)
+    elif isinstance(input_data, bytes):
+        images = convert_from_bytes(input_data)
+    else:
+        raise ValueError("input_data must be a file path (str) or bytes-like object")
+
+    text = ""
+    for image in images:
+        text += pytesseract.image_to_string(image)
+
+    return text

{langroid-0.1.211 → langroid-0.1.213}/langroid/parsing/parser.py

@@ -19,7 +19,13 @@ class Splitter(str, Enum):
 
 
 class PdfParsingConfig(BaseSettings):
-    library: Literal["fitz", "pdfplumber", "pypdf", "unstructured"] = "pdfplumber"
+    library: Literal[
+        "fitz",
+        "pdfplumber",
+        "pypdf",
+        "unstructured",
+        "pdf2image",
+    ] = "pdfplumber"
 
 
 class DocxParsingConfig(BaseSettings):

{langroid-0.1.211 → langroid-0.1.213}/langroid/parsing/repo_loader.py

@@ -19,7 +19,7 @@ from github.Repository import Repository
 from pydantic import BaseModel, BaseSettings, Field
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.document_parser import DocumentParser
+from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
 from langroid.parsing.parser import Parser, ParsingConfig
 
 logger = logging.getLogger(__name__)
@@ -550,7 +550,11 @@ class RepoLoader:
                     file_path,
                     parser.config,
                 )
-                docs.extend(doc_parser.get_doc_chunks())
+                new_chunks = doc_parser.get_doc_chunks()
+                if len(new_chunks) == 0 and file_extension.lower() == ".pdf":
+                    doc_parser = ImagePdfParser(file_path, parser.config)
+                    new_chunks = doc_parser.get_doc_chunks()
+                docs.extend(new_chunks)
             else:
                 with open(file_path, "r") as f:
                     if lines is not None:

{langroid-0.1.211 → langroid-0.1.213}/langroid/parsing/url_loader.py

@@ -12,7 +12,7 @@ from trafilatura.downloads import (
 )
 
 from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.document_parser import DocumentParser
+from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
 from langroid.parsing.parser import Parser, ParsingConfig
 
 logging.getLogger("trafilatura").setLevel(logging.ERROR)
@@ -56,7 +56,12 @@ class URLLoader:
                         url,
                         self.parser.config,
                     )
-                    docs.extend(doc_parser.get_doc_chunks())
+                    new_chunks = doc_parser.get_doc_chunks()
+                    if len(new_chunks) == 0:
+                        # If the document is empty, try to extract images
+                        img_parser = ImagePdfParser(url, self.parser.config)
+                        new_chunks = img_parser.get_doc_chunks()
+                    docs.extend(new_chunks)
                 else:
                     # Try to detect content type and handle accordingly
                     headers = requests.head(url).headers

{langroid-0.1.211 → langroid-0.1.213}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.211"
+version = "0.1.213"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"
@@ -92,6 +92,8 @@ chainlit = {version = "^1.0.400", optional = true}
 python-socketio = {version="^5.11.0", optional=true}
 duckduckgo-search = "^4.4"
 huggingface-hub = {version="^0.21.2", optional=true}
+pdf2image = "^1.17.0"
+pytesseract = "^0.3.10"
 
 [tool.poetry.extras]
 # install these using `poetry install -E [...]` where [...] is one of the extras below
@@ -129,6 +131,7 @@ python_version = "3.11"
 #check_untyped_defs = "True"
 disallow_untyped_defs = "True"
 ignore_missing_imports = "True"
+warn_unused_ignores = "False"
 strict = true
 exclude = [
     "docs", ".venv", "venv", "examples", "examples_dev", "langroid/utils/web",