langroid 0.1.253__tar.gz → 0.1.256__tar.gz

{langroid-0.1.253 → langroid-0.1.256}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.253
+Version: 0.1.256
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -12,6 +12,10 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: chainlit
 Provides-Extra: chromadb
+Provides-Extra: db
+Provides-Extra: doc-chat
+Provides-Extra: docx
+Provides-Extra: embeddings
 Provides-Extra: hf-embeddings
 Provides-Extra: lancedb
 Provides-Extra: litellm
@@ -23,6 +27,7 @@ Provides-Extra: mysql
 Provides-Extra: neo4j
 Provides-Extra: pdf-parsers
 Provides-Extra: postgres
+Provides-Extra: scrapy
 Provides-Extra: sql
 Provides-Extra: transformers
 Provides-Extra: unstructured
@@ -42,7 +47,7 @@ Requires-Dist: google-generativeai (>=0.5.2,<0.6.0)
 Requires-Dist: groq (>=0.5.0,<0.6.0)
 Requires-Dist: grpcio (>=1.62.1,<2.0.0)
 Requires-Dist: halo (>=0.0.31,<0.0.32)
-Requires-Dist: huggingface-hub (>=0.21.2,<0.22.0) ; extra == "transformers"
+Requires-Dist: huggingface-hub (>=0.21.2,<0.22.0) ; extra == "embeddings" or extra == "transformers"
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: lancedb (>=0.6.2,<0.7.0) ; extra == "lancedb"
 Requires-Dist: litellm (>=1.30.1,<2.0.0) ; extra == "litellm"
@@ -66,40 +71,41 @@ Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: onnxruntime (>=1.16.1,<2.0.0)
 Requires-Dist: openai (>=1.14.0,<2.0.0)
 Requires-Dist: pandas (>=2.0.3,<3.0.0)
-Requires-Dist: pdf2image (>=1.17.0,<2.0.0) ; extra == "pdf-parsers"
-Requires-Dist: pdfplumber (>=0.10.2,<0.11.0)
+Requires-Dist: pdf2image (>=1.17.0,<2.0.0) ; extra == "doc-chat" or extra == "pdf-parsers"
+Requires-Dist: pdfplumber (>=0.10.2,<0.11.0) ; extra == "doc-chat" or extra == "pdf-parsers"
 Requires-Dist: prettytable (>=3.8.0,<4.0.0)
-Requires-Dist: psycopg2 (>=2.9.7,<3.0.0) ; extra == "postgres" or extra == "sql"
+Requires-Dist: psycopg2 (>=2.9.7,<3.0.0) ; extra == "db" or extra == "postgres" or extra == "sql"
 Requires-Dist: pyarrow (==15.0.0) ; extra == "lancedb"
 Requires-Dist: pydantic (==1.10.13)
 Requires-Dist: pygithub (>=1.58.1,<2.0.0)
 Requires-Dist: pygments (>=2.15.1,<3.0.0)
-Requires-Dist: pymupdf (>=1.23.3,<2.0.0) ; extra == "pdf-parsers"
-Requires-Dist: pymysql (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "sql"
+Requires-Dist: pymupdf (>=1.23.3,<2.0.0) ; extra == "doc-chat" or extra == "pdf-parsers"
+Requires-Dist: pymysql (>=1.1.0,<2.0.0) ; extra == "db" or extra == "mysql" or extra == "sql"
 Requires-Dist: pyparsing (>=3.0.9,<4.0.0)
-Requires-Dist: pypdf (>=3.12.2,<4.0.0) ; extra == "pdf-parsers"
-Requires-Dist: pytesseract (>=0.3.10,<0.4.0) ; extra == "pdf-parsers"
-Requires-Dist: python-docx (>=1.1.0,<2.0.0)
+Requires-Dist: pypdf (>=3.12.2,<4.0.0) ; extra == "doc-chat" or extra == "pdf-parsers"
+Requires-Dist: pytesseract (>=0.3.10,<0.4.0) ; extra == "doc-chat" or extra == "pdf-parsers"
+Requires-Dist: python-docx (>=1.1.0,<2.0.0) ; extra == "doc-chat" or extra == "docx"
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: python-magic (>=0.4.27,<0.5.0)
 Requires-Dist: python-socketio (>=5.11.0,<6.0.0) ; extra == "chainlit"
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: qdrant-client (>=1.8.0,<2.0.0)
 Requires-Dist: rank-bm25 (>=0.2.2,<0.3.0)
 Requires-Dist: redis (>=5.0.1,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: requests-oauthlib (>=1.3.1,<2.0.0)
 Requires-Dist: rich (>=13.3.4,<14.0.0)
-Requires-Dist: scrapy (>=2.11.0,<3.0.0)
-Requires-Dist: sentence-transformers (==2.2.2) ; extra == "hf-embeddings"
-Requires-Dist: sqlalchemy (>=2.0.19,<3.0.0) ; extra == "sql"
+Requires-Dist: scrapy (>=2.11.0,<3.0.0) ; extra == "scrapy"
+Requires-Dist: sentence-transformers (==2.2.2) ; extra == "embeddings" or extra == "hf-embeddings"
+Requires-Dist: sqlalchemy (>=2.0.19,<3.0.0) ; extra == "db" or extra == "sql"
 Requires-Dist: tantivy (>=0.21.0,<0.22.0) ; extra == "lancedb"
 Requires-Dist: thefuzz (>=0.20.0,<0.21.0)
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
-Requires-Dist: torch (==2.0.0) ; extra == "hf-embeddings" or extra == "transformers"
+Requires-Dist: torch (==2.0.0) ; extra == "embeddings" or extra == "hf-embeddings" or extra == "transformers"
 Requires-Dist: trafilatura (>=1.5.0,<2.0.0)
-Requires-Dist: transformers (>=4.40.1,<5.0.0) ; extra == "transformers"
+Requires-Dist: transformers (>=4.40.1,<5.0.0) ; extra == "embeddings" or extra == "transformers"
 Requires-Dist: typer (>=0.9.0,<0.10.0)
-Requires-Dist: unstructured[docx,pdf,pptx] (>=0.10.16,<0.10.18) ; extra == "unstructured"
+Requires-Dist: unstructured[docx,pdf,pptx] (>=0.10.16,<0.10.18) ; extra == "doc-chat" or extra == "unstructured"
 Requires-Dist: wget (>=3.2,<4.0)
 Description-Content-Type: text/markdown
 
@@ -459,7 +465,8 @@ such as [LiteLLM](https://docs.litellm.ai/docs/providers) that in effect mimic t
 
 ### Install `langroid`
 Langroid requires Python 3.11+. We recommend using a virtual environment.
-Use `pip` to install `langroid` (from PyPi) to your virtual environment:
+Use `pip` to install a bare-bones slim version of `langroid` (from PyPi) to your virtual 
+environment:
 ```bash
 pip install langroid
 ```
@@ -467,17 +474,21 @@ The core Langroid package lets you use OpenAI Embeddings models via their API.
 If you instead want to use the `sentence-transformers` embedding models from HuggingFace, 
 install Langroid like this: 
 ```bash
-pip install langroid[hf-embeddings]
-```
-If using `zsh` (or similar shells), you may need to escape the square brackets, e.g.:
-```
-pip install langroid\[hf-embeddings\]
-```
-or use quotes:
-```
 pip install "langroid[hf-embeddings]"
 ```
-
+For many practical scenarios, you may need additional optional dependencies:
+- To use various document-parsers, install langroid with the `doc-chat` extra:
+    ```bash
+    pip install "langroid[doc-chat]"
+    ```
+- For "chat with databases", use the `db` extra:
+    ```bash
+    pip install "langroid[db]"
+    ``
+- You can specify multiple extras by separating them with commas, e.g.:
+    ```bash
+    pip install "langroid[doc-chat,db]"
+    ```
 
 <details>
 <summary><b>Optional Installs for using SQL Chat with a PostgreSQL DB </b></summary>

{langroid-0.1.253 → langroid-0.1.256}/README.md

@@ -354,7 +354,8 @@ such as [LiteLLM](https://docs.litellm.ai/docs/providers) that in effect mimic t
 
 ### Install `langroid`
 Langroid requires Python 3.11+. We recommend using a virtual environment.
-Use `pip` to install `langroid` (from PyPi) to your virtual environment:
+Use `pip` to install a bare-bones slim version of `langroid` (from PyPi) to your virtual 
+environment:
 ```bash
 pip install langroid
 ```
@@ -362,17 +363,21 @@ The core Langroid package lets you use OpenAI Embeddings models via their API.
 If you instead want to use the `sentence-transformers` embedding models from HuggingFace, 
 install Langroid like this: 
 ```bash
-pip install langroid[hf-embeddings]
-```
-If using `zsh` (or similar shells), you may need to escape the square brackets, e.g.:
-```
-pip install langroid\[hf-embeddings\]
-```
-or use quotes:
-```
 pip install "langroid[hf-embeddings]"
 ```
-
+For many practical scenarios, you may need additional optional dependencies:
+- To use various document-parsers, install langroid with the `doc-chat` extra:
+    ```bash
+    pip install "langroid[doc-chat]"
+    ```
+- For "chat with databases", use the `db` extra:
+    ```bash
+    pip install "langroid[db]"
+    ``
+- You can specify multiple extras by separating them with commas, e.g.:
+    ```bash
+    pip install "langroid[doc-chat,db]"
+    ```
 
 <details>
 <summary><b>Optional Installs for using SQL Chat with a PostgreSQL DB </b></summary>

{langroid-0.1.253 → langroid-0.1.256}/langroid/agent/special/__init__.py

@@ -16,7 +16,8 @@ from .table_chat_agent import (
     TableChatAgentConfig,
     PandasEvalTool,
 )
-from . import sql
+
+
 from . import relevance_extractor_agent
 from . import doc_chat_agent
 from . import retriever_agent
@@ -39,7 +40,6 @@ __all__ = [
     "TableChatAgent",
     "TableChatAgentConfig",
     "PandasEvalTool",
-    "sql",
     "relevance_extractor_agent",
     "doc_chat_agent",
     "retriever_agent",
@@ -49,3 +49,11 @@ __all__ = [
     "lance_doc_chat_agent",
     "lance_rag",
 ]
+
+try:
+    from . import sql
+
+    sql
+    __all__.append("sql")
+except ImportError:
+    pass

{langroid-0.1.253 → langroid-0.1.256}/langroid/agent/special/doc_chat_agent.py

@@ -35,6 +35,7 @@ from langroid.embedding_models.models import (
     OpenAIEmbeddingsConfig,
     SentenceTransformerEmbeddingsConfig,
 )
+from langroid.exceptions import LangroidImportError
 from langroid.language_models.base import StreamingIfAllowed
 from langroid.language_models.openai_gpt import OpenAIChatModel, OpenAIGPTConfig
 from langroid.mytypes import DocMetaData, Document, Entity
@@ -108,6 +109,9 @@ vecdb_config: VectorStoreConfig = QdrantDBConfig(
 )
 
 try:
+    import lancedb
+
+    lancedb  # appease mypy
     from langroid.vector_store.lancedb import LanceDBConfig
 
     vecdb_config = LanceDBConfig(
@@ -117,7 +121,7 @@ try:
         embedding=(hf_embed_config if has_sentence_transformers else oai_embed_config),
     )
 
-except ImportError:
+except (ImportError, LangroidImportError):
     pass
 
 

langroid-0.1.256/langroid/agent/special/sql/__init__.py

@@ -0,0 +1,17 @@
+from . import utils
+
+
+__all__ = [
+    "utils",
+]
+
+try:
+    from . import sql_chat_agent
+    from .sql_chat_agent import SQLChatAgentConfig, SQLChatAgent
+
+    sql_chat_agent
+    SQLChatAgent
+    SQLChatAgentConfig
+    __all__.extend(["SQLChatAgentConfig", "SQLChatAgent", "sql_chat_agent"])
+except ImportError:
+    pass

langroid-0.1.256/langroid/cachedb/__init__.py

@@ -0,0 +1,17 @@
+from . import base
+
+from . import redis_cachedb
+
+__all__ = [
+    "base",
+    "redis_cachedb",
+]
+
+
+try:
+    from . import momento_cachedb
+
+    momento_cachedb
+    __all__.append("momento_cachedb")
+except ImportError:
+    pass

{langroid-0.1.253 → langroid-0.1.256}/langroid/parsing/__init__.py

@@ -11,7 +11,6 @@ from . import urls
 from . import utils
 from . import search
 from . import web_search
-from . import spider
 
 from .parser import (
     Splitter,
@@ -36,7 +35,6 @@ __all__ = [
     "utils",
     "search",
     "web_search",
-    "spider",
     "Splitter",
     "PdfParsingConfig",
     "DocxParsingConfig",
@@ -44,3 +42,11 @@ __all__ = [
     "ParsingConfig",
     "Parser",
 ]
+
+try:
+    from . import spider
+
+    spider
+    __all__.append("spider")
+except ImportError:
+    pass

{langroid-0.1.253 → langroid-0.1.256}/langroid/parsing/document_parser.py

@@ -1,26 +1,37 @@
+from __future__ import annotations
+
 import itertools
 import logging
 import re
 from enum import Enum
 from io import BytesIO
-from typing import Any, Generator, List, Tuple
+from typing import TYPE_CHECKING, Any, Generator, List, Tuple
 
 from langroid.exceptions import LangroidImportError
 
 try:
     import fitz
 except ImportError:
-    raise LangroidImportError("PyMuPDF", "pdf-parsers")
+    if not TYPE_CHECKING:
+        fitz = None
 
 try:
     import pypdf
 except ImportError:
-    raise LangroidImportError("pypdf", "pdf-parsers")
+    if not TYPE_CHECKING:
+        pypdf = None
+
+try:
+    import pdfplumber
+except ImportError:
+    if not TYPE_CHECKING:
+        pdfplumber = None
 
-import pdfplumber
 import requests
 from bs4 import BeautifulSoup
-from PIL import Image
+
+if TYPE_CHECKING:
+    from PIL import Image
 
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.parser import Parser, ParsingConfig
@@ -373,19 +384,21 @@ class FitzPDFParser(DocumentParser):
     Parser for processing PDFs using the `fitz` library.
     """
 
-    def iterate_pages(self) -> Generator[Tuple[int, fitz.Page], None, None]:
+    def iterate_pages(self) -> Generator[Tuple[int, "fitz.Page"], None, None]:
         """
         Yield each page in the PDF using `fitz`.
 
         Returns:
             Generator[fitz.Page]: Generator yielding each page.
         """
+        if fitz is None:
+            raise LangroidImportError("fitz", "pdf-parsers")
         doc = fitz.open(stream=self.doc_bytes, filetype="pdf")
         for i, page in enumerate(doc):
             yield i, page
         doc.close()
 
-    def extract_text_from_page(self, page: fitz.Page) -> str:
+    def extract_text_from_page(self, page: "fitz.Page") -> str:
         """
         Extract text from a given `fitz` page.
 
@@ -410,6 +423,8 @@ class PyPDFParser(DocumentParser):
         Returns:
             Generator[pypdf.pdf.PageObject]: Generator yielding each page.
         """
+        if pypdf is None:
+            raise LangroidImportError("pypdf", "pdf-parsers")
         reader = pypdf.PdfReader(self.doc_bytes)
         for i, page in enumerate(reader.pages):
             yield i, page
@@ -441,6 +456,8 @@ class PDFPlumberParser(DocumentParser):
         Returns:
             Generator[pdfplumber.Page]: Generator yielding each page.
         """
+        if pdfplumber is None:
+            raise LangroidImportError("pdfplumber", "pdf-parsers")
         with pdfplumber.open(self.doc_bytes) as pdf:
             for i, page in enumerate(pdf.pages):
                 yield i, page
@@ -654,7 +671,10 @@ class PythonDocxParser(DocumentParser):
         In a DOCX file, pages are not explicitly defined,
         so we consider each paragraph as a separate 'page' for simplicity.
         """
-        import docx
+        try:
+            import docx
+        except ImportError:
+            raise LangroidImportError("python-docx", "docx")
 
         doc = docx.Document(self.doc_bytes)
         for i, para in enumerate(doc.paragraphs, start=1):

{langroid-0.1.253 → langroid-0.1.256}/langroid/parsing/spider.py

@@ -1,13 +1,18 @@
 from typing import List, Set, no_type_check
 from urllib.parse import urlparse
 
-from pydispatch import dispatcher
-from scrapy import signals
-from scrapy.crawler import CrawlerRunner
-from scrapy.http import Response
-from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import CrawlSpider, Rule
-from twisted.internet import defer, reactor
+from langroid.exceptions import LangroidImportError
+
+try:
+    from pydispatch import dispatcher
+    from scrapy import signals
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.http import Response
+    from scrapy.linkextractors import LinkExtractor
+    from scrapy.spiders import CrawlSpider, Rule
+    from twisted.internet import defer, reactor
+except ImportError:
+    raise LangroidImportError("scrapy", "scrapy")
 
 
 @no_type_check

{langroid-0.1.253 → langroid-0.1.256}/langroid/vector_store/__init__.py

@@ -10,14 +10,8 @@ __all__ = [
     "VectorStore",
     "VectorStoreConfig",
     "qdrantdb",
-    "meilisearch",
-    "lancedb",
     "QdrantDBConfig",
     "QdrantDB",
-    "MeiliSearch",
-    "MeiliSearchConfig",
-    "LanceDB",
-    "LanceDBConfig",
 ]
 
 

{langroid-0.1.253 → langroid-0.1.256}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.253"
+version = "0.1.256"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"
@@ -44,8 +44,12 @@ pdf2image = {version="^1.17.0", optional=true}
 pytesseract = {version="^0.3.10", optional=true}
 sqlalchemy = {version="^2.0.19", optional=true}
 pyarrow = {version="15.0.0", optional=true}
+pdfplumber = {version="^0.10.2", optional=true}
+python-docx = {version="^1.1.0", optional=true}
+scrapy = {version="^2.11.0", optional=true}
 
-# CORE
+# ====CORE================================
+pyyaml = "^6.0.1"
 onnxruntime = "^1.16.1"
 fire = "^0.5.0"
 bs4 = "^0.0.1"
@@ -77,32 +81,42 @@ lxml = "^4.9.3"
 
 rank-bm25 = "^0.2.2"
 thefuzz = "^0.20.0"
-pdfplumber = "^0.10.2"
 
 jinja2 = "^3.1.2"
-
 docstring-parser = "^0.15"
-scrapy = "^2.11.0"
-async-generator = "^1.10"
 
-python-docx = "^1.1.0"
 aiohttp = "^3.9.1"
 grpcio = "^1.62.1"
 duckduckgo-search = "^6.0.0"
 
+google-generativeai = "^0.5.2"
 groq = "^0.5.0"
 nest-asyncio = "^1.6.0"
+async-generator = "^1.10"
+
 python-magic = "^0.4.27"
 
-google-generativeai = "^0.5.2"
 
 # ================= EXTRAS =================
+
 [tool.poetry.extras]
 # install these using, e.g.,
 # `poetry install -E [...]` where [...] is one of the extras below
 # or install multiple extras using, e.g.,  `poetry install -E "litellm mysql"
+
+# Groupings for common use-cases
+doc-chat = [
+    "unstructured", "python-docx", "pdfplumber", "pypdf",
+    "pymupdf", "pdf2image", "pytesseract"
+]
+db = ["postgres", "mysql", "sqlalchemy", "psycopg2", "pymysql"]
+embeddings = ["sentence-transformers", "torch", "transformers", "huggingface-hub"]
+
+# more granular groupings
 lancedb = ["lancedb", "tantivy", "pyarrow"]
-pdf-parsers = ["pypdf", "pymupdf", "pdf2image", "pytesseract"]
+pdf-parsers = ["pdfplumber", "pypdf", "pymupdf", "pdf2image", "pytesseract"]
+docx = ["python-docx"]
+scrapy = ["scrapy"]
 hf-embeddings = ["sentence-transformers", "torch"]
 transformers = ["transformers", "huggingface-hub", "torch"]
 unstructured = ["unstructured"]

langroid-0.1.253/langroid/agent/special/sql/__init__.py

@@ -1,10 +0,0 @@
-from . import sql_chat_agent, utils
-from .sql_chat_agent import SQLChatAgentConfig, SQLChatAgent
-
-
-__all__ = [
-    "SQLChatAgentConfig",
-    "SQLChatAgent",
-    "sql_chat_agent",
-    "utils",
-]

langroid-0.1.253/langroid/cachedb/__init__.py

@@ -1,9 +0,0 @@
-from . import base
-from . import momento_cachedb
-from . import redis_cachedb
-
-__all__ = [
-    "base",
-    "momento_cachedb",
-    "redis_cachedb",
-]