langroid 0.1.225__tar.gz → 0.1.227__tar.gz

{langroid-0.1.225 → langroid-0.1.227}/PKG-INFO

@@ -1,14 +1,13 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.225
+Version: 0.1.227
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
 Author-email: pchalasani@gmail.com
-Requires-Python: >=3.9.1,<3.12
+Requires-Python: >=3.11,<3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: chainlit
 Provides-Extra: chromadb
@@ -101,6 +100,7 @@ Requires-Dist: tiktoken (>=0.5.1,<0.6.0)
 Requires-Dist: torch (==2.0.0) ; extra == "hf-embeddings"
 Requires-Dist: trafilatura (>=1.5.0,<2.0.0)
 Requires-Dist: typer (>=0.9.0,<0.10.0)
+Requires-Dist: types-pillow (>=10.2.0.20240406,<11.0.0.0)
 Requires-Dist: types-pyyaml (>=6.0.12.20240311,<7.0.0.0)
 Requires-Dist: types-redis (>=4.5.5.2,<5.0.0.0)
 Requires-Dist: types-requests (>=2.31.0.1,<3.0.0.0)

{langroid-0.1.225 → langroid-0.1.227}/langroid/agent/openai_assistant.py

@@ -8,6 +8,10 @@ from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Type, cast, no_type_check
 
 from openai.types.beta import Assistant, Thread
+from openai.types.beta.assistant_update_params import (
+    ToolResources,
+    ToolResourcesCodeInterpreter,
+)
 from openai.types.beta.threads import Message, Run
 from openai.types.beta.threads.runs import RunStep
 from pydantic import BaseModel
@@ -29,7 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class ToolType(str, Enum):
-    RETRIEVAL = "retrieval"
+    RETRIEVAL = "file_search"
     CODE_INTERPRETER = "code_interpreter"
     FUNCTION = "function"
 
@@ -127,7 +131,11 @@ class OpenAIAssistant(ChatAgent):
         self.config.files = list(set(self.config.files + files))
         self.assistant = self.assistants.update(
             self.assistant.id,
-            file_ids=[f.id for f in self.files],
+            tool_resources=ToolResources(
+                code_interpreter=ToolResourcesCodeInterpreter(
+                    file_ids=[f.id for f in self.files],
+                ),
+            ),
         )
 
     def add_assistant_tools(self, tools: List[AssistantTool]) -> None:

{langroid-0.1.225 → langroid-0.1.227}/langroid/agent/special/doc_chat_agent.py

@@ -66,7 +66,7 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_DOC_CHAT_INSTRUCTIONS = """
 Your task is to answer questions about various documents.
-You will be given various passages from these documents, and asked to answer questions 
+You will be given various passages from these documents, and asked to answer questions
 about them, or summarize them into coherent answers.
 """
 
@@ -76,7 +76,7 @@ You are a helpful assistant, helping me understand a collection of documents.
 
 has_sentence_transformers = False
 try:
-    from sentence_transformer import SentenceTransformer  # noqa: F401
+    from sentence_transformers import SentenceTransformer  # noqa: F401
 
     has_sentence_transformers = True
 except ImportError:
@@ -169,7 +169,8 @@ class DocChatAgentConfig(ChatAgentConfig):
         dims=1536,
     )
 
-    vecdb: VectorStoreConfig = LanceDBConfig(
+    # Allow vecdb to be None in case we want to explicitly set it later
+    vecdb: Optional[VectorStoreConfig] = LanceDBConfig(
         collection_name="doc-chat-lancedb",
         replace_collection=True,
         storage_path=".lancedb/data/",
@@ -209,12 +210,13 @@ class DocChatAgent(ChatAgent):
 
     def clear(self) -> None:
         """Clear the document collection and the specific collection in vecdb"""
-        if self.vecdb is None:
-            raise ValueError("VecDB not set")
         self.original_docs = []
         self.original_docs_length = 0
         self.chunked_docs = []
         self.chunked_docs_clean = []
+        if self.vecdb is None:
+            logger.warning("Attempting to clear VecDB, but VecDB not set.")
+            return
         collection_name = self.vecdb.config.collection_name
         if collection_name is None:
             return
@@ -339,7 +341,7 @@ class DocChatAgent(ChatAgent):
         n_paths = len(paths)
         print(
             f"""
-        [green]I have processed the following {n_urls} URLs 
+        [green]I have processed the following {n_urls} URLs
         and {n_paths} docs into {n_splits} parts:
         """.strip()
         )
@@ -443,7 +445,7 @@ class DocChatAgent(ChatAgent):
         if content not in df.columns:
             raise ValueError(
                 f"""
-                Content column {content} not in dataframe, 
+                Content column {content} not in dataframe,
                 so we cannot ingest into the DocChatAgent.
                 Please specify the `content` parameter as a suitable
                 text-based column in the dataframe.
@@ -816,12 +818,12 @@ class DocChatAgent(ChatAgent):
                 # Adjust this prompt depending on context.
                 answer = self.llm_response_forget(
                     f"""
-                    Give an ideal answer to the following query, 
-                    in up to 3 sentences. Do not explain yourself, 
-                    and do not apologize, just show 
+                    Give an ideal answer to the following query,
+                    in up to 3 sentences. Do not explain yourself,
+                    and do not apologize, just show
                     a good possible answer, even if you do not have any information.
                     Preface your answer with "HYPOTHETICAL ANSWER: "
-                    
+
                     QUERY: {query}
                     """
                 ).content
@@ -1293,9 +1295,9 @@ class DocChatAgent(ChatAgent):
             logger.warning(
                 """
                 No docs to summarize! Perhaps you are re-using a previously
-                defined collection? 
+                defined collection?
                 In that case, we don't have access to the original docs.
-                To create a summary, use a new collection, and specify a list of docs. 
+                To create a summary, use a new collection, and specify a list of docs.
                 """
             )
             return None
@@ -1318,7 +1320,7 @@ class DocChatAgent(ChatAgent):
             )
         prompt = f"""
         {instruction}
-        
+
         FULL TEXT:
         {full_text}
         """.strip()

{langroid-0.1.225 → langroid-0.1.227}/langroid/parsing/document_parser.py

@@ -25,7 +25,29 @@ class DocumentType(str, Enum):
     TXT = "txt"
 
 
+def find_last_full_char(possible_unicode: bytes) -> int:
+    """
+    Find the index of the last full character in a byte string.
+    Args:
+        possible_unicode (bytes): The bytes to check.
+    Returns:
+        int: The index of the last full unicode character.
+    """
+
+    for i in range(len(possible_unicode) - 1, 0, -1):
+        if (possible_unicode[i] & 0xC0) != 0x80:
+            return i
+    return 0
+
+
 def is_plain_text(path_or_bytes: str | bytes) -> bool:
+    """
+    Check if a file is plain text by attempting to decode it as UTF-8.
+    Args:
+        path_or_bytes (str|bytes): The file path or bytes object.
+    Returns:
+        bool: True if the file is plain text, False otherwise.
+    """
     if isinstance(path_or_bytes, str):
         if path_or_bytes.startswith(("http://", "https://")):
             response = requests.get(path_or_bytes)
@@ -38,6 +60,8 @@ def is_plain_text(path_or_bytes: str | bytes) -> bool:
         content = path_or_bytes[:1024]
     try:
         # Attempt to decode the content as UTF-8
+        content = content[: find_last_full_char(content)]
+
         _ = content.decode("utf-8")
         # Additional checks can go here, e.g., to verify that the content
         # doesn't contain too many unusual characters for it to be considered text
@@ -431,14 +455,14 @@ class ImagePdfParser(DocumentParser):
 
     def iterate_pages(
         self,
-    ) -> Generator[Tuple[int, Image], None, None]:
+    ) -> Generator[Tuple[int, Image], None, None]:  # type: ignore
         from pdf2image import convert_from_bytes
 
         images = convert_from_bytes(self.doc_bytes.getvalue())
         for i, image in enumerate(images):
             yield i, image
 
-    def extract_text_from_page(self, page: Image) -> str:
+    def extract_text_from_page(self, page: Image) -> str:  # type: ignore
         """
         Extract text from a given `pdf2image` page.
 
@@ -466,7 +490,7 @@ class UnstructuredPDFParser(DocumentParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """
@@ -483,7 +507,7 @@ class UnstructuredPDFParser(DocumentParser):
                 The `unstructured` library failed to parse the pdf.
                 Please try a different library by setting the `library` field
                 in the `pdf` section of the `parsing` field in the config file.
-                Supported libraries are: 
+                Supported libraries are:
                 fitz, pypdf, pdfplumber, unstructured
                 """
             )
@@ -529,7 +553,7 @@ class UnstructuredDocxParser(DocumentParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """
@@ -580,7 +604,7 @@ class UnstructuredDocParser(UnstructuredDocxParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """

{langroid-0.1.225 → langroid-0.1.227}/langroid/vector_store/qdrantdb.py

@@ -192,8 +192,7 @@ class QdrantDB(VectorStore):
                 with the same name. Defaults to False.
         """
         self.config.collection_name = collection_name
-        collections = self.list_collections()
-        if collection_name in collections:
+        if self.client.collection_exists(collection_name=collection_name):
             coll = self.client.get_collection(collection_name=collection_name)
             if (
                 coll.status == CollectionStatus.GREEN
@@ -205,7 +204,8 @@ class QdrantDB(VectorStore):
                     return
                 else:
                     logger.warning("Recreating fresh collection")
-        self.client.recreate_collection(
+            self.client.delete_collection(collection_name=collection_name)
+        self.client.create_collection(
             collection_name=collection_name,
             vectors_config=VectorParams(
                 size=self.embedding_dim,
@@ -214,7 +214,7 @@ class QdrantDB(VectorStore):
         )
         collection_info = self.client.get_collection(collection_name=collection_name)
         assert collection_info.status == CollectionStatus.GREEN
-        assert collection_info.vectors_count == 0
+        assert collection_info.vectors_count in [0, None]
         if settings.debug:
             level = logger.getEffectiveLevel()
             logger.setLevel(logging.INFO)

{langroid-0.1.225 → langroid-0.1.227}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.225"
+version = "0.1.227"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"
 license = "MIT"
 
 [tool.poetry.dependencies]
-python = ">=3.9.1,<3.12"
+python = ">=3.11,<3.12"
 mkdocs = {version="^1.4.2", optional=true}
 mkdocs-material = {version="^9.1.5", optional=true}
 mkdocstrings = {extras = ["python"], version = "^0.21.2", optional=true}
@@ -96,6 +96,7 @@ huggingface-hub = {version="^0.21.2", optional=true}
 pdf2image = "^1.17.0"
 pytesseract = "^0.3.10"
 types-pyyaml = "^6.0.12.20240311"
+types-pillow = "^10.2.0.20240406"
 
 [tool.poetry.extras]
 # install these using `poetry install -E [...]` where [...] is one of the extras below