langroid 0.1.225__tar.gz → 0.1.226__tar.gz

{langroid-0.1.225 → langroid-0.1.226}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.1.225
+Version: 0.1.226
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -101,6 +101,7 @@ Requires-Dist: tiktoken (>=0.5.1,<0.6.0)
 Requires-Dist: torch (==2.0.0) ; extra == "hf-embeddings"
 Requires-Dist: trafilatura (>=1.5.0,<2.0.0)
 Requires-Dist: typer (>=0.9.0,<0.10.0)
+Requires-Dist: types-pillow (>=10.2.0.20240406,<11.0.0.0)
 Requires-Dist: types-pyyaml (>=6.0.12.20240311,<7.0.0.0)
 Requires-Dist: types-redis (>=4.5.5.2,<5.0.0.0)
 Requires-Dist: types-requests (>=2.31.0.1,<3.0.0.0)

{langroid-0.1.225 → langroid-0.1.226}/langroid/agent/special/doc_chat_agent.py

@@ -66,7 +66,7 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_DOC_CHAT_INSTRUCTIONS = """
 Your task is to answer questions about various documents.
-You will be given various passages from these documents, and asked to answer questions 
+You will be given various passages from these documents, and asked to answer questions
 about them, or summarize them into coherent answers.
 """
 
@@ -76,7 +76,7 @@ You are a helpful assistant, helping me understand a collection of documents.
 
 has_sentence_transformers = False
 try:
-    from sentence_transformer import SentenceTransformer  # noqa: F401
+    from sentence_transformers import SentenceTransformer  # noqa: F401
 
     has_sentence_transformers = True
 except ImportError:
@@ -339,7 +339,7 @@ class DocChatAgent(ChatAgent):
         n_paths = len(paths)
         print(
             f"""
-        [green]I have processed the following {n_urls} URLs 
+        [green]I have processed the following {n_urls} URLs
         and {n_paths} docs into {n_splits} parts:
         """.strip()
         )
@@ -443,7 +443,7 @@ class DocChatAgent(ChatAgent):
         if content not in df.columns:
             raise ValueError(
                 f"""
-                Content column {content} not in dataframe, 
+                Content column {content} not in dataframe,
                 so we cannot ingest into the DocChatAgent.
                 Please specify the `content` parameter as a suitable
                 text-based column in the dataframe.
@@ -816,12 +816,12 @@ class DocChatAgent(ChatAgent):
                 # Adjust this prompt depending on context.
                 answer = self.llm_response_forget(
                     f"""
-                    Give an ideal answer to the following query, 
-                    in up to 3 sentences. Do not explain yourself, 
-                    and do not apologize, just show 
+                    Give an ideal answer to the following query,
+                    in up to 3 sentences. Do not explain yourself,
+                    and do not apologize, just show
                     a good possible answer, even if you do not have any information.
                     Preface your answer with "HYPOTHETICAL ANSWER: "
-                    
+
                     QUERY: {query}
                     """
                 ).content
@@ -1293,9 +1293,9 @@ class DocChatAgent(ChatAgent):
             logger.warning(
                 """
                 No docs to summarize! Perhaps you are re-using a previously
-                defined collection? 
+                defined collection?
                 In that case, we don't have access to the original docs.
-                To create a summary, use a new collection, and specify a list of docs. 
+                To create a summary, use a new collection, and specify a list of docs.
                 """
             )
             return None
@@ -1318,7 +1318,7 @@ class DocChatAgent(ChatAgent):
             )
         prompt = f"""
         {instruction}
-        
+
         FULL TEXT:
         {full_text}
         """.strip()

{langroid-0.1.225 → langroid-0.1.226}/langroid/parsing/document_parser.py

@@ -25,7 +25,29 @@ class DocumentType(str, Enum):
     TXT = "txt"
 
 
+def find_last_full_char(possible_unicode: bytes) -> int:
+    """
+    Find the index of the last full character in a byte string.
+    Args:
+        possible_unicode (bytes): The bytes to check.
+    Returns:
+        int: The index of the last full unicode character.
+    """
+
+    for i in range(len(possible_unicode) - 1, 0, -1):
+        if (possible_unicode[i] & 0xC0) != 0x80:
+            return i
+    return 0
+
+
 def is_plain_text(path_or_bytes: str | bytes) -> bool:
+    """
+    Check if a file is plain text by attempting to decode it as UTF-8.
+    Args:
+        path_or_bytes (str|bytes): The file path or bytes object.
+    Returns:
+        bool: True if the file is plain text, False otherwise.
+    """
     if isinstance(path_or_bytes, str):
         if path_or_bytes.startswith(("http://", "https://")):
             response = requests.get(path_or_bytes)
@@ -38,6 +60,8 @@ def is_plain_text(path_or_bytes: str | bytes) -> bool:
         content = path_or_bytes[:1024]
     try:
         # Attempt to decode the content as UTF-8
+        content = content[: find_last_full_char(content)]
+
         _ = content.decode("utf-8")
         # Additional checks can go here, e.g., to verify that the content
         # doesn't contain too many unusual characters for it to be considered text
@@ -431,14 +455,14 @@ class ImagePdfParser(DocumentParser):
 
     def iterate_pages(
         self,
-    ) -> Generator[Tuple[int, Image], None, None]:
+    ) -> Generator[Tuple[int, Image], None, None]:  # type: ignore
         from pdf2image import convert_from_bytes
 
         images = convert_from_bytes(self.doc_bytes.getvalue())
         for i, image in enumerate(images):
             yield i, image
 
-    def extract_text_from_page(self, page: Image) -> str:
+    def extract_text_from_page(self, page: Image) -> str:  # type: ignore
         """
         Extract text from a given `pdf2image` page.
 
@@ -466,7 +490,7 @@ class UnstructuredPDFParser(DocumentParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """
@@ -483,7 +507,7 @@ class UnstructuredPDFParser(DocumentParser):
                 The `unstructured` library failed to parse the pdf.
                 Please try a different library by setting the `library` field
                 in the `pdf` section of the `parsing` field in the config file.
-                Supported libraries are: 
+                Supported libraries are:
                 fitz, pypdf, pdfplumber, unstructured
                 """
             )
@@ -529,7 +553,7 @@ class UnstructuredDocxParser(DocumentParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """
@@ -580,7 +604,7 @@ class UnstructuredDocParser(UnstructuredDocxParser):
             raise ImportError(
                 """
                 The `unstructured` library is not installed by default with langroid.
-                To include this library, please install langroid with the 
+                To include this library, please install langroid with the
                 `unstructured` extra by running `pip install "langroid[unstructured]"`
                 or equivalent.
                 """

{langroid-0.1.225 → langroid-0.1.226}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langroid"
-version = "0.1.225"
+version = "0.1.226"
 description = "Harness LLMs with Multi-Agent Programming"
 authors = ["Prasad Chalasani <pchalasani@gmail.com>"]
 readme = "README.md"
@@ -96,6 +96,7 @@ huggingface-hub = {version="^0.21.2", optional=true}
 pdf2image = "^1.17.0"
 pytesseract = "^0.3.10"
 types-pyyaml = "^6.0.12.20240311"
+types-pillow = "^10.2.0.20240406"
 
 [tool.poetry.extras]
 # install these using `poetry install -E [...]` where [...] is one of the extras below