langroid 0.44.0__tar.gz → 0.45.1__tar.gz

{langroid-0.44.0 → langroid-0.45.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.44.0
+Version: 0.45.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -12,7 +12,6 @@ Requires-Dist: async-generator<2.0,>=1.10
 Requires-Dist: bs4<1.0.0,>=0.0.1
 Requires-Dist: cerebras-cloud-sdk<2.0.0,>=1.1.0
 Requires-Dist: colorlog<7.0.0,>=6.7.0
-Requires-Dist: docling<3.0.0,>=2.20.0
 Requires-Dist: docstring-parser<1.0,>=0.16
 Requires-Dist: duckduckgo-search<7.0.0,>=6.0.0
 Requires-Dist: exa-py>=1.8.7
@@ -49,7 +48,6 @@ Requires-Dist: redis<6.0.0,>=5.0.1
 Requires-Dist: requests-oauthlib<2.0.0,>=1.3.1
 Requires-Dist: requests<3.0.0,>=2.31.0
 Requires-Dist: rich<14.0.0,>=13.3.4
-Requires-Dist: tavily-python>=0.5.0
 Requires-Dist: thefuzz<1.0.0,>=0.20.0
 Requires-Dist: tiktoken<1.0.0,>=0.7.0
 Requires-Dist: trafilatura<2.0.0,>=1.5.0
@@ -63,6 +61,7 @@ Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'all'
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'all'
 Requires-Dist: huggingface-hub<1.0.0,>=0.21.2; extra == 'all'
 Requires-Dist: litellm<2.0.0,>=1.30.1; extra == 'all'
+Requires-Dist: marker-pdf; extra == 'all'
 Requires-Dist: metaphor-python<0.2.0,>=0.1.23; extra == 'all'
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'all'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'all'
@@ -99,6 +98,7 @@ Requires-Dist: pymysql<2.0.0,>=1.1.0; extra == 'db'
 Requires-Dist: sqlalchemy<3.0.0,>=2.0.19; extra == 'db'
 Provides-Extra: doc-chat
 Requires-Dist: docling<3.0.0,>=2.20.0; extra == 'doc-chat'
+Requires-Dist: marker-pdf; extra == 'doc-chat'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'doc-chat'
 Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'doc-chat'
 Requires-Dist: pymupdf<2.0.0,>=1.23.3; extra == 'doc-chat'
@@ -138,6 +138,9 @@ Requires-Dist: pyarrow<16.0.0,>=15.0.0; extra == 'lancedb'
 Requires-Dist: tantivy<0.22.0,>=0.21.0; extra == 'lancedb'
 Provides-Extra: litellm
 Requires-Dist: litellm<2.0.0,>=1.30.1; extra == 'litellm'
+Provides-Extra: marker-pdf
+Requires-Dist: marker-pdf[full]>=1.6.0; (sys_platform != 'darwin' or platform_machine != 'x86_64') and extra == 'marker-pdf'
+Requires-Dist: opencv-python>=4.11.0.86; extra == 'marker-pdf'
 Provides-Extra: meilisearch
 Requires-Dist: meilisearch-python-sdk<3.0.0,>=2.2.3; extra == 'meilisearch'
 Provides-Extra: metaphor
@@ -150,6 +153,7 @@ Provides-Extra: neo4j
 Requires-Dist: neo4j<6.0.0,>=5.14.1; extra == 'neo4j'
 Provides-Extra: pdf-parsers
 Requires-Dist: docling<3.0.0,>=2.16.0; extra == 'pdf-parsers'
+Requires-Dist: marker-pdf; extra == 'pdf-parsers'
 Requires-Dist: markitdown>=0.0.1a3; extra == 'pdf-parsers'
 Requires-Dist: pdf2image<2.0.0,>=1.17.0; extra == 'pdf-parsers'
 Requires-Dist: pymupdf4llm<0.1.0,>=0.0.17; extra == 'pdf-parsers'
@@ -791,8 +795,8 @@ wget -O .env https://raw.githubusercontent.com/langroid/langroid/main/.env-templ
 # Edit the .env file with your favorite editor (here nano), and remove any un-used settings. E.g. there are "dummy" values like "your-redis-port" etc -- if you are not using them, you MUST remove them.
 nano .env
 
-# launch the container
-docker run -it --rm  -v ./.env:/langroid/.env langroid/langroid
+# launch the container (the appropriate image for your architecture will be pulled automatically)
+docker run -it --rm  -v ./.env:/langroid/.env langroid/langroid:latest
 
 # Use this command to run any of the scripts in the `examples` directory
 python examples/<Path/To/Example.py> 

{langroid-0.44.0 → langroid-0.45.1}/README.md

@@ -599,8 +599,8 @@ wget -O .env https://raw.githubusercontent.com/langroid/langroid/main/.env-templ
 # Edit the .env file with your favorite editor (here nano), and remove any un-used settings. E.g. there are "dummy" values like "your-redis-port" etc -- if you are not using them, you MUST remove them.
 nano .env
 
-# launch the container
-docker run -it --rm  -v ./.env:/langroid/.env langroid/langroid
+# launch the container (the appropriate image for your architecture will be pulled automatically)
+docker run -it --rm  -v ./.env:/langroid/.env langroid/langroid:latest
 
 # Use this command to run any of the scripts in the `examples` directory
 python examples/<Path/To/Example.py> 

{langroid-0.44.0 → langroid-0.45.1}/langroid/agent/base.py

@@ -1016,7 +1016,7 @@ class Agent(ABC):
             # we would have already displayed the msg "live" ONLY if
             # streaming was enabled, AND we did not find a cached response
             # If we are here, it means the response has not yet been displayed.
-            cached = f"[red]{self.indent}(cached)[/red]" if response.cached else ""
+            cached = "[red](cached)[/red]" if response.cached else ""
             console.print(f"[green]{self.indent}", end="")
             print(cached + "[green]" + escape(response.message))
         self.update_token_usage(

{langroid-0.44.0 → langroid-0.45.1}/langroid/parsing/document_parser.py

@@ -150,6 +150,8 @@ class DocumentParser(Parser):
                 return ImagePdfParser(source, config)
             elif config.pdf.library == "gemini":
                 return GeminiPdfParser(source, config)
+            elif config.pdf.library == "marker":
+                return MarkerPdfParser(source, config)
             else:
                 raise ValueError(
                     f"Unsupported PDF library specified: {config.pdf.library}"
@@ -1356,3 +1358,85 @@ class GeminiPdfParser(DocumentParser):
             content=page,
             metadata=DocMetaData(source=self.source),
         )
+
+
+class MarkerPdfParser(DocumentParser):
+    DEFAULT_CONFIG = {"paginate_output": True, "output_format": "markdown"}
+
+    def __init__(self, source: Union[str, bytes], config: ParsingConfig):
+        super().__init__(source, config)
+        user_config = (
+            config.pdf.marker_config.config_dict if config.pdf.marker_config else {}
+        )
+
+        self.config_dict = {**MarkerPdfParser.DEFAULT_CONFIG, **user_config}
+
+    def iterate_pages(self) -> Generator[Tuple[int, Any], None, None]:
+        """
+        Yield each page in the PDF using `marker`.
+        """
+        try:
+            import marker  # noqa
+        except ImportError:
+            raise LangroidImportError(
+                "marker-pdf", ["marker-pdf", "pdf-parsers", "all", "doc-chat"]
+            )
+
+        import re
+
+        from marker.config.parser import ConfigParser
+        from marker.converters.pdf import PdfConverter
+        from marker.models import create_model_dict
+        from marker.output import save_output
+
+        config_parser = ConfigParser(self.config_dict)
+        converter = PdfConverter(
+            config=config_parser.generate_config_dict(),
+            artifact_dict=create_model_dict(),
+            processor_list=config_parser.get_processors(),
+            renderer=config_parser.get_renderer(),
+            llm_service=config_parser.get_llm_service(),
+        )
+        doc_path = self.source
+        if doc_path == "bytes":
+            # write to tmp file, then use that path
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
+                temp_file.write(self.doc_bytes.getvalue())
+                doc_path = temp_file.name
+
+        output_dir = Path(str(Path(doc_path).with_suffix("")) + "-pages")
+        os.makedirs(output_dir, exist_ok=True)
+        filename = Path(doc_path).stem + "_converted"
+
+        rendered = converter(doc_path)
+        save_output(rendered, output_dir=output_dir, fname_base=filename)
+        file_path = output_dir / f"{filename}.md"
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            full_markdown = f.read()
+
+        # Regex for splitting pages
+        pages = re.split(r"\{\d+\}----+", full_markdown)
+
+        page_no = 0
+        for page in pages:
+            if page.strip():
+                yield page_no, page
+            page_no += 1
+
+    def get_document_from_page(self, page: str) -> Document:
+        """
+        Get Document object from a given 1-page markdown file,
+        possibly containing image refs.
+
+        Args:
+            page (str): The page we get by splitting large md file from
+            marker
+
+        Returns:
+            Document: Document object, with content and possible metadata.
+        """
+        return Document(
+            content=self.fix_text(page),
+            metadata=DocMetaData(source=self.source),
+        )

{langroid-0.44.0 → langroid-0.45.1}/langroid/parsing/parser.py

@@ -38,8 +38,13 @@ class GeminiConfig(BaseSettings):
     requests_per_minute: Optional[int] = 5
 
 
-class PdfParsingConfig(BaseParsingConfig):
+class MarkerConfig(BaseSettings):
+    """Configuration for Markitdown-based parsing."""
+
+    config_dict: Dict[str, Any] = {}
 
+
+class PdfParsingConfig(BaseParsingConfig):
     library: Literal[
         "fitz",
         "pymupdf4llm",
@@ -49,16 +54,26 @@ class PdfParsingConfig(BaseParsingConfig):
         "pdf2image",
         "markitdown",
         "gemini",
+        "marker",
     ] = "pymupdf4llm"
     gemini_config: Optional[GeminiConfig] = None
+    marker_config: Optional[MarkerConfig] = None
 
     @root_validator(pre=True)
-    def enable_gemini_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        """Ensure GeminiConfig is set only when library is 'gemini'."""
-        if values.get("library") == "gemini":
-            values["gemini_config"] = values.get("gemini_config") or GeminiConfig()
+    def enable_configs(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Ensure correct config is set based on library selection."""
+        library = values.get("library")
+
+        if library == "gemini":
+            values.setdefault("gemini_config", GeminiConfig())
         else:
             values["gemini_config"] = None
+
+        if library == "marker":
+            values.setdefault("marker_config", MarkerConfig())
+        else:
+            values["marker_config"] = None
+
         return values
 
 

{langroid-0.44.0 → langroid-0.45.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.44.0"
+version = "0.45.1"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]
@@ -15,7 +15,6 @@ dependencies = [
     "bs4<1.0.0,>=0.0.1",
     "cerebras-cloud-sdk<2.0.0,>=1.1.0",
     "colorlog<7.0.0,>=6.7.0",
-    "docling<3.0.0,>=2.20.0",
     "docstring-parser<1.0,>=0.16",
     "duckduckgo-search<7.0.0,>=6.0.0",
     "exa-py>=1.8.7",
@@ -52,7 +51,6 @@ dependencies = [
     "requests<3.0.0,>=2.31.0",
     "requests-oauthlib<2.0.0,>=1.3.1",
     "rich<14.0.0,>=13.3.4",
-    "tavily-python>=0.5.0",
     "thefuzz<1.0.0,>=0.20.0",
     "tiktoken<1.0.0,>=0.7.0",
     "trafilatura<2.0.0,>=1.5.0",
@@ -70,6 +68,7 @@ doc-chat = [
     "pytesseract<0.4.0,>=0.3.10",
     "python-docx<2.0.0,>=1.1.0",
     "unstructured[docx,pdf,pptx]<1.0.0,>=0.16.15",
+    "marker-pdf"
 ]
 
 hf-transformers = [
@@ -122,6 +121,7 @@ all = [
     "fastembed<0.4.0,>=0.3.1",
     "pgvector>=0.3.6",
     "psycopg2-binary>=2.9.10",
+    "marker-pdf",
 ]
 
 # More granular groupings
@@ -147,12 +147,18 @@ pdf-parsers = [
     "pdf2image<2.0.0,>=1.17.0",
     "pytesseract<0.4.0,>=0.3.10",
     "markitdown>=0.0.1a3",
+    "marker-pdf",
 ]
 
 docx = [
     "python-docx<2.0.0,>=1.1.0",
 ]
 
+marker-pdf = [
+    "marker-pdf[full]>=1.6.0; sys_platform != 'darwin' or platform_machine != 'x86_64'",
+    "opencv-python>=4.11.0.86",
+]
+
 scrapy = [
     "scrapy<3.0.0,>=2.11.0",
 ]