langroid 0.46.0__py3-none-any.whl → 0.47.1__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -50,7 +50,7 @@ from langroid.parsing.search import (
     preprocess_text,
 )
 from langroid.parsing.table_loader import describe_dataframe
-from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.url_loader import BaseCrawlerConfig, TrafilaturaConfig, URLLoader
 from langroid.parsing.urls import get_list_from_user, get_urls_paths_bytes_indices
 from langroid.prompts.prompts_config import PromptsConfig
 from langroid.prompts.templates import SUMMARY_ANSWER_PROMPT_GPT4
@@ -192,6 +192,7 @@ class DocChatAgentConfig(ChatAgentConfig):
             library="pymupdf4llm",
         ),
     )
+    crawler_config: Optional[BaseCrawlerConfig] = TrafilaturaConfig()
 
     # Allow vecdb to be None in case we want to explicitly set it later
     vecdb: Optional[VectorStoreConfig] = QdrantDBConfig(
@@ -236,9 +237,7 @@ class DocChatAgent(ChatAgent):
         self.chunked_docs: List[Document] = []
         self.chunked_docs_clean: List[Document] = []
         self.response: None | Document = None
-
-        if len(config.doc_paths) > 0:
-            self.ingest()
+        self.ingest()
 
     def clear(self) -> None:
         """Clear the document collection and the specific collection in vecdb"""
@@ -274,8 +273,9 @@ class DocChatAgent(ChatAgent):
             # But let's get all the chunked docs so we can
             # do keyword and other non-vector searches
             if self.vecdb is None:
-                raise ValueError("VecDB not set")
-            self.setup_documents(filter=self.config.filter)
+                logger.warning("VecDB not set: cannot ingest docs.")
+            else:
+                self.setup_documents(filter=self.config.filter)
             return
         self.ingest_doc_paths(self.config.doc_paths)  # type: ignore
 
@@ -336,11 +336,15 @@ class DocChatAgent(ChatAgent):
             urls_meta = {u: idx2meta[u] for u in url_idxs}
             paths_meta = {p: idx2meta[p] for p in path_idxs}
         docs: List[Document] = []
-        parser = Parser(self.config.parsing)
+        parser: Parser = Parser(self.config.parsing)
         if len(urls) > 0:
             for ui in url_idxs:
                 meta = urls_meta.get(ui, {})
-                loader = URLLoader(urls=[all_paths[ui]], parser=parser)  # type: ignore
+                loader = URLLoader(
+                    urls=[all_paths[ui]],
+                    parsing_config=self.config.parsing,
+                    crawler_config=self.config.crawler_config,
+                )  # type: ignore
                 url_docs = loader.load()
                 # update metadata of each doc with meta
                 for d in url_docs:
@@ -466,6 +470,11 @@ class DocChatAgent(ChatAgent):
         docs = docs[: self.config.parsing.max_chunks]
         # vecdb should take care of adding docs in batches;
         # batching can be controlled via vecdb.config.batch_size
+        if not docs:
+            logging.warning(
+                "No documents to ingest after processing. Skipping VecDB addition."
+            )
+            return 0  # Return 0 since no documents were added
         self.vecdb.add_documents(docs)
         self.original_docs_length = self.doc_length(docs)
         self.setup_documents(docs, filter=self.config.filter)

langroid/agent/special/retriever_agent.py

@@ -1,5 +1,5 @@
 """
-Deprecated: use DocChatAgent instead, with DocChatAgentConfig.retrieve_only=True,
+DEPRECATED: use DocChatAgent instead, with DocChatAgentConfig.retrieve_only=True,
 and if you want to retrieve FULL relevant doc-contents rather than just extracts,
 then set DocChatAgentConfig.extraction_granularity=-1
 
@@ -47,10 +47,11 @@ class RetrieverAgent(DocChatAgent):
         )
 
     def get_records(self) -> Sequence[Document]:
-        raise NotImplementedError
+        # subclasses should override
+        return []
 
     def ingest(self) -> None:
         records = self.get_records()
         if self.vecdb is None:
-            raise ValueError("No vector store specified")
+            logger.warning("Vector store not configured. Cannot ingest records.")
         self.vecdb.add_documents(records)

langroid/parsing/url_loader.py

@@ -1,120 +1,340 @@
 import logging
 import os
+from abc import ABC, abstractmethod
 from tempfile import NamedTemporaryFile
-from typing import List, no_type_check
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
-import requests
+from dotenv import load_dotenv
 
+from langroid.exceptions import LangroidImportError
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
 from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.pydantic_v1 import BaseSettings
 
-logging.getLogger("trafilatura").setLevel(logging.ERROR)
+if TYPE_CHECKING:
+    from firecrawl import FirecrawlApp
 
+load_dotenv()
 
-class URLLoader:
-    """
-    Load a list of URLs and extract the text content.
-    Alternative approaches could use `bs4` or `scrapy`.
-
-    TODO - this currently does not handle cookie dialogs,
-     i.e. if there is a cookie pop-up, most/all of the extracted
-     content could be cookie policy text.
-     We could use `playwright` to simulate a user clicking
-     the "accept" button on the cookie dialog.
-    """
-
-    def __init__(self, urls: List[str], parser: Parser = Parser(ParsingConfig())):
-        self.urls = urls
-        self.parser = parser
+logging.getLogger("url_loader").setLevel(logging.WARNING)
 
-    @no_type_check
-    def load(self) -> List[Document]:
-        import trafilatura
-        from trafilatura.downloads import (
-            add_to_compressed_dict,
-            buffered_downloads,
-            load_download_buffer,
-        )
 
-        docs = []
-        threads = 4
-        # converted the input list to an internal format
-        dl_dict = add_to_compressed_dict(self.urls)
-        # processing loop
-        while not dl_dict.done:
-            buffer, dl_dict = load_download_buffer(
-                dl_dict,
-                sleep_time=5,
-            )
-            for url, result in buffered_downloads(buffer, threads):
-                if (
-                    url.lower().endswith(".pdf")
-                    or url.lower().endswith(".docx")
-                    or url.lower().endswith(".doc")
-                ):
-                    try:
-                        doc_parser = DocumentParser.create(
-                            url,
-                            self.parser.config,
-                        )
-                    except Exception as e:
-                        logging.error(f"Error parsing {url}: {e}")
-                        continue
+# Base crawler config and specific configurations
+class BaseCrawlerConfig(BaseSettings):
+    """Base configuration for web crawlers."""
+
+    parser: Optional[Parser] = None
+
+
+class TrafilaturaConfig(BaseCrawlerConfig):
+    """Configuration for Trafilatura crawler."""
+
+    threads: int = 4
+
+
+class FirecrawlConfig(BaseCrawlerConfig):
+    """Configuration for Firecrawl crawler."""
+
+    api_key: str = ""
+    mode: str = "scrape"
+    params: Dict[str, Any] = {}
+    timeout: Optional[int] = None
+
+    class Config:
+        # Leverage Pydantic's BaseSettings to
+        # allow setting of fields via env vars,
+        # e.g. FIRECRAWL_MODE=scrape and FIRECRAWL_API_KEY=...
+        env_prefix = "FIRECRAWL_"
+
+
+class BaseCrawler(ABC):
+    """Abstract base class for web crawlers."""
+
+    def __init__(self, config: BaseCrawlerConfig):
+        """Initialize the base crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        self.parser = config.parser if self.needs_parser else None
+        self.config: BaseCrawlerConfig = config
+
+    @property
+    @abstractmethod
+    def needs_parser(self) -> bool:
+        """Indicates whether the crawler requires a parser."""
+        pass
+
+    @abstractmethod
+    def crawl(self, urls: List[str]) -> List[Document]:
+        pass
+
+    def _process_document(self, url: str) -> List[Document]:
+        if self.parser:
+            import requests
+            from requests.structures import CaseInsensitiveDict
+
+            if self._is_document_url(url):
+                try:
+                    doc_parser = DocumentParser.create(url, self.parser.config)
                     new_chunks = doc_parser.get_doc_chunks()
-                    if len(new_chunks) == 0:
+                    if not new_chunks:
                         # If the document is empty, try to extract images
                         img_parser = ImagePdfParser(url, self.parser.config)
                         new_chunks = img_parser.get_doc_chunks()
-                    docs.extend(new_chunks)
-                else:
-                    # Try to detect content type and handle accordingly
+                    return new_chunks
+                except Exception as e:
+                    logging.error(f"Error parsing {url}: {e}")
+                    return []
+
+            else:
+                try:
+                    headers = requests.head(url).headers
+                except Exception as e:
+                    logging.warning(f"Error getting headers for {url}: {e}")
+                    headers = CaseInsensitiveDict()
+
+                content_type = headers.get("Content-Type", "").lower()
+                temp_file_suffix = None
+                if "application/pdf" in content_type:
+                    temp_file_suffix = ".pdf"
+                elif (
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+                    in content_type
+                ):
+                    temp_file_suffix = ".docx"
+                elif "application/msword" in content_type:
+                    temp_file_suffix = ".doc"
+
+                if temp_file_suffix:
                     try:
-                        headers = requests.head(url).headers
-                    except Exception as e:
-                        logging.warning(f"Error getting headers for {url}: {e}")
-                        headers = {}
-                    content_type = headers.get("Content-Type", "").lower()
-                    temp_file_suffix = None
-                    if "application/pdf" in content_type:
-                        temp_file_suffix = ".pdf"
-                    elif (
-                        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-                        in content_type
-                    ):
-                        temp_file_suffix = ".docx"
-                    elif "application/msword" in content_type:
-                        temp_file_suffix = ".doc"
-
-                    if temp_file_suffix:
-                        # Download the document content
                         response = requests.get(url)
                         with NamedTemporaryFile(
                             delete=False, suffix=temp_file_suffix
                         ) as temp_file:
                             temp_file.write(response.content)
                             temp_file_path = temp_file.name
-                        # Process the downloaded document
                         doc_parser = DocumentParser.create(
                             temp_file_path, self.parser.config
                         )
-                        docs.extend(doc_parser.get_doc_chunks())
-                        # Clean up the temporary file
+                        docs = doc_parser.get_doc_chunks()
                         os.remove(temp_file_path)
-                    else:
-                        text = trafilatura.extract(
-                            result,
-                            no_fallback=False,
-                            favor_recall=True,
+                        return docs
+                    except Exception as e:
+                        logging.error(f"Error downloading/parsing {url}: {e}")
+                        return []
+        return []
+
+    def _is_document_url(self, url: str) -> bool:
+        return any(url.lower().endswith(ext) for ext in [".pdf", ".docx", ".doc"])
+
+
+class CrawlerFactory:
+    """Factory for creating web crawlers."""
+
+    @staticmethod
+    def create_crawler(config: BaseCrawlerConfig) -> BaseCrawler:
+        """Create a crawler instance based on configuration type.
+
+        Args:
+            config: Configuration for the crawler
+
+        Returns:
+            A BaseCrawler instance
+
+        Raises:
+            ValueError: If config type is not supported
+        """
+        if isinstance(config, TrafilaturaConfig):
+            return TrafilaturaCrawler(config)
+        elif isinstance(config, FirecrawlConfig):
+            return FirecrawlCrawler(config)
+        else:
+            raise ValueError(f"Unsupported crawler configuration type: {type(config)}")
+
+
+class TrafilaturaCrawler(BaseCrawler):
+    """Crawler implementation using Trafilatura."""
+
+    def __init__(self, config: TrafilaturaConfig):
+        """Initialize the Trafilatura crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: TrafilaturaConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return True
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        import trafilatura
+        from trafilatura.downloads import (
+            add_to_compressed_dict,
+            buffered_downloads,
+            load_download_buffer,
+        )
+
+        docs = []
+        dl_dict = add_to_compressed_dict(urls)
+
+        while not dl_dict.done:
+            buffer, dl_dict = load_download_buffer(dl_dict, sleep_time=5)
+            for url, result in buffered_downloads(buffer, self.config.threads):
+                parsed_doc = self._process_document(url)
+                if parsed_doc:
+                    docs.extend(parsed_doc)
+                else:
+                    text = trafilatura.extract(
+                        result, no_fallback=False, favor_recall=True
+                    )
+                    if text is None and result is not None and isinstance(result, str):
+                        text = result
+                    if text:
+                        docs.append(
+                            Document(content=text, metadata=DocMetaData(source=url))
                         )
-                        if (
-                            text is None
-                            and result is not None
-                            and isinstance(result, str)
-                        ):
-                            text = result
-                        if text is not None and text != "":
-                            docs.append(
-                                Document(content=text, metadata=DocMetaData(source=url))
+
+        return docs
+
+
+class FirecrawlCrawler(BaseCrawler):
+    """Crawler implementation using Firecrawl."""
+
+    def __init__(self, config: FirecrawlConfig) -> None:
+        """Initialize the Firecrawl crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: FirecrawlConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return False
+
+    def _return_save_incremental_results(
+        self, app: "FirecrawlApp", crawl_id: str, output_dir: str = "firecrawl_output"
+    ) -> List[Document]:
+        # Code used verbatim from firecrawl blog with few modifications
+        # https://www.firecrawl.dev/blog/mastering-the-crawl-endpoint-in-firecrawl
+        import json
+        import time
+        from pathlib import Path
+
+        from tqdm import tqdm
+
+        pbar = tqdm(desc="Pages saved", unit=" pages", dynamic_ncols=True)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        processed_urls: set[str] = set()
+        docs = []
+
+        while True:
+            # Check current status
+            status = app.check_crawl_status(crawl_id)
+            new_pages = 0
+
+            # Save new pages
+            for page in status["data"]:
+                url = page["metadata"]["url"]
+                if url not in processed_urls:
+                    content = page.get("markdown", "")
+                    filename = f"{output_dir}/{len(processed_urls)}.md"
+                    with open(filename, "w") as f:
+                        f.write(content)
+                    docs.append(
+                        Document(content=content, metadata=DocMetaData(source=url))
+                    )
+                    processed_urls.add(url)
+                    new_pages += 1
+            pbar.update(new_pages)  # Update progress bar with new pages
+
+            # Break if crawl is complete
+            if status["status"] == "completed":
+                print(f"Saved {len(processed_urls)} pages.")
+                with open(f"{output_dir}/full_results.json", "w") as f:
+                    json.dump(status, f, indent=2)
+                break
+
+            time.sleep(5)  # Wait before checking again
+        return docs
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError:
+            raise LangroidImportError("firecrawl", "firecrawl")
+
+        app = FirecrawlApp(api_key=self.config.api_key)
+        docs = []
+        params = self.config.params.copy()  # Create a copy of the existing params
+
+        if self.config.timeout is not None:
+            params["timeout"] = self.config.timeout  # Add/override timeout in params
+
+        if self.config.mode == "scrape":
+            for url in urls:
+                try:
+                    result = app.scrape_url(url, params=params)
+                    metadata = result.get(
+                        "metadata", {}
+                    )  # Default to empty dict if missing
+                    status_code = metadata.get("statusCode")
+
+                    if status_code == 200:
+                        docs.append(
+                            Document(
+                                content=result["markdown"],
+                                metadata=DocMetaData(source=url),
                             )
+                        )
+                except Exception as e:
+                    logging.warning(
+                        f"Firecrawl encountered an error for {url}: {e}. "
+                        "Skipping but continuing."
+                    )
+        elif self.config.mode == "crawl":
+            if not isinstance(urls, list) or len(urls) != 1:
+                raise ValueError(
+                    "Crawl mode expects 'urls' to be a list containing a single URL."
+                )
+
+            # Start the crawl
+            crawl_status = app.async_crawl_url(url=urls[0], params=params)
+
+            # Save results incrementally
+            docs = self._return_save_incremental_results(app, crawl_status["id"])
         return docs
+
+
+class URLLoader:
+    """Loads URLs and extracts text using a specified crawler."""
+
+    def __init__(
+        self,
+        urls: List[Any],
+        parsing_config: ParsingConfig = ParsingConfig(),
+        crawler_config: Optional[BaseCrawlerConfig] = None,
+    ):
+        """Initialize the URL loader.
+
+        Args:
+            urls: List of URLs to load
+            parsing_config: Configuration for parsing
+            crawler_config: Configuration for the crawler
+        """
+        self.urls = urls
+        self.parsing_config = parsing_config
+
+        if crawler_config is None:
+            crawler_config = TrafilaturaConfig(parser=Parser(parsing_config))
+
+        self.crawler = CrawlerFactory.create_crawler(crawler_config)
+
+    def load(self) -> List[Document]:
+        """Load the URLs using the specified crawler."""
+        return self.crawler.crawl(self.urls)

{langroid-0.46.0.dist-info → langroid-0.47.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.46.0
+Version: 0.47.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -121,6 +121,8 @@ Provides-Extra: exa
 Requires-Dist: exa-py>=1.8.7; extra == 'exa'
 Provides-Extra: fastembed
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'fastembed'
+Provides-Extra: firecrawl
+Requires-Dist: firecrawl-py>=1.13.5; extra == 'firecrawl'
 Provides-Extra: google-genai
 Requires-Dist: google-genai>=1.0.0; extra == 'google-genai'
 Provides-Extra: google-generativeai

{langroid-0.46.0.dist-info → langroid-0.47.1.dist-info}/RECORD

@@ -14,11 +14,11 @@ langroid/agent/xml_tool_message.py,sha256=6SshYZJKIfi4mkE-gIoSwjkEYekQ8GwcSiCv7a
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=UHB6P_J40vsVnssosqkpkOVWRf9NK4TOY0_G2g_Arsg,20900
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=nEiHzU5Ztb0Y7rPMg4kSf2M6bGS5s1Av_y5w0idAGIE,64763
+langroid/agent/special/doc_chat_agent.py,sha256=SrotZ0qw51fKDXlDP2lwTho0PPTuqUogFAT4jjq0ne0,65231
 langroid/agent/special/lance_doc_chat_agent.py,sha256=s8xoRs0gGaFtDYFUSIRchsgDVbS5Q3C2b2mr3V1Fd-Q,10419
 langroid/agent/special/lance_tools.py,sha256=qS8x4wi8mrqfbYV2ztFzrcxyhHQ0ZWOc-zkYiH7awj0,2105
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
-langroid/agent/special/retriever_agent.py,sha256=lvMvf-u9rSosg4YASuFdUbGLgkzLPknXAbJZfZ1LZCc,1868
+langroid/agent/special/retriever_agent.py,sha256=i5TvwDRN5XtT8lSPkoq3Sv8jpoBylisEB2ivRbpunyc,1913
 langroid/agent/special/table_chat_agent.py,sha256=d9v2wsblaRx7oMnKhLV7uO_ujvk9gh59pSGvBXyeyNc,9659
 langroid/agent/special/arangodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/arangodb/arangodb_agent.py,sha256=12Y54c84c9qXV-YXRBcI5HaqyiY75JR4TmqlURYKJAM,25851
@@ -91,7 +91,7 @@ langroid/parsing/routing.py,sha256=-FcnlqldzL4ZoxuDwXjQPNHgBe9F9-F4R6q7b_z9CvI,1
 langroid/parsing/search.py,sha256=0NJ5-Rou_BbrHAD7O9b20bKjZJnbadjObvGm4Zq8Kis,9818
 langroid/parsing/spider.py,sha256=hAVM6wxh1pQ0EN4tI5wMBtAjIk0T-xnpi-ZUzWybhos,3258
 langroid/parsing/table_loader.py,sha256=qNM4obT_0Y4tjrxNBCNUYjKQ9oETCZ7FbolKBTcz-GM,3410
-langroid/parsing/url_loader.py,sha256=obi_kj6ehBkdh5mXNtYCXpm3KCuExoy2D1ODVlFbXbQ,4895
+langroid/parsing/url_loader.py,sha256=tNLyCo8A08GcB8KFr04YKDO9KFHyqNacKU0-DuWlu4I,11721
 langroid/parsing/urls.py,sha256=Tjzr64YsCusiYkY0LEGB5-rSuX8T2P_4DVoOFKAeKuI,8081
 langroid/parsing/utils.py,sha256=WwqzOhbQRlorbVvddDIZKv9b1KqZCBDm955lgIHDXRw,12828
 langroid/parsing/web_search.py,sha256=sARV1Tku4wiInhuCz0kRaMHcoF6Ok6CLu7vapLS8hjs,8222
@@ -127,7 +127,7 @@ langroid/vector_store/pineconedb.py,sha256=otxXZNaBKb9f_H75HTaU3lMHiaR2NUp5MqwLZ
 langroid/vector_store/postgres.py,sha256=wHPtIi2qM4fhO4pMQr95pz1ZCe7dTb2hxl4VYspGZoA,16104
 langroid/vector_store/qdrantdb.py,sha256=O6dSBoDZ0jzfeVBd7LLvsXu083xs2fxXtPa9gGX3JX4,18443
 langroid/vector_store/weaviatedb.py,sha256=Yn8pg139gOy3zkaPfoTbMXEEBCiLiYa1MU5d_3UA1K4,11847
-langroid-0.46.0.dist-info/METADATA,sha256=S5uBIAjkQEV4KbQSZ2OH-YMoMfGIJtnulBdaYNouSpw,63389
-langroid-0.46.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.46.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.46.0.dist-info/RECORD,,
+langroid-0.47.1.dist-info/METADATA,sha256=EI6zx7u_-h8K_P_e9BXM1gF8rnEPB1ENObZ-USOXSlU,63473
+langroid-0.47.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.47.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.47.1.dist-info/RECORD,,