langroid 0.46.0__tar.gz → 0.47.0__tar.gz

{langroid-0.46.0 → langroid-0.47.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.46.0
+Version: 0.47.0
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -121,6 +121,8 @@ Provides-Extra: exa
 Requires-Dist: exa-py>=1.8.7; extra == 'exa'
 Provides-Extra: fastembed
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'fastembed'
+Provides-Extra: firecrawl
+Requires-Dist: firecrawl-py>=1.13.5; extra == 'firecrawl'
 Provides-Extra: google-genai
 Requires-Dist: google-genai>=1.0.0; extra == 'google-genai'
 Provides-Extra: google-generativeai

{langroid-0.46.0 → langroid-0.47.0}/langroid/agent/special/doc_chat_agent.py

@@ -50,7 +50,7 @@ from langroid.parsing.search import (
     preprocess_text,
 )
 from langroid.parsing.table_loader import describe_dataframe
-from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.url_loader import BaseCrawlerConfig, TrafilaturaConfig, URLLoader
 from langroid.parsing.urls import get_list_from_user, get_urls_paths_bytes_indices
 from langroid.prompts.prompts_config import PromptsConfig
 from langroid.prompts.templates import SUMMARY_ANSWER_PROMPT_GPT4
@@ -192,6 +192,7 @@ class DocChatAgentConfig(ChatAgentConfig):
             library="pymupdf4llm",
         ),
     )
+    crawler_config: Optional[BaseCrawlerConfig] = TrafilaturaConfig()
 
     # Allow vecdb to be None in case we want to explicitly set it later
     vecdb: Optional[VectorStoreConfig] = QdrantDBConfig(
@@ -336,11 +337,15 @@ class DocChatAgent(ChatAgent):
             urls_meta = {u: idx2meta[u] for u in url_idxs}
             paths_meta = {p: idx2meta[p] for p in path_idxs}
         docs: List[Document] = []
-        parser = Parser(self.config.parsing)
+        parser: Parser = Parser(self.config.parsing)
         if len(urls) > 0:
             for ui in url_idxs:
                 meta = urls_meta.get(ui, {})
-                loader = URLLoader(urls=[all_paths[ui]], parser=parser)  # type: ignore
+                loader = URLLoader(
+                    urls=[all_paths[ui]],
+                    parsing_config=self.config.parsing,
+                    crawler_config=self.config.crawler_config,
+                )  # type: ignore
                 url_docs = loader.load()
                 # update metadata of each doc with meta
                 for d in url_docs:
@@ -466,6 +471,11 @@ class DocChatAgent(ChatAgent):
         docs = docs[: self.config.parsing.max_chunks]
         # vecdb should take care of adding docs in batches;
         # batching can be controlled via vecdb.config.batch_size
+        if not docs:
+            logging.warning(
+                "No documents to ingest after processing. Skipping VecDB addition."
+            )
+            return 0  # Return 0 since no documents were added
         self.vecdb.add_documents(docs)
         self.original_docs_length = self.doc_length(docs)
         self.setup_documents(docs, filter=self.config.filter)

langroid-0.47.0/langroid/parsing/url_loader.py

@@ -0,0 +1,340 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from dotenv import load_dotenv
+
+from langroid.exceptions import LangroidImportError
+from langroid.mytypes import DocMetaData, Document
+from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
+from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.pydantic_v1 import BaseSettings
+
+if TYPE_CHECKING:
+    from firecrawl import FirecrawlApp
+
+load_dotenv()
+
+logging.getLogger("url_loader").setLevel(logging.WARNING)
+
+
+# Base crawler config and specific configurations
+class BaseCrawlerConfig(BaseSettings):
+    """Base configuration for web crawlers."""
+
+    parser: Optional[Parser] = None
+
+
+class TrafilaturaConfig(BaseCrawlerConfig):
+    """Configuration for Trafilatura crawler."""
+
+    threads: int = 4
+
+
+class FirecrawlConfig(BaseCrawlerConfig):
+    """Configuration for Firecrawl crawler."""
+
+    api_key: str = ""
+    mode: str = "scrape"
+    params: Dict[str, Any] = {}
+    timeout: Optional[int] = None
+
+    class Config:
+        # Leverage Pydantic's BaseSettings to
+        # allow setting of fields via env vars,
+        # e.g. FIRECRAWL_MODE=scrape and FIRECRAWL_API_KEY=...
+        env_prefix = "FIRECRAWL_"
+
+
+class BaseCrawler(ABC):
+    """Abstract base class for web crawlers."""
+
+    def __init__(self, config: BaseCrawlerConfig):
+        """Initialize the base crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        self.parser = config.parser if self.needs_parser else None
+        self.config: BaseCrawlerConfig = config
+
+    @property
+    @abstractmethod
+    def needs_parser(self) -> bool:
+        """Indicates whether the crawler requires a parser."""
+        pass
+
+    @abstractmethod
+    def crawl(self, urls: List[str]) -> List[Document]:
+        pass
+
+    def _process_document(self, url: str) -> List[Document]:
+        if self.parser:
+            import requests
+            from requests.structures import CaseInsensitiveDict
+
+            if self._is_document_url(url):
+                try:
+                    doc_parser = DocumentParser.create(url, self.parser.config)
+                    new_chunks = doc_parser.get_doc_chunks()
+                    if not new_chunks:
+                        # If the document is empty, try to extract images
+                        img_parser = ImagePdfParser(url, self.parser.config)
+                        new_chunks = img_parser.get_doc_chunks()
+                    return new_chunks
+                except Exception as e:
+                    logging.error(f"Error parsing {url}: {e}")
+                    return []
+
+            else:
+                try:
+                    headers = requests.head(url).headers
+                except Exception as e:
+                    logging.warning(f"Error getting headers for {url}: {e}")
+                    headers = CaseInsensitiveDict()
+
+                content_type = headers.get("Content-Type", "").lower()
+                temp_file_suffix = None
+                if "application/pdf" in content_type:
+                    temp_file_suffix = ".pdf"
+                elif (
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+                    in content_type
+                ):
+                    temp_file_suffix = ".docx"
+                elif "application/msword" in content_type:
+                    temp_file_suffix = ".doc"
+
+                if temp_file_suffix:
+                    try:
+                        response = requests.get(url)
+                        with NamedTemporaryFile(
+                            delete=False, suffix=temp_file_suffix
+                        ) as temp_file:
+                            temp_file.write(response.content)
+                            temp_file_path = temp_file.name
+                        doc_parser = DocumentParser.create(
+                            temp_file_path, self.parser.config
+                        )
+                        docs = doc_parser.get_doc_chunks()
+                        os.remove(temp_file_path)
+                        return docs
+                    except Exception as e:
+                        logging.error(f"Error downloading/parsing {url}: {e}")
+                        return []
+        return []
+
+    def _is_document_url(self, url: str) -> bool:
+        return any(url.lower().endswith(ext) for ext in [".pdf", ".docx", ".doc"])
+
+
+class CrawlerFactory:
+    """Factory for creating web crawlers."""
+
+    @staticmethod
+    def create_crawler(config: BaseCrawlerConfig) -> BaseCrawler:
+        """Create a crawler instance based on configuration type.
+
+        Args:
+            config: Configuration for the crawler
+
+        Returns:
+            A BaseCrawler instance
+
+        Raises:
+            ValueError: If config type is not supported
+        """
+        if isinstance(config, TrafilaturaConfig):
+            return TrafilaturaCrawler(config)
+        elif isinstance(config, FirecrawlConfig):
+            return FirecrawlCrawler(config)
+        else:
+            raise ValueError(f"Unsupported crawler configuration type: {type(config)}")
+
+
+class TrafilaturaCrawler(BaseCrawler):
+    """Crawler implementation using Trafilatura."""
+
+    def __init__(self, config: TrafilaturaConfig):
+        """Initialize the Trafilatura crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: TrafilaturaConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return True
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        import trafilatura
+        from trafilatura.downloads import (
+            add_to_compressed_dict,
+            buffered_downloads,
+            load_download_buffer,
+        )
+
+        docs = []
+        dl_dict = add_to_compressed_dict(urls)
+
+        while not dl_dict.done:
+            buffer, dl_dict = load_download_buffer(dl_dict, sleep_time=5)
+            for url, result in buffered_downloads(buffer, self.config.threads):
+                parsed_doc = self._process_document(url)
+                if parsed_doc:
+                    docs.extend(parsed_doc)
+                else:
+                    text = trafilatura.extract(
+                        result, no_fallback=False, favor_recall=True
+                    )
+                    if text is None and result is not None and isinstance(result, str):
+                        text = result
+                    if text:
+                        docs.append(
+                            Document(content=text, metadata=DocMetaData(source=url))
+                        )
+
+        return docs
+
+
+class FirecrawlCrawler(BaseCrawler):
+    """Crawler implementation using Firecrawl."""
+
+    def __init__(self, config: FirecrawlConfig) -> None:
+        """Initialize the Firecrawl crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: FirecrawlConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return False
+
+    def _return_save_incremental_results(
+        self, app: "FirecrawlApp", crawl_id: str, output_dir: str = "firecrawl_output"
+    ) -> List[Document]:
+        # Code used verbatim from firecrawl blog with few modifications
+        # https://www.firecrawl.dev/blog/mastering-the-crawl-endpoint-in-firecrawl
+        import json
+        import time
+        from pathlib import Path
+
+        from tqdm import tqdm
+
+        pbar = tqdm(desc="Pages saved", unit=" pages", dynamic_ncols=True)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        processed_urls: set[str] = set()
+        docs = []
+
+        while True:
+            # Check current status
+            status = app.check_crawl_status(crawl_id)
+            new_pages = 0
+
+            # Save new pages
+            for page in status["data"]:
+                url = page["metadata"]["url"]
+                if url not in processed_urls:
+                    content = page.get("markdown", "")
+                    filename = f"{output_dir}/{len(processed_urls)}.md"
+                    with open(filename, "w") as f:
+                        f.write(content)
+                    docs.append(
+                        Document(content=content, metadata=DocMetaData(source=url))
+                    )
+                    processed_urls.add(url)
+                    new_pages += 1
+            pbar.update(new_pages)  # Update progress bar with new pages
+
+            # Break if crawl is complete
+            if status["status"] == "completed":
+                print(f"Saved {len(processed_urls)} pages.")
+                with open(f"{output_dir}/full_results.json", "w") as f:
+                    json.dump(status, f, indent=2)
+                break
+
+            time.sleep(5)  # Wait before checking again
+        return docs
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError:
+            raise LangroidImportError("firecrawl", "firecrawl")
+
+        app = FirecrawlApp(api_key=self.config.api_key)
+        docs = []
+        params = self.config.params.copy()  # Create a copy of the existing params
+
+        if self.config.timeout is not None:
+            params["timeout"] = self.config.timeout  # Add/override timeout in params
+
+        if self.config.mode == "scrape":
+            for url in urls:
+                try:
+                    result = app.scrape_url(url, params=params)
+                    metadata = result.get(
+                        "metadata", {}
+                    )  # Default to empty dict if missing
+                    status_code = metadata.get("statusCode")
+
+                    if status_code == 200:
+                        docs.append(
+                            Document(
+                                content=result["markdown"],
+                                metadata=DocMetaData(source=url),
+                            )
+                        )
+                except Exception as e:
+                    logging.warning(
+                        f"Firecrawl encountered an error for {url}: {e}. "
+                        "Skipping but continuing."
+                    )
+        elif self.config.mode == "crawl":
+            if not isinstance(urls, list) or len(urls) != 1:
+                raise ValueError(
+                    "Crawl mode expects 'urls' to be a list containing a single URL."
+                )
+
+            # Start the crawl
+            crawl_status = app.async_crawl_url(url=urls[0], params=params)
+
+            # Save results incrementally
+            docs = self._return_save_incremental_results(app, crawl_status["id"])
+        return docs
+
+
+class URLLoader:
+    """Loads URLs and extracts text using a specified crawler."""
+
+    def __init__(
+        self,
+        urls: List[Any],
+        parsing_config: ParsingConfig = ParsingConfig(),
+        crawler_config: Optional[BaseCrawlerConfig] = None,
+    ):
+        """Initialize the URL loader.
+
+        Args:
+            urls: List of URLs to load
+            parsing_config: Configuration for parsing
+            crawler_config: Configuration for the crawler
+        """
+        self.urls = urls
+        self.parsing_config = parsing_config
+
+        if crawler_config is None:
+            crawler_config = TrafilaturaConfig(parser=Parser(parsing_config))
+
+        self.crawler = CrawlerFactory.create_crawler(crawler_config)
+
+    def load(self) -> List[Document]:
+        """Load the URLs using the specified crawler."""
+        return self.crawler.crawl(self.urls)

{langroid-0.46.0 → langroid-0.47.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.46.0"
+version = "0.47.0"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]
@@ -265,6 +265,9 @@ pinecone = [
 asyncio = [
     "asyncio>=3.4.3",
 ]
+firecrawl = [
+    "firecrawl-py>=1.13.5",
+]
 
 
 [dependency-groups]

langroid-0.46.0/langroid/parsing/url_loader.py

@@ -1,120 +0,0 @@
-import logging
-import os
-from tempfile import NamedTemporaryFile
-from typing import List, no_type_check
-
-import requests
-
-from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
-from langroid.parsing.parser import Parser, ParsingConfig
-
-logging.getLogger("trafilatura").setLevel(logging.ERROR)
-
-
-class URLLoader:
-    """
-    Load a list of URLs and extract the text content.
-    Alternative approaches could use `bs4` or `scrapy`.
-
-    TODO - this currently does not handle cookie dialogs,
-     i.e. if there is a cookie pop-up, most/all of the extracted
-     content could be cookie policy text.
-     We could use `playwright` to simulate a user clicking
-     the "accept" button on the cookie dialog.
-    """
-
-    def __init__(self, urls: List[str], parser: Parser = Parser(ParsingConfig())):
-        self.urls = urls
-        self.parser = parser
-
-    @no_type_check
-    def load(self) -> List[Document]:
-        import trafilatura
-        from trafilatura.downloads import (
-            add_to_compressed_dict,
-            buffered_downloads,
-            load_download_buffer,
-        )
-
-        docs = []
-        threads = 4
-        # converted the input list to an internal format
-        dl_dict = add_to_compressed_dict(self.urls)
-        # processing loop
-        while not dl_dict.done:
-            buffer, dl_dict = load_download_buffer(
-                dl_dict,
-                sleep_time=5,
-            )
-            for url, result in buffered_downloads(buffer, threads):
-                if (
-                    url.lower().endswith(".pdf")
-                    or url.lower().endswith(".docx")
-                    or url.lower().endswith(".doc")
-                ):
-                    try:
-                        doc_parser = DocumentParser.create(
-                            url,
-                            self.parser.config,
-                        )
-                    except Exception as e:
-                        logging.error(f"Error parsing {url}: {e}")
-                        continue
-                    new_chunks = doc_parser.get_doc_chunks()
-                    if len(new_chunks) == 0:
-                        # If the document is empty, try to extract images
-                        img_parser = ImagePdfParser(url, self.parser.config)
-                        new_chunks = img_parser.get_doc_chunks()
-                    docs.extend(new_chunks)
-                else:
-                    # Try to detect content type and handle accordingly
-                    try:
-                        headers = requests.head(url).headers
-                    except Exception as e:
-                        logging.warning(f"Error getting headers for {url}: {e}")
-                        headers = {}
-                    content_type = headers.get("Content-Type", "").lower()
-                    temp_file_suffix = None
-                    if "application/pdf" in content_type:
-                        temp_file_suffix = ".pdf"
-                    elif (
-                        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-                        in content_type
-                    ):
-                        temp_file_suffix = ".docx"
-                    elif "application/msword" in content_type:
-                        temp_file_suffix = ".doc"
-
-                    if temp_file_suffix:
-                        # Download the document content
-                        response = requests.get(url)
-                        with NamedTemporaryFile(
-                            delete=False, suffix=temp_file_suffix
-                        ) as temp_file:
-                            temp_file.write(response.content)
-                            temp_file_path = temp_file.name
-                        # Process the downloaded document
-                        doc_parser = DocumentParser.create(
-                            temp_file_path, self.parser.config
-                        )
-                        docs.extend(doc_parser.get_doc_chunks())
-                        # Clean up the temporary file
-                        os.remove(temp_file_path)
-                    else:
-                        text = trafilatura.extract(
-                            result,
-                            no_fallback=False,
-                            favor_recall=True,
-                        )
-                        if (
-                            text is None
-                            and result is not None
-                            and isinstance(result, str)
-                        ):
-                            text = result
-                        if text is not None and text != "":
-                            docs.append(
-                                Document(content=text, metadata=DocMetaData(source=url))
-                            )
-        return docs