langroid 0.57.0__py3-none-any.whl → 0.58.1__py3-none-any.whl

langroid/parsing/url_loader.py

@@ -1,8 +1,9 @@
+import asyncio
 import logging
 import os
 from abc import ABC, abstractmethod
 from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional
 
 import markdownify as md
 from dotenv import load_dotenv
@@ -16,6 +17,16 @@ from langroid.pydantic_v1 import BaseSettings
 if TYPE_CHECKING:
     from firecrawl import FirecrawlApp
 
+    try:
+        from crawl4ai import CrawlResult
+        from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+        from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+        from crawl4ai.deep_crawling import DeepCrawlStrategy
+        from crawl4ai.extraction_strategy import ExtractionStrategy
+        from crawl4ai.markdown_generation_strategy import MarkdownGenerationStrategy
+    except ImportError:
+        raise LangroidImportError("crawl4ai", "crawl-4-ai")
+
 load_dotenv()
 
 logging.getLogger("url_loader").setLevel(logging.WARNING)
@@ -59,6 +70,62 @@ class ExaCrawlerConfig(BaseCrawlerConfig):
         env_prefix = "EXA_"
 
 
+class Crawl4aiConfig(BaseCrawlerConfig):
+    """Configuration for the Crawl4aiCrawler."""
+
+    crawl_mode: Literal["simple", "deep"] = "simple"
+    extraction_strategy: Optional["ExtractionStrategy"] = None
+    markdown_strategy: Optional["MarkdownGenerationStrategy"] = None
+    deep_crawl_strategy: Optional["DeepCrawlStrategy"] = None
+    scraping_strategy: Optional["ContentScrapingStrategy"] = None
+    browser_config: Optional["BrowserConfig"] = None
+    run_config: Optional["CrawlerRunConfig"] = None
+
+    _refs_resolved: bool = False
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """Resolve forward references when class is first subclassed or instantiated."""
+        super().__init_subclass__(**kwargs)
+        cls._resolve_forward_refs()
+
+    @classmethod
+    def _resolve_forward_refs(cls) -> None:
+        """Resolve forward references only when needed."""
+        if not cls._refs_resolved:
+            try:
+                from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+                from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+                from crawl4ai.deep_crawling import DeepCrawlStrategy
+                from crawl4ai.extraction_strategy import ExtractionStrategy
+                from crawl4ai.markdown_generation_strategy import (
+                    MarkdownGenerationStrategy,
+                )
+
+                # Create namespace for update_forward_refs
+                namespace = {
+                    "BrowserConfig": BrowserConfig,
+                    "CrawlerRunConfig": CrawlerRunConfig,
+                    "ContentScrapingStrategy": ContentScrapingStrategy,
+                    "DeepCrawlStrategy": DeepCrawlStrategy,
+                    "ExtractionStrategy": ExtractionStrategy,
+                    "MarkdownGenerationStrategy": MarkdownGenerationStrategy,
+                }
+
+                cls.update_forward_refs(**namespace)
+                cls._refs_resolved = True
+            except ImportError:
+                # If crawl4ai is not installed, leave forward refs as strings
+                pass
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize and ensure forward refs are resolved."""
+        self._resolve_forward_refs()
+        super().__init__(**kwargs)
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
 class BaseCrawler(ABC):
     """Abstract base class for web crawlers."""
 
@@ -163,6 +230,8 @@ class CrawlerFactory:
             return FirecrawlCrawler(config)
         elif isinstance(config, ExaCrawlerConfig):
             return ExaCrawler(config)
+        elif isinstance(config, Crawl4aiConfig):
+            return Crawl4aiCrawler(config)
         else:
             raise ValueError(f"Unsupported crawler configuration type: {type(config)}")
 
@@ -419,6 +488,180 @@ class ExaCrawler(BaseCrawler):
         return docs
 
 
+class Crawl4aiCrawler(BaseCrawler):
+    """
+    Crawler implementation using the crawl4ai library.
+
+    This crawler intelligently dispatches URLs. Standard web pages are rendered
+    and scraped using the crawl4ai browser engine. Direct links to documents
+    (PDF, DOCX, etc.) are delegated to the framework's internal DocumentParser.
+    """
+
+    def __init__(self, config: Crawl4aiConfig) -> None:
+        """Initialize the Crawl4ai crawler."""
+        super().__init__(config)
+        self.config: Crawl4aiConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        """
+        Indicates that this crawler relies on the framework's DocumentParser
+        for handling specific file types like PDF, DOCX, etc., which
+        the browser engine cannot parse directly.
+        """
+        return True
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        """
+        Executes the crawl by separating document URLs from web page URLs.
+
+        - Document URLs (.pdf, .docx, etc.) are processed using `_process_document`.
+        - Web page URLs are handled using the async crawl4ai engine.
+        """
+        all_documents: List[Document] = []
+        webpage_urls: List[str] = []
+
+        # Step 1: Separate URLs into documents and web pages
+        for url in urls:
+            parsed_doc_chunks = self._process_document(url)
+            if parsed_doc_chunks:
+                all_documents.extend(parsed_doc_chunks)
+            else:
+                webpage_urls.append(url)
+
+        # Step 2: Process web page URLs asynchronously
+        if webpage_urls:
+            try:
+                loop = asyncio.get_running_loop()
+                if loop.is_running():
+                    import nest_asyncio
+
+                    nest_asyncio.apply()
+                web_docs = asyncio.run(self._async_crawl(webpage_urls))
+            except RuntimeError:
+                web_docs = asyncio.run(self._async_crawl(webpage_urls))
+
+            all_documents.extend(web_docs)
+
+        return all_documents
+
+    def _translate_result_to_document(
+        self, result: "CrawlResult"
+    ) -> Optional[Document]:
+        """Converts a crawl4ai CrawlResult into the framework's Document format."""
+        if not result.success:
+            logging.warning(
+                f"Crawl4ai failed for URL {result.url}: {result.error_message}"
+            )
+            return None
+
+        content = ""
+        if result.extracted_content:
+            content = result.extracted_content
+        elif result.markdown:
+            if (
+                hasattr(result.markdown, "fit_markdown")
+                and result.markdown.fit_markdown
+            ):
+                content = result.markdown.fit_markdown
+            elif hasattr(result.markdown, "raw_markdown"):
+                content = result.markdown.raw_markdown
+            else:
+                content = str(result.markdown)
+
+        if not content:
+            logging.warning(f"Crawl4ai returned no content for URL {result.url}")
+            return None
+
+        # Extract metadata safely
+        title = "Unknown Title"
+        published_date = "Unknown Date"
+
+        if result.metadata:
+            title = result.metadata.get("title", "Unknown Title")
+            # Try common date field names
+            for date_field in [
+                "published_date",
+                "datePublished",
+                "article:published_time",
+                "pubdate",
+            ]:
+                if date_field in result.metadata:
+                    published_date = result.metadata.get(date_field)
+                    break
+
+        meta = DocMetaData(
+            source=result.url,
+            title=title,
+            published_date=published_date,
+            # Note: source_content is meant for reference content, not metadata
+            # Keeping it minimal as other crawlers don't populate it
+        )
+        return Document(content=content, metadata=meta)
+
+    async def _async_crawl(self, urls: List[str]) -> List[Document]:
+        try:
+            from crawl4ai import AsyncWebCrawler
+
+            # Import configs here for lazy loading
+            from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+        except ImportError:
+            raise LangroidImportError(
+                "crawl4ai", "pip install 'crawl4ai[all]' or 'crawl4ai'"
+            )
+
+        # CHANGE 2: Handle the new optional config fields.
+        # Use the user-provided config if it exists, otherwise create a default one.
+        browser_config = self.config.browser_config or BrowserConfig()
+        run_config = self.config.run_config or CrawlerRunConfig()
+
+        if self.config.extraction_strategy:
+            run_config.extraction_strategy = self.config.extraction_strategy
+        if self.config.markdown_strategy:
+            run_config.markdown_generator = self.config.markdown_strategy
+        if self.config.deep_crawl_strategy:
+            run_config.deep_crawl_strategy = self.config.deep_crawl_strategy
+        if self.config.scraping_strategy:
+            run_config.scraping_strategy = self.config.scraping_strategy
+
+        crawled_documents: List[Document] = []
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            if self.config.crawl_mode == "simple":
+                for url in urls:
+                    result = await crawler.arun(url, config=run_config)
+                    doc = self._translate_result_to_document(result)
+                    if doc:
+                        crawled_documents.append(doc)
+
+            elif self.config.crawl_mode == "deep":
+                if not urls:
+                    return []
+                if not run_config.deep_crawl_strategy:
+                    logging.warning(
+                        "Deep crawl mode requires a deep_crawl_strategy in the config."
+                    )
+                    return []
+
+                # In deep crawl mode, `crawl4ai` will discover and crawl pages
+                # starting from the seed URL. It will not process direct document links
+                # found during the deep crawl; it is designed to follow hyperlinks.
+                crawl_results = await crawler.arun(urls[0], config=run_config)
+
+                if isinstance(crawl_results, list):
+                    for result in crawl_results:
+                        doc = self._translate_result_to_document(result)
+                        if doc:
+                            crawled_documents.append(doc)
+                else:
+                    async for result in crawl_results:
+                        doc = self._translate_result_to_document(result)
+                        if doc:
+                            crawled_documents.append(doc)
+
+        return crawled_documents
+
+
 class URLLoader:
     """Loads URLs and extracts text using a specified crawler."""
 

{langroid-0.57.0.dist-info → langroid-0.58.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.57.0
+Version: 0.58.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -94,6 +94,8 @@ Requires-Dist: chainlit<3.0.0,>=2.0.1; extra == 'chainlit'
 Requires-Dist: python-socketio<6.0.0,>=5.11.0; extra == 'chainlit'
 Provides-Extra: chromadb
 Requires-Dist: chromadb<=0.4.23,>=0.4.21; extra == 'chromadb'
+Provides-Extra: crawl-4-ai
+Requires-Dist: crawl4ai>=0.6.3; extra == 'crawl-4-ai'
 Provides-Extra: db
 Requires-Dist: psycopg2-binary>=2.9.10; extra == 'db'
 Requires-Dist: psycopg2<3.0.0,>=2.9.7; extra == 'db'
@@ -344,6 +346,11 @@ teacher_task.run()
 <details>
 <summary> <b>Click to expand</b></summary>
 
+- **Jul 2025:**
+  - [0.58.0](https://github.com/langroid/langroid/releases/tag/0.58.0) Crawl4AI integration - 
+    browser-based web crawling with Playwright for JavaScript-heavy sites, no API key required (thank you @abab-dev!).
+  - [0.57.0](https://github.com/langroid/langroid/releases/tag/0.57.0) HTML Logger for interactive task visualization - 
+    self-contained HTML logs with collapsible entries, auto-refresh, and persistent UI state.
 - **Jun 2025:**
   - [0.56.0](https://github.com/langroid/langroid/releases/tag/0.56.0) `TaskTool` for delegating tasks to sub-agents - 
     enables agents to spawn sub-agents with specific tools and configurations.

{langroid-0.57.0.dist-info → langroid-0.58.1.dist-info}/RECORD

@@ -102,7 +102,7 @@ langroid/parsing/routing.py,sha256=-FcnlqldzL4ZoxuDwXjQPNHgBe9F9-F4R6q7b_z9CvI,1
 langroid/parsing/search.py,sha256=0NJ5-Rou_BbrHAD7O9b20bKjZJnbadjObvGm4Zq8Kis,9818
 langroid/parsing/spider.py,sha256=hAVM6wxh1pQ0EN4tI5wMBtAjIk0T-xnpi-ZUzWybhos,3258
 langroid/parsing/table_loader.py,sha256=qNM4obT_0Y4tjrxNBCNUYjKQ9oETCZ7FbolKBTcz-GM,3410
-langroid/parsing/url_loader.py,sha256=NQuCxa-hTOuxLZDq4xKLvPfGVB4IWFzh2ItqWq297DI,15675
+langroid/parsing/url_loader.py,sha256=ELLSimgdf0-oscmtnDxXJcL-W9jVdLDNfQ90ouCDfHE,25402
 langroid/parsing/urls.py,sha256=Tjzr64YsCusiYkY0LEGB5-rSuX8T2P_4DVoOFKAeKuI,8081
 langroid/parsing/utils.py,sha256=WwqzOhbQRlorbVvddDIZKv9b1KqZCBDm955lgIHDXRw,12828
 langroid/parsing/web_search.py,sha256=atk8wIpOfiGTvW8yL_26TvjvyY2zD24xIHIi0QjEklI,8599
@@ -139,7 +139,7 @@ langroid/vector_store/pineconedb.py,sha256=otxXZNaBKb9f_H75HTaU3lMHiaR2NUp5MqwLZ
 langroid/vector_store/postgres.py,sha256=wHPtIi2qM4fhO4pMQr95pz1ZCe7dTb2hxl4VYspGZoA,16104
 langroid/vector_store/qdrantdb.py,sha256=ZYrT9mxoUCx_67Qzb5xnkWuFG12rfe30yAg4NgG2ueA,19168
 langroid/vector_store/weaviatedb.py,sha256=Yn8pg139gOy3zkaPfoTbMXEEBCiLiYa1MU5d_3UA1K4,11847
-langroid-0.57.0.dist-info/METADATA,sha256=y3GSqdrxEQMaV2iMTmioT9KkTGBIo4oHoPRQexgCXdw,65744
-langroid-0.57.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.57.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.57.0.dist-info/RECORD,,
+langroid-0.58.1.dist-info/METADATA,sha256=mkfwCAdL_zIZxy4v12_ikK7AZLMmbHwwEkh_RJNt3QE,66270
+langroid-0.58.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.58.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.58.1.dist-info/RECORD,,