langroid 0.45.10__py3-none-any.whl → 0.47.0__py3-none-any.whl

langroid/agent/special/doc_chat_agent.py

@@ -50,7 +50,7 @@ from langroid.parsing.search import (
     preprocess_text,
 )
 from langroid.parsing.table_loader import describe_dataframe
-from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.url_loader import BaseCrawlerConfig, TrafilaturaConfig, URLLoader
 from langroid.parsing.urls import get_list_from_user, get_urls_paths_bytes_indices
 from langroid.prompts.prompts_config import PromptsConfig
 from langroid.prompts.templates import SUMMARY_ANSWER_PROMPT_GPT4
@@ -192,6 +192,7 @@ class DocChatAgentConfig(ChatAgentConfig):
             library="pymupdf4llm",
         ),
     )
+    crawler_config: Optional[BaseCrawlerConfig] = TrafilaturaConfig()
 
     # Allow vecdb to be None in case we want to explicitly set it later
     vecdb: Optional[VectorStoreConfig] = QdrantDBConfig(
@@ -336,11 +337,15 @@ class DocChatAgent(ChatAgent):
             urls_meta = {u: idx2meta[u] for u in url_idxs}
             paths_meta = {p: idx2meta[p] for p in path_idxs}
         docs: List[Document] = []
-        parser = Parser(self.config.parsing)
+        parser: Parser = Parser(self.config.parsing)
         if len(urls) > 0:
             for ui in url_idxs:
                 meta = urls_meta.get(ui, {})
-                loader = URLLoader(urls=[all_paths[ui]], parser=parser)  # type: ignore
+                loader = URLLoader(
+                    urls=[all_paths[ui]],
+                    parsing_config=self.config.parsing,
+                    crawler_config=self.config.crawler_config,
+                )  # type: ignore
                 url_docs = loader.load()
                 # update metadata of each doc with meta
                 for d in url_docs:
@@ -466,6 +471,11 @@ class DocChatAgent(ChatAgent):
         docs = docs[: self.config.parsing.max_chunks]
         # vecdb should take care of adding docs in batches;
         # batching can be controlled via vecdb.config.batch_size
+        if not docs:
+            logging.warning(
+                "No documents to ingest after processing. Skipping VecDB addition."
+            )
+            return 0  # Return 0 since no documents were added
         self.vecdb.add_documents(docs)
         self.original_docs_length = self.doc_length(docs)
         self.setup_documents(docs, filter=self.config.filter)

langroid/embedding_models/models.py

@@ -10,6 +10,7 @@ from openai import AzureOpenAI, OpenAI
 
 from langroid.embedding_models.base import EmbeddingModel, EmbeddingModelsConfig
 from langroid.exceptions import LangroidImportError
+from langroid.language_models.openai_gpt import LangDBParams
 from langroid.mytypes import Embeddings
 from langroid.parsing.utils import batched
 
@@ -24,6 +25,7 @@ class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     organization: str = ""
     dims: int = 1536
     context_length: int = 8192
+    langdb_params: LangDBParams = LangDBParams()
 
     class Config:
         # enable auto-loading of env vars with OPENAI_ prefix, e.g.
@@ -136,11 +138,13 @@ class EmbeddingFunctionCallable:
         """
         embeds = []
         if isinstance(self.embed_model, (OpenAIEmbeddings, AzureOpenAIEmbeddings)):
-            tokenized_texts = self.embed_model.truncate_texts(input)
+            # Truncate texts to context length while preserving text format
+            truncated_texts = self.embed_model.truncate_texts(input)
 
-            for batch in batched(tokenized_texts, self.batch_size):
+            # Process in batches
+            for batch in batched(truncated_texts, self.batch_size):
                 result = self.embed_model.client.embeddings.create(
-                    input=batch, model=self.embed_model.config.model_name
+                    input=batch, model=self.embed_model.config.model_name  # type: ignore
                 )
                 batch_embeds = [d.embedding for d in result.data]
                 embeds.extend(batch_embeds)
@@ -183,30 +187,66 @@ class OpenAIEmbeddings(EmbeddingModel):
         super().__init__()
         self.config = config
         load_dotenv()
-        self.config.api_key = os.getenv("OPENAI_API_KEY", "")
+
+        # Check if using LangDB
+        self.is_langdb = self.config.model_name.startswith("langdb/")
+
+        if self.is_langdb:
+            self.config.model_name = self.config.model_name.replace("langdb/", "")
+            self.config.api_base = self.config.langdb_params.base_url
+            project_id = self.config.langdb_params.project_id
+            if project_id:
+                self.config.api_base += "/" + project_id + "/v1"
+            self.config.api_key = self.config.langdb_params.api_key
+
+        if not self.config.api_key:
+            self.config.api_key = os.getenv("OPENAI_API_KEY", "")
+
         self.config.organization = os.getenv("OPENAI_ORGANIZATION", "")
+
         if self.config.api_key == "":
-            raise ValueError(
-                """OPENAI_API_KEY env variable must be set to use 
-                OpenAIEmbeddings. Please set the OPENAI_API_KEY value 
-                in your .env file.
-                """
-            )
-        self.client = OpenAI(base_url=self.config.api_base, api_key=self.config.api_key)
+            if self.is_langdb:
+                raise ValueError(
+                    """
+                    LANGDB_API_KEY must be set in .env or your environment 
+                    to use OpenAIEmbeddings via LangDB.
+                    """
+                )
+            else:
+                raise ValueError(
+                    """
+                    OPENAI_API_KEY must be set in .env or your environment 
+                    to use OpenAIEmbeddings.
+                    """
+                )
+
+        self.client = OpenAI(
+            base_url=self.config.api_base,
+            api_key=self.config.api_key,
+            organization=self.config.organization,
+        )
+        model_for_tokenizer = self.config.model_name
+        if model_for_tokenizer.startswith("openai/"):
+            self.config.model_name = model_for_tokenizer.replace("openai/", "")
         self.tokenizer = tiktoken.encoding_for_model(self.config.model_name)
 
-    def truncate_texts(self, texts: List[str]) -> List[List[int]]:
+    def truncate_texts(self, texts: List[str]) -> List[str] | List[List[int]]:
         """
         Truncate texts to the embedding model's context length.
         TODO: Maybe we should show warning, and consider doing T5 summarization?
         """
-        return [
+        truncated_tokens = [
             self.tokenizer.encode(text, disallowed_special=())[
                 : self.config.context_length
             ]
             for text in texts
         ]
 
+        if self.is_langdb:
+            # LangDB embedding endpt only works with strings, not tokens
+            return [self.tokenizer.decode(tokens) for tokens in truncated_tokens]
+        return truncated_tokens
+
     def embedding_fn(self) -> Callable[[List[str]], Embeddings]:
         return EmbeddingFunctionCallable(self, self.config.batch_size)
 
@@ -256,7 +296,7 @@ class AzureOpenAIEmbeddings(EmbeddingModel):
         )
         self.tokenizer = tiktoken.encoding_for_model(self.config.model_name)
 
-    def truncate_texts(self, texts: List[str]) -> List[List[int]]:
+    def truncate_texts(self, texts: List[str]) -> List[str] | List[List[int]]:
         """
         Truncate texts to the embedding model's context length.
         TODO: Maybe we should show warning, and consider doing T5 summarization?

langroid/language_models/openai_gpt.py

@@ -66,7 +66,7 @@ from langroid.language_models.utils import (
     retry_with_exponential_backoff,
 )
 from langroid.parsing.parse_json import parse_imperfect_json
-from langroid.pydantic_v1 import BaseModel
+from langroid.pydantic_v1 import BaseModel, BaseSettings
 from langroid.utils.configuration import settings
 from langroid.utils.constants import Colors
 from langroid.utils.system import friendly_error
@@ -82,9 +82,13 @@ DEEPSEEK_BASE_URL = "https://api.deepseek.com/v1"
 OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
 GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai"
 GLHF_BASE_URL = "https://glhf.chat/api/openai/v1"
+LANGDB_BASE_URL = "https://api.us-east-1.langdb.ai"
 OLLAMA_API_KEY = "ollama"
 DUMMY_API_KEY = "xxx"
 
+VLLM_API_KEY = os.environ.get("VLLM_API_KEY", DUMMY_API_KEY)
+LLAMACPP_API_KEY = os.environ.get("LLAMA_API_KEY", DUMMY_API_KEY)
+
 
 openai_chat_model_pref_list = [
     OpenAIChatModel.GPT4o,
@@ -177,6 +181,24 @@ def noop() -> None:
     return None
 
 
+class LangDBParams(BaseSettings):
+    """
+    Parameters specific to LangDB integration.
+    """
+
+    api_key: str = DUMMY_API_KEY
+    project_id: str = ""
+    label: Optional[str] = None
+    run_id: Optional[str] = None
+    thread_id: Optional[str] = None
+    base_url: str = LANGDB_BASE_URL
+
+    class Config:
+        # allow setting of fields via env vars,
+        # e.g. LANGDB_PROJECT_ID=1234
+        env_prefix = "LANGDB_"
+
+
 class OpenAICallParams(BaseModel):
     """
     Various params that can be sent to an OpenAI API chat-completion call.
@@ -253,6 +275,8 @@ class OpenAIGPTConfig(LLMConfig):
     # e.g. "mistral-instruct-v0.2 (a fuzzy search is done to find the closest match)
     formatter: str | None = None
     hf_formatter: HFFormatter | None = None
+    langdb_params: LangDBParams = LangDBParams()
+    headers: Dict[str, str] = {}
 
     def __init__(self, **kwargs) -> None:  # type: ignore
         local_model = "api_base" in kwargs and kwargs["api_base"] is not None
@@ -496,6 +520,7 @@ class OpenAIGPT(LanguageModel):
         self.is_deepseek = self.is_deepseek_model()
         self.is_glhf = self.config.chat_model.startswith("glhf/")
         self.is_openrouter = self.config.chat_model.startswith("openrouter/")
+        self.is_langdb = self.config.chat_model.startswith("langdb/")
 
         if self.is_groq:
             # use groq-specific client
@@ -544,18 +569,39 @@ class OpenAIGPT(LanguageModel):
                 self.api_base = DEEPSEEK_BASE_URL
                 if self.api_key == OPENAI_API_KEY:
                     self.api_key = os.getenv("DEEPSEEK_API_KEY", DUMMY_API_KEY)
+            elif self.is_langdb:
+                self.config.chat_model = self.config.chat_model.replace("langdb/", "")
+                self.api_base = self.config.langdb_params.base_url
+                project_id = self.config.langdb_params.project_id
+                if project_id:
+                    self.api_base += "/" + project_id + "/v1"
+                if self.api_key == OPENAI_API_KEY:
+                    self.api_key = self.config.langdb_params.api_key or DUMMY_API_KEY
+
+                if self.config.langdb_params:
+                    params = self.config.langdb_params
+                    if params.project_id:
+                        self.config.headers["x-project-id"] = params.project_id
+                    if params.label:
+                        self.config.headers["x-label"] = params.label
+                    if params.run_id:
+                        self.config.headers["x-run-id"] = params.run_id
+                    if params.thread_id:
+                        self.config.headers["x-thread-id"] = params.thread_id
 
             self.client = OpenAI(
                 api_key=self.api_key,
                 base_url=self.api_base,
                 organization=self.config.organization,
                 timeout=Timeout(self.config.timeout),
+                default_headers=self.config.headers,
             )
             self.async_client = AsyncOpenAI(
                 api_key=self.api_key,
                 organization=self.config.organization,
                 base_url=self.api_base,
                 timeout=Timeout(self.config.timeout),
+                default_headers=self.config.headers,
             )
 
         self.cache: CacheDB | None = None
@@ -1028,6 +1074,7 @@ class OpenAIGPT(LanguageModel):
                 OpenAIResponse object (with choices, usage)
 
         """
+
         completion = ""
         reasoning = ""
         function_args = ""
@@ -1075,7 +1122,9 @@ class OpenAIGPT(LanguageModel):
         )
 
     @staticmethod
-    def tool_deltas_to_tools(tools: List[Dict[str, Any]]) -> Tuple[
+    def tool_deltas_to_tools(
+        tools: List[Dict[str, Any]],
+    ) -> Tuple[
         str,
         List[OpenAIToolCall],
         List[Dict[str, Any]],

langroid/parsing/url_loader.py

@@ -1,120 +1,340 @@
 import logging
 import os
+from abc import ABC, abstractmethod
 from tempfile import NamedTemporaryFile
-from typing import List, no_type_check
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
-import requests
+from dotenv import load_dotenv
 
+from langroid.exceptions import LangroidImportError
 from langroid.mytypes import DocMetaData, Document
 from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
 from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.pydantic_v1 import BaseSettings
 
-logging.getLogger("trafilatura").setLevel(logging.ERROR)
+if TYPE_CHECKING:
+    from firecrawl import FirecrawlApp
 
+load_dotenv()
 
-class URLLoader:
-    """
-    Load a list of URLs and extract the text content.
-    Alternative approaches could use `bs4` or `scrapy`.
-
-    TODO - this currently does not handle cookie dialogs,
-     i.e. if there is a cookie pop-up, most/all of the extracted
-     content could be cookie policy text.
-     We could use `playwright` to simulate a user clicking
-     the "accept" button on the cookie dialog.
-    """
-
-    def __init__(self, urls: List[str], parser: Parser = Parser(ParsingConfig())):
-        self.urls = urls
-        self.parser = parser
+logging.getLogger("url_loader").setLevel(logging.WARNING)
 
-    @no_type_check
-    def load(self) -> List[Document]:
-        import trafilatura
-        from trafilatura.downloads import (
-            add_to_compressed_dict,
-            buffered_downloads,
-            load_download_buffer,
-        )
 
-        docs = []
-        threads = 4
-        # converted the input list to an internal format
-        dl_dict = add_to_compressed_dict(self.urls)
-        # processing loop
-        while not dl_dict.done:
-            buffer, dl_dict = load_download_buffer(
-                dl_dict,
-                sleep_time=5,
-            )
-            for url, result in buffered_downloads(buffer, threads):
-                if (
-                    url.lower().endswith(".pdf")
-                    or url.lower().endswith(".docx")
-                    or url.lower().endswith(".doc")
-                ):
-                    try:
-                        doc_parser = DocumentParser.create(
-                            url,
-                            self.parser.config,
-                        )
-                    except Exception as e:
-                        logging.error(f"Error parsing {url}: {e}")
-                        continue
+# Base crawler config and specific configurations
+class BaseCrawlerConfig(BaseSettings):
+    """Base configuration for web crawlers."""
+
+    parser: Optional[Parser] = None
+
+
+class TrafilaturaConfig(BaseCrawlerConfig):
+    """Configuration for Trafilatura crawler."""
+
+    threads: int = 4
+
+
+class FirecrawlConfig(BaseCrawlerConfig):
+    """Configuration for Firecrawl crawler."""
+
+    api_key: str = ""
+    mode: str = "scrape"
+    params: Dict[str, Any] = {}
+    timeout: Optional[int] = None
+
+    class Config:
+        # Leverage Pydantic's BaseSettings to
+        # allow setting of fields via env vars,
+        # e.g. FIRECRAWL_MODE=scrape and FIRECRAWL_API_KEY=...
+        env_prefix = "FIRECRAWL_"
+
+
+class BaseCrawler(ABC):
+    """Abstract base class for web crawlers."""
+
+    def __init__(self, config: BaseCrawlerConfig):
+        """Initialize the base crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        self.parser = config.parser if self.needs_parser else None
+        self.config: BaseCrawlerConfig = config
+
+    @property
+    @abstractmethod
+    def needs_parser(self) -> bool:
+        """Indicates whether the crawler requires a parser."""
+        pass
+
+    @abstractmethod
+    def crawl(self, urls: List[str]) -> List[Document]:
+        pass
+
+    def _process_document(self, url: str) -> List[Document]:
+        if self.parser:
+            import requests
+            from requests.structures import CaseInsensitiveDict
+
+            if self._is_document_url(url):
+                try:
+                    doc_parser = DocumentParser.create(url, self.parser.config)
                     new_chunks = doc_parser.get_doc_chunks()
-                    if len(new_chunks) == 0:
+                    if not new_chunks:
                         # If the document is empty, try to extract images
                         img_parser = ImagePdfParser(url, self.parser.config)
                         new_chunks = img_parser.get_doc_chunks()
-                    docs.extend(new_chunks)
-                else:
-                    # Try to detect content type and handle accordingly
+                    return new_chunks
+                except Exception as e:
+                    logging.error(f"Error parsing {url}: {e}")
+                    return []
+
+            else:
+                try:
+                    headers = requests.head(url).headers
+                except Exception as e:
+                    logging.warning(f"Error getting headers for {url}: {e}")
+                    headers = CaseInsensitiveDict()
+
+                content_type = headers.get("Content-Type", "").lower()
+                temp_file_suffix = None
+                if "application/pdf" in content_type:
+                    temp_file_suffix = ".pdf"
+                elif (
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+                    in content_type
+                ):
+                    temp_file_suffix = ".docx"
+                elif "application/msword" in content_type:
+                    temp_file_suffix = ".doc"
+
+                if temp_file_suffix:
                     try:
-                        headers = requests.head(url).headers
-                    except Exception as e:
-                        logging.warning(f"Error getting headers for {url}: {e}")
-                        headers = {}
-                    content_type = headers.get("Content-Type", "").lower()
-                    temp_file_suffix = None
-                    if "application/pdf" in content_type:
-                        temp_file_suffix = ".pdf"
-                    elif (
-                        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-                        in content_type
-                    ):
-                        temp_file_suffix = ".docx"
-                    elif "application/msword" in content_type:
-                        temp_file_suffix = ".doc"
-
-                    if temp_file_suffix:
-                        # Download the document content
                         response = requests.get(url)
                         with NamedTemporaryFile(
                             delete=False, suffix=temp_file_suffix
                         ) as temp_file:
                             temp_file.write(response.content)
                             temp_file_path = temp_file.name
-                        # Process the downloaded document
                         doc_parser = DocumentParser.create(
                             temp_file_path, self.parser.config
                         )
-                        docs.extend(doc_parser.get_doc_chunks())
-                        # Clean up the temporary file
+                        docs = doc_parser.get_doc_chunks()
                         os.remove(temp_file_path)
-                    else:
-                        text = trafilatura.extract(
-                            result,
-                            no_fallback=False,
-                            favor_recall=True,
+                        return docs
+                    except Exception as e:
+                        logging.error(f"Error downloading/parsing {url}: {e}")
+                        return []
+        return []
+
+    def _is_document_url(self, url: str) -> bool:
+        return any(url.lower().endswith(ext) for ext in [".pdf", ".docx", ".doc"])
+
+
+class CrawlerFactory:
+    """Factory for creating web crawlers."""
+
+    @staticmethod
+    def create_crawler(config: BaseCrawlerConfig) -> BaseCrawler:
+        """Create a crawler instance based on configuration type.
+
+        Args:
+            config: Configuration for the crawler
+
+        Returns:
+            A BaseCrawler instance
+
+        Raises:
+            ValueError: If config type is not supported
+        """
+        if isinstance(config, TrafilaturaConfig):
+            return TrafilaturaCrawler(config)
+        elif isinstance(config, FirecrawlConfig):
+            return FirecrawlCrawler(config)
+        else:
+            raise ValueError(f"Unsupported crawler configuration type: {type(config)}")
+
+
+class TrafilaturaCrawler(BaseCrawler):
+    """Crawler implementation using Trafilatura."""
+
+    def __init__(self, config: TrafilaturaConfig):
+        """Initialize the Trafilatura crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: TrafilaturaConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return True
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        import trafilatura
+        from trafilatura.downloads import (
+            add_to_compressed_dict,
+            buffered_downloads,
+            load_download_buffer,
+        )
+
+        docs = []
+        dl_dict = add_to_compressed_dict(urls)
+
+        while not dl_dict.done:
+            buffer, dl_dict = load_download_buffer(dl_dict, sleep_time=5)
+            for url, result in buffered_downloads(buffer, self.config.threads):
+                parsed_doc = self._process_document(url)
+                if parsed_doc:
+                    docs.extend(parsed_doc)
+                else:
+                    text = trafilatura.extract(
+                        result, no_fallback=False, favor_recall=True
+                    )
+                    if text is None and result is not None and isinstance(result, str):
+                        text = result
+                    if text:
+                        docs.append(
+                            Document(content=text, metadata=DocMetaData(source=url))
                         )
-                        if (
-                            text is None
-                            and result is not None
-                            and isinstance(result, str)
-                        ):
-                            text = result
-                        if text is not None and text != "":
-                            docs.append(
-                                Document(content=text, metadata=DocMetaData(source=url))
+
+        return docs
+
+
+class FirecrawlCrawler(BaseCrawler):
+    """Crawler implementation using Firecrawl."""
+
+    def __init__(self, config: FirecrawlConfig) -> None:
+        """Initialize the Firecrawl crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: FirecrawlConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return False
+
+    def _return_save_incremental_results(
+        self, app: "FirecrawlApp", crawl_id: str, output_dir: str = "firecrawl_output"
+    ) -> List[Document]:
+        # Code used verbatim from firecrawl blog with few modifications
+        # https://www.firecrawl.dev/blog/mastering-the-crawl-endpoint-in-firecrawl
+        import json
+        import time
+        from pathlib import Path
+
+        from tqdm import tqdm
+
+        pbar = tqdm(desc="Pages saved", unit=" pages", dynamic_ncols=True)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        processed_urls: set[str] = set()
+        docs = []
+
+        while True:
+            # Check current status
+            status = app.check_crawl_status(crawl_id)
+            new_pages = 0
+
+            # Save new pages
+            for page in status["data"]:
+                url = page["metadata"]["url"]
+                if url not in processed_urls:
+                    content = page.get("markdown", "")
+                    filename = f"{output_dir}/{len(processed_urls)}.md"
+                    with open(filename, "w") as f:
+                        f.write(content)
+                    docs.append(
+                        Document(content=content, metadata=DocMetaData(source=url))
+                    )
+                    processed_urls.add(url)
+                    new_pages += 1
+            pbar.update(new_pages)  # Update progress bar with new pages
+
+            # Break if crawl is complete
+            if status["status"] == "completed":
+                print(f"Saved {len(processed_urls)} pages.")
+                with open(f"{output_dir}/full_results.json", "w") as f:
+                    json.dump(status, f, indent=2)
+                break
+
+            time.sleep(5)  # Wait before checking again
+        return docs
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError:
+            raise LangroidImportError("firecrawl", "firecrawl")
+
+        app = FirecrawlApp(api_key=self.config.api_key)
+        docs = []
+        params = self.config.params.copy()  # Create a copy of the existing params
+
+        if self.config.timeout is not None:
+            params["timeout"] = self.config.timeout  # Add/override timeout in params
+
+        if self.config.mode == "scrape":
+            for url in urls:
+                try:
+                    result = app.scrape_url(url, params=params)
+                    metadata = result.get(
+                        "metadata", {}
+                    )  # Default to empty dict if missing
+                    status_code = metadata.get("statusCode")
+
+                    if status_code == 200:
+                        docs.append(
+                            Document(
+                                content=result["markdown"],
+                                metadata=DocMetaData(source=url),
                             )
+                        )
+                except Exception as e:
+                    logging.warning(
+                        f"Firecrawl encountered an error for {url}: {e}. "
+                        "Skipping but continuing."
+                    )
+        elif self.config.mode == "crawl":
+            if not isinstance(urls, list) or len(urls) != 1:
+                raise ValueError(
+                    "Crawl mode expects 'urls' to be a list containing a single URL."
+                )
+
+            # Start the crawl
+            crawl_status = app.async_crawl_url(url=urls[0], params=params)
+
+            # Save results incrementally
+            docs = self._return_save_incremental_results(app, crawl_status["id"])
         return docs
+
+
+class URLLoader:
+    """Loads URLs and extracts text using a specified crawler."""
+
+    def __init__(
+        self,
+        urls: List[Any],
+        parsing_config: ParsingConfig = ParsingConfig(),
+        crawler_config: Optional[BaseCrawlerConfig] = None,
+    ):
+        """Initialize the URL loader.
+
+        Args:
+            urls: List of URLs to load
+            parsing_config: Configuration for parsing
+            crawler_config: Configuration for the crawler
+        """
+        self.urls = urls
+        self.parsing_config = parsing_config
+
+        if crawler_config is None:
+            crawler_config = TrafilaturaConfig(parser=Parser(parsing_config))
+
+        self.crawler = CrawlerFactory.create_crawler(crawler_config)
+
+    def load(self) -> List[Document]:
+        """Load the URLs using the specified crawler."""
+        return self.crawler.crawl(self.urls)

{langroid-0.45.10.dist-info → langroid-0.47.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.45.10
+Version: 0.47.0
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -121,6 +121,8 @@ Provides-Extra: exa
 Requires-Dist: exa-py>=1.8.7; extra == 'exa'
 Provides-Extra: fastembed
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'fastembed'
+Provides-Extra: firecrawl
+Requires-Dist: firecrawl-py>=1.13.5; extra == 'firecrawl'
 Provides-Extra: google-genai
 Requires-Dist: google-genai>=1.0.0; extra == 'google-genai'
 Provides-Extra: google-generativeai

{langroid-0.45.10.dist-info → langroid-0.47.0.dist-info}/RECORD

@@ -14,7 +14,7 @@ langroid/agent/xml_tool_message.py,sha256=6SshYZJKIfi4mkE-gIoSwjkEYekQ8GwcSiCv7a
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=UHB6P_J40vsVnssosqkpkOVWRf9NK4TOY0_G2g_Arsg,20900
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=nEiHzU5Ztb0Y7rPMg4kSf2M6bGS5s1Av_y5w0idAGIE,64763
+langroid/agent/special/doc_chat_agent.py,sha256=_CwxBx2gLs5BoUI_4CpqHyzua5ljFfqytV9mwjHdmbY,65233
 langroid/agent/special/lance_doc_chat_agent.py,sha256=s8xoRs0gGaFtDYFUSIRchsgDVbS5Q3C2b2mr3V1Fd-Q,10419
 langroid/agent/special/lance_tools.py,sha256=qS8x4wi8mrqfbYV2ztFzrcxyhHQ0ZWOc-zkYiH7awj0,2105
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -59,7 +59,7 @@ langroid/cachedb/momento_cachedb.py,sha256=YEOJ62hEcV6iIeMr5aGgRYgWQqFYaej9gEDEc
 langroid/cachedb/redis_cachedb.py,sha256=7kgnbf4b5CKsCrlL97mHWKvdvlLt8zgn7lc528jEpiE,5141
 langroid/embedding_models/__init__.py,sha256=KyYxR3jDFUCfYjSuCL86qjAmrq6mXXjOT4lFNOKVj6Y,955
 langroid/embedding_models/base.py,sha256=Ml7oA6PzQm0wZmIYn3fhF7dvZCi-amviWUwOeBegH3A,2562
-langroid/embedding_models/models.py,sha256=kYGGG-FkmGxNr0msXM3ANZ2eU-C85iTcoWNo00E9F_4,19151
+langroid/embedding_models/models.py,sha256=iGRrQR7ehDunA_7cPMu3CiHFugYWDkauOsiqHH-bv9s,20725
 langroid/embedding_models/remote_embeds.py,sha256=6_kjXByVbqhY9cGwl9R83ZcYC2km-nGieNNAo1McHaY,5151
 langroid/embedding_models/protoc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/embedding_models/protoc/embeddings.proto,sha256=_O-SgFpTaylQeOTgSpxhEJ7CUw7PeCQQJLaPqpPYKJg,321
@@ -72,7 +72,7 @@ langroid/language_models/base.py,sha256=mDYmFCBCLdq8_Uvws4MiewwEgcOCP8Qb0e5yUXr3
 langroid/language_models/config.py,sha256=9Q8wk5a7RQr8LGMT_0WkpjY8S4ywK06SalVRjXlfCiI,378
 langroid/language_models/mock_lm.py,sha256=5BgHKDVRWFbUwDT_PFgTZXz9-k8wJSA2e3PZmyDgQ1k,4022
 langroid/language_models/model_info.py,sha256=tfBBxL0iUf2mVN6CjcvqflzFUVg2oZqOJZexZ8jHTYA,12216
-langroid/language_models/openai_gpt.py,sha256=FMi4rQsdJETof5eSQrxIaBdu-5cOHFg8fFy7Hx7twOQ,77691
+langroid/language_models/openai_gpt.py,sha256=Re4T1my9rhOPI-w4JCluhAZUVUIbW2AZJ3MIJMYjRuk,79633
 langroid/language_models/utils.py,sha256=L4_CbihDMTGcsg0TOG1Yd5JFEto46--h7CX_14m89sQ,5016
 langroid/language_models/prompt_formatter/__init__.py,sha256=2-5cdE24XoFDhifOLl8yiscohil1ogbP1ECkYdBlBsk,372
 langroid/language_models/prompt_formatter/base.py,sha256=eDS1sgRNZVnoajwV_ZIha6cba5Dt8xjgzdRbPITwx3Q,1221
@@ -91,7 +91,7 @@ langroid/parsing/routing.py,sha256=-FcnlqldzL4ZoxuDwXjQPNHgBe9F9-F4R6q7b_z9CvI,1
 langroid/parsing/search.py,sha256=0NJ5-Rou_BbrHAD7O9b20bKjZJnbadjObvGm4Zq8Kis,9818
 langroid/parsing/spider.py,sha256=hAVM6wxh1pQ0EN4tI5wMBtAjIk0T-xnpi-ZUzWybhos,3258
 langroid/parsing/table_loader.py,sha256=qNM4obT_0Y4tjrxNBCNUYjKQ9oETCZ7FbolKBTcz-GM,3410
-langroid/parsing/url_loader.py,sha256=obi_kj6ehBkdh5mXNtYCXpm3KCuExoy2D1ODVlFbXbQ,4895
+langroid/parsing/url_loader.py,sha256=tNLyCo8A08GcB8KFr04YKDO9KFHyqNacKU0-DuWlu4I,11721
 langroid/parsing/urls.py,sha256=Tjzr64YsCusiYkY0LEGB5-rSuX8T2P_4DVoOFKAeKuI,8081
 langroid/parsing/utils.py,sha256=WwqzOhbQRlorbVvddDIZKv9b1KqZCBDm955lgIHDXRw,12828
 langroid/parsing/web_search.py,sha256=sARV1Tku4wiInhuCz0kRaMHcoF6Ok6CLu7vapLS8hjs,8222
@@ -127,7 +127,7 @@ langroid/vector_store/pineconedb.py,sha256=otxXZNaBKb9f_H75HTaU3lMHiaR2NUp5MqwLZ
 langroid/vector_store/postgres.py,sha256=wHPtIi2qM4fhO4pMQr95pz1ZCe7dTb2hxl4VYspGZoA,16104
 langroid/vector_store/qdrantdb.py,sha256=O6dSBoDZ0jzfeVBd7LLvsXu083xs2fxXtPa9gGX3JX4,18443
 langroid/vector_store/weaviatedb.py,sha256=Yn8pg139gOy3zkaPfoTbMXEEBCiLiYa1MU5d_3UA1K4,11847
-langroid-0.45.10.dist-info/METADATA,sha256=wprB9hS3tbbIaV9FvHE7VTrTZq0Xv6b0TLu3YG21k6A,63390
-langroid-0.45.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.45.10.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.45.10.dist-info/RECORD,,
+langroid-0.47.0.dist-info/METADATA,sha256=kI3V76rm2kss0FLvAbM1XvinKBBxsovU386BTBDs2Ss,63473
+langroid-0.47.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.47.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.47.0.dist-info/RECORD,,