langroid 0.45.10__tar.gz → 0.47.0__tar.gz

{langroid-0.45.10 → langroid-0.47.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.45.10
+Version: 0.47.0
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT
@@ -121,6 +121,8 @@ Provides-Extra: exa
 Requires-Dist: exa-py>=1.8.7; extra == 'exa'
 Provides-Extra: fastembed
 Requires-Dist: fastembed<0.4.0,>=0.3.1; extra == 'fastembed'
+Provides-Extra: firecrawl
+Requires-Dist: firecrawl-py>=1.13.5; extra == 'firecrawl'
 Provides-Extra: google-genai
 Requires-Dist: google-genai>=1.0.0; extra == 'google-genai'
 Provides-Extra: google-generativeai

{langroid-0.45.10 → langroid-0.47.0}/langroid/agent/special/doc_chat_agent.py

@@ -50,7 +50,7 @@ from langroid.parsing.search import (
     preprocess_text,
 )
 from langroid.parsing.table_loader import describe_dataframe
-from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.url_loader import BaseCrawlerConfig, TrafilaturaConfig, URLLoader
 from langroid.parsing.urls import get_list_from_user, get_urls_paths_bytes_indices
 from langroid.prompts.prompts_config import PromptsConfig
 from langroid.prompts.templates import SUMMARY_ANSWER_PROMPT_GPT4
@@ -192,6 +192,7 @@ class DocChatAgentConfig(ChatAgentConfig):
             library="pymupdf4llm",
         ),
     )
+    crawler_config: Optional[BaseCrawlerConfig] = TrafilaturaConfig()
 
     # Allow vecdb to be None in case we want to explicitly set it later
     vecdb: Optional[VectorStoreConfig] = QdrantDBConfig(
@@ -336,11 +337,15 @@ class DocChatAgent(ChatAgent):
             urls_meta = {u: idx2meta[u] for u in url_idxs}
             paths_meta = {p: idx2meta[p] for p in path_idxs}
         docs: List[Document] = []
-        parser = Parser(self.config.parsing)
+        parser: Parser = Parser(self.config.parsing)
         if len(urls) > 0:
             for ui in url_idxs:
                 meta = urls_meta.get(ui, {})
-                loader = URLLoader(urls=[all_paths[ui]], parser=parser)  # type: ignore
+                loader = URLLoader(
+                    urls=[all_paths[ui]],
+                    parsing_config=self.config.parsing,
+                    crawler_config=self.config.crawler_config,
+                )  # type: ignore
                 url_docs = loader.load()
                 # update metadata of each doc with meta
                 for d in url_docs:
@@ -466,6 +471,11 @@ class DocChatAgent(ChatAgent):
         docs = docs[: self.config.parsing.max_chunks]
         # vecdb should take care of adding docs in batches;
         # batching can be controlled via vecdb.config.batch_size
+        if not docs:
+            logging.warning(
+                "No documents to ingest after processing. Skipping VecDB addition."
+            )
+            return 0  # Return 0 since no documents were added
         self.vecdb.add_documents(docs)
         self.original_docs_length = self.doc_length(docs)
         self.setup_documents(docs, filter=self.config.filter)

{langroid-0.45.10 → langroid-0.47.0}/langroid/embedding_models/models.py

@@ -10,6 +10,7 @@ from openai import AzureOpenAI, OpenAI
 
 from langroid.embedding_models.base import EmbeddingModel, EmbeddingModelsConfig
 from langroid.exceptions import LangroidImportError
+from langroid.language_models.openai_gpt import LangDBParams
 from langroid.mytypes import Embeddings
 from langroid.parsing.utils import batched
 
@@ -24,6 +25,7 @@ class OpenAIEmbeddingsConfig(EmbeddingModelsConfig):
     organization: str = ""
     dims: int = 1536
     context_length: int = 8192
+    langdb_params: LangDBParams = LangDBParams()
 
     class Config:
         # enable auto-loading of env vars with OPENAI_ prefix, e.g.
@@ -136,11 +138,13 @@ class EmbeddingFunctionCallable:
         """
         embeds = []
         if isinstance(self.embed_model, (OpenAIEmbeddings, AzureOpenAIEmbeddings)):
-            tokenized_texts = self.embed_model.truncate_texts(input)
+            # Truncate texts to context length while preserving text format
+            truncated_texts = self.embed_model.truncate_texts(input)
 
-            for batch in batched(tokenized_texts, self.batch_size):
+            # Process in batches
+            for batch in batched(truncated_texts, self.batch_size):
                 result = self.embed_model.client.embeddings.create(
-                    input=batch, model=self.embed_model.config.model_name
+                    input=batch, model=self.embed_model.config.model_name  # type: ignore
                 )
                 batch_embeds = [d.embedding for d in result.data]
                 embeds.extend(batch_embeds)
@@ -183,30 +187,66 @@ class OpenAIEmbeddings(EmbeddingModel):
         super().__init__()
         self.config = config
         load_dotenv()
-        self.config.api_key = os.getenv("OPENAI_API_KEY", "")
+
+        # Check if using LangDB
+        self.is_langdb = self.config.model_name.startswith("langdb/")
+
+        if self.is_langdb:
+            self.config.model_name = self.config.model_name.replace("langdb/", "")
+            self.config.api_base = self.config.langdb_params.base_url
+            project_id = self.config.langdb_params.project_id
+            if project_id:
+                self.config.api_base += "/" + project_id + "/v1"
+            self.config.api_key = self.config.langdb_params.api_key
+
+        if not self.config.api_key:
+            self.config.api_key = os.getenv("OPENAI_API_KEY", "")
+
         self.config.organization = os.getenv("OPENAI_ORGANIZATION", "")
+
         if self.config.api_key == "":
-            raise ValueError(
-                """OPENAI_API_KEY env variable must be set to use 
-                OpenAIEmbeddings. Please set the OPENAI_API_KEY value 
-                in your .env file.
-                """
-            )
-        self.client = OpenAI(base_url=self.config.api_base, api_key=self.config.api_key)
+            if self.is_langdb:
+                raise ValueError(
+                    """
+                    LANGDB_API_KEY must be set in .env or your environment 
+                    to use OpenAIEmbeddings via LangDB.
+                    """
+                )
+            else:
+                raise ValueError(
+                    """
+                    OPENAI_API_KEY must be set in .env or your environment 
+                    to use OpenAIEmbeddings.
+                    """
+                )
+
+        self.client = OpenAI(
+            base_url=self.config.api_base,
+            api_key=self.config.api_key,
+            organization=self.config.organization,
+        )
+        model_for_tokenizer = self.config.model_name
+        if model_for_tokenizer.startswith("openai/"):
+            self.config.model_name = model_for_tokenizer.replace("openai/", "")
         self.tokenizer = tiktoken.encoding_for_model(self.config.model_name)
 
-    def truncate_texts(self, texts: List[str]) -> List[List[int]]:
+    def truncate_texts(self, texts: List[str]) -> List[str] | List[List[int]]:
         """
         Truncate texts to the embedding model's context length.
         TODO: Maybe we should show warning, and consider doing T5 summarization?
         """
-        return [
+        truncated_tokens = [
             self.tokenizer.encode(text, disallowed_special=())[
                 : self.config.context_length
             ]
             for text in texts
         ]
 
+        if self.is_langdb:
+            # LangDB embedding endpt only works with strings, not tokens
+            return [self.tokenizer.decode(tokens) for tokens in truncated_tokens]
+        return truncated_tokens
+
     def embedding_fn(self) -> Callable[[List[str]], Embeddings]:
         return EmbeddingFunctionCallable(self, self.config.batch_size)
 
@@ -256,7 +296,7 @@ class AzureOpenAIEmbeddings(EmbeddingModel):
         )
         self.tokenizer = tiktoken.encoding_for_model(self.config.model_name)
 
-    def truncate_texts(self, texts: List[str]) -> List[List[int]]:
+    def truncate_texts(self, texts: List[str]) -> List[str] | List[List[int]]:
         """
         Truncate texts to the embedding model's context length.
         TODO: Maybe we should show warning, and consider doing T5 summarization?

{langroid-0.45.10 → langroid-0.47.0}/langroid/language_models/openai_gpt.py

@@ -66,7 +66,7 @@ from langroid.language_models.utils import (
     retry_with_exponential_backoff,
 )
 from langroid.parsing.parse_json import parse_imperfect_json
-from langroid.pydantic_v1 import BaseModel
+from langroid.pydantic_v1 import BaseModel, BaseSettings
 from langroid.utils.configuration import settings
 from langroid.utils.constants import Colors
 from langroid.utils.system import friendly_error
@@ -82,9 +82,13 @@ DEEPSEEK_BASE_URL = "https://api.deepseek.com/v1"
 OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
 GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai"
 GLHF_BASE_URL = "https://glhf.chat/api/openai/v1"
+LANGDB_BASE_URL = "https://api.us-east-1.langdb.ai"
 OLLAMA_API_KEY = "ollama"
 DUMMY_API_KEY = "xxx"
 
+VLLM_API_KEY = os.environ.get("VLLM_API_KEY", DUMMY_API_KEY)
+LLAMACPP_API_KEY = os.environ.get("LLAMA_API_KEY", DUMMY_API_KEY)
+
 
 openai_chat_model_pref_list = [
     OpenAIChatModel.GPT4o,
@@ -177,6 +181,24 @@ def noop() -> None:
     return None
 
 
+class LangDBParams(BaseSettings):
+    """
+    Parameters specific to LangDB integration.
+    """
+
+    api_key: str = DUMMY_API_KEY
+    project_id: str = ""
+    label: Optional[str] = None
+    run_id: Optional[str] = None
+    thread_id: Optional[str] = None
+    base_url: str = LANGDB_BASE_URL
+
+    class Config:
+        # allow setting of fields via env vars,
+        # e.g. LANGDB_PROJECT_ID=1234
+        env_prefix = "LANGDB_"
+
+
 class OpenAICallParams(BaseModel):
     """
     Various params that can be sent to an OpenAI API chat-completion call.
@@ -253,6 +275,8 @@ class OpenAIGPTConfig(LLMConfig):
     # e.g. "mistral-instruct-v0.2 (a fuzzy search is done to find the closest match)
     formatter: str | None = None
     hf_formatter: HFFormatter | None = None
+    langdb_params: LangDBParams = LangDBParams()
+    headers: Dict[str, str] = {}
 
     def __init__(self, **kwargs) -> None:  # type: ignore
         local_model = "api_base" in kwargs and kwargs["api_base"] is not None
@@ -496,6 +520,7 @@ class OpenAIGPT(LanguageModel):
         self.is_deepseek = self.is_deepseek_model()
         self.is_glhf = self.config.chat_model.startswith("glhf/")
         self.is_openrouter = self.config.chat_model.startswith("openrouter/")
+        self.is_langdb = self.config.chat_model.startswith("langdb/")
 
         if self.is_groq:
             # use groq-specific client
@@ -544,18 +569,39 @@ class OpenAIGPT(LanguageModel):
                 self.api_base = DEEPSEEK_BASE_URL
                 if self.api_key == OPENAI_API_KEY:
                     self.api_key = os.getenv("DEEPSEEK_API_KEY", DUMMY_API_KEY)
+            elif self.is_langdb:
+                self.config.chat_model = self.config.chat_model.replace("langdb/", "")
+                self.api_base = self.config.langdb_params.base_url
+                project_id = self.config.langdb_params.project_id
+                if project_id:
+                    self.api_base += "/" + project_id + "/v1"
+                if self.api_key == OPENAI_API_KEY:
+                    self.api_key = self.config.langdb_params.api_key or DUMMY_API_KEY
+
+                if self.config.langdb_params:
+                    params = self.config.langdb_params
+                    if params.project_id:
+                        self.config.headers["x-project-id"] = params.project_id
+                    if params.label:
+                        self.config.headers["x-label"] = params.label
+                    if params.run_id:
+                        self.config.headers["x-run-id"] = params.run_id
+                    if params.thread_id:
+                        self.config.headers["x-thread-id"] = params.thread_id
 
             self.client = OpenAI(
                 api_key=self.api_key,
                 base_url=self.api_base,
                 organization=self.config.organization,
                 timeout=Timeout(self.config.timeout),
+                default_headers=self.config.headers,
             )
             self.async_client = AsyncOpenAI(
                 api_key=self.api_key,
                 organization=self.config.organization,
                 base_url=self.api_base,
                 timeout=Timeout(self.config.timeout),
+                default_headers=self.config.headers,
             )
 
         self.cache: CacheDB | None = None
@@ -1028,6 +1074,7 @@ class OpenAIGPT(LanguageModel):
                 OpenAIResponse object (with choices, usage)
 
         """
+
         completion = ""
         reasoning = ""
         function_args = ""
@@ -1075,7 +1122,9 @@ class OpenAIGPT(LanguageModel):
         )
 
     @staticmethod
-    def tool_deltas_to_tools(tools: List[Dict[str, Any]]) -> Tuple[
+    def tool_deltas_to_tools(
+        tools: List[Dict[str, Any]],
+    ) -> Tuple[
         str,
         List[OpenAIToolCall],
         List[Dict[str, Any]],

langroid-0.47.0/langroid/parsing/url_loader.py

@@ -0,0 +1,340 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from dotenv import load_dotenv
+
+from langroid.exceptions import LangroidImportError
+from langroid.mytypes import DocMetaData, Document
+from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
+from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.pydantic_v1 import BaseSettings
+
+if TYPE_CHECKING:
+    from firecrawl import FirecrawlApp
+
+load_dotenv()
+
+logging.getLogger("url_loader").setLevel(logging.WARNING)
+
+
+# Base crawler config and specific configurations
+class BaseCrawlerConfig(BaseSettings):
+    """Base configuration for web crawlers."""
+
+    parser: Optional[Parser] = None
+
+
+class TrafilaturaConfig(BaseCrawlerConfig):
+    """Configuration for Trafilatura crawler."""
+
+    threads: int = 4
+
+
+class FirecrawlConfig(BaseCrawlerConfig):
+    """Configuration for Firecrawl crawler."""
+
+    api_key: str = ""
+    mode: str = "scrape"
+    params: Dict[str, Any] = {}
+    timeout: Optional[int] = None
+
+    class Config:
+        # Leverage Pydantic's BaseSettings to
+        # allow setting of fields via env vars,
+        # e.g. FIRECRAWL_MODE=scrape and FIRECRAWL_API_KEY=...
+        env_prefix = "FIRECRAWL_"
+
+
+class BaseCrawler(ABC):
+    """Abstract base class for web crawlers."""
+
+    def __init__(self, config: BaseCrawlerConfig):
+        """Initialize the base crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        self.parser = config.parser if self.needs_parser else None
+        self.config: BaseCrawlerConfig = config
+
+    @property
+    @abstractmethod
+    def needs_parser(self) -> bool:
+        """Indicates whether the crawler requires a parser."""
+        pass
+
+    @abstractmethod
+    def crawl(self, urls: List[str]) -> List[Document]:
+        pass
+
+    def _process_document(self, url: str) -> List[Document]:
+        if self.parser:
+            import requests
+            from requests.structures import CaseInsensitiveDict
+
+            if self._is_document_url(url):
+                try:
+                    doc_parser = DocumentParser.create(url, self.parser.config)
+                    new_chunks = doc_parser.get_doc_chunks()
+                    if not new_chunks:
+                        # If the document is empty, try to extract images
+                        img_parser = ImagePdfParser(url, self.parser.config)
+                        new_chunks = img_parser.get_doc_chunks()
+                    return new_chunks
+                except Exception as e:
+                    logging.error(f"Error parsing {url}: {e}")
+                    return []
+
+            else:
+                try:
+                    headers = requests.head(url).headers
+                except Exception as e:
+                    logging.warning(f"Error getting headers for {url}: {e}")
+                    headers = CaseInsensitiveDict()
+
+                content_type = headers.get("Content-Type", "").lower()
+                temp_file_suffix = None
+                if "application/pdf" in content_type:
+                    temp_file_suffix = ".pdf"
+                elif (
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+                    in content_type
+                ):
+                    temp_file_suffix = ".docx"
+                elif "application/msword" in content_type:
+                    temp_file_suffix = ".doc"
+
+                if temp_file_suffix:
+                    try:
+                        response = requests.get(url)
+                        with NamedTemporaryFile(
+                            delete=False, suffix=temp_file_suffix
+                        ) as temp_file:
+                            temp_file.write(response.content)
+                            temp_file_path = temp_file.name
+                        doc_parser = DocumentParser.create(
+                            temp_file_path, self.parser.config
+                        )
+                        docs = doc_parser.get_doc_chunks()
+                        os.remove(temp_file_path)
+                        return docs
+                    except Exception as e:
+                        logging.error(f"Error downloading/parsing {url}: {e}")
+                        return []
+        return []
+
+    def _is_document_url(self, url: str) -> bool:
+        return any(url.lower().endswith(ext) for ext in [".pdf", ".docx", ".doc"])
+
+
+class CrawlerFactory:
+    """Factory for creating web crawlers."""
+
+    @staticmethod
+    def create_crawler(config: BaseCrawlerConfig) -> BaseCrawler:
+        """Create a crawler instance based on configuration type.
+
+        Args:
+            config: Configuration for the crawler
+
+        Returns:
+            A BaseCrawler instance
+
+        Raises:
+            ValueError: If config type is not supported
+        """
+        if isinstance(config, TrafilaturaConfig):
+            return TrafilaturaCrawler(config)
+        elif isinstance(config, FirecrawlConfig):
+            return FirecrawlCrawler(config)
+        else:
+            raise ValueError(f"Unsupported crawler configuration type: {type(config)}")
+
+
+class TrafilaturaCrawler(BaseCrawler):
+    """Crawler implementation using Trafilatura."""
+
+    def __init__(self, config: TrafilaturaConfig):
+        """Initialize the Trafilatura crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: TrafilaturaConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return True
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        import trafilatura
+        from trafilatura.downloads import (
+            add_to_compressed_dict,
+            buffered_downloads,
+            load_download_buffer,
+        )
+
+        docs = []
+        dl_dict = add_to_compressed_dict(urls)
+
+        while not dl_dict.done:
+            buffer, dl_dict = load_download_buffer(dl_dict, sleep_time=5)
+            for url, result in buffered_downloads(buffer, self.config.threads):
+                parsed_doc = self._process_document(url)
+                if parsed_doc:
+                    docs.extend(parsed_doc)
+                else:
+                    text = trafilatura.extract(
+                        result, no_fallback=False, favor_recall=True
+                    )
+                    if text is None and result is not None and isinstance(result, str):
+                        text = result
+                    if text:
+                        docs.append(
+                            Document(content=text, metadata=DocMetaData(source=url))
+                        )
+
+        return docs
+
+
+class FirecrawlCrawler(BaseCrawler):
+    """Crawler implementation using Firecrawl."""
+
+    def __init__(self, config: FirecrawlConfig) -> None:
+        """Initialize the Firecrawl crawler.
+
+        Args:
+            config: Configuration for the crawler
+        """
+        super().__init__(config)
+        self.config: FirecrawlConfig = config
+
+    @property
+    def needs_parser(self) -> bool:
+        return False
+
+    def _return_save_incremental_results(
+        self, app: "FirecrawlApp", crawl_id: str, output_dir: str = "firecrawl_output"
+    ) -> List[Document]:
+        # Code used verbatim from firecrawl blog with few modifications
+        # https://www.firecrawl.dev/blog/mastering-the-crawl-endpoint-in-firecrawl
+        import json
+        import time
+        from pathlib import Path
+
+        from tqdm import tqdm
+
+        pbar = tqdm(desc="Pages saved", unit=" pages", dynamic_ncols=True)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        processed_urls: set[str] = set()
+        docs = []
+
+        while True:
+            # Check current status
+            status = app.check_crawl_status(crawl_id)
+            new_pages = 0
+
+            # Save new pages
+            for page in status["data"]:
+                url = page["metadata"]["url"]
+                if url not in processed_urls:
+                    content = page.get("markdown", "")
+                    filename = f"{output_dir}/{len(processed_urls)}.md"
+                    with open(filename, "w") as f:
+                        f.write(content)
+                    docs.append(
+                        Document(content=content, metadata=DocMetaData(source=url))
+                    )
+                    processed_urls.add(url)
+                    new_pages += 1
+            pbar.update(new_pages)  # Update progress bar with new pages
+
+            # Break if crawl is complete
+            if status["status"] == "completed":
+                print(f"Saved {len(processed_urls)} pages.")
+                with open(f"{output_dir}/full_results.json", "w") as f:
+                    json.dump(status, f, indent=2)
+                break
+
+            time.sleep(5)  # Wait before checking again
+        return docs
+
+    def crawl(self, urls: List[str]) -> List[Document]:
+        try:
+            from firecrawl import FirecrawlApp
+        except ImportError:
+            raise LangroidImportError("firecrawl", "firecrawl")
+
+        app = FirecrawlApp(api_key=self.config.api_key)
+        docs = []
+        params = self.config.params.copy()  # Create a copy of the existing params
+
+        if self.config.timeout is not None:
+            params["timeout"] = self.config.timeout  # Add/override timeout in params
+
+        if self.config.mode == "scrape":
+            for url in urls:
+                try:
+                    result = app.scrape_url(url, params=params)
+                    metadata = result.get(
+                        "metadata", {}
+                    )  # Default to empty dict if missing
+                    status_code = metadata.get("statusCode")
+
+                    if status_code == 200:
+                        docs.append(
+                            Document(
+                                content=result["markdown"],
+                                metadata=DocMetaData(source=url),
+                            )
+                        )
+                except Exception as e:
+                    logging.warning(
+                        f"Firecrawl encountered an error for {url}: {e}. "
+                        "Skipping but continuing."
+                    )
+        elif self.config.mode == "crawl":
+            if not isinstance(urls, list) or len(urls) != 1:
+                raise ValueError(
+                    "Crawl mode expects 'urls' to be a list containing a single URL."
+                )
+
+            # Start the crawl
+            crawl_status = app.async_crawl_url(url=urls[0], params=params)
+
+            # Save results incrementally
+            docs = self._return_save_incremental_results(app, crawl_status["id"])
+        return docs
+
+
+class URLLoader:
+    """Loads URLs and extracts text using a specified crawler."""
+
+    def __init__(
+        self,
+        urls: List[Any],
+        parsing_config: ParsingConfig = ParsingConfig(),
+        crawler_config: Optional[BaseCrawlerConfig] = None,
+    ):
+        """Initialize the URL loader.
+
+        Args:
+            urls: List of URLs to load
+            parsing_config: Configuration for parsing
+            crawler_config: Configuration for the crawler
+        """
+        self.urls = urls
+        self.parsing_config = parsing_config
+
+        if crawler_config is None:
+            crawler_config = TrafilaturaConfig(parser=Parser(parsing_config))
+
+        self.crawler = CrawlerFactory.create_crawler(crawler_config)
+
+    def load(self) -> List[Document]:
+        """Load the URLs using the specified crawler."""
+        return self.crawler.crawl(self.urls)

{langroid-0.45.10 → langroid-0.47.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.45.10"
+version = "0.47.0"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]
@@ -265,6 +265,9 @@ pinecone = [
 asyncio = [
     "asyncio>=3.4.3",
 ]
+firecrawl = [
+    "firecrawl-py>=1.13.5",
+]
 
 
 [dependency-groups]