langroid 0.59.0b3__py3-none-any.whl → 0.59.1__py3-none-any.whl

langroid/parsing/urls.py-e

@@ -1,279 +0,0 @@
-import logging
-import os
-import tempfile
-import urllib.parse
-import urllib.robotparser
-from typing import List, Optional, Set, Tuple
-from urllib.parse import urldefrag, urljoin, urlparse
-
-import fire
-import requests
-from bs4 import BeautifulSoup
-from rich import print
-from rich.prompt import Prompt
-
-from pydantic import BaseModel, HttpUrl, ValidationError, parse_obj_as
-
-logger = logging.getLogger(__name__)
-
-
-def url_to_tempfile(url: str) -> str:
-    """
-    Fetch content from the given URL and save it to a temporary local file.
-
-    Args:
-        url (str): The URL of the content to fetch.
-
-    Returns:
-        str: The path to the temporary file where the content is saved.
-
-    Raises:
-        HTTPError: If there's any issue fetching the content.
-    """
-
-    response = requests.get(url)
-    response.raise_for_status()  # Raise an exception for HTTP errors
-
-    # Create a temporary file and write the content
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") as temp_file:
-        temp_file.write(response.content)
-        return temp_file.name
-
-
-def get_user_input(msg: str, color: str = "blue") -> str:
-    """
-    Prompt the user for input.
-    Args:
-        msg: printed prompt
-        color: color of the prompt
-    Returns:
-        user input
-    """
-    color_str = f"[{color}]{msg} " if color else msg + " "
-    print(color_str, end="")
-    return input("")
-
-
-def get_list_from_user(
-    prompt: str = "Enter input (type 'done' or hit return to finish)",
-    n: int | None = None,
-) -> List[str]:
-    """
-    Prompt the user for inputs.
-    Args:
-        prompt: printed prompt
-        n: how many inputs to prompt for. If None, then prompt until done, otherwise
-            quit after n inputs.
-    Returns:
-        list of input strings
-    """
-    # Create an empty set to store the URLs.
-    input_set = set()
-
-    # Use a while loop to continuously ask the user for URLs.
-    for _ in range(n or 1000):
-        # Prompt the user for input.
-        input_str = Prompt.ask(f"[blue]{prompt}")
-
-        # Check if the user wants to exit the loop.
-        if input_str.lower() == "done" or input_str == "":
-            break
-
-        # if it is a URL, ask how many to crawl
-        if is_url(input_str):
-            url = input_str
-            input_str = Prompt.ask("[blue] How many new URLs to crawl?", default="0")
-            max_urls = int(input_str) + 1
-            tot_urls = list(find_urls(url, max_links=max_urls, max_depth=2))
-            tot_urls_str = "\n".join(tot_urls)
-            print(
-                f"""
-                Found these {len(tot_urls)} links upto depth 2:
-                {tot_urls_str}
-                """
-            )
-
-            input_set.update(tot_urls)
-        else:
-            input_set.add(input_str.strip())
-
-    return list(input_set)
-
-
-class Url(BaseModel):
-    url: HttpUrl
-
-
-def is_url(s: str) -> bool:
-    try:
-        Url(url=parse_obj_as(HttpUrl, s))
-        return True
-    except ValidationError:
-        return False
-
-
-def get_urls_paths_bytes_indices(
-    inputs: List[str | bytes],
-) -> Tuple[List[int], List[int], List[int]]:
-    """
-    Given a list of inputs, return a
-    list of indices of URLs, list of indices of paths, list of indices of byte-contents.
-    Args:
-        inputs: list of strings or bytes
-    Returns:
-        list of Indices of URLs,
-        list of indices of paths,
-        list of indices of byte-contents
-    """
-    urls = []
-    paths = []
-    byte_list = []
-    for i, item in enumerate(inputs):
-        if isinstance(item, bytes):
-            byte_list.append(i)
-            continue
-        try:
-            Url(url=parse_obj_as(HttpUrl, item))
-            urls.append(i)
-        except ValidationError:
-            if os.path.exists(item):
-                paths.append(i)
-            else:
-                logger.warning(f"{item} is neither a URL nor a path.")
-    return urls, paths, byte_list
-
-
-def crawl_url(url: str, max_urls: int = 1) -> List[str]:
-    """
-    Crawl starting at the url and return a list of URLs to be parsed,
-    up to a maximum of `max_urls`.
-    This has not been tested to work as intended. Ignore.
-    """
-    from trafilatura.spider import focused_crawler
-
-    if max_urls == 1:
-        # no need to crawl, just return the original list
-        return [url]
-
-    to_visit = None
-    known_urls = None
-
-    # Create a RobotFileParser object
-    robots = urllib.robotparser.RobotFileParser()
-    while True:
-        if known_urls is not None and len(known_urls) >= max_urls:
-            break
-        # Set the RobotFileParser object to the website's robots.txt file
-        robots.set_url(url + "/robots.txt")
-        robots.read()
-
-        if robots.can_fetch("*", url):
-            # Start or resume the crawl
-            to_visit, known_urls = focused_crawler(
-                url,
-                max_seen_urls=max_urls,
-                max_known_urls=max_urls,
-                todo=to_visit,
-                known_links=known_urls,
-                rules=robots,
-            )
-        if to_visit is None:
-            break
-
-    if known_urls is None:
-        return [url]
-    final_urls = [s.strip() for s in known_urls]
-    return list(final_urls)[:max_urls]
-
-
-def find_urls(
-    url: str = "https://en.wikipedia.org/wiki/Generative_pre-trained_transformer",
-    max_links: int = 20,
-    visited: Optional[Set[str]] = None,
-    depth: int = 0,
-    max_depth: int = 2,
-    match_domain: bool = True,
-) -> Set[str]:
-    """
-    Recursively find all URLs on a given page.
-
-    Args:
-        url (str): The URL to start from.
-        max_links (int): The maximum number of links to find.
-        visited (set): A set of URLs that have already been visited.
-        depth (int): The current depth of the recursion.
-        max_depth (int): The maximum depth of the recursion.
-        match_domain (bool): Whether to only return URLs that are on the same domain.
-
-    Returns:
-        set: A set of URLs found on the page.
-    """
-
-    if visited is None:
-        visited = set()
-
-    if url in visited or depth > max_depth:
-        return visited
-
-    visited.add(url)
-    base_domain = urlparse(url).netloc
-
-    try:
-        response = requests.get(url, timeout=5)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.text, "html.parser")
-        links = [
-            urljoin(url, a["href"])  # type: ignore
-            for a in soup.find_all("a", href=True)
-        ]
-
-        # Defrag links: discard links that are to portions of same page
-        defragged_links = list(
-            set(urldefrag(link).url for link in links)  # type: ignore
-        )
-
-        # Filter links based on domain matching requirement
-        domain_matching_links = [
-            link for link in defragged_links if urlparse(link).netloc == base_domain
-        ]
-
-        # ensure url is first, since below we are taking first max_links urls
-        domain_matching_links = [url] + [x for x in domain_matching_links if x != url]
-
-        # If found links exceed max_links, return immediately
-        if len(domain_matching_links) >= max_links:
-            return set(domain_matching_links[:max_links])
-
-        for link in domain_matching_links:
-            if len(visited) >= max_links:
-                break
-
-            if link not in visited:
-                visited.update(
-                    find_urls(
-                        link,
-                        max_links,
-                        visited,
-                        depth + 1,
-                        max_depth,
-                        match_domain,
-                    )
-                )
-
-    except (requests.RequestException, Exception) as e:
-        print(f"Error fetching {url}. Error: {e}")
-
-    return set(list(visited)[:max_links])
-
-
-def org_user_from_github(url: str) -> str:
-    parsed = urllib.parse.urlparse(url)
-    org, user = parsed.path.lstrip("/").split("/")
-    return f"{org}-{user}"
-
-
-if __name__ == "__main__":
-    # Example usage
-    found_urls = set(fire.Fire(find_urls))
-    for url in found_urls:
-        print(url)

langroid/pydantic_v1/__init__.py-e

@@ -1,36 +0,0 @@
-"""
-Compatibility layer for Pydantic v2 migration.
-
-This module now imports directly from Pydantic v2 since all internal code
-has been migrated to use Pydantic v2 patterns.
-"""
-
-# Import everything from pydantic v2
-from pydantic import *  # noqa: F403, F401
-
-# Import BaseSettings from pydantic-settings v2
-from pydantic_settings import BaseSettings  # noqa: F401
-
-# Explicitly re-export commonly used items for better IDE support and type checking
-from pydantic import (  # noqa: F401
-    BaseModel,
-    Field,
-    ConfigDict,
-    ValidationError,
-    field_validator,
-    model_validator,
-    create_model,
-    HttpUrl,
-    AnyUrl,
-    TypeAdapter,
-)
-
-# Legacy names that map to v2 equivalents
-validator = field_validator  # noqa: F401
-root_validator = model_validator  # noqa: F401
-
-
-# For parse_obj_as, we need to create a wrapper function
-def parse_obj_as(type_, obj):
-    """Compatibility wrapper for parse_obj_as which was removed in Pydantic v2."""
-    return TypeAdapter(type_).validate_python(obj)

langroid/pydantic_v1/main.py-e

@@ -1,11 +0,0 @@
-"""
-Compatibility layer for Pydantic v2 migration.
-
-This module now imports directly from Pydantic v2 since all internal code
-has been migrated to use Pydantic v2 patterns.
-"""
-
-# Explicitly export BaseModel for better type checking
-from pydantic.main import *  # noqa: F403, F401
-
-from pydantic import BaseModel  # noqa: F401

langroid/utils/configuration.py-e

@@ -1,141 +0,0 @@
-import os
-import threading
-from contextlib import contextmanager
-from typing import Any, Dict, Iterator, List, Literal, cast
-
-from dotenv import find_dotenv, load_dotenv
-from pydantic_settings import BaseSettings
-
-from pydantic import ConfigDict
-
-# Global reentrant lock to serialize any modifications to the global settings.
-_global_lock = threading.RLock()
-
-
-class Settings(BaseSettings):
-    debug: bool = False  # show debug messages?
-    max_turns: int = -1  # maximum number of turns in a task (to avoid inf loop)
-    progress: bool = False  # show progress spinners/bars?
-    stream: bool = True  # stream output?
-    cache: bool = True  # use cache?
-    cache_type: Literal["redis", "fakeredis", "none"] = "redis"  # cache type
-    chat_model: str = ""  # language model name, e.g. litellm/ollama/llama2
-    quiet: bool = False  # quiet mode (i.e. suppress all output)?
-    notebook: bool = False  # running in a notebook?
-
-    model_config = ConfigDict(extra="forbid")
-
-
-# Load environment variables from .env file.
-load_dotenv(find_dotenv(usecwd=True))
-
-# The global (default) settings instance.
-# This is updated by update_global_settings() and set_global().
-_global_settings = Settings()
-
-# Thread-local storage for temporary (per-thread) settings overrides.
-_thread_local = threading.local()
-
-
-class SettingsProxy:
-    """
-    A proxy for the settings that returns a thread‐local override if set,
-    or else falls back to the global settings.
-    """
-
-    def __getattr__(self, name: str) -> Any:
-        # If the calling thread has set an override, use that.
-        if hasattr(_thread_local, "override"):
-            return getattr(_thread_local.override, name)
-        return getattr(_global_settings, name)
-
-    def __setattr__(self, name: str, value: Any) -> None:
-        # All writes go to the global settings.
-        setattr(_global_settings, name, value)
-
-    def update(self, new_settings: Settings) -> None:
-        _global_settings.__dict__.update(new_settings.__dict__)
-
-    def dict(self) -> Dict[str, Any]:
-        # Return a dict view of the settings as seen by the caller.
-        # Note that temporary overrides are not “merged” with global settings.
-        if hasattr(_thread_local, "override"):
-            return cast(
-                Dict[str, Any], cast(Settings, _thread_local.override.model_dump())
-            )
-        return _global_settings.model_dump()
-
-
-settings = SettingsProxy()
-
-
-def update_global_settings(cfg: BaseSettings, keys: List[str]) -> None:
-    """
-    Update global settings so that modules can later access them via, e.g.,
-
-        from langroid.utils.configuration import settings
-        if settings.debug: ...
-
-    This updates the global default.
-    """
-    config_dict = cfg.model_dump()
-    filtered_config = {key: config_dict[key] for key in keys if key in config_dict}
-    new_settings = Settings(**filtered_config)
-    _global_settings.__dict__.update(new_settings.__dict__)
-
-
-def set_global(key_vals: Settings) -> None:
-    """
-    Update the global settings object.
-    """
-    _global_settings.__dict__.update(key_vals.__dict__)
-
-
-@contextmanager
-def temporary_settings(temp_settings: Settings) -> Iterator[None]:
-    """
-    Temporarily override the settings for the calling thread.
-
-    Within the context, any access to "settings" will use the provided temporary
-    settings. Once the context is exited, the thread reverts to the global settings.
-    """
-    saved = getattr(_thread_local, "override", None)
-    _thread_local.override = temp_settings
-    try:
-        yield
-    finally:
-        if saved is not None:
-            _thread_local.override = saved
-        else:
-            del _thread_local.override
-
-
-@contextmanager
-def quiet_mode(quiet: bool = True) -> Iterator[None]:
-    """
-    Temporarily override settings.quiet for the current thread.
-    This implementation builds on the thread‑local temporary_settings context manager.
-    The effective quiet mode is merged:
-    if quiet is already True (from an outer context),
-    then it remains True even if a nested context passes quiet=False.
-    """
-    current_effective = (
-        settings.model_dump()
-    )  # get the current thread's effective settings
-    # Create a new settings instance from the current effective state.
-    temp = Settings(**current_effective)
-    # Merge the new flag: once quiet is enabled, it stays enabled.
-    temp.quiet = settings.quiet or quiet
-    with temporary_settings(temp):
-        yield
-
-
-def set_env(settings_instance: BaseSettings) -> None:
-    """
-    Set environment variables from a BaseSettings instance.
-
-    Each field in the settings is written to os.environ.
-    """
-    for field_name, field in settings_instance.__class__.__fields__.items():
-        env_var_name = field.field_info.extra.get("env", field_name).upper()
-        os.environ[env_var_name] = str(settings_instance.model_dump()[field_name])

langroid/utils/constants.py-e

@@ -1,32 +0,0 @@
-from pydantic import BaseModel
-
-
-# Define the ANSI escape sequences for various colors and reset
-class Colors(BaseModel):
-    RED: str = "\033[31m"
-    BLUE: str = "\033[34m"
-    GREEN: str = "\033[32m"
-    GREEN_DIMMER: str = "\033[38;5;22m"  # very dark green
-    GREEN_DIM: str = "\033[38;5;28m"  # medium-dim green
-    ORANGE: str = "\033[33m"  # no standard ANSI color for orange; using yellow
-    CYAN: str = "\033[36m"
-    MAGENTA: str = "\033[35m"
-    YELLOW: str = "\033[33m"
-    RESET: str = "\033[0m"
-
-
-NO_ANSWER = "DO-NOT-KNOW"
-DONE = "DONE"
-USER_QUIT_STRINGS = ["q", "x", "quit", "exit", "bye", DONE]
-PASS = "__PASS__"
-PASS_TO = PASS + ":"
-SEND_TO = "__SEND__:"
-TOOL = "TOOL"
-# This is a recommended setting for TaskConfig.addressing_prefix if using it at all;
-# prefer to use `RecipientTool` to allow agents addressing others.
-# Caution the AT string should NOT contain any 'word' characters, i.e.
-# it no letters, digits or underscores.
-# See tests/main/test_msg_routing for example usage
-AT = "|@|"
-TOOL_BEGIN = "TOOL_BEGIN"
-TOOL_END = "TOOL_END"

langroid/utils/globals.py-e

@@ -1,49 +0,0 @@
-from typing import Any, Dict, Optional, Type, TypeVar
-
-from pydantic import BaseModel
-
-T = TypeVar("T", bound="GlobalState")
-
-
-class GlobalState(BaseModel):
-    """A base Pydantic model for global states."""
-
-    _instance: Optional["GlobalState"] = None
-
-    @classmethod
-    def get_instance(cls: Type["GlobalState"]) -> "GlobalState":
-        """
-        Get the global instance of the specific subclass.
-
-        Returns:
-            The global instance of the subclass.
-        """
-        if cls._instance is None:
-            cls._instance = cls()
-        return cls._instance
-
-    @classmethod
-    def set_values(cls: Type[T], **kwargs: Dict[str, Any]) -> None:
-        """
-        Set values on the global instance of the specific subclass.
-
-        Args:
-            **kwargs: The fields and their values to set.
-        """
-        instance = cls.get_instance()
-        for key, value in kwargs.items():
-            setattr(instance, key, value)
-
-    @classmethod
-    def get_value(cls: Type[T], name: str) -> Any:
-        """
-        Retrieve the value of a specific field from the global instance.
-
-        Args:
-            name (str): The name of the field to retrieve.
-
-        Returns:
-            str: The value of the specified field.
-        """
-        instance = cls.get_instance()
-        return getattr(instance, name)