langroid 0.58.2__py3-none-any.whl → 0.59.0b1__py3-none-any.whl

langroid/parsing/urls.py

@@ -9,11 +9,10 @@ from urllib.parse import urldefrag, urljoin, urlparse
 import fire
 import requests
 from bs4 import BeautifulSoup
+from pydantic import BaseModel, HttpUrl, TypeAdapter, ValidationError
 from rich import print
 from rich.prompt import Prompt
 
-from langroid.pydantic_v1 import BaseModel, HttpUrl, ValidationError, parse_obj_as
-
 logger = logging.getLogger(__name__)
 
 
@@ -106,7 +105,8 @@ class Url(BaseModel):
 
 def is_url(s: str) -> bool:
     try:
-        Url(url=parse_obj_as(HttpUrl, s))
+        url_adapter = TypeAdapter(HttpUrl)
+        Url(url=url_adapter.validate_python(s))
         return True
     except ValidationError:
         return False
@@ -133,7 +133,8 @@ def get_urls_paths_bytes_indices(
             byte_list.append(i)
             continue
         try:
-            Url(url=parse_obj_as(HttpUrl, item))
+            url_adapter = TypeAdapter(HttpUrl)
+            Url(url=url_adapter.validate_python(item))
             urls.append(i)
         except ValidationError:
             if os.path.exists(item):

langroid/parsing/urls.py-e

@@ -0,0 +1,279 @@
+import logging
+import os
+import tempfile
+import urllib.parse
+import urllib.robotparser
+from typing import List, Optional, Set, Tuple
+from urllib.parse import urldefrag, urljoin, urlparse
+
+import fire
+import requests
+from bs4 import BeautifulSoup
+from rich import print
+from rich.prompt import Prompt
+
+from pydantic import BaseModel, HttpUrl, ValidationError, parse_obj_as
+
+logger = logging.getLogger(__name__)
+
+
+def url_to_tempfile(url: str) -> str:
+    """
+    Fetch content from the given URL and save it to a temporary local file.
+
+    Args:
+        url (str): The URL of the content to fetch.
+
+    Returns:
+        str: The path to the temporary file where the content is saved.
+
+    Raises:
+        HTTPError: If there's any issue fetching the content.
+    """
+
+    response = requests.get(url)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+
+    # Create a temporary file and write the content
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") as temp_file:
+        temp_file.write(response.content)
+        return temp_file.name
+
+
+def get_user_input(msg: str, color: str = "blue") -> str:
+    """
+    Prompt the user for input.
+    Args:
+        msg: printed prompt
+        color: color of the prompt
+    Returns:
+        user input
+    """
+    color_str = f"[{color}]{msg} " if color else msg + " "
+    print(color_str, end="")
+    return input("")
+
+
+def get_list_from_user(
+    prompt: str = "Enter input (type 'done' or hit return to finish)",
+    n: int | None = None,
+) -> List[str]:
+    """
+    Prompt the user for inputs.
+    Args:
+        prompt: printed prompt
+        n: how many inputs to prompt for. If None, then prompt until done, otherwise
+            quit after n inputs.
+    Returns:
+        list of input strings
+    """
+    # Create an empty set to store the URLs.
+    input_set = set()
+
+    # Use a while loop to continuously ask the user for URLs.
+    for _ in range(n or 1000):
+        # Prompt the user for input.
+        input_str = Prompt.ask(f"[blue]{prompt}")
+
+        # Check if the user wants to exit the loop.
+        if input_str.lower() == "done" or input_str == "":
+            break
+
+        # if it is a URL, ask how many to crawl
+        if is_url(input_str):
+            url = input_str
+            input_str = Prompt.ask("[blue] How many new URLs to crawl?", default="0")
+            max_urls = int(input_str) + 1
+            tot_urls = list(find_urls(url, max_links=max_urls, max_depth=2))
+            tot_urls_str = "\n".join(tot_urls)
+            print(
+                f"""
+                Found these {len(tot_urls)} links upto depth 2:
+                {tot_urls_str}
+                """
+            )
+
+            input_set.update(tot_urls)
+        else:
+            input_set.add(input_str.strip())
+
+    return list(input_set)
+
+
+class Url(BaseModel):
+    url: HttpUrl
+
+
+def is_url(s: str) -> bool:
+    try:
+        Url(url=parse_obj_as(HttpUrl, s))
+        return True
+    except ValidationError:
+        return False
+
+
+def get_urls_paths_bytes_indices(
+    inputs: List[str | bytes],
+) -> Tuple[List[int], List[int], List[int]]:
+    """
+    Given a list of inputs, return a
+    list of indices of URLs, list of indices of paths, list of indices of byte-contents.
+    Args:
+        inputs: list of strings or bytes
+    Returns:
+        list of Indices of URLs,
+        list of indices of paths,
+        list of indices of byte-contents
+    """
+    urls = []
+    paths = []
+    byte_list = []
+    for i, item in enumerate(inputs):
+        if isinstance(item, bytes):
+            byte_list.append(i)
+            continue
+        try:
+            Url(url=parse_obj_as(HttpUrl, item))
+            urls.append(i)
+        except ValidationError:
+            if os.path.exists(item):
+                paths.append(i)
+            else:
+                logger.warning(f"{item} is neither a URL nor a path.")
+    return urls, paths, byte_list
+
+
+def crawl_url(url: str, max_urls: int = 1) -> List[str]:
+    """
+    Crawl starting at the url and return a list of URLs to be parsed,
+    up to a maximum of `max_urls`.
+    This has not been tested to work as intended. Ignore.
+    """
+    from trafilatura.spider import focused_crawler
+
+    if max_urls == 1:
+        # no need to crawl, just return the original list
+        return [url]
+
+    to_visit = None
+    known_urls = None
+
+    # Create a RobotFileParser object
+    robots = urllib.robotparser.RobotFileParser()
+    while True:
+        if known_urls is not None and len(known_urls) >= max_urls:
+            break
+        # Set the RobotFileParser object to the website's robots.txt file
+        robots.set_url(url + "/robots.txt")
+        robots.read()
+
+        if robots.can_fetch("*", url):
+            # Start or resume the crawl
+            to_visit, known_urls = focused_crawler(
+                url,
+                max_seen_urls=max_urls,
+                max_known_urls=max_urls,
+                todo=to_visit,
+                known_links=known_urls,
+                rules=robots,
+            )
+        if to_visit is None:
+            break
+
+    if known_urls is None:
+        return [url]
+    final_urls = [s.strip() for s in known_urls]
+    return list(final_urls)[:max_urls]
+
+
+def find_urls(
+    url: str = "https://en.wikipedia.org/wiki/Generative_pre-trained_transformer",
+    max_links: int = 20,
+    visited: Optional[Set[str]] = None,
+    depth: int = 0,
+    max_depth: int = 2,
+    match_domain: bool = True,
+) -> Set[str]:
+    """
+    Recursively find all URLs on a given page.
+
+    Args:
+        url (str): The URL to start from.
+        max_links (int): The maximum number of links to find.
+        visited (set): A set of URLs that have already been visited.
+        depth (int): The current depth of the recursion.
+        max_depth (int): The maximum depth of the recursion.
+        match_domain (bool): Whether to only return URLs that are on the same domain.
+
+    Returns:
+        set: A set of URLs found on the page.
+    """
+
+    if visited is None:
+        visited = set()
+
+    if url in visited or depth > max_depth:
+        return visited
+
+    visited.add(url)
+    base_domain = urlparse(url).netloc
+
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, "html.parser")
+        links = [
+            urljoin(url, a["href"])  # type: ignore
+            for a in soup.find_all("a", href=True)
+        ]
+
+        # Defrag links: discard links that are to portions of same page
+        defragged_links = list(
+            set(urldefrag(link).url for link in links)  # type: ignore
+        )
+
+        # Filter links based on domain matching requirement
+        domain_matching_links = [
+            link for link in defragged_links if urlparse(link).netloc == base_domain
+        ]
+
+        # ensure url is first, since below we are taking first max_links urls
+        domain_matching_links = [url] + [x for x in domain_matching_links if x != url]
+
+        # If found links exceed max_links, return immediately
+        if len(domain_matching_links) >= max_links:
+            return set(domain_matching_links[:max_links])
+
+        for link in domain_matching_links:
+            if len(visited) >= max_links:
+                break
+
+            if link not in visited:
+                visited.update(
+                    find_urls(
+                        link,
+                        max_links,
+                        visited,
+                        depth + 1,
+                        max_depth,
+                        match_domain,
+                    )
+                )
+
+    except (requests.RequestException, Exception) as e:
+        print(f"Error fetching {url}. Error: {e}")
+
+    return set(list(visited)[:max_links])
+
+
+def org_user_from_github(url: str) -> str:
+    parsed = urllib.parse.urlparse(url)
+    org, user = parsed.path.lstrip("/").split("/")
+    return f"{org}-{user}"
+
+
+if __name__ == "__main__":
+    # Example usage
+    found_urls = set(fire.Fire(find_urls))
+    for url in found_urls:
+        print(url)

langroid/prompts/prompts_config.py

@@ -1,4 +1,4 @@
-from langroid.pydantic_v1 import BaseSettings
+from pydantic_settings import BaseSettings
 
 
 class PromptsConfig(BaseSettings):

langroid/pydantic_v1/__init__.py

@@ -1,10 +1,49 @@
 """
-If we're on Pydantic v2, use the v1 namespace, else just use the main namespace.
+Compatibility layer for Pydantic v2 migration.
 
-This allows compatibility with both Pydantic v1 and v2
+This module now imports directly from Pydantic v2 since all internal code
+has been migrated to use Pydantic v2 patterns.
 """
 
-try:
-    from pydantic.v1 import *  # noqa: F403, F401
-except ImportError:
-    from pydantic import *  # type: ignore # noqa: F403, F401
+# Import everything from pydantic v2
+from pydantic import *  # noqa: F403, F401
+
+# Import BaseSettings and SettingsConfigDict from pydantic-settings v2
+from pydantic_settings import BaseSettings, SettingsConfigDict  # noqa: F401
+
+# Explicitly re-export commonly used items for better IDE support and type checking
+from pydantic import (  # noqa: F401
+    BaseModel,
+    Field,
+    ConfigDict,
+    ValidationError,
+    field_validator,
+    model_validator,
+    create_model,
+    HttpUrl,
+    AnyUrl,
+    TypeAdapter,
+    parse_obj_as,
+)
+
+# Legacy names are already provided by pydantic v2 for backward compatibility
+# No need to redefine validator and root_validator as they are already imported above
+
+# Explicitly export all items for mypy
+__all__ = [
+    "BaseModel",
+    "BaseSettings",
+    "SettingsConfigDict",
+    "Field",
+    "ConfigDict",
+    "ValidationError",
+    "field_validator",
+    "model_validator",
+    "create_model",
+    "HttpUrl",
+    "AnyUrl",
+    "TypeAdapter",
+    "parse_obj_as",
+    "validator",
+    "root_validator",
+]

langroid/pydantic_v1/__init__.py-e

@@ -0,0 +1,36 @@
+"""
+Compatibility layer for Pydantic v2 migration.
+
+This module now imports directly from Pydantic v2 since all internal code
+has been migrated to use Pydantic v2 patterns.
+"""
+
+# Import everything from pydantic v2
+from pydantic import *  # noqa: F403, F401
+
+# Import BaseSettings from pydantic-settings v2
+from pydantic_settings import BaseSettings  # noqa: F401
+
+# Explicitly re-export commonly used items for better IDE support and type checking
+from pydantic import (  # noqa: F401
+    BaseModel,
+    Field,
+    ConfigDict,
+    ValidationError,
+    field_validator,
+    model_validator,
+    create_model,
+    HttpUrl,
+    AnyUrl,
+    TypeAdapter,
+)
+
+# Legacy names that map to v2 equivalents
+validator = field_validator  # noqa: F401
+root_validator = model_validator  # noqa: F401
+
+
+# For parse_obj_as, we need to create a wrapper function
+def parse_obj_as(type_, obj):
+    """Compatibility wrapper for parse_obj_as which was removed in Pydantic v2."""
+    return TypeAdapter(type_).validate_python(obj)

langroid/pydantic_v1/main.py

@@ -1,4 +1,11 @@
-try:
-    from pydantic.v1.main import *  # noqa: F403, F401
-except ImportError:
-    from pydantic.main import *  # type: ignore # noqa: F403, F401
+"""
+Compatibility layer for Pydantic v2 migration.
+
+This module now imports directly from Pydantic v2 since all internal code
+has been migrated to use Pydantic v2 patterns.
+"""
+
+# Explicitly export BaseModel for better type checking
+from pydantic.main import *  # noqa: F403, F401
+
+from langroid.pydantic_v1 import BaseModel  # noqa: F401

langroid/pydantic_v1/main.py-e

@@ -0,0 +1,11 @@
+"""
+Compatibility layer for Pydantic v2 migration.
+
+This module now imports directly from Pydantic v2 since all internal code
+has been migrated to use Pydantic v2 patterns.
+"""
+
+# Explicitly export BaseModel for better type checking
+from pydantic.main import *  # noqa: F403, F401
+
+from pydantic import BaseModel  # noqa: F401

langroid/utils/configuration.py

@@ -4,8 +4,7 @@ from contextlib import contextmanager
 from typing import Any, Dict, Iterator, List, Literal, cast
 
 from dotenv import find_dotenv, load_dotenv
-
-from langroid.pydantic_v1 import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 # Global reentrant lock to serialize any modifications to the global settings.
 _global_lock = threading.RLock()
@@ -22,8 +21,7 @@ class Settings(BaseSettings):
     quiet: bool = False  # quiet mode (i.e. suppress all output)?
     notebook: bool = False  # running in a notebook?
 
-    class Config:
-        extra = "forbid"
+    model_config = SettingsConfigDict(extra="forbid")
 
 
 # Load environment variables from .env file.
@@ -60,8 +58,10 @@ class SettingsProxy:
         # Return a dict view of the settings as seen by the caller.
         # Note that temporary overrides are not “merged” with global settings.
         if hasattr(_thread_local, "override"):
-            return cast(Dict[str, Any], cast(Settings, _thread_local.override.dict()))
-        return _global_settings.dict()
+            return cast(
+                Dict[str, Any], cast(Settings, _thread_local.override.model_dump())
+            )
+        return _global_settings.model_dump()
 
 
 settings = SettingsProxy()
@@ -76,7 +76,7 @@ def update_global_settings(cfg: BaseSettings, keys: List[str]) -> None:
 
     This updates the global default.
     """
-    config_dict = cfg.dict()
+    config_dict = cfg.model_dump()
     filtered_config = {key: config_dict[key] for key in keys if key in config_dict}
     new_settings = Settings(**filtered_config)
     _global_settings.__dict__.update(new_settings.__dict__)
@@ -117,7 +117,9 @@ def quiet_mode(quiet: bool = True) -> Iterator[None]:
     if quiet is already True (from an outer context),
     then it remains True even if a nested context passes quiet=False.
     """
-    current_effective = settings.dict()  # get the current thread's effective settings
+    current_effective = (
+        settings.model_dump()
+    )  # get the current thread's effective settings
     # Create a new settings instance from the current effective state.
     temp = Settings(**current_effective)
     # Merge the new flag: once quiet is enabled, it stays enabled.
@@ -132,6 +134,6 @@ def set_env(settings_instance: BaseSettings) -> None:
 
     Each field in the settings is written to os.environ.
     """
-    for field_name, field in settings_instance.__class__.__fields__.items():
-        env_var_name = field.field_info.extra.get("env", field_name).upper()
-        os.environ[env_var_name] = str(settings_instance.dict()[field_name])
+    for field_name, field in settings_instance.__class__.model_fields.items():
+        env_var_name = field.alias or field_name.upper()
+        os.environ[env_var_name] = str(settings_instance.model_dump()[field_name])

langroid/utils/configuration.py-e

@@ -0,0 +1,141 @@
+import os
+import threading
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator, List, Literal, cast
+
+from dotenv import find_dotenv, load_dotenv
+from pydantic_settings import BaseSettings
+
+from pydantic import ConfigDict
+
+# Global reentrant lock to serialize any modifications to the global settings.
+_global_lock = threading.RLock()
+
+
+class Settings(BaseSettings):
+    debug: bool = False  # show debug messages?
+    max_turns: int = -1  # maximum number of turns in a task (to avoid inf loop)
+    progress: bool = False  # show progress spinners/bars?
+    stream: bool = True  # stream output?
+    cache: bool = True  # use cache?
+    cache_type: Literal["redis", "fakeredis", "none"] = "redis"  # cache type
+    chat_model: str = ""  # language model name, e.g. litellm/ollama/llama2
+    quiet: bool = False  # quiet mode (i.e. suppress all output)?
+    notebook: bool = False  # running in a notebook?
+
+    model_config = ConfigDict(extra="forbid")
+
+
+# Load environment variables from .env file.
+load_dotenv(find_dotenv(usecwd=True))
+
+# The global (default) settings instance.
+# This is updated by update_global_settings() and set_global().
+_global_settings = Settings()
+
+# Thread-local storage for temporary (per-thread) settings overrides.
+_thread_local = threading.local()
+
+
+class SettingsProxy:
+    """
+    A proxy for the settings that returns a thread‐local override if set,
+    or else falls back to the global settings.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        # If the calling thread has set an override, use that.
+        if hasattr(_thread_local, "override"):
+            return getattr(_thread_local.override, name)
+        return getattr(_global_settings, name)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        # All writes go to the global settings.
+        setattr(_global_settings, name, value)
+
+    def update(self, new_settings: Settings) -> None:
+        _global_settings.__dict__.update(new_settings.__dict__)
+
+    def dict(self) -> Dict[str, Any]:
+        # Return a dict view of the settings as seen by the caller.
+        # Note that temporary overrides are not “merged” with global settings.
+        if hasattr(_thread_local, "override"):
+            return cast(
+                Dict[str, Any], cast(Settings, _thread_local.override.model_dump())
+            )
+        return _global_settings.model_dump()
+
+
+settings = SettingsProxy()
+
+
+def update_global_settings(cfg: BaseSettings, keys: List[str]) -> None:
+    """
+    Update global settings so that modules can later access them via, e.g.,
+
+        from langroid.utils.configuration import settings
+        if settings.debug: ...
+
+    This updates the global default.
+    """
+    config_dict = cfg.model_dump()
+    filtered_config = {key: config_dict[key] for key in keys if key in config_dict}
+    new_settings = Settings(**filtered_config)
+    _global_settings.__dict__.update(new_settings.__dict__)
+
+
+def set_global(key_vals: Settings) -> None:
+    """
+    Update the global settings object.
+    """
+    _global_settings.__dict__.update(key_vals.__dict__)
+
+
+@contextmanager
+def temporary_settings(temp_settings: Settings) -> Iterator[None]:
+    """
+    Temporarily override the settings for the calling thread.
+
+    Within the context, any access to "settings" will use the provided temporary
+    settings. Once the context is exited, the thread reverts to the global settings.
+    """
+    saved = getattr(_thread_local, "override", None)
+    _thread_local.override = temp_settings
+    try:
+        yield
+    finally:
+        if saved is not None:
+            _thread_local.override = saved
+        else:
+            del _thread_local.override
+
+
+@contextmanager
+def quiet_mode(quiet: bool = True) -> Iterator[None]:
+    """
+    Temporarily override settings.quiet for the current thread.
+    This implementation builds on the thread‑local temporary_settings context manager.
+    The effective quiet mode is merged:
+    if quiet is already True (from an outer context),
+    then it remains True even if a nested context passes quiet=False.
+    """
+    current_effective = (
+        settings.model_dump()
+    )  # get the current thread's effective settings
+    # Create a new settings instance from the current effective state.
+    temp = Settings(**current_effective)
+    # Merge the new flag: once quiet is enabled, it stays enabled.
+    temp.quiet = settings.quiet or quiet
+    with temporary_settings(temp):
+        yield
+
+
+def set_env(settings_instance: BaseSettings) -> None:
+    """
+    Set environment variables from a BaseSettings instance.
+
+    Each field in the settings is written to os.environ.
+    """
+    for field_name, field in settings_instance.__class__.__fields__.items():
+        env_var_name = field.field_info.extra.get("env", field_name).upper()
+        os.environ[env_var_name] = str(settings_instance.model_dump()[field_name])

langroid/utils/constants.py

@@ -1,4 +1,4 @@
-from langroid.pydantic_v1 import BaseModel
+from pydantic import BaseModel
 
 
 # Define the ANSI escape sequences for various colors and reset

langroid/utils/constants.py-e

@@ -0,0 +1,32 @@
+from pydantic import BaseModel
+
+
+# Define the ANSI escape sequences for various colors and reset
+class Colors(BaseModel):
+    RED: str = "\033[31m"
+    BLUE: str = "\033[34m"
+    GREEN: str = "\033[32m"
+    GREEN_DIMMER: str = "\033[38;5;22m"  # very dark green
+    GREEN_DIM: str = "\033[38;5;28m"  # medium-dim green
+    ORANGE: str = "\033[33m"  # no standard ANSI color for orange; using yellow
+    CYAN: str = "\033[36m"
+    MAGENTA: str = "\033[35m"
+    YELLOW: str = "\033[33m"
+    RESET: str = "\033[0m"
+
+
+NO_ANSWER = "DO-NOT-KNOW"
+DONE = "DONE"
+USER_QUIT_STRINGS = ["q", "x", "quit", "exit", "bye", DONE]
+PASS = "__PASS__"
+PASS_TO = PASS + ":"
+SEND_TO = "__SEND__:"
+TOOL = "TOOL"
+# This is a recommended setting for TaskConfig.addressing_prefix if using it at all;
+# prefer to use `RecipientTool` to allow agents addressing others.
+# Caution the AT string should NOT contain any 'word' characters, i.e.
+# it no letters, digits or underscores.
+# See tests/main/test_msg_routing for example usage
+AT = "|@|"
+TOOL_BEGIN = "TOOL_BEGIN"
+TOOL_END = "TOOL_END"