lionagi 0.5.4__py3-none-any.whl → 0.6.0__py3-none-any.whl

lionagi/libs/file/path.py

@@ -1,301 +0,0 @@
-import logging
-import re
-import sys
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-from ..utils import unique_hash
-
-
-def create_path(
-    directory: Path | str,
-    filename: str,
-    extension: str = None,
-    timestamp: bool = False,
-    dir_exist_ok: bool = True,
-    file_exist_ok: bool = False,
-    time_prefix: bool = False,
-    timestamp_format: str | None = None,
-    random_hash_digits: int = 0,
-) -> Path:
-    """
-    Generate a new file path with optional timestamp and random hash.
-
-    Args:
-        directory: The directory where the file will be created.
-        filename: The base name of the file to create.
-        timestamp: If True, adds a timestamp to the filename.
-        dir_exist_ok: If True, doesn't raise an error if the directory
-            exists.
-        file_exist_ok: If True, allows overwriting of existing files.
-        time_prefix: If True, adds the timestamp as a prefix instead of
-            a suffix.
-        timestamp_format: Custom format for the timestamp.
-        random_hash_digits: Number of digits for the random hash.
-
-    Returns:
-        The full path to the new or existing file.
-
-    Raises:
-        ValueError: If the filename contains illegal characters.
-        FileExistsError: If the file exists and file_exist_ok is False.
-    """
-    if "/" in filename or "\\" in filename:
-        raise ValueError("Filename cannot contain directory separators.")
-    directory = Path(directory)
-
-    name, ext = None, None
-    if "." in filename:
-        name, ext = filename.rsplit(".", 1)
-    else:
-        name = filename
-        ext = extension.strip(".").strip() if extension else None
-
-    if not ext:
-        raise ValueError("No extension provided for filename.")
-
-    ext = f".{ext}" if ext else ""
-
-    if timestamp:
-        timestamp_str = datetime.now().strftime(
-            timestamp_format or "%Y%m%d%H%M%S"
-        )
-        name = (
-            f"{timestamp_str}_{name}"
-            if time_prefix
-            else f"{name}_{timestamp_str}"
-        )
-
-    if random_hash_digits > 0:
-        random_hash = "-" + unique_hash(random_hash_digits)
-        name = f"{name}{random_hash}"
-
-    full_filename = f"{name}{ext}"
-    full_path = directory / full_filename
-
-    if full_path.exists():
-        if file_exist_ok:
-            return full_path
-        raise FileExistsError(
-            f"File {full_path} already exists and file_exist_ok is False."
-        )
-    full_path.parent.mkdir(parents=True, exist_ok=dir_exist_ok)
-    return full_path
-
-
-def is_valid_path(
-    path: str | Path,
-    *,
-    max_length: int | None = None,
-    allow_relative: bool = True,
-    allow_symlinks: bool = True,
-    custom_reserved_names: list[str] | None = None,
-    strict_mode: bool = False,
-) -> bool:
-    """
-    Validates whether the given path is syntactically valid for the current operating system.
-
-    Args:
-        path (Union[str, Path]): The filesystem path to validate.
-        max_length (Optional[int]): Maximum allowed path length. If None, uses OS default.
-        allow_relative (bool): Whether to allow relative paths. Default is True.
-        allow_symlinks (bool): Whether to allow symlinks. Default is True.
-        custom_reserved_names (Optional[List[str]]): Additional reserved names to check.
-        strict_mode (bool): If True, applies stricter validation rules. Default is False.
-
-    Returns:
-        bool: True if the path is valid, False otherwise.
-
-    Raises:
-        ValueError: If the path is invalid, with a detailed explanation.
-    """
-    if isinstance(path, Path):
-        path_str = str(path)
-    elif isinstance(path, str):
-        path_str = path
-    else:
-        raise TypeError("Path must be a string or Path object.")
-
-    if not path_str:
-        raise ValueError("Path cannot be an empty string.")
-
-    issues = []
-    is_windows = sys.platform.startswith("win")
-
-    # Common checks for both Windows and Unix-like systems
-    if "\0" in path_str:
-        issues.append("Path contains null character.")
-
-    if not max_length:
-        max_length = 260 if is_windows else 4096
-    if len(path_str) > max_length:
-        issues.append(
-            f"Path exceeds the maximum length of {max_length} characters."
-        )
-
-    if is_windows:
-        # Windows-specific validation
-        invalid_chars = r'<>:"/\\|?*'
-        if re.search(f"[{re.escape(invalid_chars)}]", path_str):
-            issues.append(f"Path contains invalid characters: {invalid_chars}")
-
-        reserved_names = {
-            "CON",
-            "PRN",
-            "AUX",
-            "NUL",
-            "COM1",
-            "COM2",
-            "COM3",
-            "COM4",
-            "COM5",
-            "COM6",
-            "COM7",
-            "COM8",
-            "COM9",
-            "LPT1",
-            "LPT2",
-            "LPT3",
-            "LPT4",
-            "LPT5",
-            "LPT6",
-            "LPT7",
-            "LPT8",
-            "LPT9",
-        }
-        if custom_reserved_names:
-            reserved_names.update(custom_reserved_names)
-
-        path = Path(path_str)
-        for part in path.parts:
-            name = part.upper().rstrip(". ")
-            if name in reserved_names:
-                issues.append(f"Path contains a reserved name: '{part}'")
-
-        if path_str.endswith(" ") or path_str.endswith("."):
-            issues.append(
-                "Path cannot end with a space or a period on Windows."
-            )
-
-        if strict_mode:
-            if not path_str.startswith("\\\\?\\") and len(path_str) > 260:
-                issues.append(
-                    "Path exceeds 260 characters without long path prefix."
-                )
-
-    else:
-        # Unix-like systems validation
-        if strict_mode:
-            if re.search(r"//+", path_str):
-                issues.append("Path contains consecutive slashes.")
-
-        if not allow_relative and not path_str.startswith("/"):
-            issues.append("Relative paths are not allowed.")
-
-    # Common additional checks
-    if not allow_symlinks and Path(path_str).is_symlink():
-        issues.append("Symlinks are not allowed.")
-
-    if strict_mode:
-        if re.search(r"\s", path_str):
-            issues.append("Path contains whitespace characters.")
-
-    if issues:
-        raise ValueError("Invalid path: " + "; ".join(issues))
-
-    return True
-
-
-def split_path(path: Path | str) -> tuple[Path, str]:
-    """
-    Split a path into its directory and filename components.
-
-    Args:
-        path: The path to split.
-
-    Returns:
-        A tuple containing the directory and filename.
-    """
-    path = Path(path)
-    return path.parent, path.name
-
-
-def _get_path_kwargs(
-    persist_path: str | Path, postfix: str, **path_kwargs: Any
-) -> dict[str, Any]:
-    """
-    Generate keyword arguments for path creation.
-
-    Args:
-        persist_path: The base path to use.
-        postfix: The file extension to use.
-        **path_kwargs: Additional keyword arguments to override defaults.
-
-    Returns:
-        A dictionary of keyword arguments for path creation.
-    """
-    persist_path = Path(persist_path)
-    postfix = f".{postfix.strip('.')}"
-
-    if persist_path.suffix != postfix:
-        dirname = persist_path
-        filename = f"new_file{postfix}"
-    else:
-        dirname, filename = persist_path.parent, persist_path.name
-
-    return {
-        "timestamp": path_kwargs.get("timestamp", False),
-        "file_exist_ok": path_kwargs.get("file_exist_ok", True),
-        "directory": path_kwargs.get("directory", dirname),
-        "filename": path_kwargs.get("filename", filename),
-    }
-
-
-def clear_path(
-    path: Path | str,
-    /,
-    recursive: bool = False,
-    exclude: list[str] | None = None,
-) -> None:
-    """
-    Clear all files and directories in the specified path.
-
-    Args:
-        path: The path to the directory to clear.
-        recursive: If True, clears directories recursively.
-        exclude: A list of string patterns to exclude from deletion.
-
-    Raises:
-        FileNotFoundError: If the specified directory does not exist.
-        PermissionError: If there are insufficient permissions to delete
-            files.
-    """
-    path = Path(path)
-    if not path.exists():
-        raise FileNotFoundError(
-            f"The specified directory {path} does not exist."
-        )
-
-    exclude = exclude or []
-    exclude_pattern = re.compile("|".join(exclude)) if exclude else None
-
-    for file_path in path.iterdir():
-        if exclude_pattern and exclude_pattern.search(file_path.name):
-            logging.info(f"Excluded from deletion: {file_path}")
-            continue
-
-        try:
-            if file_path.is_dir():
-                if recursive:
-                    clear_path(file_path, recursive=True, exclude=exclude)
-                    file_path.rmdir()
-                else:
-                    continue
-            else:
-                file_path.unlink()
-            logging.info(f"Successfully deleted {file_path}")
-        except PermissionError as e:
-            logging.error(f"Permission denied when deleting {file_path}: {e}")
-        except Exception as e:
-            logging.error(f"Failed to delete {file_path}: {e}")

lionagi/libs/file/types.py

@@ -1,22 +0,0 @@
-from .chunk import chunk_by_chars, chunk_by_tokens, chunk_content
-from .file_ops import copy_file, get_file_size, list_files, read_file
-from .path import clear_path, create_path, is_valid_path, split_path
-from .process import dir_to_files, file_to_chunks
-from .save import save_to_file
-
-__all__ = [
-    "chunk_by_chars",
-    "chunk_by_tokens",
-    "chunk_content",
-    "clear_path",
-    "copy_file",
-    "create_path",
-    "dir_to_files",
-    "file_to_chunks",
-    "get_file_size",
-    "is_valid_path",
-    "list_files",
-    "read_file",
-    "save_to_file",
-    "split_path",
-]

lionagi/libs/func/async_calls/__init__.py

@@ -1,24 +0,0 @@
-# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from .alcall import ALCallParams, alcall
-from .bcall import BCallParams, bcall
-from .mcall import mcall
-from .pcall import pcall
-from .rcall import RCallParams, rcall
-from .tcall import tcall
-from .ucall import ucall
-
-__all__ = [
-    "bcall",
-    "alcall",
-    "mcall",
-    "pcall",
-    "rcall",
-    "tcall",
-    "ucall",
-    "ALCallParams",
-    "BCallParams",
-    "RCallParams",
-]

lionagi/libs/func/async_calls/alcall.py

@@ -1,210 +0,0 @@
-import asyncio
-from collections.abc import Callable
-from dataclasses import dataclass
-from typing import Any, TypeVar
-
-from lionagi.libs.utils import DataClass
-
-from ...constants import UNDEFINED
-from ...parse import to_list
-from .ucall import ucall
-
-T = TypeVar("T")
-
-
-@dataclass
-class ALCallParams(DataClass):
-    function: Callable[..., T]
-    num_retries: int = 0
-    initial_delay: float = 0
-    retry_delay: float = 0
-    backoff_factor: float = 1
-    retry_default: Any = UNDEFINED
-    retry_timeout: float | None = None
-    retry_timing: bool = False
-    verbose: bool = True
-    error_msg: str | None = None
-    error_map: dict[type, Callable[[Exception], None]] | None = None
-    max_concurrent: int | None = None
-    throttle_period: float | None = None
-    flatten: bool = False
-    dropna: bool = False
-    unique: bool = False
-
-    async def __call__(self, input_, *args, **kwargs):
-        return await alcall(
-            input_,
-            self.function,
-            *args,
-            num_retries=self.num_retries,
-            initial_delay=self.initial_delay,
-            retry_delay=self.retry_delay,
-            backoff_factor=self.backoff_factor,
-            retry_default=self.retry_default,
-            retry_timeout=self.retry_timeout,
-            retry_timing=self.retry_timing,
-            verbose_retry=self.verbose,
-            error_msg=self.error_msg,
-            error_map=self.error_map,
-            max_concurrent=self.max_concurrent,
-            throttle_period=self.throttle_period,
-            flatten=self.flatten,
-            dropna=self.dropna,
-            unique=self.unique,
-            **kwargs,
-        )
-
-
-async def alcall(
-    input_: list[Any],
-    func: Callable[..., T],
-    /,
-    num_retries: int = 0,
-    initial_delay: float = 0,
-    retry_delay: float = 0,
-    backoff_factor: float = 1,
-    retry_default: Any = UNDEFINED,
-    retry_timeout: float | None = None,
-    retry_timing: bool = False,
-    verbose_retry: bool = True,
-    error_msg: str | None = None,
-    error_map: dict[type, Callable[[Exception], None]] | None = None,
-    max_concurrent: int | None = None,
-    throttle_period: float | None = None,
-    flatten: bool = False,
-    dropna: bool = False,
-    unique: bool = False,
-    **kwargs: Any,
-) -> list[T] | list[tuple[T, float]]:
-    """Apply a function to each element of a list asynchronously with options.
-
-    Args:
-        input_: List of inputs to be processed.
-        func: Async or sync function to apply to each input element.
-        num_retries: Number of retry attempts for each function call.
-        initial_delay: Initial delay before starting execution (seconds).
-        retry_delay: Delay between retry attempts (seconds).
-        backoff_factor: Factor by which delay increases after each attempt.
-        retry_default: Default value to return if all attempts fail.
-        retry_timeout: Timeout for each function execution (seconds).
-        retry_timing: If True, return execution duration for each call.
-        verbose_retry: If True, print retry messages.
-        error_msg: Custom error message prefix for exceptions.
-        error_map: Dict mapping exception types to error handlers.
-        max_concurrent: Maximum number of concurrent executions.
-        throttle_period: Minimum time between function executions (seconds).
-        flatten: If True, flatten the resulting list.
-        dropna: If True, remove None values from the result.
-        **kwargs: Additional keyword arguments passed to func.
-
-    Returns:
-        list[T] | list[tuple[T, float]]: List of results, optionally with
-        execution times if retry_timing is True.
-
-    Raises:
-        asyncio.TimeoutError: If execution exceeds retry_timeout.
-        Exception: Any exception raised by func if not handled by error_map.
-
-    Examples:
-        >>> async def square(x):
-        ...     return x * x
-        >>> await alcall([1, 2, 3], square)
-        [1, 4, 9]
-        >>> await alcall([1, 2, 3], square, retry_timing=True)
-        [(1, 0.001), (4, 0.001), (9, 0.001)]
-
-    Note:
-        - Uses semaphores for concurrency control if max_concurrent is set.
-        - Supports both synchronous and asynchronous functions for `func`.
-        - Results are returned in the original input order.
-    """
-    if initial_delay:
-        await asyncio.sleep(initial_delay)
-
-    semaphore = asyncio.Semaphore(max_concurrent) if max_concurrent else None
-    throttle_delay = throttle_period if throttle_period else 0
-
-    async def _task(i: Any, index: int) -> Any:
-        if semaphore:
-            async with semaphore:
-                return await _execute_task(i, index)
-        else:
-            return await _execute_task(i, index)
-
-    async def _execute_task(i: Any, index: int) -> Any:
-        attempts = 0
-        current_delay = retry_delay
-        while True:
-            try:
-                if retry_timing:
-                    start_time = asyncio.get_event_loop().time()
-                    result = await asyncio.wait_for(
-                        ucall(func, i, **kwargs), retry_timeout
-                    )
-                    end_time = asyncio.get_event_loop().time()
-                    return index, result, end_time - start_time
-                else:
-                    result = await asyncio.wait_for(
-                        ucall(func, i, **kwargs), retry_timeout
-                    )
-                    return index, result
-
-            except InterruptedError:
-                return index, None
-
-            except asyncio.CancelledError:
-                return index, None
-
-            except TimeoutError as e:
-                raise TimeoutError(
-                    f"{error_msg or ''} Timeout {retry_timeout} seconds "
-                    "exceeded"
-                ) from e
-            except Exception as e:
-                if error_map and type(e) in error_map:
-                    handler = error_map[type(e)]
-                    if asyncio.iscoroutinefunction(handler):
-                        return index, await handler(e)
-                    else:
-                        return index, handler(e)
-                attempts += 1
-                if attempts <= num_retries:
-                    if verbose_retry:
-                        print(
-                            f"Attempt {attempts}/{num_retries} failed: {e}"
-                            ", retrying..."
-                        )
-                    await asyncio.sleep(current_delay)
-                    current_delay *= backoff_factor
-                else:
-                    if retry_default is not UNDEFINED:
-                        return index, retry_default
-                    raise e
-
-    tasks = [_task(i, index) for index, i in enumerate(input_)]
-    results = []
-    for coro in asyncio.as_completed(tasks):
-        result = await coro
-        results.append(result)
-        await asyncio.sleep(throttle_delay)
-
-    results.sort(
-        key=lambda x: x[0]
-    )  # Sort results based on the original index
-
-    if retry_timing:
-        if dropna:
-            return [
-                (result[1], result[2])
-                for result in results
-                if result[1] is not None
-            ]
-        else:
-            return [(result[1], result[2]) for result in results]
-    else:
-        return to_list(
-            [result[1] for result in results],
-            flatten=flatten,
-            dropna=dropna,
-            unique=unique,
-        )

lionagi/libs/func/async_calls/bcall.py

@@ -1,130 +0,0 @@
-from collections.abc import AsyncGenerator, Callable
-from dataclasses import dataclass
-from typing import Any, TypeVar
-
-from lionagi.libs.constants import UNDEFINED
-from lionagi.libs.utils import DataClass
-
-from ...parse import to_list
-from .alcall import alcall
-
-T = TypeVar("T")
-
-
-@dataclass
-class BCallParams(DataClass):
-    function: Callable[..., T]
-    batch_size: int
-    num_retries: int = 0
-    initial_delay: float = 0
-    retry_delay: float = 0
-    backoff_factor: float = 1
-    retry_default: Any = UNDEFINED
-    retry_timeout: float | None = None
-    retry_timing: bool = False
-    verbose_retry: bool = True
-    error_msg: str | None = None
-    error_map: dict[type, Callable[[Exception], None]] | None = None
-    max_concurrent: int | None = None
-    throttle_period: float | None = None
-    flatten: bool = False
-    dropna: bool = False
-    unique: bool = False
-
-    async def __call__(self, input_, *args, **kwargs):
-        return await bcall(
-            input_,
-            self.function,
-            *args,
-            batch_size=self.batch_size,
-            num_retries=self.num_retries,
-            initial_delay=self.initial_delay,
-            retry_delay=self.retry_delay,
-            backoff_factor=self.backoff_factor,
-            default=self.retry_default,
-            timeout=self.retry_timeout,
-            timing=self.retry_timing,
-            verbose=self.verbose_retry,
-            error_msg=self.error_msg,
-            error_map=self.error_map,
-            max_concurrent=self.max_concurrent,
-            throttle_period=self.throttle_period,
-            flatten=self.flatten,
-            dropna=self.dropna,
-            unique=self.unique,
-            **kwargs,
-        )
-
-
-async def bcall(
-    input_: Any,
-    func: Callable[..., T],
-    /,
-    batch_size: int,
-    num_retries: int = 0,
-    initial_delay: float = 0,
-    retry_delay: float = 0,
-    backoff_factor: float = 1,
-    retry_default: Any = None,
-    retry_timeout: float | None = None,
-    retry_timing: bool = False,
-    verbose_retry: bool = True,
-    error_msg: str | None = None,
-    error_map: dict[type, Callable[[Exception], Any]] | None = None,
-    max_concurrent: int | None = None,
-    throttle_period: float | None = None,
-    **kwargs: Any,
-) -> AsyncGenerator[list[T | tuple[T, float]], None]:
-    """
-    Asynchronously call a function in batches with retry and timing options.
-
-    Args:
-        input_: The input data to process.
-        func: The function to call.
-        batch_size: The size of each batch.
-        retries: The number of retries.
-        initial_delay: Initial delay before the first attempt in seconds.
-        delay: The delay between retries in seconds.
-        backoff_factor: Factor by which delay increases after each retry.
-        default: Default value to return if an error occurs.
-        timeout: The timeout for the function call in seconds.
-        timing: If True, return execution time along with the result.
-        verbose: If True, print retry attempts and exceptions.
-        error_msg: Custom error message prefix.
-        error_map: Mapping of errors to handle custom error responses.
-        max_concurrent: Maximum number of concurrent calls.
-        throttle_period: Throttle period in seconds.
-        **kwargs: Additional keyword arguments to pass to the function.
-
-    Yields:
-        A list of results for each batch of inputs.
-
-    Examples:
-        >>> async def sample_func(x):
-        ...     return x * 2
-        >>> async for batch_results in bcall([1, 2, 3, 4, 5], sample_func, 2,
-        ...                                  retries=3, delay=1):
-        ...     print(batch_results)
-    """
-    input_ = to_list(input_, flatten=True, dropna=True)
-
-    for i in range(0, len(input_), batch_size):
-        batch = input_[i : i + batch_size]  # noqa: E203
-        batch_results = await alcall(
-            batch,
-            func,
-            num_retries=num_retries,
-            initial_delay=initial_delay,
-            retry_delay=retry_delay,
-            backoff_factor=backoff_factor,
-            retry_default=retry_default,
-            retry_timeout=retry_timeout,
-            retry_timing=retry_timing,
-            verbose_retry=verbose_retry,
-            error_msg=error_msg,
-            error_map=error_map,
-            max_concurrent=max_concurrent,
-            throttle_period=throttle_period,
-            **kwargs,
-        )
-        yield batch_results