lionagi 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/file/save.py

@@ -0,0 +1,90 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from .path import create_path
+
+
+def save_to_file(
+    text: str,
+    directory: Path | str,
+    filename: str,
+    extension: str = None,
+    timestamp: bool = False,
+    dir_exist_ok: bool = True,
+    file_exist_ok: bool = False,
+    time_prefix: bool = False,
+    timestamp_format: str | None = None,
+    random_hash_digits: int = 0,
+    verbose: bool = True,
+) -> Path:
+    """
+    Save text to a file within a specified directory, optionally adding a
+    timestamp, hash, and verbose logging.
+
+    Args:
+        text: The text to save.
+        directory: The directory path to save the file.
+        filename: The filename for the saved text.
+        timestamp: If True, append a timestamp to the filename.
+        dir_exist_ok: If True, creates the directory if it does not exist.
+        time_prefix: If True, prepend the timestamp instead of appending.
+        timestamp_format: A custom format for the timestamp.
+        random_hash_digits: Number of random hash digits to append
+            to filename.
+        verbose: If True, logs the file path after saving.
+
+    Returns:
+        Path: The path to the saved file.
+
+    Raises:
+        OSError: If there's an error creating the directory or
+            writing the file.
+    """
+    try:
+        file_path = create_path(
+            directory=directory,
+            filename=filename,
+            extension=extension,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            file_exist_ok=file_exist_ok,
+            time_prefix=time_prefix,
+            timestamp_format=timestamp_format,
+            random_hash_digits=random_hash_digits,
+        )
+        with file_path.open("w", encoding="utf-8") as file:
+            file.write(text)
+        if verbose:
+            logging.info(f"Text saved to: {file_path}")
+        return file_path
+
+    except OSError as e:
+        logging.error(f"Failed to save file {filename}: {e}")
+        raise
+
+
+def save_chunks(
+    chunks: list[dict[str, Any]],
+    output_dir: str | Path,
+    verbose: bool,
+    timestamp: bool,
+    random_hash_digits: int,
+) -> None:
+    """Helper function to save chunks to files."""
+    output_path = Path(output_dir)
+    for i, chunk in enumerate(chunks):
+        file_path = create_path(
+            directory=output_path,
+            filename=f"chunk_{i+1}",
+            extension="json",
+            timestamp=timestamp,
+            random_hash_digits=random_hash_digits,
+        )
+        save_to_file(
+            json.dumps(chunk, ensure_ascii=False, indent=2),
+            directory=file_path.parent,
+            filename=file_path.name,
+            verbose=verbose,
+        )

lionagi/libs/file/types.py

@@ -0,0 +1,22 @@
+from .chunk import chunk_by_chars, chunk_by_tokens, chunk_content
+from .file_ops import copy_file, get_file_size, list_files, read_file
+from .path import clear_path, create_path, is_valid_path, split_path
+from .process import dir_to_files, file_to_chunks
+from .save import save_to_file
+
+__all__ = [
+    "chunk_by_chars",
+    "chunk_by_tokens",
+    "chunk_content",
+    "clear_path",
+    "copy_file",
+    "create_path",
+    "dir_to_files",
+    "file_to_chunks",
+    "get_file_size",
+    "is_valid_path",
+    "list_files",
+    "read_file",
+    "save_to_file",
+    "split_path",
+]

lionagi/libs/func/async_calls/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .alcall import alcall
+from .bcall import bcall
+from .mcall import mcall
+from .pcall import pcall
+from .rcall import rcall
+from .tcall import tcall
+from .ucall import ucall
+
+__all__ = [
+    "bcall",
+    "alcall",
+    "mcall",
+    "pcall",
+    "rcall",
+    "tcall",
+    "ucall",
+]

lionagi/libs/func/async_calls/alcall.py

@@ -0,0 +1,157 @@
+import asyncio
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from ...constants import UNDEFINED
+from ...parse import to_list
+from .ucall import ucall
+
+T = TypeVar("T")
+
+
+async def alcall(
+    input_: list[Any],
+    func: Callable[..., T],
+    /,
+    num_retries: int = 0,
+    initial_delay: float = 0,
+    retry_delay: float = 0,
+    backoff_factor: float = 1,
+    retry_default: Any = UNDEFINED,
+    retry_timeout: float | None = None,
+    retry_timing: bool = False,
+    verbose_retry: bool = True,
+    error_msg: str | None = None,
+    error_map: dict[type, Callable[[Exception], None]] | None = None,
+    max_concurrent: int | None = None,
+    throttle_period: float | None = None,
+    flatten: bool = False,
+    dropna: bool = False,
+    unique: bool = False,
+    **kwargs: Any,
+) -> list[T] | list[tuple[T, float]]:
+    """Apply a function to each element of a list asynchronously with options.
+
+    Args:
+        input_: List of inputs to be processed.
+        func: Async or sync function to apply to each input element.
+        num_retries: Number of retry attempts for each function call.
+        initial_delay: Initial delay before starting execution (seconds).
+        retry_delay: Delay between retry attempts (seconds).
+        backoff_factor: Factor by which delay increases after each attempt.
+        retry_default: Default value to return if all attempts fail.
+        retry_timeout: Timeout for each function execution (seconds).
+        retry_timing: If True, return execution duration for each call.
+        verbose_retry: If True, print retry messages.
+        error_msg: Custom error message prefix for exceptions.
+        error_map: Dict mapping exception types to error handlers.
+        max_concurrent: Maximum number of concurrent executions.
+        throttle_period: Minimum time between function executions (seconds).
+        flatten: If True, flatten the resulting list.
+        dropna: If True, remove None values from the result.
+        **kwargs: Additional keyword arguments passed to func.
+
+    Returns:
+        list[T] | list[tuple[T, float]]: List of results, optionally with
+        execution times if retry_timing is True.
+
+    Raises:
+        asyncio.TimeoutError: If execution exceeds retry_timeout.
+        Exception: Any exception raised by func if not handled by error_map.
+
+    Examples:
+        >>> async def square(x):
+        ...     return x * x
+        >>> await alcall([1, 2, 3], square)
+        [1, 4, 9]
+        >>> await alcall([1, 2, 3], square, retry_timing=True)
+        [(1, 0.001), (4, 0.001), (9, 0.001)]
+
+    Note:
+        - Uses semaphores for concurrency control if max_concurrent is set.
+        - Supports both synchronous and asynchronous functions for `func`.
+        - Results are returned in the original input order.
+    """
+    if initial_delay:
+        await asyncio.sleep(initial_delay)
+
+    semaphore = asyncio.Semaphore(max_concurrent) if max_concurrent else None
+    throttle_delay = throttle_period if throttle_period else 0
+
+    async def _task(i: Any, index: int) -> Any:
+        if semaphore:
+            async with semaphore:
+                return await _execute_task(i, index)
+        else:
+            return await _execute_task(i, index)
+
+    async def _execute_task(i: Any, index: int) -> Any:
+        attempts = 0
+        current_delay = retry_delay
+        while True:
+            try:
+                if retry_timing:
+                    start_time = asyncio.get_event_loop().time()
+                    result = await asyncio.wait_for(
+                        ucall(func, i, **kwargs), retry_timeout
+                    )
+                    end_time = asyncio.get_event_loop().time()
+                    return index, result, end_time - start_time
+                else:
+                    result = await asyncio.wait_for(
+                        ucall(func, i, **kwargs), retry_timeout
+                    )
+                    return index, result
+            except TimeoutError as e:
+                raise TimeoutError(
+                    f"{error_msg or ''} Timeout {retry_timeout} seconds "
+                    "exceeded"
+                ) from e
+            except Exception as e:
+                if error_map and type(e) in error_map:
+                    handler = error_map[type(e)]
+                    if asyncio.iscoroutinefunction(handler):
+                        return index, await handler(e)
+                    else:
+                        return index, handler(e)
+                attempts += 1
+                if attempts <= num_retries:
+                    if verbose_retry:
+                        print(
+                            f"Attempt {attempts}/{num_retries} failed: {e}"
+                            ", retrying..."
+                        )
+                    await asyncio.sleep(current_delay)
+                    current_delay *= backoff_factor
+                else:
+                    if retry_default is not UNDEFINED:
+                        return index, retry_default
+                    raise e
+
+    tasks = [_task(i, index) for index, i in enumerate(input_)]
+    results = []
+    for coro in asyncio.as_completed(tasks):
+        result = await coro
+        results.append(result)
+        await asyncio.sleep(throttle_delay)
+
+    results.sort(
+        key=lambda x: x[0]
+    )  # Sort results based on the original index
+
+    if retry_timing:
+        if dropna:
+            return [
+                (result[1], result[2])
+                for result in results
+                if result[1] is not None
+            ]
+        else:
+            return [(result[1], result[2]) for result in results]
+    else:
+        return to_list(
+            [result[1] for result in results],
+            flatten=flatten,
+            dropna=dropna,
+            unique=unique,
+        )

lionagi/libs/func/async_calls/bcall.py

@@ -0,0 +1,82 @@
+import asyncio
+from collections.abc import AsyncGenerator, Callable
+from typing import Any, TypeVar
+
+from ...parse import to_list
+from .alcall import alcall
+
+T = TypeVar("T")
+
+
+async def bcall(
+    input_: Any,
+    func: Callable[..., T],
+    /,
+    batch_size: int,
+    num_retries: int = 0,
+    initial_delay: float = 0,
+    retry_delay: float = 0,
+    backoff_factor: float = 1,
+    retry_default: Any = None,
+    retry_timeout: float | None = None,
+    retry_timing: bool = False,
+    verbose_retry: bool = True,
+    error_msg: str | None = None,
+    error_map: dict[type, Callable[[Exception], Any]] | None = None,
+    max_concurrent: int | None = None,
+    throttle_period: float | None = None,
+    **kwargs: Any,
+) -> AsyncGenerator[list[T | tuple[T, float]], None]:
+    """
+    Asynchronously call a function in batches with retry and timing options.
+
+    Args:
+        input_: The input data to process.
+        func: The function to call.
+        batch_size: The size of each batch.
+        retries: The number of retries.
+        initial_delay: Initial delay before the first attempt in seconds.
+        delay: The delay between retries in seconds.
+        backoff_factor: Factor by which delay increases after each retry.
+        default: Default value to return if an error occurs.
+        timeout: The timeout for the function call in seconds.
+        timing: If True, return execution time along with the result.
+        verbose: If True, print retry attempts and exceptions.
+        error_msg: Custom error message prefix.
+        error_map: Mapping of errors to handle custom error responses.
+        max_concurrent: Maximum number of concurrent calls.
+        throttle_period: Throttle period in seconds.
+        **kwargs: Additional keyword arguments to pass to the function.
+
+    Yields:
+        A list of results for each batch of inputs.
+
+    Examples:
+        >>> async def sample_func(x):
+        ...     return x * 2
+        >>> async for batch_results in bcall([1, 2, 3, 4, 5], sample_func, 2,
+        ...                                  retries=3, delay=1):
+        ...     print(batch_results)
+    """
+    input_ = to_list(input_, flatten=True, dropna=True)
+
+    for i in range(0, len(input_), batch_size):
+        batch = input_[i : i + batch_size]  # noqa: E203
+        batch_results = await alcall(
+            batch,
+            func,
+            num_retries=num_retries,
+            initial_delay=initial_delay,
+            retry_delay=retry_delay,
+            backoff_factor=backoff_factor,
+            retry_default=retry_default,
+            retry_timeout=retry_timeout,
+            retry_timing=retry_timing,
+            verbose_retry=verbose_retry,
+            error_msg=error_msg,
+            error_map=error_map,
+            max_concurrent=max_concurrent,
+            throttle_period=throttle_period,
+            **kwargs,
+        )
+        yield batch_results

lionagi/libs/func/async_calls/mcall.py

@@ -0,0 +1,134 @@
+import asyncio
+from collections.abc import Callable, Sequence
+from typing import Any, TypeVar
+
+from ...constants import UNDEFINED
+from ...parse import to_list
+from .alcall import alcall
+from .rcall import rcall
+
+T = TypeVar("T")
+
+
+async def mcall(
+    input_: Any,
+    func: Callable[..., T] | Sequence[Callable[..., T]],
+    /,
+    *,
+    explode: bool = False,
+    num_retries: int = 0,
+    initial_delay: float = 0,
+    retry_delay: float = 0,
+    backoff_factor: float = 1,
+    retry_default: Any = UNDEFINED,
+    retry_timeout: float | None = None,
+    retry_timing: bool = False,
+    verbose_retry: bool = True,
+    error_msg: str | None = None,
+    error_map: dict[type, Callable[[Exception], None]] | None = None,
+    max_concurrent: int | None = None,
+    throttle_period: float | None = None,
+    dropna: bool = False,
+    **kwargs: Any,
+) -> list[T] | list[tuple[T, float]]:
+    """
+    Apply functions over inputs asynchronously with customizable options.
+
+    Args:
+        input_: The input data to be processed.
+        func: The function or sequence of functions to be applied.
+        explode: Whether to apply each function to all inputs.
+        retries: Number of retry attempts for each function call.
+        initial_delay: Initial delay before starting execution.
+        delay: Delay between retry attempts.
+        backoff_factor: Factor by which delay increases after each attempt.
+        default: Default value to return if all attempts fail.
+        timeout: Timeout for each function execution.
+        timing: Whether to return the execution duration.
+        verbose: Whether to print retry messages.
+        error_msg: Custom error message.
+        error_map: Dictionary mapping exception types to error handlers.
+        max_concurrent: Maximum number of concurrent executions.
+        throttle_period: Minimum time period between function executions.
+        dropna: Whether to drop None values from the output list.
+        **kwargs: Additional keyword arguments for the functions.
+
+    Returns:
+        List of results, optionally including execution durations if timing
+        is True.
+
+    Raises:
+        ValueError: If the length of inputs and functions don't match when
+            not exploding the function calls.
+    """
+    input_ = to_list(input_, flatten=False, dropna=False)
+    func = to_list(func, flatten=False, dropna=False)
+
+    if explode:
+        tasks = [
+            alcall(
+                input_,
+                f,
+                num_retries=num_retries,
+                initial_delay=initial_delay,
+                retry_delay=retry_delay,
+                backoff_factor=backoff_factor,
+                retry_default=retry_default,
+                retry_timeout=retry_timeout,
+                retry_timing=retry_timing,
+                verbose_retry=verbose_retry,
+                error_msg=error_msg,
+                error_map=error_map,
+                max_concurrent=max_concurrent,
+                throttle_period=throttle_period,
+                dropna=dropna,
+                **kwargs,
+            )
+            for f in func
+        ]
+        return await asyncio.gather(*tasks)
+    elif len(func) == 1:
+        tasks = [
+            rcall(
+                func[0],
+                inp,
+                num_retries=num_retries,
+                initial_delay=initial_delay,
+                retry_delay=retry_delay,
+                backoff_factor=backoff_factor,
+                retry_default=retry_default,
+                retry_timeout=retry_timeout,
+                retry_timing=retry_timing,
+                verbose_retry=verbose_retry,
+                error_msg=error_msg,
+                error_map=error_map,
+                **kwargs,
+            )
+            for inp in input_
+        ]
+        return await asyncio.gather(*tasks)
+
+    elif len(input_) == len(func):
+        tasks = [
+            rcall(
+                f,
+                inp,
+                num_retries=num_retries,
+                initial_delay=initial_delay,
+                retry_delay=retry_delay,
+                backoff_factor=backoff_factor,
+                retry_default=retry_default,
+                retry_timeout=retry_timeout,
+                retry_timing=retry_timing,
+                verbose_retry=verbose_retry,
+                error_msg=error_msg,
+                error_map=error_map,
+                **kwargs,
+            )
+            for inp, f in zip(input_, func)
+        ]
+        return await asyncio.gather(*tasks)
+    else:
+        raise ValueError(
+            "Inputs and functions must be the same length for map calling."
+        )

lionagi/libs/func/async_calls/pcall.py

@@ -0,0 +1,149 @@
+import asyncio
+from collections.abc import Callable, Sequence
+from typing import Any, TypeVar
+
+from lionagi.libs.func.utils import is_coroutine_func
+
+from ...constants import UNDEFINED
+from .ucall import ucall
+
+T = TypeVar("T")
+
+
+async def pcall(
+    funcs: Sequence[Callable[..., T]],
+    /,
+    num_retries: int = 0,
+    initial_delay: float = 0,
+    retry_delay: float = 0,
+    backoff_factor: float = 1,
+    retry_default: Any = UNDEFINED,
+    retry_timeout: float | None = None,
+    retry_timing: bool = False,
+    verbose_retry: bool = True,
+    error_msg: str | None = None,
+    error_map: dict[type, Callable[[Exception], None]] | None = None,
+    max_concurrent: int | None = None,
+    throttle_period: float | None = None,
+    **kwargs: Any,
+) -> list[T] | list[tuple[T, float]]:
+    """Execute multiple functions asynchronously in parallel with options.
+
+    Manages parallel execution of functions with retry logic, timing, and
+    error handling. Supports concurrency control and throttling.
+
+    Args:
+        funcs: Sequence of functions to execute in parallel.
+        num_retries: Number of retry attempts for each function (default: 0).
+        initial_delay: Delay before starting execution (seconds).
+        retry_delay: Initial delay between retry attempts (seconds).
+        backoff_factor: Factor to increase delay after each retry.
+        retry_default: Value to return if all attempts for a function fail.
+        retry_timeout: Timeout for each function execution (seconds).
+        retry_timing: If True, return execution duration for each function.
+        verbose_retry: If True, print retry messages.
+        error_msg: Custom error message prefix.
+        error_map: Dict mapping exception types to error handlers.
+        max_concurrent: Maximum number of functions to run concurrently.
+        throttle_period: Minimum time between function starts (seconds).
+        **kwargs: Additional keyword arguments passed to each function.
+
+    Returns:
+        list[T] | list[tuple[T, float]]: List of results, optionally with
+        execution times if retry_timing is True.
+
+    Raises:
+        asyncio.TimeoutError: If any function execution exceeds retry_timeout.
+        Exception: Any unhandled exception from function executions.
+
+    Examples:
+        >>> async def func1(x):
+        ...     await asyncio.sleep(1)
+        ...     return x * 2
+        >>> async def func2(x):
+        ...     await asyncio.sleep(0.5)
+        ...     return x + 10
+        >>> results = await pcall([func1, func2], retry_timing=True, x=5)
+        >>> for result, duration in results:
+        ...     print(f"Result: {result}, Duration: {duration:.2f}s")
+        Result: 10, Duration: 1.00s
+        Result: 15, Duration: 0.50s
+
+    Note:
+        - Executes functions in parallel, respecting max_concurrent limit.
+        - Implements exponential backoff for retries.
+        - Can return execution timing for performance analysis.
+        - Supports both coroutine and regular functions via ucall.
+        - Results are returned in the original order of input functions.
+    """
+    if initial_delay:
+        await asyncio.sleep(initial_delay)
+
+    semaphore = asyncio.Semaphore(max_concurrent) if max_concurrent else None
+    throttle_delay = throttle_period if throttle_period else 0
+
+    async def _task(func: Callable[..., Any], index: int) -> Any:
+        if semaphore:
+            async with semaphore:
+                return await _execute_task(func, index)
+        else:
+            return await _execute_task(func, index)
+
+    async def _execute_task(func: Callable[..., Any], index: int) -> Any:
+        attempts = 0
+        current_delay = retry_delay
+        while True:
+            try:
+                if retry_timing:
+                    start_time = asyncio.get_event_loop().time()
+                    result = await asyncio.wait_for(
+                        ucall(func, **kwargs), retry_timeout
+                    )
+                    end_time = asyncio.get_event_loop().time()
+                    return index, result, end_time - start_time
+                else:
+                    result = await asyncio.wait_for(
+                        ucall(func, **kwargs), retry_timeout
+                    )
+                    return index, result
+            except TimeoutError as e:
+                raise TimeoutError(
+                    f"{error_msg or ''} Timeout {retry_timeout} seconds "
+                    "exceeded"
+                ) from e
+            except Exception as e:
+                if error_map and type(e) in error_map:
+                    handler = error_map[type(e)]
+                    if is_coroutine_func(handler):
+                        return index, await handler(e)
+                    else:
+                        return index, handler(e)
+                attempts += 1
+                if attempts <= num_retries:
+                    if verbose_retry:
+                        print(
+                            f"Attempt {attempts}/{num_retries + 1} failed: {e}"
+                            ", retrying..."
+                        )
+                    await asyncio.sleep(current_delay)
+                    current_delay *= backoff_factor
+                else:
+                    if retry_default is not UNDEFINED:
+                        return index, retry_default
+                    raise e
+
+    tasks = [_task(func, index) for index, func in enumerate(funcs)]
+    results = []
+    for coro in asyncio.as_completed(tasks):
+        result = await coro
+        results.append(result)
+        await asyncio.sleep(throttle_delay)
+
+    results.sort(
+        key=lambda x: x[0]
+    )  # Sort results based on the original index
+
+    if retry_timing:
+        return [(result[1], result[2]) for result in results]
+    else:
+        return [result[1] for result in results]