lionagi 0.17.4__py3-none-any.whl → 0.17.6__py3-none-any.whl

lionagi/libs/file/concat.py

@@ -1,121 +0,0 @@
-from pathlib import Path
-from typing import Any
-
-from lionagi.utils import create_path, lcall
-
-from .process import dir_to_files
-
-
-def concat(
-    data_path: str | Path | list,
-    file_types: list[str],
-    output_dir: str | Path = None,
-    output_filename: str = None,
-    file_exist_ok: bool = True,
-    recursive: bool = True,
-    verbose: bool = True,
-    threshold: int = 0,
-    return_fps: bool = False,
-    return_files: bool = False,
-    exclude_patterns: list[str] = None,
-    **kwargs,
-) -> dict[str, Any]:
-    """
-    data_path: str or Path or list of str or Path, the directory or file paths to concatenate.
-    file_types: list of str, the file types to concatenate. [e.g. ['.txt', '.md']]
-    output_dir: str or Path, the directory to save the concatenated file. If provided, will save the file.
-    output_filename: str, the filename to save the concatenated file.
-    file_exist_ok: bool, if True, overwrite the existing file. Default is True.
-    recursive: bool, if True, search files recursively. Default is True.
-    verbose: bool, if True, print the output path. Default is True.
-    threshold: int, the minimum number of chars for the file to be considered valid to concatenate.
-    exclude_patterns: list of str, patterns to exclude files from concatenation (e.g. ['log', 'temp', '.venv']).
-    kwargs: additional keyword arguments to pass to create_path.
-    """
-    persist_path = None
-    if output_dir:
-        if not output_filename:
-            output_filename = "concatenated_text.txt"
-            kwargs["timestamp"] = kwargs.get("timestamp", True)
-            kwargs["random_hash_digits"] = kwargs.get("random_hash_digits", 6)
-        output_filename = output_filename or "concatenated_text.txt"
-        persist_path = create_path(
-            output_dir, output_filename, file_exist_ok=file_exist_ok, **kwargs
-        )
-
-    texts = []
-
-    def _check_existence(_p: str) -> Path | list[Path] | None:
-        if exclude_patterns:
-            _str_p = str(_p)
-            for pattern in exclude_patterns:
-                if pattern in _str_p:
-                    return None
-
-        if not Path(_p).exists():
-            # if the path doesn't exist, return None
-            if verbose:
-                print(f"Path {_p} does not exist, skipping...")
-            return None
-
-        p = Path(_p)
-        if p.is_dir():
-            return dir_to_files(
-                p,
-                recursive=recursive,
-                file_types=file_types,
-                ignore_errors=True,
-                max_workers=5,
-            )
-        if p.is_file():
-            return p
-
-    data_path: list[Path] = lcall(
-        data_path,
-        _check_existence,
-        sanitize_input=True,
-        unique_input=True,
-        flatten=True,
-        dropna=True,
-        unique_output=True,
-        flatten_tuple_set=True,
-    )
-
-    contents = {}
-    fps = []
-    for dp in data_path:
-        try:
-            text = dp.read_text(encoding="utf-8")
-
-        except Exception:
-            # if we cannot read the file, skip it
-            print(f"Could not read file: {dp}. Skipping...")
-            continue
-
-        if threshold > 0 and len(text) < threshold:
-            continue
-
-        fps.append(dp)
-        contents[str(dp)] = text
-
-    for k, text in sorted(contents.items(), key=lambda x: x[0]):
-        texts.extend(["---", k, "---\n", text])
-
-    text = "\n".join(texts)
-    if persist_path:
-        persist_path.write_text(text, encoding="utf-8")
-    if verbose:
-        print(
-            f"Concatenated {len(fps)} files to {persist_path}."
-            f" The file contains {len(text)} characters."
-        )
-
-    out = {"text": text}  # default output
-    if persist_path:
-        out["persist_path"] = persist_path
-    if return_files:
-        out["texts"] = texts
-    if return_fps:
-        out["fps"] = fps
-
-    return out

lionagi/libs/file/concat_files.py

@@ -1,85 +0,0 @@
-from pathlib import Path
-
-from lionagi.utils import create_path
-
-from .process import dir_to_files
-
-
-def concat_files(
-    data_path: str | Path | list,
-    file_types: list[str],
-    output_dir: str | Path = None,
-    output_filename: str = None,
-    file_exist_ok: bool = True,
-    recursive: bool = True,
-    verbose: bool = True,
-    threshold: int = 0,
-    return_fps: bool = False,
-    return_files: bool = False,
-    **kwargs,
-) -> list[str] | str | tuple[list[str], list[Path]] | tuple[str, list[Path]]:
-    """
-    data_path: str or Path or list of str or Path, the directory or file paths to concatenate.
-    file_types: list of str, the file types to concatenate. [e.g. ['.txt', '.md']]
-    output_dir: str or Path, the directory to save the concatenated file. If provided, will save the file.
-    output_filename: str, the filename to save the concatenated file.
-    file_exist_ok: bool, if True, overwrite the existing file. Default is True.
-    recursive: bool, if True, search files recursively. Default is True.
-    verbose: bool, if True, print the output path. Default is True.
-    threshold: int, the minimum number of chars for the file to be considered valid to concatenate.
-    kwargs: additional keyword arguments to pass to create_path.
-    """
-    persist_path = None
-    if output_dir:
-        if not output_filename:
-            output_filename = "concatenated_text.txt"
-            kwargs["timestamp"] = kwargs.get("timestamp", True)
-            kwargs["random_hash_digits"] = kwargs.get("random_hash_digits", 6)
-        output_filename = output_filename or "concatenated_text.txt"
-        persist_path = create_path(
-            output_dir, output_filename, file_exist_ok=file_exist_ok, **kwargs
-        )
-
-    texts = []
-    data_path = (
-        [str(data_path)] if not isinstance(data_path, list) else data_path
-    )
-    data_path = sorted(data_path)
-    data_path = [Path(dp) for dp in data_path if Path(dp).exists()]
-
-    fps = []
-    for dp in data_path:
-        _fps = dir_to_files(dp, recursive=recursive, file_types=file_types)
-
-        data_path = sorted([str(i) for i in _fps])
-        data_path: list[Path] = [
-            Path(dp) for dp in data_path if Path(dp).exists()
-        ]
-
-        for fp in data_path:
-            text = fp.read_text(encoding="utf-8")
-            if len(text) >= threshold:
-                fp_text = (
-                    "\n----------------------------------------------------\n"
-                    f"{str(fp)}"
-                    "\n----------------------------------------------------\n"
-                )
-                text = fp_text + text
-                texts.append(text)
-        fps.extend(data_path)
-
-    text = "\n".join(texts)
-    if persist_path:
-        persist_path.write_text(text, encoding="utf-8")
-    if verbose:
-        print(f"Concatenated {len(fps)} files to {persist_path}")
-        print(f"The file contains {len(text)} characters.")
-
-    if return_files:
-        if return_fps:
-            return texts, fps
-        return texts
-
-    if return_fps:
-        return text, fps
-    return text

lionagi/libs/file/file_ops.py

@@ -1,118 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-from pathlib import Path
-from shutil import copy2
-
-
-def copy_file(src: Path | str, dest: Path | str) -> None:
-    """
-    Copy a file from a source path to a destination path.
-
-    Args:
-        src: The source file path.
-        dest: The destination file path.
-
-    Raises:
-        FileNotFoundError: If the source file does not exist or is not
-            a file.
-        PermissionError: If there are insufficient permissions to copy
-            the file.
-        OSError: If there's an OS-level error during the copy operation.
-    """
-    src_path, dest_path = Path(src), Path(dest)
-    if not src_path.is_file():
-        raise FileNotFoundError(f"{src_path} does not exist or is not a file.")
-
-    try:
-        dest_path.parent.mkdir(parents=True, exist_ok=True)
-        copy2(src_path, dest_path)
-    except PermissionError as e:
-        raise PermissionError(
-            f"Permission denied when copying {src_path} to {dest_path}"
-        ) from e
-    except OSError as e:
-        raise OSError(f"Failed to copy {src_path} to {dest_path}: {e}") from e
-
-
-def get_file_size(path: Path | str) -> int:
-    """
-    Get the size of a file or total size of files in a directory.
-
-    Args:
-        path: The file or directory path.
-
-    Returns:
-        The size in bytes.
-
-    Raises:
-        FileNotFoundError: If the path does not exist.
-        PermissionError: If there are insufficient permissions
-            to access the path.
-    """
-    path = Path(path)
-    try:
-        if path.is_file():
-            return path.stat().st_size
-        elif path.is_dir():
-            return sum(
-                f.stat().st_size for f in path.rglob("*") if f.is_file()
-            )
-        else:
-            raise FileNotFoundError(f"{path} does not exist.")
-    except PermissionError as e:
-        raise PermissionError(
-            f"Permission denied when accessing {path}"
-        ) from e
-
-
-def list_files(
-    dir_path: Path | str, extension: str | None = None
-) -> list[Path]:
-    """
-    List all files in a specified directory with an optional extension
-    filter, including files in subdirectories.
-
-    Args:
-        dir_path: The directory path where files are listed.
-        extension: Filter files by extension.
-
-    Returns:
-        A list of Path objects representing files in the directory.
-
-    Raises:
-        NotADirectoryError: If the provided dir_path is not a directory.
-    """
-    dir_path = Path(dir_path)
-    if not dir_path.is_dir():
-        raise NotADirectoryError(f"{dir_path} is not a directory.")
-
-    pattern = f"*.{extension}" if extension else "*"
-    return [f for f in dir_path.rglob(pattern) if f.is_file()]
-
-
-def read_file(path: Path | str, /) -> str:
-    """
-    Read the contents of a file.
-
-    Args:
-        path: The path to the file to read.
-
-    Returns:
-        str: The contents of the file.
-
-    Raises:
-        FileNotFoundError: If the file does not exist.
-        PermissionError: If there are insufficient permissions to read
-            the file.
-    """
-    try:
-        return Path(path).read_text(encoding="utf-8")
-    except FileNotFoundError as e:
-        logging.error(f"File not found: {path}: {e}")
-        raise
-    except PermissionError as e:
-        logging.error(f"Permission denied when reading file: {path}: {e}")
-        raise

lionagi/libs/file/save.py

@@ -1,103 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-from pathlib import Path
-from typing import Any
-
-from lionagi.utils import create_path
-
-
-def save_to_file(
-    text: str,
-    directory: Path | str,
-    filename: str,
-    extension: str = None,
-    timestamp: bool = False,
-    dir_exist_ok: bool = True,
-    file_exist_ok: bool = False,
-    time_prefix: bool = False,
-    timestamp_format: str | None = None,
-    random_hash_digits: int = 0,
-    verbose: bool = True,
-) -> Path:
-    """
-    Save text to a file within a specified directory, optionally adding a
-    timestamp, hash, and verbose logging.
-
-    Args:
-        text: The text to save.
-        directory: The directory path to save the file.
-        filename: The filename for the saved text.
-        timestamp: If True, append a timestamp to the filename.
-        dir_exist_ok: If True, creates the directory if it does not exist.
-        time_prefix: If True, prepend the timestamp instead of appending.
-        timestamp_format: A custom format for the timestamp.
-        random_hash_digits: Number of random hash digits to append
-            to filename.
-        verbose: If True, logs the file path after saving.
-
-    Returns:
-        Path: The path to the saved file.
-
-    Raises:
-        OSError: If there's an error creating the directory or
-            writing the file.
-    """
-    try:
-        file_path = create_path(
-            directory=directory,
-            filename=filename,
-            extension=extension,
-            timestamp=timestamp,
-            dir_exist_ok=dir_exist_ok,
-            file_exist_ok=file_exist_ok,
-            time_prefix=time_prefix,
-            timestamp_format=timestamp_format,
-            random_hash_digits=random_hash_digits,
-        )
-        with file_path.open("w", encoding="utf-8") as file:
-            file.write(text)
-        file.close()
-        if verbose:
-            logging.info(f"Text saved to: {file_path}")
-        return file_path
-
-    except OSError as e:
-        logging.error(f"Failed to save file {filename}: {e}")
-        raise
-
-
-def save_chunks(
-    chunks: list[dict[str, Any]],
-    output_dir: str | Path,
-    verbose: bool,
-    timestamp: bool,
-    random_hash_digits: int,
-) -> None:
-    """Helper function to save chunks to files."""
-    from lionagi import ln
-
-    output_path = Path(output_dir)
-    for i, chunk in enumerate(chunks):
-        file_path = create_path(
-            directory=output_path,
-            filename=f"chunk_{i + 1}",
-            extension="json",
-            timestamp=timestamp,
-            random_hash_digits=random_hash_digits,
-        )
-        save_to_file(
-            ln.json_dumps(
-                chunk,
-                pretty=True,
-                sort_keys=True,
-                append_newline=True,
-                deterministic_sets=True,
-                decimal_as_float=True,
-            ),
-            directory=file_path.parent,
-            filename=file_path.name,
-            verbose=verbose,
-        )

lionagi/ln/concurrency/throttle.py

@@ -1,83 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import functools
-from collections.abc import Callable
-from time import sleep, time
-from typing import Any, TypeVar
-
-from typing_extensions import deprecated
-
-T = TypeVar("T")
-
-
-__all__ = ("Throttle",)
-
-
-@deprecated("Throttle is deprecated and will be removed in a future release.")
-class Throttle:
-    """
-    Provide a throttling mechanism for function calls.
-
-    When used as a decorator, it ensures that the decorated function can only
-    be called once per specified period. Subsequent calls within this period
-    are delayed to enforce this constraint.
-
-    Attributes:
-        period: The minimum time period (in seconds) between successive calls.
-    """
-
-    def __init__(self, period: float) -> None:
-        """
-        Initialize a new instance of Throttle.
-
-        Args:
-            period: The minimum time period (in seconds) between
-                successive calls.
-        """
-        self.period = period
-        self.last_called = 0
-
-    def __call__(self, func: Callable[..., T]) -> Callable[..., T]:
-        """
-        Decorate a synchronous function with the throttling mechanism.
-
-        Args:
-            func: The synchronous function to be throttled.
-
-        Returns:
-            The throttled synchronous function.
-        """
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs) -> Any:
-            elapsed = time() - self.last_called
-            if elapsed < self.period:
-                sleep(self.period - elapsed)
-            self.last_called = time()
-            return func(*args, **kwargs)
-
-        return wrapper
-
-    def __call_async__(
-        self, func: Callable[..., Callable[..., T]]
-    ) -> Callable[..., Callable[..., T]]:
-        """
-        Decorate an asynchronous function with the throttling mechanism.
-
-        Args:
-            func: The asynchronous function to be throttled.
-
-        Returns:
-            The throttled asynchronous function.
-        """
-
-        @functools.wraps(func)
-        async def wrapper(*args, **kwargs) -> Any:
-            elapsed = time() - self.last_called
-            if elapsed < self.period:
-                await asyncio.sleep(self.period - elapsed)
-            self.last_called = time()
-            return await func(*args, **kwargs)
-
-        return wrapper

lionagi/settings.py

@@ -1,71 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from datetime import timezone
-
-CACHED_CONFIG = {
-    "ttl": 300,
-    "key": None,
-    "namespace": None,
-    "key_builder": None,
-    "skip_cache_func": lambda x: False,
-    "serializer": None,
-    "plugins": None,
-    "alias": None,
-    "noself": lambda x: False,
-}
-
-CHAT_IMODEL_CONFIG = {
-    "provider": "openai",
-    "model": "gpt-4.1-nano",
-    "base_url": "https://api.openai.com/v1",
-    "endpoint": "chat/completions",
-    "api_key": "OPENAI_API_KEY",
-    "queue_capacity": 100,
-    "capacity_refresh_time": 60,
-    "interval": None,
-    "limit_requests": None,
-    "limit_tokens": None,
-}
-
-PARSE_IMODEL_CONFIG = {
-    "provider": "openai",
-    "model": "gpt-4.1-nano",
-    "base_url": "https://api.openai.com/v1",
-    "endpoint": "chat/completions",
-    "api_key": "OPENAI_API_KEY",
-    "queue_capacity": 100,
-    "capacity_refresh_time": 60,
-    "interval": None,
-    "limit_requests": None,
-    "limit_tokens": None,
-}
-
-LOG_CONFIG = {
-    "persist_dir": "./data/logs",
-    "subfolder": None,
-    "capacity": 50,
-    "extension": ".json",
-    "use_timestamp": True,
-    "hash_digits": 5,
-    "file_prefix": "log",
-    "auto_save_on_exit": True,
-    "clear_after_dump": True,
-}
-
-
-class Settings:
-    class Config:
-        TIMEZONE: timezone = timezone.utc
-        LOG: dict = LOG_CONFIG
-
-    class Action:
-        CACHED_CONFIG: dict = CACHED_CONFIG
-
-    class API:
-        CACHED_CONFIG: dict = CACHED_CONFIG
-
-    class iModel:
-        CHAT: dict = CHAT_IMODEL_CONFIG
-        PARSE: dict = PARSE_IMODEL_CONFIG