lionagi 0.10.7__py3-none-any.whl → 0.12.0__py3-none-any.whl

lionagi/adapters/__init__.py

@@ -0,0 +1 @@
+"""deprecated, will be removed in v0.13.0"""

lionagi/fields/file.py

@@ -71,7 +71,7 @@ class File(HashableModel):
         header: str | None = None,
         footer: str | None = None,
     ) -> Path:
-        from lionagi.utils import create_path
+        from lionagi.libs.file.create_path import create_path
 
         fp = create_path(
             directory=directory,

lionagi/fields/reason.py

@@ -4,8 +4,8 @@
 
 from pydantic import Field, field_validator
 
+from lionagi.libs.parse.to_num import to_num
 from lionagi.models import FieldModel, HashableModel
-from lionagi.utils import to_num
 
 __all__ = ("Reason",)
 

lionagi/libs/file/concat.py

@@ -1,7 +1,12 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 from typing import Any
 
-from lionagi.utils import create_path, lcall
+from lionagi.libs.file.create_path import create_path
+from lionagi.utils import lcall
 
 from .process import dir_to_files
 

lionagi/libs/file/concat_files.py

@@ -1,6 +1,10 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 
-from lionagi.utils import create_path
+from lionagi.libs.file.create_path import create_path
 
 from .process import dir_to_files
 

lionagi/libs/file/create_path.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+
+def create_path(
+    directory: Path | str,
+    filename: str,
+    extension: str = None,
+    timestamp: bool = False,
+    dir_exist_ok: bool = True,
+    file_exist_ok: bool = False,
+    time_prefix: bool = False,
+    timestamp_format: str | None = None,
+    random_hash_digits: int = 0,
+) -> Path:
+    """
+    Generate a new file path with optional timestamp and a random suffix.
+
+    Args:
+        directory: The directory where the file will be created.
+        filename: The base name of the file to create.
+        extension: The file extension, if not part of filename.
+        timestamp: If True, add a timestamp to the filename.
+        dir_exist_ok: If True, don't error if directory exists.
+        file_exist_ok: If True, allow overwriting existing files.
+        time_prefix: If True, timestamp is prefixed instead of suffixed.
+        timestamp_format: Custom format for timestamp (default: "%Y%m%d%H%M%S").
+        random_hash_digits: Number of hex digits for a random suffix.
+
+    Returns:
+        The full Path to the new or existing file.
+
+    Raises:
+        ValueError: If filename is invalid.
+        FileExistsError: If file exists and file_exist_ok=False.
+    """
+    if "/" in filename:
+        sub_dir, filename = filename.split("/")[:-1], filename.split("/")[-1]
+        directory = Path(directory) / "/".join(sub_dir)
+
+    if "\\" in filename:
+        raise ValueError("Filename cannot contain directory separators.")
+
+    directory = Path(directory)
+
+    # Extract name and extension from filename if present
+    if "." in filename:
+        name, ext = filename.rsplit(".", 1)
+    else:
+        name, ext = filename, extension
+
+    # Ensure extension has a single leading dot
+    ext = f".{ext.lstrip('.')}" if ext else ""
+
+    # Add timestamp if requested
+    if timestamp:
+        ts_str = datetime.now().strftime(timestamp_format or "%Y%m%d%H%M%S")
+        name = f"{ts_str}_{name}" if time_prefix else f"{name}_{ts_str}"
+
+    # Add random suffix if requested
+    if random_hash_digits > 0:
+        # Use UUID4 and truncate its hex for random suffix
+        random_suffix = uuid.uuid4().hex[:random_hash_digits]
+        name = f"{name}-{random_suffix}"
+
+    full_path = directory / f"{name}{ext}"
+
+    # Check if file or directory existence
+    full_path.parent.mkdir(parents=True, exist_ok=dir_exist_ok)
+    if full_path.exists() and not file_exist_ok:
+        raise FileExistsError(
+            f"File {full_path} already exists and file_exist_ok is False."
+        )
+
+    return full_path

lionagi/libs/file/file_util.py

@@ -0,0 +1,358 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+
+class FileUtil:
+
+    @staticmethod
+    def chunk_by_chars(
+        text: str,
+        chunk_size: int = 2048,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_by_chars
+
+        return chunk_by_chars(
+            text,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def chunk_by_tokens(
+        text: str,
+        tokenizer: Callable[[str], list[str]] = str.split,
+        chunk_size: int = 1024,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_by_tokens
+
+        return chunk_by_tokens(
+            text,
+            tokenizer=tokenizer,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def chunk_content(
+        content: str,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        tokenizer: Callable[[str], list[str]] = str.split,
+        chunk_size: int = 1024,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_content
+
+        return chunk_content(
+            content,
+            chunk_by=chunk_by,
+            tokenizer=tokenizer,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def concat_files(
+        data_path: str | Path | list,
+        file_types: list[str],
+        output_dir: str | Path = None,
+        output_filename: str = None,
+        file_exist_ok: bool = True,
+        recursive: bool = True,
+        verbose: bool = True,
+        threshold: int = 0,
+        return_fps: bool = False,
+        return_files: bool = False,
+        **kwargs,
+    ) -> (
+        list[str] | str | tuple[list[str], list[Path]] | tuple[str, list[Path]]
+    ):
+        from .concat_files import concat_files
+
+        return concat_files(
+            data_path,
+            file_types=file_types,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_exist_ok=file_exist_ok,
+            recursive=recursive,
+            verbose=verbose,
+            threshold=threshold,
+            return_fps=return_fps,
+            return_files=return_files,
+            **kwargs,
+        )
+
+    @staticmethod
+    def concat(
+        data_path: str | Path | list,
+        file_types: list[str],
+        output_dir: str | Path = None,
+        output_filename: str = None,
+        file_exist_ok: bool = True,
+        recursive: bool = True,
+        verbose: bool = True,
+        threshold: int = 0,
+        return_fps: bool = False,
+        return_files: bool = False,
+        exclude_patterns: list[str] = None,
+        **kwargs,
+    ) -> dict[str, Any]:
+        from .concat import concat
+
+        return concat(
+            data_path,
+            file_types=file_types,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_exist_ok=file_exist_ok,
+            recursive=recursive,
+            verbose=verbose,
+            threshold=threshold,
+            return_fps=return_fps,
+            return_files=return_files,
+            exclude_patterns=exclude_patterns,
+            **kwargs,
+        )
+
+    @staticmethod
+    def copy_file(src: Path | str, dest: Path | str) -> None:
+        from .file_ops import copy_file
+
+        copy_file(src, dest)
+
+    @staticmethod
+    def get_file_size(path: Path | str) -> int:
+        from .file_ops import get_file_size
+
+        return get_file_size(path)
+
+    @staticmethod
+    def list_files(
+        dir_path: Path | str, extension: str | None = None
+    ) -> list[Path]:
+        from .file_ops import list_files
+
+        return list_files(dir_path, extension=extension)
+
+    @staticmethod
+    def read_file(file_path: Path | str, encoding: str = "utf-8") -> str:
+        from .file_ops import read_file
+
+        return read_file(file_path, encoding=encoding)
+
+    @staticmethod
+    def read_image_to_base64(image_path: str | Path) -> str:
+        import base64
+
+        import cv2  # type: ignore[import]
+
+        image_path = str(image_path)
+        image = cv2.imread(image_path, cv2.COLOR_BGR2RGB)
+
+        if image is None:
+            raise ValueError(f"Could not read image from path: {image_path}")
+
+        file_extension = "." + image_path.split(".")[-1]
+
+        success, buffer = cv2.imencode(file_extension, image)
+        if not success:
+            raise ValueError(
+                f"Could not encode image to {file_extension} format."
+            )
+        encoded_image = base64.b64encode(buffer).decode("utf-8")
+        return encoded_image
+
+    @staticmethod
+    def pdf_to_images(
+        pdf_path: str, output_folder: str, dpi: int = 300, fmt: str = "jpeg"
+    ) -> list:
+        """
+        Convert a PDF file into images, one image per page.
+
+        Args:
+            pdf_path (str): Path to the input PDF file.
+            output_folder (str): Directory to save the output images.
+            dpi (int): Dots per inch (resolution) for conversion (default: 300).
+            fmt (str): Image format (default: 'jpeg'). Use 'png' if preferred.
+
+        Returns:
+            list: A list of file paths for the saved images.
+        """
+        import os
+
+        from lionagi.utils import check_import
+
+        convert_from_path = check_import(
+            "pdf2image", import_name="convert_from_path"
+        )
+
+        # Ensure the output folder exists
+        os.makedirs(output_folder, exist_ok=True)
+
+        # Convert PDF to a list of PIL Image objects
+        images = convert_from_path(pdf_path, dpi=dpi)
+
+        saved_paths = []
+        for i, image in enumerate(images):
+            # Construct the output file name
+            image_file = os.path.join(output_folder, f"page_{i + 1}.{fmt}")
+            image.save(image_file, fmt.upper())
+            saved_paths.append(image_file)
+
+        return saved_paths
+
+    @staticmethod
+    def dir_to_files(
+        directory: str | Path,
+        file_types: list[str] | None = None,
+        max_workers: int | None = None,
+        ignore_errors: bool = False,
+        verbose: bool = False,
+        recursive: bool = False,
+    ) -> list[Path]:
+        from .process import dir_to_files
+
+        return dir_to_files(
+            directory,
+            file_types=file_types,
+            max_workers=max_workers,
+            ignore_errors=ignore_errors,
+            verbose=verbose,
+            recursive=recursive,
+        )
+
+    @staticmethod
+    def file_to_chunks(
+        file_path: str | Path,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        chunk_size: int = 1500,
+        overlap: float = 0.1,
+        threshold: int = 200,
+        encoding: str = "utf-8",
+        custom_metadata: dict[str, Any] | None = None,
+        output_dir: str | Path | None = None,
+        verbose: bool = False,
+        timestamp: bool = True,
+        random_hash_digits: int = 4,
+        as_node: bool = False,
+    ) -> list[dict[str, Any]]:
+        from .file_ops import file_to_chunks
+
+        return file_to_chunks(
+            file_path,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            encoding=encoding,
+            custom_metadata=custom_metadata,
+            output_dir=output_dir,
+            verbose=verbose,
+            timestamp=timestamp,
+            random_hash_digits=random_hash_digits,
+            as_node=as_node,
+        )
+
+    @staticmethod
+    def chunk(
+        *,
+        text: str | None = None,
+        url_or_path: str | Path = None,
+        file_types: list[str] | None = None,  # only local files
+        recursive: bool = False,  # only local files
+        tokenizer: Callable[[str], list[str]] = None,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        chunk_size: int = 1500,
+        overlap: float = 0.1,
+        threshold: int = 200,
+        output_file: str | Path | None = None,
+        metadata: dict[str, Any] | None = None,
+        reader_tool: Callable = None,
+        as_node: bool = False,
+    ) -> list:
+        from .process import chunk
+
+        return chunk(
+            text=text,
+            url_or_path=url_or_path,
+            file_types=file_types,
+            recursive=recursive,
+            tokenizer=tokenizer,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            output_file=output_file,
+            metadata=metadata,
+            reader_tool=reader_tool,
+            as_node=as_node,
+        )
+
+    @staticmethod
+    def save_to_file(
+        text: str,
+        directory: str | Path,
+        filename: str,
+        extension: str = "txt",
+        timestamp: bool = True,
+        dir_exist_ok: bool = True,
+        file_exist_ok: bool = True,
+        time_prefix: bool = False,
+        timestamp_format: str = "%Y%m%d_%H%M%S",
+        random_hash_digits: int = 4,
+        verbose: bool = False,
+    ) -> Path:
+        from .save import save_to_file
+
+        return save_to_file(
+            text,
+            directory=directory,
+            filename=filename,
+            extension=extension,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            file_exist_ok=file_exist_ok,
+            time_prefix=time_prefix,
+            timestamp_format=timestamp_format,
+            random_hash_digits=random_hash_digits,
+            verbose=verbose,
+        )
+
+    @staticmethod
+    def create_path(
+        directory: Path | str,
+        filename: str,
+        extension: str = None,
+        timestamp: bool = False,
+        dir_exist_ok: bool = True,
+        file_exist_ok: bool = False,
+        time_prefix: bool = False,
+        timestamp_format: str | None = None,
+        random_hash_digits: int = 0,
+    ) -> Path:
+        from .create_path import create_path
+
+        return create_path(
+            directory=directory,
+            filename=filename,
+            extension=extension,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            file_exist_ok=file_exist_ok,
+            time_prefix=time_prefix,
+            timestamp_format=timestamp_format,
+            random_hash_digits=random_hash_digits,
+        )

lionagi/libs/file/save.py

@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 from typing import Any
 
-from lionagi.utils import create_path
+from .create_path import create_path
 
 
 def save_to_file(

lionagi/libs/package/imports.py

@@ -2,15 +2,184 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from lionagi.utils import (
-    check_import,
-    import_module,
-    install_import,
-    is_import_installed,
-    run_package_manager_command,
-)
+import logging
+import shutil
+import subprocess
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from lionagi.utils import is_import_installed
+
+
+def run_package_manager_command(
+    args: Sequence[str],
+) -> subprocess.CompletedProcess[bytes]:
+    """Run a package manager command, using uv if available, otherwise falling back to pip."""
+    # Check if uv is available in PATH
+    uv_path = shutil.which("uv")
+
+    if uv_path:
+        # Use uv if available
+        try:
+            return subprocess.run(
+                [uv_path] + list(args),
+                check=True,
+                capture_output=True,
+            )
+        except subprocess.CalledProcessError:
+            # If uv fails, fall back to pip
+            print("uv command failed, falling back to pip...")
+
+    # Fall back to pip
+    return subprocess.run(
+        [sys.executable, "-m", "pip"] + list(args),
+        check=True,
+        capture_output=True,
+    )
+
+
+def check_import(
+    package_name: str,
+    module_name: str | None = None,
+    import_name: str | None = None,
+    pip_name: str | None = None,
+    attempt_install: bool = True,
+    error_message: str = "",
+):
+    """
+    Check if a package is installed, attempt to install if not.
+
+    Args:
+        package_name: The name of the package to check.
+        module_name: The specific module to import (if any).
+        import_name: The specific name to import from the module (if any).
+        pip_name: The name to use for pip installation (if different).
+        attempt_install: Whether to attempt installation if not found.
+        error_message: Custom error message to use if package not found.
+
+    Raises:
+        ImportError: If the package is not found and not installed.
+        ValueError: If the import fails after installation attempt.
+    """
+    if not is_import_installed(package_name):
+        if attempt_install:
+            logging.info(
+                f"Package {package_name} not found. Attempting " "to install.",
+            )
+            try:
+                return install_import(
+                    package_name=package_name,
+                    module_name=module_name,
+                    import_name=import_name,
+                    pip_name=pip_name,
+                )
+            except ImportError as e:
+                raise ValueError(
+                    f"Failed to install {package_name}: {e}"
+                ) from e
+        else:
+            logging.info(
+                f"Package {package_name} not found. {error_message}",
+            )
+            raise ImportError(
+                f"Package {package_name} not found. {error_message}",
+            )
+
+    return import_module(
+        package_name=package_name,
+        module_name=module_name,
+        import_name=import_name,
+    )
+
+
+def import_module(
+    package_name: str,
+    module_name: str = None,
+    import_name: str | list = None,
+) -> Any:
+    """
+    Import a module by its path.
+
+    Args:
+        module_path: The path of the module to import.
+
+    Returns:
+        The imported module.
+
+    Raises:
+        ImportError: If the module cannot be imported.
+    """
+    try:
+        full_import_path = (
+            f"{package_name}.{module_name}" if module_name else package_name
+        )
+
+        if import_name:
+            import_name = (
+                [import_name]
+                if not isinstance(import_name, list)
+                else import_name
+            )
+            a = __import__(
+                full_import_path,
+                fromlist=import_name,
+            )
+            if len(import_name) == 1:
+                return getattr(a, import_name[0])
+            return [getattr(a, name) for name in import_name]
+        else:
+            return __import__(full_import_path)
+
+    except ImportError as e:
+        raise ImportError(
+            f"Failed to import module {full_import_path}: {e}"
+        ) from e
+
+
+def install_import(
+    package_name: str,
+    module_name: str | None = None,
+    import_name: str | None = None,
+    pip_name: str | None = None,
+):
+    """
+    Attempt to import a package, installing it if not found.
+
+    Args:
+        package_name: The name of the package to import.
+        module_name: The specific module to import (if any).
+        import_name: The specific name to import from the module (if any).
+        pip_name: The name to use for pip installation (if different).
+
+    Raises:
+        ImportError: If the package cannot be imported or installed.
+        subprocess.CalledProcessError: If pip installation fails.
+    """
+    pip_name = pip_name or package_name
+
+    try:
+        return import_module(
+            package_name=package_name,
+            module_name=module_name,
+            import_name=import_name,
+        )
+    except ImportError:
+        logging.info(f"Installing {pip_name}...")
+        try:
+            run_package_manager_command(["install", pip_name])
+            return import_module(
+                package_name=package_name,
+                module_name=module_name,
+                import_name=import_name,
+            )
+        except subprocess.CalledProcessError as e:
+            raise ImportError(f"Failed to install {pip_name}: {e}") from e
+        except ImportError as e:
+            raise ImportError(
+                f"Failed to import {pip_name} after installation: {e}"
+            ) from e
 
-# backward compatibility
 
 __all__ = (
     "run_package_manager_command",