lionagi 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/file/params.py

@@ -0,0 +1,212 @@
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class ChunkContentParams(BaseModel):
+    """Parameters for chunking text content into smaller pieces.
+
+    This model defines parameters used by the chunk_content function to split text
+    content into chunks, with support for both character-based and token-based chunking.
+    """
+
+    content: str = Field(description="The text content to be chunked")
+    chunk_by: Literal["chars", "tokens"] = Field(
+        default="chars",
+        description="Method to use for chunking: 'chars' for character-based or 'tokens' for token-based",
+    )
+    tokenizer: Callable[[str], list[str]] | None = Field(
+        default=None,
+        description="Function to use for tokenization. Defaults to str.split if None",
+    )
+    chunk_size: int = Field(
+        default=1024, ge=1, description="Target size for each chunk"
+    )
+    overlap: float = Field(
+        default=0.0,
+        ge=0.0,
+        le=1.0,
+        description="Fraction of overlap between chunks (0.0 to 1.0)",
+    )
+    threshold: int = Field(
+        default=256,
+        ge=0,
+        description="Minimum size threshold for the last chunk",
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata to include with each chunk",
+    )
+    return_tokens: bool = Field(
+        default=False,
+        description="If True, return chunks as token lists; if False, return as joined strings",
+    )
+
+
+class CreatePathParams(BaseModel):
+    """Parameters for creating file paths with various options.
+
+    This model defines parameters used by the create_path function to generate
+    file paths with support for timestamps, random hashes, and existence checks.
+    """
+
+    directory: Path | str = Field(
+        description="Directory where the file will be created"
+    )
+    filename: str = Field(description="Base name of the file to create")
+    extension: str | None = Field(
+        default=None,
+        description="File extension (with or without leading dot)",
+    )
+    timestamp: bool = Field(
+        default=False, description="If True, adds a timestamp to the filename"
+    )
+    dir_exist_ok: bool = Field(
+        default=True,
+        description="If True, doesn't raise an error if the directory exists",
+    )
+    file_exist_ok: bool = Field(
+        default=False,
+        description="If True, allows overwriting of existing files",
+    )
+    time_prefix: bool = Field(
+        default=False,
+        description="If True, adds timestamp as prefix instead of suffix",
+    )
+    timestamp_format: str | None = Field(
+        default=None,
+        description="Custom format for the timestamp (strftime format)",
+    )
+    random_hash_digits: int = Field(
+        default=0,
+        ge=0,
+        description="Number of random hash digits to append to filename",
+    )
+
+
+class DirToFilesParams(BaseModel):
+    """Parameters for processing directories and collecting file paths.
+
+    This model defines parameters used by the dir_to_files function to recursively
+    process directories and collect matching file paths.
+    """
+
+    directory: Path | str = Field(
+        description="Directory to process recursively"
+    )
+    file_types: list[str] | None = Field(
+        default=None,
+        description="List of file extensions to include (e.g., ['.txt', '.pdf']). If None, includes all types",
+    )
+    max_workers: int | None = Field(
+        default=None,
+        description="Maximum number of worker threads for concurrent processing",
+    )
+    ignore_errors: bool = Field(
+        default=False,
+        description="If True, log warnings for errors instead of raising exceptions",
+    )
+    verbose: bool = Field(
+        default=False,
+        description="If True, print verbose output during processing",
+    )
+
+
+class FileToChunksParams(BaseModel):
+    """Parameters for splitting file content into chunks.
+
+    This model defines parameters used by the file_to_chunks function to read
+    and split file content into chunks with metadata.
+    """
+
+    file_path: Path | str = Field(
+        description="Path to the file to be processed"
+    )
+    chunk_func: Callable[[str, int, float, int], list[str]] = Field(
+        description="Function to use for chunking the content"
+    )
+    chunk_size: int = Field(
+        default=1500, ge=1, description="Target size for each chunk"
+    )
+    overlap: float = Field(
+        default=0.1,
+        ge=0.0,
+        le=1.0,
+        description="Fraction of overlap between chunks (0.0 to 1.0)",
+    )
+    threshold: int = Field(
+        default=200,
+        ge=0,
+        description="Minimum size threshold for the last chunk",
+    )
+    encoding: str = Field(
+        default="utf-8",
+        description="File encoding to use when reading the file",
+    )
+    custom_metadata: dict[str, Any] | None = Field(
+        default=None,
+        description="Additional metadata to include with each chunk",
+    )
+    output_dir: Path | str | None = Field(
+        default=None,
+        description="Directory to save output chunks (if provided)",
+    )
+    verbose: bool = Field(
+        default=False,
+        description="If True, print verbose output during processing",
+    )
+    timestamp: bool = Field(
+        default=True,
+        description="If True, include timestamp in output filenames",
+    )
+    random_hash_digits: int = Field(
+        default=4,
+        ge=0,
+        description="Number of random hash digits to include in output filenames",
+    )
+
+
+class SaveToFileParams(BaseModel):
+    """Parameters for saving text content to a file.
+
+    This model defines parameters used by the save_to_file function to save
+    text content to a file with various options for naming and handling.
+    """
+
+    text: str = Field(description="The text content to save to file")
+    directory: Path | str = Field(
+        description="Directory where the file will be saved"
+    )
+    filename: str = Field(description="Name of the file to create")
+    extension: str | None = Field(
+        default=None,
+        description="File extension (with or without leading dot)",
+    )
+    timestamp: bool = Field(
+        default=False, description="If True, append timestamp to filename"
+    )
+    dir_exist_ok: bool = Field(
+        default=True,
+        description="If True, creates directory if it doesn't exist",
+    )
+    file_exist_ok: bool = Field(
+        default=False, description="If True, allows overwriting existing files"
+    )
+    time_prefix: bool = Field(
+        default=False,
+        description="If True, prepend timestamp instead of append",
+    )
+    timestamp_format: str | None = Field(
+        default=None,
+        description="Custom format for timestamp (strftime format)",
+    )
+    random_hash_digits: int = Field(
+        default=0,
+        ge=0,
+        description="Number of random hash digits to append to filename",
+    )
+    verbose: bool = Field(
+        default=True, description="If True, logs the file path after saving"
+    )

lionagi/libs/file/path.py

@@ -0,0 +1,301 @@
+import logging
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from ..utils import unique_hash
+
+
+def create_path(
+    directory: Path | str,
+    filename: str,
+    extension: str = None,
+    timestamp: bool = False,
+    dir_exist_ok: bool = True,
+    file_exist_ok: bool = False,
+    time_prefix: bool = False,
+    timestamp_format: str | None = None,
+    random_hash_digits: int = 0,
+) -> Path:
+    """
+    Generate a new file path with optional timestamp and random hash.
+
+    Args:
+        directory: The directory where the file will be created.
+        filename: The base name of the file to create.
+        timestamp: If True, adds a timestamp to the filename.
+        dir_exist_ok: If True, doesn't raise an error if the directory
+            exists.
+        file_exist_ok: If True, allows overwriting of existing files.
+        time_prefix: If True, adds the timestamp as a prefix instead of
+            a suffix.
+        timestamp_format: Custom format for the timestamp.
+        random_hash_digits: Number of digits for the random hash.
+
+    Returns:
+        The full path to the new or existing file.
+
+    Raises:
+        ValueError: If the filename contains illegal characters.
+        FileExistsError: If the file exists and file_exist_ok is False.
+    """
+    if "/" in filename or "\\" in filename:
+        raise ValueError("Filename cannot contain directory separators.")
+    directory = Path(directory)
+
+    name, ext = None, None
+    if "." in filename:
+        name, ext = filename.rsplit(".", 1)
+    else:
+        name = filename
+        ext = extension.strip(".").strip() if extension else None
+
+    if not ext:
+        raise ValueError("No extension provided for filename.")
+
+    ext = f".{ext}" if ext else ""
+
+    if timestamp:
+        timestamp_str = datetime.now().strftime(
+            timestamp_format or "%Y%m%d%H%M%S"
+        )
+        name = (
+            f"{timestamp_str}_{name}"
+            if time_prefix
+            else f"{name}_{timestamp_str}"
+        )
+
+    if random_hash_digits > 0:
+        random_hash = "-" + unique_hash(random_hash_digits)
+        name = f"{name}{random_hash}"
+
+    full_filename = f"{name}{ext}"
+    full_path = directory / full_filename
+
+    if full_path.exists():
+        if file_exist_ok:
+            return full_path
+        raise FileExistsError(
+            f"File {full_path} already exists and file_exist_ok is False."
+        )
+    full_path.parent.mkdir(parents=True, exist_ok=dir_exist_ok)
+    return full_path
+
+
+def is_valid_path(
+    path: str | Path,
+    *,
+    max_length: int | None = None,
+    allow_relative: bool = True,
+    allow_symlinks: bool = True,
+    custom_reserved_names: list[str] | None = None,
+    strict_mode: bool = False,
+) -> bool:
+    """
+    Validates whether the given path is syntactically valid for the current operating system.
+
+    Args:
+        path (Union[str, Path]): The filesystem path to validate.
+        max_length (Optional[int]): Maximum allowed path length. If None, uses OS default.
+        allow_relative (bool): Whether to allow relative paths. Default is True.
+        allow_symlinks (bool): Whether to allow symlinks. Default is True.
+        custom_reserved_names (Optional[List[str]]): Additional reserved names to check.
+        strict_mode (bool): If True, applies stricter validation rules. Default is False.
+
+    Returns:
+        bool: True if the path is valid, False otherwise.
+
+    Raises:
+        ValueError: If the path is invalid, with a detailed explanation.
+    """
+    if isinstance(path, Path):
+        path_str = str(path)
+    elif isinstance(path, str):
+        path_str = path
+    else:
+        raise TypeError("Path must be a string or Path object.")
+
+    if not path_str:
+        raise ValueError("Path cannot be an empty string.")
+
+    issues = []
+    is_windows = sys.platform.startswith("win")
+
+    # Common checks for both Windows and Unix-like systems
+    if "\0" in path_str:
+        issues.append("Path contains null character.")
+
+    if not max_length:
+        max_length = 260 if is_windows else 4096
+    if len(path_str) > max_length:
+        issues.append(
+            f"Path exceeds the maximum length of {max_length} characters."
+        )
+
+    if is_windows:
+        # Windows-specific validation
+        invalid_chars = r'<>:"/\\|?*'
+        if re.search(f"[{re.escape(invalid_chars)}]", path_str):
+            issues.append(f"Path contains invalid characters: {invalid_chars}")
+
+        reserved_names = {
+            "CON",
+            "PRN",
+            "AUX",
+            "NUL",
+            "COM1",
+            "COM2",
+            "COM3",
+            "COM4",
+            "COM5",
+            "COM6",
+            "COM7",
+            "COM8",
+            "COM9",
+            "LPT1",
+            "LPT2",
+            "LPT3",
+            "LPT4",
+            "LPT5",
+            "LPT6",
+            "LPT7",
+            "LPT8",
+            "LPT9",
+        }
+        if custom_reserved_names:
+            reserved_names.update(custom_reserved_names)
+
+        path = Path(path_str)
+        for part in path.parts:
+            name = part.upper().rstrip(". ")
+            if name in reserved_names:
+                issues.append(f"Path contains a reserved name: '{part}'")
+
+        if path_str.endswith(" ") or path_str.endswith("."):
+            issues.append(
+                "Path cannot end with a space or a period on Windows."
+            )
+
+        if strict_mode:
+            if not path_str.startswith("\\\\?\\") and len(path_str) > 260:
+                issues.append(
+                    "Path exceeds 260 characters without long path prefix."
+                )
+
+    else:
+        # Unix-like systems validation
+        if strict_mode:
+            if re.search(r"//+", path_str):
+                issues.append("Path contains consecutive slashes.")
+
+        if not allow_relative and not path_str.startswith("/"):
+            issues.append("Relative paths are not allowed.")
+
+    # Common additional checks
+    if not allow_symlinks and Path(path_str).is_symlink():
+        issues.append("Symlinks are not allowed.")
+
+    if strict_mode:
+        if re.search(r"\s", path_str):
+            issues.append("Path contains whitespace characters.")
+
+    if issues:
+        raise ValueError("Invalid path: " + "; ".join(issues))
+
+    return True
+
+
+def split_path(path: Path | str) -> tuple[Path, str]:
+    """
+    Split a path into its directory and filename components.
+
+    Args:
+        path: The path to split.
+
+    Returns:
+        A tuple containing the directory and filename.
+    """
+    path = Path(path)
+    return path.parent, path.name
+
+
+def _get_path_kwargs(
+    persist_path: str | Path, postfix: str, **path_kwargs: Any
+) -> dict[str, Any]:
+    """
+    Generate keyword arguments for path creation.
+
+    Args:
+        persist_path: The base path to use.
+        postfix: The file extension to use.
+        **path_kwargs: Additional keyword arguments to override defaults.
+
+    Returns:
+        A dictionary of keyword arguments for path creation.
+    """
+    persist_path = Path(persist_path)
+    postfix = f".{postfix.strip('.')}"
+
+    if persist_path.suffix != postfix:
+        dirname = persist_path
+        filename = f"new_file{postfix}"
+    else:
+        dirname, filename = persist_path.parent, persist_path.name
+
+    return {
+        "timestamp": path_kwargs.get("timestamp", False),
+        "file_exist_ok": path_kwargs.get("file_exist_ok", True),
+        "directory": path_kwargs.get("directory", dirname),
+        "filename": path_kwargs.get("filename", filename),
+    }
+
+
+def clear_path(
+    path: Path | str,
+    /,
+    recursive: bool = False,
+    exclude: list[str] | None = None,
+) -> None:
+    """
+    Clear all files and directories in the specified path.
+
+    Args:
+        path: The path to the directory to clear.
+        recursive: If True, clears directories recursively.
+        exclude: A list of string patterns to exclude from deletion.
+
+    Raises:
+        FileNotFoundError: If the specified directory does not exist.
+        PermissionError: If there are insufficient permissions to delete
+            files.
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"The specified directory {path} does not exist."
+        )
+
+    exclude = exclude or []
+    exclude_pattern = re.compile("|".join(exclude)) if exclude else None
+
+    for file_path in path.iterdir():
+        if exclude_pattern and exclude_pattern.search(file_path.name):
+            logging.info(f"Excluded from deletion: {file_path}")
+            continue
+
+        try:
+            if file_path.is_dir():
+                if recursive:
+                    clear_path(file_path, recursive=True, exclude=exclude)
+                    file_path.rmdir()
+                else:
+                    continue
+            else:
+                file_path.unlink()
+            logging.info(f"Successfully deleted {file_path}")
+        except PermissionError as e:
+            logging.error(f"Permission denied when deleting {file_path}: {e}")
+        except Exception as e:
+            logging.error(f"Failed to delete {file_path}: {e}")

lionagi/libs/file/process.py

@@ -0,0 +1,139 @@
+import logging
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+from .chunk import chunk_content
+from .save import save_chunks
+
+
+def dir_to_files(
+    directory: str | Path,
+    file_types: list[str] | None = None,
+    max_workers: int | None = None,
+    ignore_errors: bool = False,
+    verbose: bool = False,
+) -> list[Path]:
+    """
+    Recursively process a directory and return a list of file paths.
+
+    This function walks through the given directory and its subdirectories,
+    collecting file paths that match the specified file types (if any).
+
+    Args:
+        directory (Union[str, Path]): The directory to process.
+        file_types (Optional[List[str]]): List of file extensions to include (e.g., ['.txt', '.pdf']).
+                                          If None, include all file types.
+        max_workers (Optional[int]): Maximum number of worker threads for concurrent processing.
+                                     If None, uses the default ThreadPoolExecutor behavior.
+        ignore_errors (bool): If True, log warnings for errors instead of raising exceptions.
+        verbose (bool): If True, print verbose output.
+
+    Returns:
+        List[Path]: A list of Path objects representing the files found.
+
+    Raises:
+        ValueError: If the provided directory doesn't exist or isn't a directory.
+    """
+    directory_path = Path(directory)
+    if not directory_path.is_dir():
+        raise ValueError(
+            f"The provided path is not a valid directory: {directory}"
+        )
+
+    def process_file(file_path: Path) -> Path | None:
+        try:
+            if file_types is None or file_path.suffix in file_types:
+                return file_path
+        except Exception as e:
+            if ignore_errors:
+                if verbose:
+                    logging.warning(f"Error processing {file_path}: {e}")
+            else:
+                raise ValueError(f"Error processing {file_path}: {e}") from e
+        return None
+
+    try:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [
+                executor.submit(process_file, f)
+                for f in directory_path.rglob("*")
+                if f.is_file()
+            ]
+            files = [
+                future.result()
+                for future in as_completed(futures)
+                if future.result() is not None
+            ]
+
+        if verbose:
+            logging.info(f"Processed {len(files)} files from {directory}")
+
+        return files
+    except Exception as e:
+        raise ValueError(f"Error processing directory {directory}: {e}") from e
+
+
+def file_to_chunks(
+    file_path: str | Path,
+    chunk_func: Callable[[str, int, float, int], list[str]],
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    encoding: str = "utf-8",
+    custom_metadata: dict[str, Any] | None = None,
+    output_dir: str | Path | None = None,
+    verbose: bool = False,
+    timestamp: bool = True,
+    random_hash_digits: int = 4,
+) -> list[dict[str, Any]]:
+    """
+    Process a file and split its content into chunks.
+
+    This function reads a file, splits its content into chunks using the provided
+    chunking function, and optionally saves the chunks to separate files.
+
+    Args:
+        file_path (Union[str, Path]): Path to the file to be processed.
+        chunk_func (Callable): Function to use for chunking the content.
+        chunk_size (int): The target size for each chunk.
+        overlap (float): The fraction of overlap between chunks.
+        threshold (int): The minimum size for the last chunk.
+        encoding (str): File encoding to use when reading the file.
+        custom_metadata (Optional[Dict[str, Any]]): Additional metadata to include with each chunk.
+        output_dir (Optional[Union[str, Path]]): Directory to save output chunks (if provided).
+        verbose (bool): If True, print verbose output.
+        timestamp (bool): If True, include timestamp in output filenames.
+        random_hash_digits (int): Number of random hash digits to include in output filenames.
+
+    Returns:
+        List[Dict[str, Any]]: A list of dictionaries, each representing a chunk with metadata.
+
+    Raises:
+        ValueError: If there's an error processing the file.
+    """
+    try:
+        file_path = Path(file_path)
+        with open(file_path, encoding=encoding) as f:
+            content = f.read()
+
+        metadata = {
+            "file_path": str(file_path),
+            "file_name": file_path.name,
+            "file_size": file_path.stat().st_size,
+            **(custom_metadata or {}),
+        }
+
+        chunks = chunk_content(
+            content, chunk_func, chunk_size, overlap, threshold, metadata
+        )
+
+        if output_dir:
+            save_chunks(
+                chunks, output_dir, verbose, timestamp, random_hash_digits
+            )
+
+        return chunks
+    except Exception as e:
+        raise ValueError(f"Error processing file {file_path}: {e}") from e