lionagi 0.3.8__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/compress/utils.py

@@ -0,0 +1,81 @@
+"""
+Copyright 2024 HaiyangLi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+"""
+
+import asyncio
+
+from lionagi.integrations.litellm_.imodel import iModel
+
+from .models import PerplexityScores
+
+
+async def compute_perplexity(
+    imodel: iModel,
+    initial_context: str = None,
+    tokens: list[str] = None,
+    system_msg: str = None,
+    n_samples: int = 1,  # number of samples used for the computation
+    use_residue: bool = True,  # whether to use residue for the last sample
+    **kwargs,
+) -> list[PerplexityScores]:
+    tasks = []
+    context = initial_context or ""
+
+    n_samples = n_samples or len(tokens)
+    sample_token_len, residue = divmod(len(tokens), n_samples)
+    samples = []
+
+    if n_samples == 1:
+        samples = [tokens]
+    else:
+        samples = [
+            tokens[: (i + 1) * sample_token_len] for i in range(n_samples)
+        ]
+
+        if use_residue and residue != 0:
+            samples.append(tokens[-residue:])
+
+    sampless = [context + " ".join(sample) for sample in samples]
+
+    for sample in sampless:
+        messages = (
+            [{"role": "system", "content": system_msg}] if system_msg else []
+        )
+        messages.append(
+            {"role": "user", "content": sample},
+        )
+        task = asyncio.create_task(
+            imodel.invoke(
+                messages=messages,
+                logprobs=True,
+                max_tokens=sample_token_len,
+                **kwargs,
+            )
+        )
+        tasks.append(task)
+
+    results = await asyncio.gather(*tasks)
+
+    outs = []
+
+    for idx, item in enumerate(results):
+        p = PerplexityScores(
+            completion_response=item,
+            original_tokens=samples[idx],
+            n_samples=n_samples,
+        )
+        outs.append(p)
+
+    return outs

lionagi/libs/constants.py

@@ -0,0 +1,98 @@
+from typing import Any, Final, Literal, TypedDict, TypeVar, Union
+
+
+class UndefinedType:
+    def __init__(self) -> None:
+        self.undefined = True
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    def __deepcopy__(self, memo):
+        # Ensure UNDEFINED is universal
+        return self
+
+    def __repr__(self) -> Literal["UNDEFINED"]:
+        return "UNDEFINED"
+
+    __slots__ = ["undefined"]
+
+
+UNDEFINED = UndefinedType()
+
+# Type definitions
+NUM_TYPE_LITERAL = Literal["int", "float", "complex"]
+NUM_TYPES = Union[type[int], type[float], type[complex], NUM_TYPE_LITERAL]
+NumericType = TypeVar("NumericType", int, float, complex)
+
+# Type mapping
+TYPE_MAP = {"int": int, "float": float, "complex": complex}
+
+# Regex patterns for different numeric formats
+PATTERNS = {
+    "scientific": r"[-+]?(?:\d*\.)?\d+[eE][-+]?\d+",
+    "complex_sci": r"[-+]?(?:\d*\.)?\d+(?:[eE][-+]?\d+)?[-+](?:\d*\.)?\d+(?:[eE][-+]?\d+)?[jJ]",
+    "complex": r"[-+]?(?:\d*\.)?\d+[-+](?:\d*\.)?\d+[jJ]",
+    "pure_imaginary": r"[-+]?(?:\d*\.)?\d*[jJ]",
+    "percentage": r"[-+]?(?:\d*\.)?\d+%",
+    "fraction": r"[-+]?\d+/\d+",
+    "decimal": r"[-+]?(?:\d*\.)?\d+",
+    "special": r"[-+]?(?:inf|infinity|nan)",
+}
+
+
+md_json_char_map = {"'": '\\"', "\n": "\\n", "\r": "\\r", "\t": "\\t"}
+
+py_json_msp = {
+    "str": "string",
+    "int": "number",
+    "float": "number",
+    "list": "array",
+    "tuple": "array",
+    "bool": "boolean",
+    "dict": "object",
+}
+
+
+# Define constants for valid boolean string representations
+TRUE_VALUES: Final[frozenset[str]] = frozenset(
+    [
+        "true",
+        "1",
+        "yes",
+        "y",
+        "on",
+        "correct",
+        "t",
+        "enabled",
+        "enable",
+        "active",
+        "activated",
+    ]
+)
+
+FALSE_VALUES: Final[frozenset[str]] = frozenset(
+    [
+        "false",
+        "0",
+        "no",
+        "n",
+        "off",
+        "incorrect",
+        "f",
+        "disabled",
+        "disable",
+        "inactive",
+        "deactivated",
+        "none",
+        "null",
+        "n/a",
+        "na",
+    ]
+)
+
+
+class KeysDict(TypedDict, total=False):
+    """TypedDict for keys dictionary."""
+
+    key: Any  # Represents any key-type pair

lionagi/libs/file/chunk.py

@@ -0,0 +1,265 @@
+import math
+from collections.abc import Callable
+from typing import Any, Literal
+
+
+def chunk_by_chars(
+    text: str, chunk_size: int = 2048, overlap: float = 0, threshold: int = 256
+) -> list[str]:
+    """
+    Split a text into chunks of approximately equal size, with optional overlap.
+
+    This function divides the input text into chunks based on the specified
+    chunk size. It handles different scenarios based on the number of chunks
+    required and provides options for overlap between chunks.
+
+    Args:
+        text (str): The input text to be chunked.
+        chunk_size (int, optional): The target size for each chunk. Defaults to 2048.
+        overlap (float, optional): The fraction of overlap between chunks. Defaults to 0.
+        threshold (int, optional): The minimum size for the last chunk. Defaults to 256.
+
+    Returns:
+        List[str]: A list of text chunks.
+
+    Raises:
+        ValueError: If an error occurs during the chunking process.
+
+    Examples:
+        >>> text = "This is a sample text for chunking."
+        >>> chunks = chunk_by_chars(text, chunk_size=10, overlap=0.2)
+        >>> print(chunks)
+        ['This is a ', 'a sample ', 'le text fo', 'for chunki', 'king.']
+    """
+    try:
+        n_chunks = math.ceil(len(text) / chunk_size)
+        overlap_size = int(chunk_size * overlap / 2)
+
+        if n_chunks == 1:
+            return [text]
+        elif n_chunks == 2:
+            return _chunk_two_parts(text, chunk_size, overlap_size, threshold)
+        else:
+            return _chunk_multiple_parts(
+                text, chunk_size, overlap_size, n_chunks, threshold
+            )
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the text: {e}")
+
+
+def _chunk_two_parts(
+    text: str, chunk_size: int, overlap_size: int, threshold: int
+) -> list[str]:
+    """Handle chunking for two parts."""
+    first_chunk = text[: chunk_size + overlap_size]
+    if len(text) - chunk_size > threshold:
+        return [first_chunk, text[chunk_size - overlap_size :]]
+    return [text]
+
+
+def _chunk_multiple_parts(
+    text: str,
+    chunk_size: int,
+    overlap_size: int,
+    n_chunks: int,
+    threshold: int,
+) -> list[str]:
+    """Handle chunking for more than two parts."""
+    chunks = [text[: chunk_size + overlap_size]]
+
+    for i in range(1, n_chunks - 1):
+        start_idx = chunk_size * i - overlap_size
+        end_idx = chunk_size * (i + 1) + overlap_size
+        chunks.append(text[start_idx:end_idx])
+
+    last_chunk_start = chunk_size * (n_chunks - 1) - overlap_size
+    if len(text) - last_chunk_start > threshold:
+        chunks.append(text[last_chunk_start:])
+    else:
+        chunks[-1] += text[chunk_size * (n_chunks - 1) + overlap_size :]
+
+    return chunks
+
+
+def chunk_by_tokens(
+    tokens: list[str],
+    chunk_size: int = 1024,
+    overlap: float = 0,
+    threshold: int = 128,
+    return_tokens: bool = False,
+) -> list[str | list[str]]:
+    """
+    Split a list of tokens into chunks of approximately equal size, with optional overlap.
+
+    This function divides the input tokens into chunks based on the specified
+    chunk size. It handles different scenarios based on the number of chunks
+    required and provides options for overlap between chunks.
+
+    Args:
+        tokens (list[str]): The input list of tokens to be chunked.
+        chunk_size (int, optional): The target size for each chunk. Defaults to 1024.
+        overlap (float, optional): The fraction of overlap between chunks. Defaults to 0.
+        threshold (int, optional): The minimum size for the last chunk. Defaults to 128.
+        return_tokens (bool, optional): If True, return chunks as lists of tokens;
+                                        if False, return as joined strings. Defaults to False.
+
+    Returns:
+        list[Union[str, list[str]]]: A list of chunked tokens, either as strings or token lists.
+
+    Raises:
+        ValueError: If an error occurs during the chunking process.
+
+    Examples:
+        >>> tokens = ["This", "is", "a", "sample", "text", "for", "chunking."]
+        >>> chunks = chunk_by_tokens(tokens, chunk_size=3, overlap=0.2)
+        >>> print(chunks)
+        ['This is a', 'a sample text', 'text for chunking.']
+    """
+    try:
+        n_chunks = math.ceil(len(tokens) / chunk_size)
+        overlap_size = int(overlap * chunk_size / 2)
+        residue = len(tokens) % chunk_size
+
+        if n_chunks == 1:
+            return _process_single_chunk(tokens, return_tokens)
+        elif n_chunks == 2:
+            return _chunk_token_two_parts(
+                tokens,
+                chunk_size,
+                overlap_size,
+                threshold,
+                residue,
+                return_tokens,
+            )
+        else:
+            return _chunk_token_multiple_parts(
+                tokens,
+                chunk_size,
+                overlap_size,
+                n_chunks,
+                threshold,
+                residue,
+                return_tokens,
+            )
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the tokens: {e}")
+
+
+def _process_single_chunk(
+    tokens: list[str], return_tokens: bool
+) -> list[str | list[str]]:
+    """Handle processing for a single chunk."""
+    return [tokens] if return_tokens else [" ".join(tokens).strip()]
+
+
+def _chunk_token_two_parts(
+    tokens: list[str],
+    chunk_size: int,
+    overlap_size: int,
+    threshold: int,
+    residue: int,
+    return_tokens: bool,
+) -> list[str | list[str]]:
+    """Handle chunking for two parts."""
+    chunks = [tokens[: chunk_size + overlap_size]]
+    if residue > threshold:
+        chunks.append(tokens[chunk_size - overlap_size :])
+    else:
+        return _process_single_chunk(tokens, return_tokens)
+    return _format_chunks(chunks, return_tokens)
+
+
+def _chunk_token_multiple_parts(
+    tokens: list[str],
+    chunk_size: int,
+    overlap_size: int,
+    n_chunks: int,
+    threshold: int,
+    residue: int,
+    return_tokens: bool,
+) -> list[str | list[str]]:
+    """Handle chunking for more than two parts."""
+    chunks = [tokens[: chunk_size + overlap_size]]
+    for i in range(1, n_chunks - 1):
+        start_idx = chunk_size * i - overlap_size
+        end_idx = chunk_size * (i + 1) + overlap_size
+        chunks.append(tokens[start_idx:end_idx])
+
+    last_chunk_start = chunk_size * (n_chunks - 1) - overlap_size
+    if len(tokens) - last_chunk_start > threshold:
+        chunks.append(tokens[last_chunk_start:])
+    else:
+        chunks[-1] += tokens[-residue:]
+
+    return _format_chunks(chunks, return_tokens)
+
+
+def _format_chunks(
+    chunks: list[list[str]], return_tokens: bool
+) -> list[str | list[str]]:
+    """Format chunks based on the return_tokens flag."""
+    return (
+        chunks
+        if return_tokens
+        else [" ".join(chunk).strip() for chunk in chunks]
+    )
+
+
+def chunk_content(
+    content: str,
+    chunk_by: Literal["chars", "tokens"] = "chars",
+    tokenizer: Callable[[str], list[str]] = str.split,
+    chunk_size: int = 1024,
+    overlap: float = 0,
+    threshold: int = 256,
+    metadata: dict[str, Any] = {},
+    return_tokens: bool = False,
+    **kwargs: Any,
+) -> list[dict[str, Any]]:
+    """
+    Split content into chunks and add metadata.
+
+    This function takes a string content, splits it into chunks using the provided
+    chunking function, and adds metadata to each chunk.
+
+    Args:
+        content (str): The content to be chunked.
+        chunk_by(str): The method to use for chunking: "chars" or "tokens".
+        tokenizer (Callable): The function to use for tokenization. defaults to str.split.
+        chunk_size (int): The target size for each chunk.
+        overlap (float): The fraction of overlap between chunks.
+        threshold (int): The minimum size for the last chunk.
+        metadata (Dict[str, Any]): Metadata to be included with each chunk.
+        kwargs for tokenizer, if needed.
+
+
+    Returns:
+        List[Dict[str, Any]]: A list of dictionaries, each representing a chunk with metadata.
+    """
+
+    if chunk_by == "tokens":
+        chunks = chunk_by_tokens(
+            tokens=tokenizer(content, **kwargs),
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            return_tokens=return_tokens,
+        )
+    else:
+        chunks = chunk_by_chars(
+            text=content,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    return [
+        {
+            "chunk_content": chunk,
+            "chunk_id": i + 1,
+            "total_chunks": len(chunks),
+            "chunk_size": len(chunk),
+            **metadata,
+        }
+        for i, chunk in enumerate(chunks)
+    ]

lionagi/libs/file/file_ops.py

@@ -0,0 +1,114 @@
+import logging
+from pathlib import Path
+from shutil import copy2
+
+
+def copy_file(src: Path | str, dest: Path | str) -> None:
+    """
+    Copy a file from a source path to a destination path.
+
+    Args:
+        src: The source file path.
+        dest: The destination file path.
+
+    Raises:
+        FileNotFoundError: If the source file does not exist or is not
+            a file.
+        PermissionError: If there are insufficient permissions to copy
+            the file.
+        OSError: If there's an OS-level error during the copy operation.
+    """
+    src_path, dest_path = Path(src), Path(dest)
+    if not src_path.is_file():
+        raise FileNotFoundError(f"{src_path} does not exist or is not a file.")
+
+    try:
+        dest_path.parent.mkdir(parents=True, exist_ok=True)
+        copy2(src_path, dest_path)
+    except PermissionError as e:
+        raise PermissionError(
+            f"Permission denied when copying {src_path} to {dest_path}"
+        ) from e
+    except OSError as e:
+        raise OSError(f"Failed to copy {src_path} to {dest_path}: {e}") from e
+
+
+def get_file_size(path: Path | str) -> int:
+    """
+    Get the size of a file or total size of files in a directory.
+
+    Args:
+        path: The file or directory path.
+
+    Returns:
+        The size in bytes.
+
+    Raises:
+        FileNotFoundError: If the path does not exist.
+        PermissionError: If there are insufficient permissions
+            to access the path.
+    """
+    path = Path(path)
+    try:
+        if path.is_file():
+            return path.stat().st_size
+        elif path.is_dir():
+            return sum(
+                f.stat().st_size for f in path.rglob("*") if f.is_file()
+            )
+        else:
+            raise FileNotFoundError(f"{path} does not exist.")
+    except PermissionError as e:
+        raise PermissionError(
+            f"Permission denied when accessing {path}"
+        ) from e
+
+
+def list_files(
+    dir_path: Path | str, extension: str | None = None
+) -> list[Path]:
+    """
+    List all files in a specified directory with an optional extension
+    filter, including files in subdirectories.
+
+    Args:
+        dir_path: The directory path where files are listed.
+        extension: Filter files by extension.
+
+    Returns:
+        A list of Path objects representing files in the directory.
+
+    Raises:
+        NotADirectoryError: If the provided dir_path is not a directory.
+    """
+    dir_path = Path(dir_path)
+    if not dir_path.is_dir():
+        raise NotADirectoryError(f"{dir_path} is not a directory.")
+
+    pattern = f"*.{extension}" if extension else "*"
+    return [f for f in dir_path.rglob(pattern) if f.is_file()]
+
+
+def read_file(path: Path | str, /) -> str:
+    """
+    Read the contents of a file.
+
+    Args:
+        path: The path to the file to read.
+
+    Returns:
+        str: The contents of the file.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        PermissionError: If there are insufficient permissions to read
+            the file.
+    """
+    try:
+        return Path(path).read_text(encoding="utf-8")
+    except FileNotFoundError as e:
+        logging.error(f"File not found: {path}: {e}")
+        raise
+    except PermissionError as e:
+        logging.error(f"Permission denied when reading file: {path}: {e}")
+        raise