lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/libs/ln_queue.py

@@ -1,107 +0,0 @@
-"""
-A class that manages asynchronous task processing with controlled concurrency.
-"""
-
-import asyncio
-from collections.abc import Callable
-from typing import Any
-
-from lionagi.libs import func_call
-
-
-class AsyncQueue:
-    """
-    This class handles the enqueueing and processing of tasks with a limit on
-    how many can run simultaneously, using an asyncio.Queue for task storage and
-    an asyncio.Semaphore to limit concurrency.
-
-    Attributes:
-        queue (asyncio.Queue): The queue to store tasks.
-        _stop_event (asyncio.Event): Event to signal processing should stop.
-        max_concurrent_tasks (int): Maximum number of tasks processed concurrently.
-        semaphore (asyncio.Semaphore): Controls concurrent access to task execution.
-    """
-
-    def __init__(self, max_concurrent_tasks=5):
-        """
-        Initializes the AsyncQueue with a concurrency limit.
-
-        Args:
-            max_concurrent_tasks (int): The maximum number of concurrent tasks
-                                        allowed. Default is 5.
-        """
-        self.queue = asyncio.Queue()
-        self._stop_event = asyncio.Event()
-        self.max_concurrent_tasks = max_concurrent_tasks
-        self.semaphore = asyncio.Semaphore(max_concurrent_tasks)
-
-    async def enqueue(self, input_: Any) -> None:
-        """
-        Enqueues an item to be processed asynchronously.
-
-        Args:
-            input_ (Any): The item to be enqueued.
-        """
-        await self.queue.put(input_)
-
-    async def dequeue(self) -> Any:
-        """
-        Dequeues an item for processing.
-
-        Returns:
-            Any: The dequeued item.
-        """
-        return await self.queue.get()
-
-    async def join(self) -> None:
-        """Waits for all items in the queue to be processed."""
-        await self.queue.join()
-
-    async def stop(self) -> None:
-        """Signals the queue to stop processing new items."""
-        self._stop_event.set()
-
-    def stopped(self) -> bool:
-        """
-        Checks if the stop signal has been issued.
-
-        Returns:
-            bool: True if the queue has been stopped, otherwise False.
-        """
-        return self._stop_event.is_set()
-
-    async def process_requests(
-        self, func: Callable, retry_kwargs: dict = {}
-    ) -> None:
-        """
-        Processes tasks from the queue using the provided function with retries.
-
-        This method continuously processes tasks from the queue using the specified
-        function until a stop event is triggered. Handles concurrency using a
-        semaphore and manages task completion.
-
-        Args:
-            func (Callable): The function to process each task.
-            retry_kwargs (dict): Keyword arguments for retry behavior. Default is
-                                 an empty dictionary.
-        """
-        tasks = set()
-        while not self.stopped():
-            if len(tasks) >= self.max_concurrent_tasks:
-                _, done = await asyncio.wait(
-                    tasks, return_when=asyncio.FIRST_COMPLETED
-                )
-                tasks.difference_update(done)
-
-            async with self.semaphore:
-                input_ = await self.dequeue()
-                if input_ is None:
-                    await self.stop()
-                    break
-                task = asyncio.create_task(
-                    func_call.rcall(func, input_, **retry_kwargs)
-                )
-                tasks.add(task)
-
-        if tasks:
-            await asyncio.wait(tasks)

lionagi/libs/ln_tokenize.py

@@ -1,179 +0,0 @@
-import math
-
-import tiktoken
-
-from .ln_convert import to_str
-from .special_tokens import disallowed_tokens
-
-
-class TokenizeUtil:
-
-    @staticmethod
-    def tokenize(
-        text,
-        encoding_model=None,
-        encoding_name=None,
-        return_byte=False,
-        disallowed_tokens=disallowed_tokens,
-    ):
-        encoding = None
-
-        if encoding_model:
-            try:
-                encoding_name = tiktoken.encoding_name_for_model(
-                    encoding_model
-                )
-            except:
-                encoding_name = encoding_name or "cl100k_base"
-
-        if (
-            not encoding_name
-            or encoding_name in tiktoken.list_encoding_names()
-        ):
-            encoding_name = encoding_name or "cl100k_base"
-            encoding = tiktoken.get_encoding(encoding_name)
-
-        special_encodings = (
-            [encoding.encode(token) for token in disallowed_tokens]
-            if disallowed_tokens
-            else []
-        )
-        codes = encoding.encode(text)
-        if special_encodings and len(special_encodings) > 0:
-            codes = [code for code in codes if code not in special_encodings]
-
-        if return_byte:
-            return codes
-
-        return [encoding.decode([code]) for code in codes]
-
-    @staticmethod
-    def chunk_by_chars(
-        text: str, chunk_size: int, overlap: float, threshold: int
-    ) -> list[str | None]:
-        """
-        Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
-
-        Parameters:
-            text (str): The input text to chunk.
-
-            chunk_size (int): The size of each chunk.
-
-            overlap (float): The amount of overlap between chunks.
-
-            threshold (int): The minimum size of the final chunk.
-
-        Returns:
-            List[Union[str, None]]: A list of text chunks.
-
-        Raises:
-            ValueError: If an error occurs during chunking.
-        """
-
-        def _chunk_n1():
-            return [text]
-
-        def _chunk_n2():
-            chunks = []
-            chunks.append(text[: chunk_size + overlap_size])
-
-            if len(text) - chunk_size > threshold:
-                chunks.append(text[chunk_size - overlap_size :])
-            else:
-                return _chunk_n1()
-
-            return chunks
-
-        def _chunk_n3():
-            chunks = []
-            chunks.append(text[: chunk_size + overlap_size])
-            for i in range(1, n_chunks - 1):
-                start_idx = chunk_size * i - overlap_size
-                end_idx = chunk_size * (i + 1) + overlap_size
-                chunks.append(text[start_idx:end_idx])
-
-            if len(text) - chunk_size * (n_chunks - 1) > threshold:
-                chunks.append(
-                    text[chunk_size * (n_chunks - 1) - overlap_size :]
-                )
-            else:
-                chunks[-1] += text[
-                    chunk_size * (n_chunks - 1) + overlap_size :
-                ]
-
-            return chunks
-
-        try:
-            if not isinstance(text, str):
-                text = to_str(text)
-
-            n_chunks = math.ceil(len(text) / chunk_size)
-            overlap_size = int(overlap / 2)
-
-            if n_chunks == 1:
-                return _chunk_n1()
-
-            elif n_chunks == 2:
-                return _chunk_n2()
-
-            elif n_chunks > 2:
-                return _chunk_n3()
-
-        except Exception as e:
-            raise ValueError(f"An error occurred while chunking the text. {e}")
-
-    @staticmethod
-    def chunk_by_tokens(
-        text: str,
-        chunk_size: int,
-        overlap: float,
-        threshold: int,  # minimum size of the final chunk in number of tokens
-        encoding_model=None,
-        encoding_name=None,
-        return_tokens=False,
-        return_byte=False,
-    ) -> list[str | None]:
-
-        tokens = TokenizeUtil.tokenize(
-            text, encoding_model, encoding_name, return_byte=return_byte
-        )
-
-        n_chunks = math.ceil(len(tokens) / chunk_size)
-        overlap_size = int(overlap * chunk_size / 2)
-        residue = len(tokens) % chunk_size
-
-        if n_chunks == 1:
-            return text if not return_tokens else [tokens]
-
-        elif n_chunks == 2:
-            chunks = [tokens[: chunk_size + overlap_size]]
-            if residue > threshold:
-                chunks.append(tokens[chunk_size - overlap_size :])
-                return (
-                    [" ".join(chunk).strip() for chunk in chunks]
-                    if not return_tokens
-                    else chunks
-                )
-            else:
-                return text if not return_tokens else [tokens]
-
-        elif n_chunks > 2:
-            chunks = []
-            chunks.append(tokens[: chunk_size + overlap_size])
-            for i in range(1, n_chunks - 1):
-                start_idx = chunk_size * i - overlap_size
-                end_idx = chunk_size * (i + 1) + overlap_size
-                chunks.append(tokens[start_idx:end_idx])
-
-            if len(tokens) - chunk_size * (n_chunks - 1) > threshold:
-                chunks.append(
-                    tokens[chunk_size * (n_chunks - 1) - overlap_size :]
-                )
-            else:
-                chunks[-1] += tokens[-residue:]
-
-            return (
-                [" ".join(chunk) for chunk in chunks]
-                if not return_tokens
-                else chunks
-            )

lionagi/libs/ln_validate.py

@@ -1,299 +0,0 @@
-"""
-This module provides functions for validating and fixing field values based on their data types.
-
-The module defines several functions for checking and fixing field values of different data types,
-including numeric, boolean, string, and enum. It also provides a dictionary `validation_funcs` that
-maps data types to their corresponding validation functions.
-"""
-
-from .ln_convert import (
-    is_same_dtype,
-    strip_lower,
-    to_dict,
-    to_list,
-    to_num,
-    to_str,
-)
-from .ln_parse import ParseUtil, StringMatch
-
-
-def check_dict_field(x, keys: list[str] | dict, fix_=True, **kwargs):
-    if isinstance(x, dict):
-        return x
-    if fix_:
-        try:
-            x = to_str(x)
-            return StringMatch.force_validate_dict(x, keys=keys, **kwargs)
-        except Exception as e:
-            raise ValueError("Invalid dict field type.") from e
-    raise ValueError(
-        f"Default value for DICT must be a dict, got {type(x).__name__}"
-    )
-
-
-def check_action_field(x, fix_=True, **kwargs):
-    if (
-        isinstance(x, list)
-        and is_same_dtype(x, dict)
-        and all(_has_action_keys(y) for y in x)
-    ):
-        return x
-    try:
-        x = _fix_action_field(x, fix_)
-        return x
-    except Exception as e:
-        raise ValueError("Invalid action field type.") from e
-
-
-def check_number_field(x, fix_=True, **kwargs):
-    """
-    Checks if the given value is a valid numeric field.
-
-    Args:
-        x: The value to check.
-        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
-        **kwargs: Additional keyword arguments for fixing the value.
-
-    Returns:
-        The original value if it's valid, or the fixed value if `fix_` is True.
-
-    Raises:
-        ValueError: If the value is not a valid numeric field and cannot be fixed.
-    """
-    if not isinstance(x, (int, float)):
-        if fix_:
-            try:
-                return _fix_number_field(x, **kwargs)
-            except Exception as e:
-                raise e
-
-        raise ValueError(
-            f"Default value for NUMERIC must be an int or float, got {type(x).__name__}"
-        )
-    return x
-
-
-def check_bool_field(x, fix_=True):
-    """
-    Checks if the given value is a valid boolean field.
-
-    Args:
-        x: The value to check.
-        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
-
-    Returns:
-        The original value if it's valid, or the fixed value if `fix_` is True.
-
-    Raises:
-        ValueError: If the value is not a valid boolean field and cannot be fixed.
-    """
-    if not isinstance(x, bool):
-        if fix_:
-            try:
-                return _fix_bool_field(x)
-            except Exception as e:
-                raise e
-
-        raise ValueError(
-            f"Default value for BOOLEAN must be a bool, got {type(x).__name__}"
-        )
-    return x
-
-
-def check_str_field(x, *args, fix_=True, **kwargs):
-    """
-    Checks if the given value is a valid string field.
-
-    Args:
-        x: The value to check.
-        *args: Additional positional arguments for fixing the value.
-        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
-        **kwargs: Additional keyword arguments for fixing the value.
-
-    Returns:
-        The original value if it's valid, or the fixed value if `fix_` is True.
-
-    Raises:
-        ValueError: If the value is not a valid string field and cannot be fixed.
-    """
-    if not isinstance(x, str):
-        if fix_:
-            try:
-                return _fix_str_field(x, *args, **kwargs)
-            except Exception as e:
-                raise e
-
-        raise ValueError(
-            f"Default value for STRING must be a str, got {type(x).__name__}"
-        )
-    return x
-
-
-def check_enum_field(x, choices, fix_=True, **kwargs):
-    """
-    Checks if the given value is a valid enum field.
-
-    Args:
-        x: The value to check.
-        choices: The list of valid choices for the enum field.
-        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
-        **kwargs: Additional keyword arguments for fixing the value.
-
-    Returns:
-        The original value if it's valid, or the fixed value if `fix_` is True.
-
-    Raises:
-        ValueError: If the value is not a valid enum field and cannot be fixed.
-    """
-    same_dtype, dtype_ = is_same_dtype(choices, return_dtype=True)
-    if not same_dtype:
-        raise ValueError(
-            f"Field type ENUM requires all choices to be of the same type, got {choices}"
-        )
-
-    if not isinstance(x, dtype_):
-        raise ValueError(
-            f"Default value for ENUM must be an instance of the {dtype_.__name__}, got {type(x).__name__}"
-        )
-
-    if x not in choices:
-        if fix_:
-            try:
-                return _fix_enum_field(x, choices, **kwargs)
-            except Exception as e:
-                raise e
-        raise ValueError(
-            f"Default value for ENUM must be one of the {choices}, got {x}"
-        )
-
-    return x
-
-
-def _has_action_keys(dict_):
-    return list(dict_.keys()) >= ["function", "arguments"]
-
-
-def _fix_action_field(x, discard_=True):
-    corrected = []
-    if isinstance(x, str):
-        x = ParseUtil.fuzzy_parse_json(x)
-
-    try:
-        x = to_list(x)
-
-        for i in x:
-            i = to_dict(i)
-            if _has_action_keys(i):
-                corrected.append(i)
-            elif not discard_:
-                raise ValueError(f"Invalid action field: {i}")
-    except Exception as e:
-        raise ValueError(f"Invalid action field: {e}") from e
-
-    return corrected
-
-
-def _fix_number_field(x, *args, **kwargs):
-    """
-    Attempts to fix an invalid numeric field value.
-
-    Args:
-        x: The value to fix.
-        *args: Additional positional arguments for fixing the value.
-        **kwargs: Additional keyword arguments for fixing the value.
-
-    Returns:
-        The fixed numeric value.
-
-    Raises:
-        ValueError: If the value cannot be converted into a valid numeric value.
-    """
-    try:
-        x = to_num(x, *args, **kwargs)
-        if isinstance(x, (int, float)):
-            return x
-        raise ValueError(f"Failed to convert {x} into a numeric value")
-    except Exception as e:
-        raise ValueError(f"Failed to convert {x} into a numeric value") from e
-
-
-def _fix_bool_field(x):
-    """
-    Attempts to fix an invalid boolean field value.
-
-    Args:
-        x: The value to fix.
-
-    Returns:
-        The fixed boolean value.
-
-    Raises:
-        ValueError: If the value cannot be converted into a valid boolean value.
-    """
-    try:
-        x = strip_lower(to_str(x))
-        if x in ["true", "1", "correct", "yes"]:
-            return True
-
-        elif x in ["false", "0", "incorrect", "no", "none", "n/a"]:
-            return False
-
-        raise ValueError(f"Failed to convert {x} into a boolean value")
-    except Exception as e:
-        raise ValueError(f"Failed to convert {x} into a boolean value") from e
-
-
-def _fix_str_field(x):
-    """
-    Attempts to fix an invalid string field value.
-
-    Args:
-        x: The value to fix.
-
-    Returns:
-        The fixed string value.
-
-    Raises:
-        ValueError: If the value cannot be converted into a valid string value.
-    """
-    try:
-        x = to_str(x)
-        if isinstance(x, str):
-            return x
-        raise ValueError(f"Failed to convert {x} into a string value")
-    except Exception as e:
-        raise ValueError(f"Failed to convert {x} into a string value") from e
-
-
-def _fix_enum_field(x, choices, **kwargs):
-    """
-    Attempts to fix an invalid enum field value.
-
-    Args:
-        x: The value to fix.
-        choices: The list of valid choices for the enum field.
-        **kwargs: Additional keyword arguments for fixing the value.
-
-    Returns:
-        The fixed enum value.
-
-    Raises:
-        ValueError: If the value cannot be converted into a valid enum value.
-    """
-    try:
-        x = to_str(x)
-        return StringMatch.choose_most_similar(x, choices, **kwargs)
-    except Exception as e:
-        raise ValueError(
-            f"Failed to convert {x} into one of the choices"
-        ) from e
-
-
-validation_funcs = {
-    "number": check_number_field,
-    "bool": check_bool_field,
-    "str": check_str_field,
-    "enum": check_enum_field,
-    "action": check_action_field,
-    "dict": check_dict_field,
-}