lionagi 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/libs/ln_convert.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import json
 import re
 from functools import singledispatch
@@ -361,6 +377,8 @@ def _(
     reset_index: bool = True,
     **kwargs,
 ) -> pd.DataFrame:
+    if not input_:
+        return pd.DataFrame()
     if not isinstance(input_[0], (pd.DataFrame, pd.Series, pd.core.generic.NDFrame)):
         if drop_kwargs is None:
             drop_kwargs = {}
@@ -400,7 +418,7 @@ def to_num(
     *,
     upper_bound: int | float | None = None,
     lower_bound: int | float | None = None,
-    num_type: Type[int | float] = int,
+    num_type: Type[int | float] = float,
     precision: int | None = None,
 ) -> int | float:
     """
@@ -423,15 +441,9 @@ def to_num(
     return _str_to_num(str_, upper_bound, lower_bound, num_type, precision)
 
 
-def to_readable_dict(input_: Any | list[Any]) -> str | list[Any]:
+def to_readable_dict(input_: Any) -> str:
     """
-    Converts a given input to a readable dictionary format, either as a string or a list of dictionaries.
-
-    Args:
-            input_ (Any | list[Any]): The input to convert to a readable dictionary format.
-
-    Returns:
-            str | list[str]: The readable dictionary format of the input.
+    Converts a given input to a readable dictionary format
     """
 
     try:

lionagi/libs/ln_func_call.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from __future__ import annotations
 
 import functools
@@ -115,12 +131,12 @@ async def alcall(
     tasks = []
     if input_ is not None:
         lst = to_list(input_)
-        tasks = [AsyncUtil.handle_async_sync(func, i, **kwargs) for i in lst]
+        tasks = [call_handler(func, i, **kwargs) for i in lst]
 
     else:
-        tasks = [AsyncUtil.handle_async_sync(func, **kwargs)]
+        tasks = [call_handler(func, **kwargs)]
 
-    outs = await AsyncUtil.execute_tasks(*tasks)
+    outs = await asyncio.gather(*tasks)
     outs_ = []
     for i in outs:
         outs_.append(await i if isinstance(i, (Coroutine, asyncio.Future)) else i)
@@ -128,6 +144,11 @@ async def alcall(
     return to_list(outs_, flatten=flatten, dropna=dropna)
 
 
+async def pcall(funcs):
+    task = [call_handler(func) for func in funcs]
+    return await asyncio.gather(*task)
+
+
 async def mcall(
     input_: Any, /, func: Any, *, explode: bool = False, **kwargs
 ) -> tuple[Any]:
@@ -290,10 +311,12 @@ async def rcall(
     func: Callable,
     *args,
     retries: int = 0,
-    delay: float = 1.0,
-    backoff_factor: float = 2.0,
+    delay: float = 0.1,
+    backoff_factor: float = 2,
     default: Any = None,
     timeout: float | None = None,
+    timing: bool = False,
+    verbose: bool = True,
     **kwargs,
 ) -> Any:
     """
@@ -339,20 +362,34 @@ async def rcall(
     last_exception = None
     result = None
 
+    start = SysUtil.get_now(datetime_=False)
     for attempt in range(retries + 1) if retries == 0 else range(retries):
         try:
+            err_msg = f"Attempt {attempt + 1}/{retries}: " if retries > 0 else None
+            if timing:
+                return (
+                    await _tcall(
+                        func, *args, err_msg=err_msg, timeout=timeout, **kwargs
+                    ),
+                    SysUtil.get_now(datetime_=False) - start,
+                )
+
             return await _tcall(func, *args, timeout=timeout, **kwargs)
         except Exception as e:
             last_exception = e
             if attempt < retries:
-                await AsyncUtil.sleep(delay)
+                if verbose:
+                    print(f"Attempt {attempt + 1}/{retries} failed: {e}, retrying...")
+                await asyncio.sleep(delay)
                 delay *= backoff_factor
             else:
                 break
     if result is None and default is not None:
         return default
     elif last_exception is not None:
-        raise last_exception
+        raise RuntimeError(
+            f"Operation failed after {retries+1} attempts: {last_exception}"
+        ) from last_exception
     else:
         raise RuntimeError("rcall failed without catching an exception")
 
@@ -395,8 +432,8 @@ async def _alcall(
             [1, 4, 9]
     """
     lst = to_list(input_)
-    tasks = [AsyncUtil.handle_async_sync(func, i, **kwargs) for i in lst]
-    outs = await AsyncUtil.execute_tasks(*tasks)
+    tasks = [call_handler(func, i, **kwargs) for i in lst]
+    outs = await asyncio.gather(*tasks)
     return to_list(outs, flatten=flatten)
 
 
@@ -435,11 +472,11 @@ async def _tcall(
     """
     start_time = SysUtil.get_now(datetime_=False)
     try:
-        await AsyncUtil.sleep(delay)
+        await asyncio.sleep(delay)
         # Apply timeout to the function call
         if timeout is not None:
             coro = ""
-            if AsyncUtil.is_coroutine_func(func):
+            if is_coroutine_func(func):
                 coro = func(*args, **kwargs)
             else:
 
@@ -451,14 +488,13 @@ async def _tcall(
             result = await asyncio.wait_for(coro, timeout)
 
         else:
-            if AsyncUtil.is_coroutine_func(func):
+            if is_coroutine_func(func):
                 return await func(*args, **kwargs)
             return func(*args, **kwargs)
         duration = SysUtil.get_now(datetime_=False) - start_time
         return (result, duration) if timing else result
     except asyncio.TimeoutError as e:
-        err_msg = f"{err_msg} Error: {e}" if err_msg else f"An error occurred: {e}"
-        print(err_msg)
+        err_msg = f"{err_msg or ''}Timeout {timeout} seconds exceeded"
         if ignore_err:
             return (
                 (default, SysUtil.get_now(datetime_=False) - start_time)
@@ -466,10 +502,9 @@ async def _tcall(
                 else default
             )
         else:
-            raise e  # Re-raise the timeout exception
+            raise asyncio.TimeoutError(err_msg)  # Re-raise the timeout exception
     except Exception as e:
         err_msg = f"{err_msg} Error: {e}" if err_msg else f"An error occurred: {e}"
-        print(err_msg)
         if ignore_err:
             return (
                 (default, SysUtil.get_now(datetime_=False) - start_time)
@@ -537,7 +572,11 @@ class CallDecorator:
 
     @staticmethod
     def retry(
-        retries: int = 3, delay: float = 2.0, backoff_factor: float = 2.0
+        retries: int = 3,
+        delay: float = 2.0,
+        backoff_factor: float = 2.0,
+        default=...,
+        verbose=True,
     ) -> Callable:
         """
         Decorates an asynchronous function to automatically retry on failure,
@@ -581,6 +620,8 @@ class CallDecorator:
                     retries=retries,
                     delay=delay,
                     backoff_factor=backoff_factor,
+                    default=default,
+                    verbose=verbose,
                     **kwargs,
                 )
 
@@ -690,7 +731,7 @@ class CallDecorator:
         """
 
         def decorator(func: Callable[..., list[Any]]) -> Callable:
-            if AsyncUtil.is_coroutine_func(func):
+            if is_coroutine_func(func):
 
                 @functools.wraps(func)
                 async def async_wrapper(*args, **kwargs) -> list[Any]:
@@ -748,7 +789,7 @@ class CallDecorator:
         """
 
         def decorator(func: Callable) -> Callable:
-            if not any(AsyncUtil.is_coroutine_func(f) for f in functions):
+            if not any(is_coroutine_func(f) for f in functions):
 
                 @functools.wraps(func)
                 def sync_wrapper(*args, **kwargs):
@@ -763,7 +804,7 @@ class CallDecorator:
                     return value
 
                 return sync_wrapper
-            elif all(AsyncUtil.is_coroutine_func(f) for f in functions):
+            elif all(is_coroutine_func(f) for f in functions):
 
                 @functools.wraps(func)
                 async def async_wrapper(*args, **kwargs):
@@ -827,7 +868,7 @@ class CallDecorator:
         """
 
         def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-            if AsyncUtil.is_coroutine_func(func):
+            if is_coroutine_func(func):
 
                 @functools.wraps(func)
                 async def async_wrapper(*args, **kwargs) -> Any:
@@ -889,7 +930,7 @@ class CallDecorator:
                 ... # will return the cached result without re-executing the function body.
         """
 
-        if AsyncUtil.is_coroutine_func(func):
+        if is_coroutine_func(func):
             # Asynchronous function handling
             @AsyncUtil.cached(ttl=ttl)
             async def cached_async(*args, **kwargs) -> Any:
@@ -993,7 +1034,7 @@ class CallDecorator:
         """
 
         def decorator(func: Callable[..., list[Any]]) -> Callable:
-            if AsyncUtil.is_coroutine_func(func):
+            if is_coroutine_func(func):
 
                 @functools.wraps(func)
                 async def async_wrapper(*args, **kwargs) -> Any:
@@ -1039,11 +1080,11 @@ class CallDecorator:
         """
 
         def decorator(func: Callable) -> Callable:
-            if not AsyncUtil.is_coroutine_func(func):
+            if not is_coroutine_func(func):
                 raise TypeError(
                     "max_concurrency decorator can only be used with async functions."
                 )
-            semaphore = AsyncUtil.semaphore(limit)
+            semaphore = asyncio.Semaphore(limit)
 
             @functools.wraps(func)
             async def wrapper(*args, **kwargs):
@@ -1089,7 +1130,7 @@ class CallDecorator:
         @functools.wraps(fn)
         def wrapper(*args, **kwargs):
             future = pool.submit(fn, *args, **kwargs)
-            return AsyncUtil.wrap_future(future)  # make it awaitable
+            return asyncio.wrap_future(future)  # make it awaitable
 
         return wrapper
 
@@ -1233,8 +1274,8 @@ async def call_handler(
     except Exception as e:
         if error_map:
             _custom_error_handler(e, error_map)
-        else:
-            logging.error(f"Error in call_handler: {e}")
+        # else:
+        #     logging.error(f"Error in call_handler: {e}")
         raise
 
 

lionagi/libs/ln_image.py

@@ -0,0 +1,107 @@
+import base64
+import numpy as np
+from typing import Optional
+from .sys_util import SysUtil
+
+
+class ImageUtil:
+
+    @staticmethod
+    def preprocess_image(
+        image: np.ndarray, color_conversion_code: Optional[int] = None
+    ) -> np.ndarray:
+        SysUtil.check_import("cv2", pip_name="opencv-python")
+        import cv2
+
+        color_conversion_code = color_conversion_code or cv2.COLOR_BGR2RGB
+        return cv2.cvtColor(image, color_conversion_code)
+
+    @staticmethod
+    def encode_image_to_base64(image: np.ndarray, file_extension: str = ".jpg") -> str:
+        SysUtil.check_import("cv2", pip_name="opencv-python")
+        import cv2
+
+        success, buffer = cv2.imencode(file_extension, image)
+        if not success:
+            raise ValueError(f"Could not encode image to {file_extension} format.")
+        encoded_image = base64.b64encode(buffer).decode("utf-8")
+        return encoded_image
+
+    @staticmethod
+    def read_image_to_array(
+        image_path: str, color_flag: Optional[int] = None
+    ) -> np.ndarray:
+        SysUtil.check_import("cv2", pip_name="opencv-python")
+        import cv2
+
+        image = cv2.imread(image_path, color_flag)
+        color_flag = color_flag or cv2.IMREAD_COLOR
+        if image is None:
+            raise ValueError(f"Could not read image from path: {image_path}")
+        return image
+
+    @staticmethod
+    def read_image_to_base64(
+        image_path: str,
+        color_flag: Optional[int] = None,
+    ) -> str:
+        image_path = str(image_path)
+        image = ImageUtil.read_image_to_array(image_path, color_flag)
+
+        file_extension = "." + image_path.split(".")[-1]
+        return ImageUtil.encode_image_to_base64(image, file_extension)
+
+    # @staticmethod
+    # def encode_image(image_path):
+    #     with open(image_path, "rb") as image_file:
+    #         return base64.b64encode(image_file.read()).decode("utf-8")
+
+    @staticmethod
+    def calculate_image_token_usage_from_base64(image_base64: str, detail):
+        """
+        Calculate the token usage for processing OpenAI images from a base64-encoded string.
+
+        Parameters:
+        image_base64 (str): The base64-encoded string of the image.
+        detail (str): The detail level of the image, either 'low' or 'high'.
+
+        Returns:
+        int: The total token cost for processing the image.
+        """
+        import base64
+        from io import BytesIO
+        from PIL import Image
+
+        # Decode the base64 string to get image data
+        if "data:image/jpeg;base64," in image_base64:
+            image_base64 = image_base64.split("data:image/jpeg;base64,")[1]
+            image_base64.strip("{}")
+
+        image_data = base64.b64decode(image_base64)
+        image = Image.open(BytesIO(image_data))
+
+        # Get image dimensions
+        width, height = image.size
+
+        if detail == "low":
+            return 85
+
+        # Scale to fit within a 2048 x 2048 square
+        max_dimension = 2048
+        if width > max_dimension or height > max_dimension:
+            scale_factor = max_dimension / max(width, height)
+            width = int(width * scale_factor)
+            height = int(height * scale_factor)
+
+        # Scale such that the shortest side is 768px
+        min_side = 768
+        if min(width, height) > min_side:
+            scale_factor = min_side / min(width, height)
+            width = int(width * scale_factor)
+            height = int(height * scale_factor)
+
+        # Calculate the number of 512px squares
+        num_squares = (width // 512) * (height // 512)
+        token_cost = 170 * num_squares + 85
+
+        return token_cost

lionagi/libs/ln_nested.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from collections import defaultdict
 from itertools import chain
 from typing import Any, Generator, Callable
@@ -52,7 +68,7 @@ def nset(nested_structure: dict | list, indices: list[int | str], value: Any) ->
 def nget(
     nested_structure: dict | list,
     indices: list[int | str],
-    default: Any | None = None,
+    default=...,
 ) -> Any:
     """
     retrieves a value from a nested list or dictionary structure, with an option to
@@ -98,12 +114,12 @@ def nget(
             return target_container[last_index]
         elif isinstance(target_container, dict) and last_index in target_container:
             return target_container[last_index]
-        elif default is not None:
+        elif default is not ...:
             return default
         else:
             raise LookupError("Target not found and no default value provided.")
     except (IndexError, KeyError, TypeError):
-        if default is not None:
+        if default is not ...:
             return default
         else:
             raise LookupError("Target not found and no default value provided.")
@@ -116,7 +132,7 @@ def nmerge(
     *,
     overwrite: bool = False,
     dict_sequence: bool = False,
-    sequence_separator: str = "_",
+    sequence_separator: str = "[^_^]",
     sort_list: bool = False,
     custom_sort: Callable[[Any], Any] | None = None,
 ) -> dict | list:
@@ -176,7 +192,7 @@ def flatten(
     /,
     *,
     parent_key: str = "",
-    sep: str = "_",
+    sep: str = "[^_^]",
     max_depth: int | None = None,
     inplace: bool = False,
     dict_only: bool = False,
@@ -238,7 +254,7 @@ def unflatten(
     flat_dict: dict[str, Any],
     /,
     *,
-    sep: str = "_",
+    sep: str = "[^_^]",
     custom_logic: Callable[[str], Any] | None = None,
     max_depth: int | None = None,
 ) -> dict | list:
@@ -330,7 +346,7 @@ def ninsert(
     indices: list[str | int],
     value: Any,
     *,
-    sep: str = "_",
+    sep: str = "[^_^]",
     max_depth: int | None = None,
     current_depth: int = 0,
 ) -> None:
@@ -393,12 +409,11 @@ def ninsert(
         nested_structure[last_part] = value
 
 
-# noinspection PyDecorator
 def get_flattened_keys(
     nested_structure: Any,
     /,
     *,
-    sep: str = "_",
+    sep: str = "[^_^]",
     max_depth: int | None = None,
     dict_only: bool = False,
     inplace: bool = False,
@@ -448,7 +463,7 @@ def _dynamic_flatten_in_place(
     /,
     *,
     parent_key: str = "",
-    sep: str = "_",
+    sep: str = "[^_^]",
     max_depth: int | None = None,
     current_depth: int = 0,
     dict_only: bool = False,
@@ -581,7 +596,7 @@ def _deep_update(original: dict, update: dict) -> dict:
 def _dynamic_flatten_generator(
     nested_structure: Any,
     parent_key: tuple[str, ...],
-    sep: str = "_",
+    sep: str = "[^_^]",
     max_depth: int | None = None,
     current_depth: int = 0,
     dict_only: bool = False,