lionagi 0.0.312__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/libs/ln_parse.py

@@ -1,11 +1,28 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from collections.abc import Callable
 import re
 import inspect
 import itertools
-from collections.abc import Callable
 from typing import Any
 import numpy as np
 import lionagi.libs.ln_convert as convert
 
+
 md_json_char_map = {"\n": "\\n", "\r": "\\r", "\t": "\\t", '"': '\\"'}
 
 
@@ -100,12 +117,12 @@ class ParseUtil:
     # inspired by langchain_core.output_parsers.json (MIT License)
     # https://github.com/langchain-ai/langchain/blob/master/libs/core/langchain_core/output_parsers/json.py
     @staticmethod
-    def extract_code_block(
+    def extract_json_block(
         str_to_parse: str,
         language: str | None = None,
         regex_pattern: str | None = None,
         *,
-        parser: Callable[[str], Any],
+        parser: Callable[[str], Any] = None,
     ) -> Any:
         """
         Extracts and parses a code block from Markdown content.
@@ -145,9 +162,37 @@ class ParseUtil:
             raise ValueError(
                 f"No {language or 'specified'} code block found in the Markdown content."
             )
+        if not match:
+            str_to_parse = str_to_parse.strip()
+            if str_to_parse.startswith("```json\n") and str_to_parse.endswith("\n```"):
+                str_to_parse = str_to_parse[8:-4].strip()
 
+        parser = parser or ParseUtil.fuzzy_parse_json
         return parser(code_str)
 
+    @staticmethod
+    def extract_code_blocks(code):
+        code_blocks = []
+        lines = code.split("\n")
+        inside_code_block = False
+        current_block = []
+
+        for line in lines:
+            if line.startswith("```"):
+                if inside_code_block:
+                    code_blocks.append("\n".join(current_block))
+                    current_block = []
+                    inside_code_block = False
+                else:
+                    inside_code_block = True
+            elif inside_code_block:
+                current_block.append(line)
+
+        if current_block:
+            code_blocks.append("\n".join(current_block))
+
+        return "\n\n".join(code_blocks)
+
     @staticmethod
     def md_to_json(
         str_to_parse: str,
@@ -178,7 +223,7 @@ class ParseUtil:
                 >>> md_to_json('```json\\n{"key": "value"}\\n```', expected_keys=['key'])
                 {'key': 'value'}
         """
-        json_obj = ParseUtil.extract_code_block(
+        json_obj = ParseUtil.extract_json_block(
             str_to_parse, language="json", parser=parser or ParseUtil.fuzzy_parse_json
         )
 
@@ -382,7 +427,9 @@ class ParseUtil:
         return type_mapping.get(py_type, "object")
 
     @staticmethod
-    def _func_to_schema(func, style="google"):
+    def _func_to_schema(
+        func, style="google", func_description=None, params_description=None
+    ):
         """
         Generates a schema description for a given function, using typing hints and
         docstrings. The schema includes the function's name, description, and parameters.
@@ -409,9 +456,11 @@ class ParseUtil:
         """
         # Extracting function name and docstring details
         func_name = func.__name__
-        func_description, params_description = ParseUtil._extract_docstring_details(
-            func, style
-        )
+
+        if not func_description:
+            func_description, _ = ParseUtil._extract_docstring_details(func, style)
+        if not params_description:
+            _, params_description = ParseUtil._extract_docstring_details(func, style)
 
         # Extracting parameters with typing hints
         sig = inspect.signature(func)
@@ -590,14 +639,15 @@ class StringMatch:
         return d[m][n]
 
     @staticmethod
-    def correct_keys(output_fields, out_, score_func=None):
+    def correct_dict_keys(keys: dict | list[str], dict_, score_func=None):
         if score_func is None:
             score_func = StringMatch.jaro_winkler_similarity
-        fields_set = set(output_fields.keys())
+
+        fields_set = set(keys if isinstance(keys, list) else keys.keys())
         corrected_out = {}
         used_keys = set()
 
-        for k, v in out_.items():
+        for k, v in dict_.items():
             if k in fields_set:
                 corrected_out[k] = v
                 fields_set.remove(k)  # Remove the matched key
@@ -614,8 +664,8 @@ class StringMatch:
                 fields_set.remove(best_match)  # Remove the matched key
                 used_keys.add(best_match)
 
-        if len(used_keys) < len(out_):
-            for k, v in out_.items():
+        if len(used_keys) < len(dict_):
+            for k, v in dict_.items():
                 if k not in used_keys:
                     corrected_out[k] = v
 
@@ -630,10 +680,56 @@ class StringMatch:
         # Calculate Jaro-Winkler similarity scores for each potential match
         scores = np.array(
             [
-                score_func(convert.to_str(word), correct_word)
+                score_func(str(word), str(correct_word))
                 for correct_word in correct_words_list
             ]
         )
         # Find the index of the highest score
         max_score_index = np.argmax(scores)
         return correct_words_list[max_score_index]
+
+    @staticmethod
+    def force_validate_dict(x, keys: dict | list[str]) -> dict:
+        out_ = x
+
+        if isinstance(out_, str):
+            # first try to parse it straight as a fuzzy json
+            
+            try:
+                out_ = ParseUtil.fuzzy_parse_json(out_)
+                return StringMatch.correct_dict_keys(keys, out_)
+            
+            except:
+                try:
+                    out_ = ParseUtil.md_to_json(out_)
+                    return StringMatch.correct_dict_keys(keys, out_)
+
+                except Exception:
+                    try:
+                        # if failed we try to extract the json block and parse it
+                        out_ = ParseUtil.md_to_json(out_)
+                        return StringMatch.correct_dict_keys(keys, out_)
+
+                    except Exception:
+                        # if still failed we try to extract the json block using re and parse it again
+                        match = re.search(r"```json\n({.*?})\n```", out_, re.DOTALL)
+                        if match:
+                            out_ = match.group(1)
+                            try:
+                                out_ = ParseUtil.fuzzy_parse_json(out_)
+                                return StringMatch.correct_dict_keys(keys, out_)
+
+                            except:
+                                try:
+                                    out_ = ParseUtil.fuzzy_parse_json(
+                                        out_.replace("'", '"')
+                                    )
+                                    return StringMatch.correct_dict_keys(keys, out_)
+                                except:
+                                    pass
+
+        if isinstance(out_, dict):
+            try:
+                return StringMatch.correct_dict_keys(keys, out_)
+            except Exception as e:
+                raise ValueError(f"Failed to force_validate_dict for input: {x}") from e

lionagi/libs/ln_queue.py

@@ -0,0 +1,117 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+"""
+A class that manages asynchronous task processing with controlled concurrency.
+"""
+
+from typing import Any, Callable
+import asyncio
+from lionagi.libs import func_call
+
+
+class AsyncQueue:
+    """
+    This class handles the enqueueing and processing of tasks with a limit on
+    how many can run simultaneously, using an asyncio.Queue for task storage and
+    an asyncio.Semaphore to limit concurrency.
+
+    Attributes:
+        queue (asyncio.Queue): The queue to store tasks.
+        _stop_event (asyncio.Event): Event to signal processing should stop.
+        max_concurrent_tasks (int): Maximum number of tasks processed concurrently.
+        semaphore (asyncio.Semaphore): Controls concurrent access to task execution.
+    """
+
+    def __init__(self, max_concurrent_tasks=5):
+        """
+        Initializes the AsyncQueue with a concurrency limit.
+
+        Args:
+            max_concurrent_tasks (int): The maximum number of concurrent tasks
+                                        allowed. Default is 5.
+        """
+        self.queue = asyncio.Queue()
+        self._stop_event = asyncio.Event()
+        self.max_concurrent_tasks = max_concurrent_tasks
+        self.semaphore = asyncio.Semaphore(max_concurrent_tasks)
+
+    async def enqueue(self, input_: Any) -> None:
+        """
+        Enqueues an item to be processed asynchronously.
+
+        Args:
+            input_ (Any): The item to be enqueued.
+        """
+        await self.queue.put(input_)
+
+    async def dequeue(self) -> Any:
+        """
+        Dequeues an item for processing.
+
+        Returns:
+            Any: The dequeued item.
+        """
+        return await self.queue.get()
+
+    async def join(self) -> None:
+        """Waits for all items in the queue to be processed."""
+        await self.queue.join()
+
+    async def stop(self) -> None:
+        """Signals the queue to stop processing new items."""
+        self._stop_event.set()
+
+    def stopped(self) -> bool:
+        """
+        Checks if the stop signal has been issued.
+
+        Returns:
+            bool: True if the queue has been stopped, otherwise False.
+        """
+        return self._stop_event.is_set()
+
+    async def process_requests(self, func: Callable, retry_kwargs: dict = {}) -> None:
+        """
+        Processes tasks from the queue using the provided function with retries.
+
+        This method continuously processes tasks from the queue using the specified
+        function until a stop event is triggered. Handles concurrency using a
+        semaphore and manages task completion.
+
+        Args:
+            func (Callable): The function to process each task.
+            retry_kwargs (dict): Keyword arguments for retry behavior. Default is
+                                 an empty dictionary.
+        """
+        tasks = set()
+        while not self.stopped():
+            if len(tasks) >= self.max_concurrent_tasks:
+                _, done = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+                tasks.difference_update(done)
+
+            async with self.semaphore:
+                input_ = await self.dequeue()
+                if input_ is None:
+                    await self.stop()
+                    break
+                task = asyncio.create_task(
+                    func_call.rcall(func, input_, **retry_kwargs)
+                )
+                tasks.add(task)
+
+        if tasks:
+            await asyncio.wait(tasks)

lionagi/libs/ln_tokenize.py

@@ -0,0 +1,164 @@
+import tiktoken
+import math
+from .ln_convert import to_str
+from .special_tokens import disallowed_tokens
+
+
+class TokenizeUtil:
+
+    @staticmethod
+    def tokenize(
+        text,
+        encoding_model=None,
+        encoding_name=None,
+        return_byte=False,
+        disallowed_tokens=disallowed_tokens,
+    ):
+        encoding = None
+
+        if encoding_model:
+            try:
+                encoding_name = tiktoken.encoding_name_for_model(encoding_model)
+            except:
+                encoding_name = encoding_name or "cl100k_base"
+
+        if not encoding_name or encoding_name in tiktoken.list_encoding_names():
+            encoding_name = encoding_name or "cl100k_base"
+            encoding = tiktoken.get_encoding(encoding_name)
+
+        special_encodings = (
+            [encoding.encode(token) for token in disallowed_tokens]
+            if disallowed_tokens
+            else []
+        )
+        codes = encoding.encode(text)
+        if special_encodings and len(special_encodings) > 0:
+            codes = [code for code in codes if code not in special_encodings]
+
+        if return_byte:
+            return codes
+
+        return [encoding.decode([code]) for code in codes]
+
+    @staticmethod
+    def chunk_by_chars(
+        text: str, chunk_size: int, overlap: float, threshold: int
+    ) -> list[str | None]:
+        """
+        Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
+
+        Parameters:
+            text (str): The input text to chunk.
+
+            chunk_size (int): The size of each chunk.
+
+            overlap (float): The amount of overlap between chunks.
+
+            threshold (int): The minimum size of the final chunk.
+
+        Returns:
+            List[Union[str, None]]: A list of text chunks.
+
+        Raises:
+            ValueError: If an error occurs during chunking.
+        """
+
+        def _chunk_n1():
+            return [text]
+
+        def _chunk_n2():
+            chunks = []
+            chunks.append(text[: chunk_size + overlap_size])
+
+            if len(text) - chunk_size > threshold:
+                chunks.append(text[chunk_size - overlap_size :])
+            else:
+                return _chunk_n1()
+
+            return chunks
+
+        def _chunk_n3():
+            chunks = []
+            chunks.append(text[: chunk_size + overlap_size])
+            for i in range(1, n_chunks - 1):
+                start_idx = chunk_size * i - overlap_size
+                end_idx = chunk_size * (i + 1) + overlap_size
+                chunks.append(text[start_idx:end_idx])
+
+            if len(text) - chunk_size * (n_chunks - 1) > threshold:
+                chunks.append(text[chunk_size * (n_chunks - 1) - overlap_size :])
+            else:
+                chunks[-1] += text[chunk_size * (n_chunks - 1) + overlap_size :]
+
+            return chunks
+
+        try:
+            if not isinstance(text, str):
+                text = to_str(text)
+
+            n_chunks = math.ceil(len(text) / chunk_size)
+            overlap_size = int(overlap / 2)
+
+            if n_chunks == 1:
+                return _chunk_n1()
+
+            elif n_chunks == 2:
+                return _chunk_n2()
+
+            elif n_chunks > 2:
+                return _chunk_n3()
+
+        except Exception as e:
+            raise ValueError(f"An error occurred while chunking the text. {e}")
+
+    @staticmethod
+    def chunk_by_tokens(
+        text: str,
+        chunk_size: int,
+        overlap: float,
+        threshold: int,  # minimum size of the final chunk in number of tokens
+        encoding_model=None,
+        encoding_name=None,
+        return_tokens=False,
+        return_byte=False,
+    ) -> list[str | None]:
+
+        tokens = TokenizeUtil.tokenize(
+            text, encoding_model, encoding_name, return_byte=return_byte
+        )
+
+        n_chunks = math.ceil(len(tokens) / chunk_size)
+        overlap_size = int(overlap * chunk_size / 2)
+        residue = len(tokens) % chunk_size
+
+        if n_chunks == 1:
+            return text if not return_tokens else [tokens]
+
+        elif n_chunks == 2:
+            chunks = [tokens[: chunk_size + overlap_size]]
+            if residue > threshold:
+                chunks.append(tokens[chunk_size - overlap_size :])
+                return (
+                    [" ".join(chunk).strip() for chunk in chunks]
+                    if not return_tokens
+                    else chunks
+                )
+            else:
+                return text if not return_tokens else [tokens]
+
+        elif n_chunks > 2:
+            chunks = []
+            chunks.append(tokens[: chunk_size + overlap_size])
+            for i in range(1, n_chunks - 1):
+                start_idx = chunk_size * i - overlap_size
+                end_idx = chunk_size * (i + 1) + overlap_size
+                chunks.append(tokens[start_idx:end_idx])
+
+            if len(tokens) - chunk_size * (n_chunks - 1) > threshold:
+                chunks.append(tokens[chunk_size * (n_chunks - 1) - overlap_size :])
+            else:
+                chunks[-1] += tokens[-residue:]
+
+            return (
+                [" ".join(chunk) for chunk in chunks] if not return_tokens else chunks
+            )

lionagi/{core/prompt/field_validator.py → libs/ln_validate.py}

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 """
 This module provides functions for validating and fixing field values based on their data types.
 
@@ -6,7 +22,34 @@ including numeric, boolean, string, and enum. It also provides a dictionary `val
 maps data types to their corresponding validation functions.
 """
 
-from lionagi.libs import convert, StringMatch
+from .ln_convert import to_str, is_same_dtype, to_list, to_dict, to_num, strip_lower
+from .ln_parse import StringMatch, ParseUtil
+
+
+def check_dict_field(x, keys: list[str] | dict, fix_=True, **kwargs):
+    if isinstance(x, dict):
+        return x
+    if fix_:
+        try:
+            x = to_str(x)
+            return StringMatch.force_validate_dict(x, keys=keys, **kwargs)
+        except Exception as e:
+            raise ValueError("Invalid dict field type.") from e
+    raise ValueError(f"Default value for DICT must be a dict, got {type(x).__name__}")
+
+
+def check_action_field(x, fix_=True, **kwargs):
+    if (
+        isinstance(x, list)
+        and is_same_dtype(x, dict)
+        and all(_has_action_keys(y) for y in x)
+    ):
+        return x
+    try:
+        x = _fix_action_field(x, fix_)
+        return x
+    except Exception as e:
+        raise ValueError("Invalid action field type.") from e
 
 
 def check_number_field(x, fix_=True, **kwargs):
@@ -109,7 +152,7 @@ def check_enum_field(x, choices, fix_=True, **kwargs):
     Raises:
         ValueError: If the value is not a valid enum field and cannot be fixed.
     """
-    same_dtype, dtype_ = convert.is_same_dtype(choices, return_dtype=True)
+    same_dtype, dtype_ = is_same_dtype(choices, return_dtype=True)
     if not same_dtype:
         raise ValueError(
             f"Field type ENUM requires all choices to be of the same type, got {choices}"
@@ -133,6 +176,30 @@ def check_enum_field(x, choices, fix_=True, **kwargs):
     return x
 
 
+def _has_action_keys(dict_):
+    return list(dict_.keys()) >= ["function", "arguments"]
+
+
+def _fix_action_field(x, discard_=True):
+    corrected = []
+    if isinstance(x, str):
+        x = ParseUtil.fuzzy_parse_json(x)
+
+    try:
+        x = to_list(x)
+
+        for i in x:
+            i = to_dict(i)
+            if _has_action_keys(i):
+                corrected.append(i)
+            elif not discard_:
+                raise ValueError(f"Invalid action field: {i}")
+    except Exception as e:
+        raise ValueError(f"Invalid action field: {e}") from e
+
+    return corrected
+
+
 def _fix_number_field(x, *args, **kwargs):
     """
     Attempts to fix an invalid numeric field value.
@@ -149,7 +216,7 @@ def _fix_number_field(x, *args, **kwargs):
         ValueError: If the value cannot be converted into a valid numeric value.
     """
     try:
-        x = convert.to_num(x, *args, **kwargs)
+        x = to_num(x, *args, **kwargs)
         if isinstance(x, (int, float)):
             return x
         raise ValueError(f"Failed to convert {x} into a numeric value")
@@ -171,17 +238,13 @@ def _fix_bool_field(x):
         ValueError: If the value cannot be converted into a valid boolean value.
     """
     try:
-        x = convert.to_str(x)
-        if (
-            convert.strip_lower(x) in ["true", "1", "correct", "yes"]
-            or convert.to_num(x) == 1
-        ):
+        x = strip_lower(to_str(x))
+        if x in ["true", "1", "correct", "yes"]:
             return True
-        elif (
-            convert.strip_lower(x) in ["false", "0", "incorrect", "no", "none", "n/a"]
-            or convert.to_num(x) == 0
-        ):
+
+        elif x in ["false", "0", "incorrect", "no", "none", "n/a"]:
             return False
+
         raise ValueError(f"Failed to convert {x} into a boolean value")
     except Exception as e:
         raise ValueError(f"Failed to convert {x} into a boolean value") from e
@@ -201,7 +264,7 @@ def _fix_str_field(x):
         ValueError: If the value cannot be converted into a valid string value.
     """
     try:
-        x = convert.to_str(x)
+        x = to_str(x)
         if isinstance(x, str):
             return x
         raise ValueError(f"Failed to convert {x} into a string value")
@@ -225,7 +288,7 @@ def _fix_enum_field(x, choices, **kwargs):
         ValueError: If the value cannot be converted into a valid enum value.
     """
     try:
-        x = convert.to_str(x)
+        x = to_str(x)
         return StringMatch.choose_most_similar(x, choices, **kwargs)
     except Exception as e:
         raise ValueError(f"Failed to convert {x} into one of the choices") from e
@@ -236,4 +299,6 @@ validation_funcs = {
     "bool": check_bool_field,
     "str": check_str_field,
     "enum": check_enum_field,
+    "action": check_action_field,
+    "dict": check_dict_field,
 }