lionagi 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/libs/ln_parse.py

@@ -1,9 +1,23 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from collections.abc import Callable
 import re
 import inspect
 import itertools
-import contextlib
-from functools import singledispatchmethod
 from typing import Any
 import numpy as np
 import lionagi.libs.ln_convert as convert
@@ -103,12 +117,12 @@ class ParseUtil:
     # inspired by langchain_core.output_parsers.json (MIT License)
     # https://github.com/langchain-ai/langchain/blob/master/libs/core/langchain_core/output_parsers/json.py
     @staticmethod
-    def extract_code_block(
+    def extract_json_block(
         str_to_parse: str,
         language: str | None = None,
         regex_pattern: str | None = None,
         *,
-        parser: Callable[[str], Any],
+        parser: Callable[[str], Any] = None,
     ) -> Any:
         """
         Extracts and parses a code block from Markdown content.
@@ -148,9 +162,37 @@ class ParseUtil:
             raise ValueError(
                 f"No {language or 'specified'} code block found in the Markdown content."
             )
+        if not match:
+            str_to_parse = str_to_parse.strip()
+            if str_to_parse.startswith("```json\n") and str_to_parse.endswith("\n```"):
+                str_to_parse = str_to_parse[8:-4].strip()
 
+        parser = parser or ParseUtil.fuzzy_parse_json
         return parser(code_str)
 
+    @staticmethod
+    def extract_code_blocks(code):
+        code_blocks = []
+        lines = code.split("\n")
+        inside_code_block = False
+        current_block = []
+
+        for line in lines:
+            if line.startswith("```"):
+                if inside_code_block:
+                    code_blocks.append("\n".join(current_block))
+                    current_block = []
+                    inside_code_block = False
+                else:
+                    inside_code_block = True
+            elif inside_code_block:
+                current_block.append(line)
+
+        if current_block:
+            code_blocks.append("\n".join(current_block))
+
+        return "\n\n".join(code_blocks)
+
     @staticmethod
     def md_to_json(
         str_to_parse: str,
@@ -181,7 +223,7 @@ class ParseUtil:
                 >>> md_to_json('```json\\n{"key": "value"}\\n```', expected_keys=['key'])
                 {'key': 'value'}
         """
-        json_obj = ParseUtil.extract_code_block(
+        json_obj = ParseUtil.extract_json_block(
             str_to_parse, language="json", parser=parser or ParseUtil.fuzzy_parse_json
         )
 
@@ -385,7 +427,9 @@ class ParseUtil:
         return type_mapping.get(py_type, "object")
 
     @staticmethod
-    def _func_to_schema(func, style="google"):
+    def _func_to_schema(
+        func, style="google", func_description=None, params_description=None
+    ):
         """
         Generates a schema description for a given function, using typing hints and
         docstrings. The schema includes the function's name, description, and parameters.
@@ -412,9 +456,11 @@ class ParseUtil:
         """
         # Extracting function name and docstring details
         func_name = func.__name__
-        func_description, params_description = ParseUtil._extract_docstring_details(
-            func, style
-        )
+
+        if not func_description:
+            func_description, _ = ParseUtil._extract_docstring_details(func, style)
+        if not params_description:
+            _, params_description = ParseUtil._extract_docstring_details(func, style)
 
         # Extracting parameters with typing hints
         sig = inspect.signature(func)
@@ -634,7 +680,7 @@ class StringMatch:
         # Calculate Jaro-Winkler similarity scores for each potential match
         scores = np.array(
             [
-                score_func(convert.to_str(word), correct_word)
+                score_func(str(word), str(correct_word))
                 for correct_word in correct_words_list
             ]
         )
@@ -648,26 +694,39 @@ class StringMatch:
 
         if isinstance(out_, str):
             # first try to parse it straight as a fuzzy json
+            
             try:
                 out_ = ParseUtil.fuzzy_parse_json(out_)
-            except Exception:
+                return StringMatch.correct_dict_keys(keys, out_)
+            
+            except:
                 try:
-                    # if failed we try to extract the json block and parse it
                     out_ = ParseUtil.md_to_json(out_)
+                    return StringMatch.correct_dict_keys(keys, out_)
+
                 except Exception:
-                    # if still failed we try to extract the json block using re and parse it again
-                    match = re.search(r"```json\n({.*?})\n```", out_, re.DOTALL)
-                    if match:
-                        out_ = match.group(1)
-                        try:
-                            out_ = ParseUtil.fuzzy_parse_json(out_)
-                        except:
+                    try:
+                        # if failed we try to extract the json block and parse it
+                        out_ = ParseUtil.md_to_json(out_)
+                        return StringMatch.correct_dict_keys(keys, out_)
+
+                    except Exception:
+                        # if still failed we try to extract the json block using re and parse it again
+                        match = re.search(r"```json\n({.*?})\n```", out_, re.DOTALL)
+                        if match:
+                            out_ = match.group(1)
                             try:
-                                out_ = ParseUtil.fuzzy_parse_json(
-                                    out_.replace("'", '"')
-                                )
+                                out_ = ParseUtil.fuzzy_parse_json(out_)
+                                return StringMatch.correct_dict_keys(keys, out_)
+
                             except:
-                                pass
+                                try:
+                                    out_ = ParseUtil.fuzzy_parse_json(
+                                        out_.replace("'", '"')
+                                    )
+                                    return StringMatch.correct_dict_keys(keys, out_)
+                                except:
+                                    pass
 
         if isinstance(out_, dict):
             try:

lionagi/libs/ln_queue.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 """
 A class that manages asynchronous task processing with controlled concurrency.
 """

lionagi/libs/ln_tokenize.py

@@ -0,0 +1,164 @@
+import tiktoken
+import math
+from .ln_convert import to_str
+from .special_tokens import disallowed_tokens
+
+
+class TokenizeUtil:
+
+    @staticmethod
+    def tokenize(
+        text,
+        encoding_model=None,
+        encoding_name=None,
+        return_byte=False,
+        disallowed_tokens=disallowed_tokens,
+    ):
+        encoding = None
+
+        if encoding_model:
+            try:
+                encoding_name = tiktoken.encoding_name_for_model(encoding_model)
+            except:
+                encoding_name = encoding_name or "cl100k_base"
+
+        if not encoding_name or encoding_name in tiktoken.list_encoding_names():
+            encoding_name = encoding_name or "cl100k_base"
+            encoding = tiktoken.get_encoding(encoding_name)
+
+        special_encodings = (
+            [encoding.encode(token) for token in disallowed_tokens]
+            if disallowed_tokens
+            else []
+        )
+        codes = encoding.encode(text)
+        if special_encodings and len(special_encodings) > 0:
+            codes = [code for code in codes if code not in special_encodings]
+
+        if return_byte:
+            return codes
+
+        return [encoding.decode([code]) for code in codes]
+
+    @staticmethod
+    def chunk_by_chars(
+        text: str, chunk_size: int, overlap: float, threshold: int
+    ) -> list[str | None]:
+        """
+        Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
+
+        Parameters:
+            text (str): The input text to chunk.
+
+            chunk_size (int): The size of each chunk.
+
+            overlap (float): The amount of overlap between chunks.
+
+            threshold (int): The minimum size of the final chunk.
+
+        Returns:
+            List[Union[str, None]]: A list of text chunks.
+
+        Raises:
+            ValueError: If an error occurs during chunking.
+        """
+
+        def _chunk_n1():
+            return [text]
+
+        def _chunk_n2():
+            chunks = []
+            chunks.append(text[: chunk_size + overlap_size])
+
+            if len(text) - chunk_size > threshold:
+                chunks.append(text[chunk_size - overlap_size :])
+            else:
+                return _chunk_n1()
+
+            return chunks
+
+        def _chunk_n3():
+            chunks = []
+            chunks.append(text[: chunk_size + overlap_size])
+            for i in range(1, n_chunks - 1):
+                start_idx = chunk_size * i - overlap_size
+                end_idx = chunk_size * (i + 1) + overlap_size
+                chunks.append(text[start_idx:end_idx])
+
+            if len(text) - chunk_size * (n_chunks - 1) > threshold:
+                chunks.append(text[chunk_size * (n_chunks - 1) - overlap_size :])
+            else:
+                chunks[-1] += text[chunk_size * (n_chunks - 1) + overlap_size :]
+
+            return chunks
+
+        try:
+            if not isinstance(text, str):
+                text = to_str(text)
+
+            n_chunks = math.ceil(len(text) / chunk_size)
+            overlap_size = int(overlap / 2)
+
+            if n_chunks == 1:
+                return _chunk_n1()
+
+            elif n_chunks == 2:
+                return _chunk_n2()
+
+            elif n_chunks > 2:
+                return _chunk_n3()
+
+        except Exception as e:
+            raise ValueError(f"An error occurred while chunking the text. {e}")
+
+    @staticmethod
+    def chunk_by_tokens(
+        text: str,
+        chunk_size: int,
+        overlap: float,
+        threshold: int,  # minimum size of the final chunk in number of tokens
+        encoding_model=None,
+        encoding_name=None,
+        return_tokens=False,
+        return_byte=False,
+    ) -> list[str | None]:
+
+        tokens = TokenizeUtil.tokenize(
+            text, encoding_model, encoding_name, return_byte=return_byte
+        )
+
+        n_chunks = math.ceil(len(tokens) / chunk_size)
+        overlap_size = int(overlap * chunk_size / 2)
+        residue = len(tokens) % chunk_size
+
+        if n_chunks == 1:
+            return text if not return_tokens else [tokens]
+
+        elif n_chunks == 2:
+            chunks = [tokens[: chunk_size + overlap_size]]
+            if residue > threshold:
+                chunks.append(tokens[chunk_size - overlap_size :])
+                return (
+                    [" ".join(chunk).strip() for chunk in chunks]
+                    if not return_tokens
+                    else chunks
+                )
+            else:
+                return text if not return_tokens else [tokens]
+
+        elif n_chunks > 2:
+            chunks = []
+            chunks.append(tokens[: chunk_size + overlap_size])
+            for i in range(1, n_chunks - 1):
+                start_idx = chunk_size * i - overlap_size
+                end_idx = chunk_size * (i + 1) + overlap_size
+                chunks.append(tokens[start_idx:end_idx])
+
+            if len(tokens) - chunk_size * (n_chunks - 1) > threshold:
+                chunks.append(tokens[chunk_size * (n_chunks - 1) - overlap_size :])
+            else:
+                chunks[-1] += tokens[-residue:]
+
+            return (
+                [" ".join(chunk) for chunk in chunks] if not return_tokens else chunks
+            )

lionagi/libs/ln_validate.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 """
 This module provides functions for validating and fixing field values based on their data types.
 

lionagi/libs/special_tokens.py

@@ -0,0 +1,172 @@
+# disallowed special tokens
+
+disallowed_tokens = [
+    "▄",
+    "▅",
+    "▆",
+    "▇",
+    "█",
+    "▏",
+    "▎",
+    "▍",
+    "▌",
+    "▋",
+    "▊",
+    "▉",
+    "▔",
+    "▕",
+    "▁▁",
+    "▁▂",
+    "▁▃",
+    "▁▄",
+    "▁▅",
+    "▁▆",
+    "▁▇",
+    "▁█",
+    "▁▏",
+    "▁▎",
+    "▁▍",
+    "▁▌",
+    "▁▋",
+    "▁▊",
+    "▁▉",
+    "▁▔",
+    "▁▕",
+    "▄▄",
+    "▄▅",
+    "▄▆",
+    "▄▇",
+    "▄█",
+    "▄▏",
+    "▄▎",
+    "▄▍",
+    "▄▌",
+    "▄▋",
+    "▄▊",
+    "▄▉",
+    "▄▔",
+    "▄▕",
+    "▅▅",
+    "▅▆",
+    "▅▇",
+    "▅█",
+    "▅▏",
+    "▅▎",
+    "▅▍",
+    "▅▌",
+    "▅▋",
+    "▅▊",
+    "▅▉",
+    "▅▔",
+    "▅▕",
+    "▆▆",
+    "▆▇",
+    "▆█",
+    "▆▏",
+    "▆▎",
+    "▆▍",
+    "▆▌",
+    "▆▋",
+    "▆▊",
+    "▆▉",
+    "▆▔",
+    "▆▕",
+    "▇▇",
+    "▇█",
+    "▇▏",
+    "▇▎",
+    "▇▍",
+    "▇▌",
+    "▇▋",
+    "▇▊",
+    "▇▉",
+    "▇▔",
+    "▇▕",
+    "██",
+    "█▏",
+    "█▎",
+    "█▍",
+    "█▌",
+    "█▋",
+    "█▊",
+    "█▉",
+    "█",
+    "█▔",
+    "█▕",
+    "▏▏",
+    "▏▎",
+    "▏▍",
+    "▏▌",
+    "▏▋",
+    "▏▊",
+    "▏▉",
+    "▏",
+    "▏▔",
+    "▏▕",
+    "▎▎",
+    "▎▍",
+    "▎▌",
+    "▎▋",
+    "▎▊",
+    "▎▉",
+    "▎",
+    "▎▔",
+    "▎▕",
+    "▍▍",
+    "▍▌",
+    "▍▋",
+    "▍▊",
+    "▍▉",
+    "▍",
+    "▍▔",
+    "▍▕",
+    "▌▌",
+    "▌▋",
+    "▌▊",
+    "▌▉",
+    "▌",
+    "▌▔",
+    "▌▕",
+    "▋▋",
+    "▋▊",
+    "▋▉",
+    "▋",
+    "▋▔",
+    "▋▕",
+    "▊▊",
+    "▊▉",
+    "▊",
+    "▊▔",
+    "▊▕",
+    "▉▉",
+    "▉",
+    "▉▔",
+    "▉▕",
+    "▔▔",
+    "▔▕",
+    "▕▕",
+    "▁▁▁",
+    "▁▁▂",
+    "▁▁▃",
+    "▁▁▄",
+    "▁▁▅",
+    "▁▁▆",
+    "▁▁▇",
+    "▁▁█",
+    "▁▁▏",
+    "▁▁▎",
+    "▁▁▍",
+    "▁▁▌",
+    "▁▁▋",
+    "▁▁▊",
+    "▁▁▉",
+    "▁▁▔",
+    "▁▁▕",
+    "▁▂▂",
+    "▁▂▃",
+    "▁▂▄",
+    "▁▂▅",
+    "▁▂▆",
+    "▁▂▇",
+    "▁▂█",
+]

lionagi/libs/sys_util.py

@@ -1,28 +1,17 @@
 """
-MIT License
-
-Copyright (c) 2023 HaiyangLi quantocean.li@gmail.com
-
-Permission is hereby granted, free of charge, to any person 
-obtaining a copy of this software and associated documentation 
-files (the "Software"), to deal in the Software without 
-restriction, including without limitation the rights to use, 
-copy, modify, merge, publish, distribute, sublicense, and/or 
-sell copies of the Software, and to permit persons to whom 
-the Software is furnished to do so, subject to the following 
-conditions:
-
-The above copyright notice and this permission notice shall be 
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 
-BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 
-ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
-SOFTWARE.
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
 """
 
 import copy
@@ -430,6 +419,19 @@ class SysUtil:
 
     @staticmethod
     def list_files(dir_path: Path | str, extension: str = None) -> list[Path]:
+        """
+        Lists all files in a specified directory with an optional filter for file extensions.
+
+        Args:
+            dir_path (Path | str): The directory path where files are listed.
+            extension (str, optional): Filter files by extension. Default is None, which lists all files.
+
+        Returns:
+            list[Path]: A list of Path objects representing files in the directory.
+
+        Raises:
+            NotADirectoryError: If the provided dir_path is not a directory.
+        """
         dir_path = Path(dir_path)
         if not dir_path.is_dir():
             raise NotADirectoryError(f"{dir_path} is not a directory.")
@@ -440,6 +442,16 @@ class SysUtil:
 
     @staticmethod
     def copy_file(src: Path | str, dest: Path | str) -> None:
+        """
+        Copies a file from a source path to a destination path.
+
+        Args:
+            src (Path | str): The source file path.
+            dest (Path | str): The destination file path.
+
+        Raises:
+            FileNotFoundError: If the source file does not exist or is not a file.
+        """
         from shutil import copy2
 
         src, dest = Path(src), Path(dest)
@@ -450,6 +462,18 @@ class SysUtil:
 
     @staticmethod
     def get_size(path: Path | str) -> int:
+        """
+        Gets the size of a file or total size of files in a directory.
+
+        Args:
+            path (Path | str): The file or directory path.
+
+        Returns:
+            int: The size in bytes.
+
+        Raises:
+            FileNotFoundError: If the path does not exist.
+        """
         path = Path(path)
         if path.is_file():
             return path.stat().st_size
@@ -457,3 +481,50 @@ class SysUtil:
             return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file())
         else:
             raise FileNotFoundError(f"{path} does not exist.")
+
+    @staticmethod
+    def save_to_file(
+        text,
+        directory: Path | str,
+        filename: str,
+        timestamp: bool = True,
+        dir_exist_ok: bool = True,
+        time_prefix: bool = False,
+        custom_timestamp_format: str | None = None,
+        random_hash_digits=0,
+        verbose=True,
+    ):
+        """
+        Saves text to a file within a specified directory, optionally adding a timestamp, hash, and verbose logging.
+
+        Args:
+            text (str): The text to save.
+            directory (Path | str): The directory path to save the file.
+            filename (str): The filename for the saved text.
+            timestamp (bool): If True, append a timestamp to the filename. Default is True.
+            dir_exist_ok (bool): If True, creates the directory if it does not exist. Default is True.
+            time_prefix (bool): If True, prepend the timestamp instead of appending. Default is False.
+            custom_timestamp_format (str | None): A custom format for the timestamp, if None uses default format. Default is None.
+            random_hash_digits (int): Number of random hash digits to append to filename. Default is 0.
+            verbose (bool): If True, prints the file path after saving. Default is True.
+
+        Returns:
+            bool: True if the text was successfully saved.
+        """
+        file_path = SysUtil.create_path(
+            directory=directory,
+            filename=filename,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            time_prefix=time_prefix,
+            custom_timestamp_format=custom_timestamp_format,
+            random_hash_digits=random_hash_digits,
+        )
+
+        with open(file_path, "w") as file:
+            file.write(text)
+
+        if verbose:
+            print(f"Text saved to: {file_path}")
+
+        return True