lionagi 0.0.315__py3-none-any.whl → 0.1.0__py3-none-any.whl

lionagi/libs/ln_parse.py

@@ -1,11 +1,14 @@
+from collections.abc import Callable
 import re
 import inspect
 import itertools
-from collections.abc import Callable
+import contextlib
+from functools import singledispatchmethod
 from typing import Any
 import numpy as np
 import lionagi.libs.ln_convert as convert
 
+
 md_json_char_map = {"\n": "\\n", "\r": "\\r", "\t": "\\t", '"': '\\"'}
 
 
@@ -590,14 +593,15 @@ class StringMatch:
         return d[m][n]
 
     @staticmethod
-    def correct_keys(output_fields, out_, score_func=None):
+    def correct_dict_keys(keys: dict | list[str], dict_, score_func=None):
         if score_func is None:
             score_func = StringMatch.jaro_winkler_similarity
-        fields_set = set(output_fields.keys())
+
+        fields_set = set(keys if isinstance(keys, list) else keys.keys())
         corrected_out = {}
         used_keys = set()
 
-        for k, v in out_.items():
+        for k, v in dict_.items():
             if k in fields_set:
                 corrected_out[k] = v
                 fields_set.remove(k)  # Remove the matched key
@@ -614,8 +618,8 @@ class StringMatch:
                 fields_set.remove(best_match)  # Remove the matched key
                 used_keys.add(best_match)
 
-        if len(used_keys) < len(out_):
-            for k, v in out_.items():
+        if len(used_keys) < len(dict_):
+            for k, v in dict_.items():
                 if k not in used_keys:
                     corrected_out[k] = v
 
@@ -637,3 +641,36 @@ class StringMatch:
         # Find the index of the highest score
         max_score_index = np.argmax(scores)
         return correct_words_list[max_score_index]
+
+    @staticmethod
+    def force_validate_dict(x, keys: dict | list[str]) -> dict:
+        out_ = x
+
+        if isinstance(out_, str):
+            # first try to parse it straight as a fuzzy json
+            try:
+                out_ = ParseUtil.fuzzy_parse_json(out_)
+            except Exception:
+                try:
+                    # if failed we try to extract the json block and parse it
+                    out_ = ParseUtil.md_to_json(out_)
+                except Exception:
+                    # if still failed we try to extract the json block using re and parse it again
+                    match = re.search(r"```json\n({.*?})\n```", out_, re.DOTALL)
+                    if match:
+                        out_ = match.group(1)
+                        try:
+                            out_ = ParseUtil.fuzzy_parse_json(out_)
+                        except:
+                            try:
+                                out_ = ParseUtil.fuzzy_parse_json(
+                                    out_.replace("'", '"')
+                                )
+                            except:
+                                pass
+
+        if isinstance(out_, dict):
+            try:
+                return StringMatch.correct_dict_keys(keys, out_)
+            except Exception as e:
+                raise ValueError(f"Failed to force_validate_dict for input: {x}") from e

lionagi/libs/ln_validate.py

@@ -0,0 +1,288 @@
+"""
+This module provides functions for validating and fixing field values based on their data types.
+
+The module defines several functions for checking and fixing field values of different data types,
+including numeric, boolean, string, and enum. It also provides a dictionary `validation_funcs` that
+maps data types to their corresponding validation functions.
+"""
+
+from .ln_convert import to_str, is_same_dtype, to_list, to_dict, to_num, strip_lower
+from .ln_parse import StringMatch, ParseUtil
+
+
+def check_dict_field(x, keys: list[str] | dict, fix_=True, **kwargs):
+    if isinstance(x, dict):
+        return x
+    if fix_:
+        try:
+            x = to_str(x)
+            return StringMatch.force_validate_dict(x, keys=keys, **kwargs)
+        except Exception as e:
+            raise ValueError("Invalid dict field type.") from e
+    raise ValueError(f"Default value for DICT must be a dict, got {type(x).__name__}")
+
+
+def check_action_field(x, fix_=True, **kwargs):
+    if (
+        isinstance(x, list)
+        and is_same_dtype(x, dict)
+        and all(_has_action_keys(y) for y in x)
+    ):
+        return x
+    try:
+        x = _fix_action_field(x, fix_)
+        return x
+    except Exception as e:
+        raise ValueError("Invalid action field type.") from e
+
+
+def check_number_field(x, fix_=True, **kwargs):
+    """
+    Checks if the given value is a valid numeric field.
+
+    Args:
+        x: The value to check.
+        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
+        **kwargs: Additional keyword arguments for fixing the value.
+
+    Returns:
+        The original value if it's valid, or the fixed value if `fix_` is True.
+
+    Raises:
+        ValueError: If the value is not a valid numeric field and cannot be fixed.
+    """
+    if not isinstance(x, (int, float)):
+        if fix_:
+            try:
+                return _fix_number_field(x, **kwargs)
+            except Exception as e:
+                raise e
+
+        raise ValueError(
+            f"Default value for NUMERIC must be an int or float, got {type(x).__name__}"
+        )
+    return x
+
+
+def check_bool_field(x, fix_=True):
+    """
+    Checks if the given value is a valid boolean field.
+
+    Args:
+        x: The value to check.
+        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
+
+    Returns:
+        The original value if it's valid, or the fixed value if `fix_` is True.
+
+    Raises:
+        ValueError: If the value is not a valid boolean field and cannot be fixed.
+    """
+    if not isinstance(x, bool):
+        if fix_:
+            try:
+                return _fix_bool_field(x)
+            except Exception as e:
+                raise e
+
+        raise ValueError(
+            f"Default value for BOOLEAN must be a bool, got {type(x).__name__}"
+        )
+    return x
+
+
+def check_str_field(x, *args, fix_=True, **kwargs):
+    """
+    Checks if the given value is a valid string field.
+
+    Args:
+        x: The value to check.
+        *args: Additional positional arguments for fixing the value.
+        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
+        **kwargs: Additional keyword arguments for fixing the value.
+
+    Returns:
+        The original value if it's valid, or the fixed value if `fix_` is True.
+
+    Raises:
+        ValueError: If the value is not a valid string field and cannot be fixed.
+    """
+    if not isinstance(x, str):
+        if fix_:
+            try:
+                return _fix_str_field(x, *args, **kwargs)
+            except Exception as e:
+                raise e
+
+        raise ValueError(
+            f"Default value for STRING must be a str, got {type(x).__name__}"
+        )
+    return x
+
+
+def check_enum_field(x, choices, fix_=True, **kwargs):
+    """
+    Checks if the given value is a valid enum field.
+
+    Args:
+        x: The value to check.
+        choices: The list of valid choices for the enum field.
+        fix_ (bool): Flag indicating whether to attempt fixing the value if it's invalid (default: True).
+        **kwargs: Additional keyword arguments for fixing the value.
+
+    Returns:
+        The original value if it's valid, or the fixed value if `fix_` is True.
+
+    Raises:
+        ValueError: If the value is not a valid enum field and cannot be fixed.
+    """
+    same_dtype, dtype_ = is_same_dtype(choices, return_dtype=True)
+    if not same_dtype:
+        raise ValueError(
+            f"Field type ENUM requires all choices to be of the same type, got {choices}"
+        )
+
+    if not isinstance(x, dtype_):
+        raise ValueError(
+            f"Default value for ENUM must be an instance of the {dtype_.__name__}, got {type(x).__name__}"
+        )
+
+    if x not in choices:
+        if fix_:
+            try:
+                return _fix_enum_field(x, choices, **kwargs)
+            except Exception as e:
+                raise e
+        raise ValueError(
+            f"Default value for ENUM must be one of the {choices}, got {x}"
+        )
+
+    return x
+
+
+def _has_action_keys(dict_):
+    return list(dict_.keys()) >= ["function", "arguments"]
+
+
+def _fix_action_field(x, discard_=True):
+    corrected = []
+    if isinstance(x, str):
+        x = ParseUtil.fuzzy_parse_json(x)
+
+    try:
+        x = to_list(x)
+
+        for i in x:
+            i = to_dict(i)
+            if _has_action_keys(i):
+                corrected.append(i)
+            elif not discard_:
+                raise ValueError(f"Invalid action field: {i}")
+    except Exception as e:
+        raise ValueError(f"Invalid action field: {e}") from e
+
+    return corrected
+
+
+def _fix_number_field(x, *args, **kwargs):
+    """
+    Attempts to fix an invalid numeric field value.
+
+    Args:
+        x: The value to fix.
+        *args: Additional positional arguments for fixing the value.
+        **kwargs: Additional keyword arguments for fixing the value.
+
+    Returns:
+        The fixed numeric value.
+
+    Raises:
+        ValueError: If the value cannot be converted into a valid numeric value.
+    """
+    try:
+        x = to_num(x, *args, **kwargs)
+        if isinstance(x, (int, float)):
+            return x
+        raise ValueError(f"Failed to convert {x} into a numeric value")
+    except Exception as e:
+        raise ValueError(f"Failed to convert {x} into a numeric value") from e
+
+
+def _fix_bool_field(x):
+    """
+    Attempts to fix an invalid boolean field value.
+
+    Args:
+        x: The value to fix.
+
+    Returns:
+        The fixed boolean value.
+
+    Raises:
+        ValueError: If the value cannot be converted into a valid boolean value.
+    """
+    try:
+        x = strip_lower(to_str(x))
+        if x in ["true", "1", "correct", "yes"]:
+            return True
+
+        elif x in ["false", "0", "incorrect", "no", "none", "n/a"]:
+            return False
+
+        raise ValueError(f"Failed to convert {x} into a boolean value")
+    except Exception as e:
+        raise ValueError(f"Failed to convert {x} into a boolean value") from e
+
+
+def _fix_str_field(x):
+    """
+    Attempts to fix an invalid string field value.
+
+    Args:
+        x: The value to fix.
+
+    Returns:
+        The fixed string value.
+
+    Raises:
+        ValueError: If the value cannot be converted into a valid string value.
+    """
+    try:
+        x = to_str(x)
+        if isinstance(x, str):
+            return x
+        raise ValueError(f"Failed to convert {x} into a string value")
+    except Exception as e:
+        raise ValueError(f"Failed to convert {x} into a string value") from e
+
+
+def _fix_enum_field(x, choices, **kwargs):
+    """
+    Attempts to fix an invalid enum field value.
+
+    Args:
+        x: The value to fix.
+        choices: The list of valid choices for the enum field.
+        **kwargs: Additional keyword arguments for fixing the value.
+
+    Returns:
+        The fixed enum value.
+
+    Raises:
+        ValueError: If the value cannot be converted into a valid enum value.
+    """
+    try:
+        x = to_str(x)
+        return StringMatch.choose_most_similar(x, choices, **kwargs)
+    except Exception as e:
+        raise ValueError(f"Failed to convert {x} into one of the choices") from e
+
+
+validation_funcs = {
+    "number": check_number_field,
+    "bool": check_bool_field,
+    "str": check_str_field,
+    "enum": check_enum_field,
+    "action": check_action_field,
+    "dict": check_dict_field,
+}

lionagi/libs/sys_util.py

@@ -3,11 +3,26 @@ MIT License
 
 Copyright (c) 2023 HaiyangLi quantocean.li@gmail.com
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Permission is hereby granted, free of charge, to any person 
+obtaining a copy of this software and associated documentation 
+files (the "Software"), to deal in the Software without 
+restriction, including without limitation the rights to use, 
+copy, modify, merge, publish, distribute, sublicense, and/or 
+sell copies of the Software, and to permit persons to whom 
+the Software is furnished to do so, subject to the following 
+conditions:
+
+The above copyright notice and this permission notice shall be 
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
+SOFTWARE.
 """
 
 import copy
@@ -363,6 +378,7 @@ class SysUtil:
         dir_exist_ok: bool = True,
         time_prefix: bool = False,
         custom_timestamp_format: str | None = None,
+        random_hash_digits=0,
     ) -> Path:
         """
         Creates a path with an optional timestamp in the specified directory.
@@ -400,7 +416,13 @@ class SysUtil:
         else:
             filename = name
 
-        full_filename = f"{filename}{ext}"
+        random_hash = (
+            "-" + SysUtil.create_id(random_hash_digits)
+            if random_hash_digits > 0
+            else ""
+        )
+
+        full_filename = f"{filename}{random_hash}{ext}"
         full_path = directory / full_filename
         full_path.parent.mkdir(parents=True, exist_ok=dir_exist_ok)