lionagi 0.5.4__py3-none-any.whl → 0.6.0__py3-none-any.whl

lionagi/libs/{parse/json/schema.py → schema/json_schema.py}

@@ -1,6 +1,10 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Literal
 
-from ..flatten.flatten import flatten
+from ..nested.flatten import flatten
 
 
 def extract_json_schema(

lionagi/libs/validate/common_field_validators.py

@@ -0,0 +1,170 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pydantic import BaseModel, JsonValue
+
+from lionagi.utils import UNDEFINED, copy, to_list
+
+from .validate_boolean import validate_boolean
+
+
+def validate_boolean_field(cls, value, default=None) -> bool | None:
+    """Validate boolean field value.
+
+    Args:
+        cls: The validator class method.
+        value: The value to validate as boolean.
+
+    Returns:
+        bool | None: The validated boolean value or None if invalid.
+    """
+    try:
+        return validate_boolean(value)
+    except Exception:
+        return default
+
+
+def validate_same_dtype_flat_list(
+    cls,
+    value,
+    dtype: type,
+    default=[],
+    dropna: bool = True,
+) -> list:
+    """Validate list of same data type.
+
+    Args:
+        cls: The validator class method.
+        value: The value to validate as a list.
+        dtype: The data type to validate.
+
+    Returns:
+        list: The validated list of same data type.
+
+    Raises:
+        ValueError: If value is not a list or contains different data types.
+    """
+    if value in [None, UNDEFINED, {}]:
+        return default
+
+    to_list_kwargs = {}
+    to_list_kwargs["flatten"] = True
+    to_list_kwargs["use_values"] = True
+    if dropna:
+        to_list_kwargs["dropna"] = True
+    value = to_list(value, **to_list_kwargs)
+
+    if not all(isinstance(i, dtype) for i in value):
+        raise ValueError(f"List must contain only {dtype.__name__} values.")
+
+    return value
+
+
+def validate_nullable_string_field(
+    cls, value, field_name: str = None, strict=True
+) -> str | None:
+    """Validate nullable string field.
+
+    Args:
+        cls: The validator class method.
+        value: The value to validate as a string.
+
+    Returns:
+        str | None: The validated string value or None if invalid.
+    """
+    if value is None or (isinstance(value, str) and not value.strip()):
+        return None
+
+    if not isinstance(value, str):
+        if strict:
+            raise ValueError(f"{field_name or 'Field'} must be a string.")
+        return None
+
+    return value
+
+
+def validate_nullable_jsonvalue_field(cls, value) -> JsonValue | None:
+    """Validates that the instruction is not empty or None and is in the correct format.
+
+    Args:
+        cls: The validator class method.
+        value (JsonValue | None): The instruction value to validate.
+
+    Returns:
+        JsonValue | None: The validated instruction or None if invalid.
+    """
+    if value is None or (isinstance(value, str) and not value.strip()):
+        return None
+    return value
+
+
+def validate_dict_kwargs_params(cls, value) -> dict:
+    """Validate validator kwargs."""
+    if value in [None, UNDEFINED, []]:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError("Validator kwargs must be a dictionary")
+    return value
+
+
+def validate_callable(
+    cls, value, undefind_able: bool = True, check_name: bool = False
+) -> callable:
+    """Validate strict callable function.
+
+    Args:
+        cls: The validator class method.
+        value: The value to validate as a callable function.
+
+    Returns:
+        callable: The validated callable function.
+
+    Raises:
+        ValueError: If value is not callable.
+    """
+    if not callable(value):
+        if undefind_able and value in [None, UNDEFINED]:
+            pass
+        else:
+            raise ValueError("Value must be a callable function")
+    if check_name and not hasattr(value, "__name__"):
+        raise ValueError("Function must have a name.")
+    return value
+
+
+def validate_model_to_type(cls, value):
+    if value is None:
+        return BaseModel
+    if isinstance(value, type) and issubclass(value, BaseModel):
+        return value
+    if isinstance(value, BaseModel):
+        return value.__class__
+    raise ValueError("Base must be a BaseModel subclass or instance.")
+
+
+def validate_list_dict_str_keys(cls, value):
+    if value is None:
+        return []
+    if isinstance(value, dict):
+        value = list(value.keys())
+    if isinstance(value, set | tuple):
+        value = list(value)
+    if isinstance(value, list):
+        if not all(isinstance(i, str) for i in value):
+            raise ValueError("Field names must be strings.")
+        return copy(value)
+    raise ValueError("Fields must be a list, set, or dictionary.")
+
+
+def validate_str_str_dict(cls, value):
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError("Field must be a dictionary.")
+    for k, v in value.items():
+        if not isinstance(k, str):
+            raise ValueError("Field names must be strings.")
+        if not isinstance(v, str):
+            raise ValueError("Field value must be strings.")
+    return value

lionagi/libs/{parse/validate/keys.py → validate/fuzzy_match_keys.py}

@@ -1,20 +1,54 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from collections.abc import Callable, Sequence
-from typing import Any, Literal, TypedDict
+from typing import Any, Literal
+
+from lionagi.utils import KeysDict, Params
 
-from ...string_similarity import (
+from .string_similarity import (
     SIMILARITY_ALGO_MAP,
     SIMILARITY_TYPE,
     string_similarity,
 )
 
-
-class KeysDict(TypedDict):
-    """Dictionary mapping keys to their expected types."""
-
-    pass
+__all__ = (
+    "fuzzy_match_keys",
+    "FuzzyMatchKeysParams",
+)
 
 
-def validate_keys(
+class FuzzyMatchKeysParams(Params):
+    similarity_algo: SIMILARITY_TYPE | Callable[[str, str], float] = (
+        "jaro_winkler"
+    )
+    similarity_threshold: float = 0.85
+    fuzzy_match: bool = True
+    handle_unmatched: Literal["ignore", "raise", "remove", "fill", "force"] = (
+        "ignore"
+    )
+    fill_value: Any = None
+    fill_mapping: dict[str, Any] | None = None
+    strict: bool = False
+
+    def __call__(
+        self, d_: dict[str, Any], keys: Sequence[str] | KeysDict
+    ) -> dict[str, Any]:
+        return fuzzy_match_keys(
+            d_,
+            keys,
+            similarity_algo=self.similarity_algo,
+            similarity_threshold=self.similarity_threshold,
+            fuzzy_match=self.fuzzy_match,
+            handle_unmatched=self.handle_unmatched,
+            fill_value=self.fill_value,
+            fill_mapping=self.fill_mapping,
+            strict=self.strict,
+        )
+
+
+def fuzzy_match_keys(
     d_: dict[str, Any],
     keys: Sequence[str] | KeysDict,
     /,

lionagi/libs/{parse/validate/mapping.py → validate/fuzzy_validate_mapping.py}

@@ -1,13 +1,48 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from collections.abc import Callable, Sequence
 from typing import Any, Literal
 
-from ...string_similarity import SIMILARITY_TYPE
-from ..json.to_json import to_json
-from ..type_convert.to_dict import to_dict
-from .keys import KeysDict, validate_keys
+from lionagi.utils import KeysDict, Params, to_dict, to_json
+
+from .fuzzy_match_keys import fuzzy_match_keys
+from .string_similarity import SIMILARITY_TYPE
+
+
+class FuzzyValidateMappingParams(Params):
+    similarity_algo: SIMILARITY_TYPE | Callable[[str, str], float] = (
+        "jaro_winkler"
+    )
+    similarity_threshold: float = 0.85
+    fuzzy_match: bool = True
+    handle_unmatched: Literal["ignore", "raise", "remove", "fill", "force"] = (
+        "ignore"
+    )
+    fill_value: Any = None
+    fill_mapping: dict[str, Any] | None = None
+    strict: bool = False
+    suppress_conversion_errors: bool = False
+
+    def __call__(
+        self, d_: dict[str, Any], keys: Sequence[str] | KeysDict
+    ) -> dict[str, Any]:
+        return fuzzy_validate_mapping(
+            d_,
+            keys,
+            similarity_algo=self.similarity_algo,
+            similarity_threshold=self.similarity_threshold,
+            fuzzy_match=self.fuzzy_match,
+            handle_unmatched=self.handle_unmatched,
+            fill_value=self.fill_value,
+            fill_mapping=self.fill_mapping,
+            strict=self.strict,
+            suppress_conversion_errors=self.suppress_conversion_errors,
+        )
 
 
-def validate_mapping(
+def fuzzy_validate_mapping(
     d: Any,
     keys: Sequence[str] | KeysDict,
     /,
@@ -96,7 +131,7 @@ def validate_mapping(
             raise ValueError(f"Failed to convert input to dictionary: {e}")
 
     # Validate the dictionary
-    return validate_keys(
+    return fuzzy_match_keys(
         dict_input,
         keys,
         similarity_algo=similarity_algo,

lionagi/libs/{string_similarity/algorithms.py → validate/string_similarity.py}

@@ -1,7 +1,14 @@
-"""String similarity calculation algorithms."""
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
 
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
 from difflib import SequenceMatcher
 from itertools import product
+from typing import Literal
+
+__all__ = ("string_similarity",)
 
 
 def cosine_similarity(s1: str, s2: str) -> float:
@@ -217,3 +224,110 @@ SIMILARITY_ALGO_MAP = {
     "hamming": hamming_similarity,
     "cosine": cosine_similarity,
 }
+
+
+SIMILARITY_TYPE = Literal[
+    "jaro_winkler",
+    "levenshtein",
+    "sequence_matcher",
+    "hamming",
+    "cosine",
+]
+
+# Type alias for similarity functions
+SimilarityFunc = Callable[[str, str], float]
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    """Represents a string matching result."""
+
+    word: str
+    score: float
+    index: int
+
+
+def string_similarity(
+    word: str,
+    correct_words: Sequence[str],
+    algorithm: SIMILARITY_TYPE | Callable[[str, str], float] = "jaro_winkler",
+    threshold: float = 0.0,
+    case_sensitive: bool = False,
+    return_most_similar: bool = False,
+) -> str | list[str] | None:
+    """Find similar strings using specified similarity algorithm.
+
+    Args:
+        word: The input string to find matches for
+        correct_words: List of strings to compare against
+        algorithm: Similarity algorithm to use
+        threshold: Minimum similarity score (0.0 to 1.0)
+        case_sensitive: Whether to consider case when matching
+        return_most_similar: Return only the most similar match
+
+    Returns:
+        Matching string(s) or None if no matches found
+
+    Raises:
+        ValueError: If correct_words is empty or threshold is invalid
+    """
+    if not correct_words:
+        raise ValueError("correct_words must not be empty")
+
+    if not 0.0 <= threshold <= 1.0:
+        raise ValueError("threshold must be between 0.0 and 1.0")
+
+    # Convert inputs to strings
+    compare_word = str(word)
+    original_words = [str(w) for w in correct_words]
+
+    # Handle case sensitivity
+    if not case_sensitive:
+        compare_word = compare_word.lower()
+        compare_words = [w.lower() for w in original_words]
+    else:
+        compare_words = original_words.copy()
+
+    # Get scoring function
+    if isinstance(algorithm, str):
+        score_func = SIMILARITY_ALGO_MAP.get(algorithm)
+        if score_func is None:
+            raise ValueError(f"Unsupported algorithm: {algorithm}")
+    elif callable(algorithm):
+        score_func = algorithm
+    else:
+        raise ValueError(
+            "algorithm must be a string specifying a built-in algorithm or "
+            "a callable"
+        )
+
+    # Calculate similarities
+    results = []
+    for idx, (orig_word, comp_word) in enumerate(
+        zip(original_words, compare_words)
+    ):
+        # Skip different length strings for hamming similarity
+        if algorithm == "hamming" and len(comp_word) != len(compare_word):
+            continue
+
+        score = score_func(compare_word, comp_word)
+        if score >= threshold:
+            results.append(MatchResult(orig_word, score, idx))
+
+    # Return None if no matches
+    if not results:
+        return None
+
+    # Sort by score (descending) and index (ascending) for stable ordering
+    results.sort(key=lambda x: (-x.score, x.index))
+
+    # Return results
+    if return_most_similar:
+        return results[0].word
+
+    # Filter exact matches for case sensitive comparisons
+    if case_sensitive:
+        max_score = results[0].score
+        results = [r for r in results if r.score == max_score]
+
+    return [r.word for r in results]

lionagi/libs/{parse/validate/boolean.py → validate/validate_boolean.py}

@@ -1,7 +1,46 @@
-from numbers import Complex
-from typing import Any
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
 
-from lionagi.libs.constants import FALSE_VALUES, TRUE_VALUES
+from numbers import Complex
+from typing import Any, Final
+
+# Define constants for valid boolean string representations
+TRUE_VALUES: Final[frozenset[str]] = frozenset(
+    [
+        "true",
+        "1",
+        "yes",
+        "y",
+        "on",
+        "correct",
+        "t",
+        "enabled",
+        "enable",
+        "active",
+        "activated",
+    ]
+)
+
+FALSE_VALUES: Final[frozenset[str]] = frozenset(
+    [
+        "false",
+        "0",
+        "no",
+        "n",
+        "off",
+        "incorrect",
+        "f",
+        "disabled",
+        "disable",
+        "inactive",
+        "deactivated",
+        "none",
+        "null",
+        "n/a",
+        "na",
+    ]
+)
 
 
 def validate_boolean(x: Any, /) -> bool:

lionagi/operations/__init__.py

@@ -1,3 +1,13 @@
-from .brainstorm import *
-from .plan import *
-from .select import *
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .brainstorm.brainstorm import brainstorm
+from .plan.plan import plan
+from .select.select import select
+
+__all__ = (
+    "brainstorm",
+    "plan",
+    "select",
+)

lionagi/operations/brainstorm/__init__.py

@@ -1,3 +1,3 @@
-from .brainstorm import brainstorm
-
-__all__ = ["brainstorm"]
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0

lionagi/operations/brainstorm/brainstorm.py

@@ -3,18 +3,19 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
-from typing import Literal
-
-from lionagi.core.session.branch import Branch
-from lionagi.core.session.session import Session
-from lionagi.core.typing import ID, Any, BaseModel
-from lionagi.libs.func.types import alcall
-from lionagi.libs.parse import to_flat_list
-from lionagi.protocols.operatives.instruct import (
-    INSTRUCT_FIELD_MODEL,
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+from lionagi.operatives.instruct.instruct import (
+    LIST_INSTRUCT_FIELD_MODEL,
     Instruct,
     InstructResponse,
 )
+from lionagi.protocols.generic.element import ID
+from lionagi.session.branch import Branch
+from lionagi.session.session import Session
+from lionagi.utils import alcall, to_list
 
 from ..utils import prepare_instruct, prepare_session
 from .prompt import PROMPT
@@ -86,7 +87,7 @@ async def run_instruct(
     # Prepare config for the branch operation
     config = {**ins.model_dump(), **kwargs}
     result = await branch.operate(**config)
-    branch.msgs.logger.dump()
+    branch._log_manager.dump()
 
     # Extract any newly generated instructions
     instructs = []
@@ -126,6 +127,7 @@ async def brainstorm(
     return_session: bool = False,
     verbose: bool = False,
     branch_as_default: bool = True,
+    operative_model: type[BaseModel] | None = None,
     **kwargs: Any,
 ) -> Any:
     """
@@ -137,6 +139,13 @@ async def brainstorm(
       3. Optionally explore the resulting instructions (auto_explore=True)
          using the chosen strategy (concurrent, sequential, etc.).
     """
+    if operative_model:
+        logging.warning(
+            "The 'operative_model' parameter is deprecated and will be removed in a future version.use 'response_format' instead."
+        )
+        kwargs["response_format"] = kwargs.get(
+            "response_format", operative_model
+        )
 
     # -----------------------------------------------------------------
     # Basic Validations and Setup
@@ -149,8 +158,8 @@ async def brainstorm(
 
     # Make sure the correct field model is present
     field_models: list = kwargs.get("field_models", [])
-    if INSTRUCT_FIELD_MODEL not in field_models:
-        field_models.append(INSTRUCT_FIELD_MODEL)
+    if LIST_INSTRUCT_FIELD_MODEL not in field_models:
+        field_models.append(LIST_INSTRUCT_FIELD_MODEL)
     kwargs["field_models"] = field_models
 
     # Prepare session, branch, and the instruction
@@ -199,14 +208,18 @@ async def brainstorm(
             brainstorm_results = await alcall(
                 instructs, run_brainstorm_instruction
             )
-            brainstorm_results = to_flat_list(brainstorm_results, dropna=True)
+            brainstorm_results = to_list(
+                brainstorm_results, dropna=True, flatten=True
+            )
 
             # Filter out plain str/dict responses, keep model-based
             filtered = [
                 r if not isinstance(r, (str, dict)) else None
                 for r in brainstorm_results
             ]
-            filtered = to_flat_list(filtered, unique=True, dropna=True)
+            filtered = to_list(
+                filtered, unique=True, dropna=True, flatten=True
+            )
 
             out.brainstorm = (
                 filtered if isinstance(filtered, list) else [filtered]
@@ -220,7 +233,7 @@ async def brainstorm(
         # -----------------------------------------------------------------
         if response_ and auto_explore:
             # Gather all newly generated instructions
-            all_explore_instructs = to_flat_list(
+            all_explore_instructs = to_list(
                 [
                     r.instruct_models
                     for r in response_
@@ -228,6 +241,7 @@ async def brainstorm(
                 ],
                 dropna=True,
                 unique=True,
+                flatten=True,
             )
 
             # Decide how to explore based on the strategy
@@ -246,7 +260,7 @@ async def brainstorm(
                             )
                             print(f"\n-----Exploring Idea-----\n{snippet}")
                         new_branch = session.split(branch)
-                        resp = await new_branch.instruct(
+                        resp = await new_branch._instruct(
                             ins_, **(explore_kwargs or {})
                         )
                         return InstructResponse(instruct=ins_, response=resp)
@@ -293,7 +307,7 @@ async def brainstorm(
                                 else i.guidance
                             )
                             print(f"\n-----Exploring Idea-----\n{snippet}")
-                        seq_res = await branch.instruct(
+                        seq_res = await branch._instruct(
                             i, **(explore_kwargs or {})
                         )
                         explore_results.append(
@@ -323,7 +337,7 @@ async def brainstorm(
 
                         async def _explore(ins_: Instruct):
                             child_branch = session.split(base_branch)
-                            child_resp = await child_branch.instruct(
+                            child_resp = await child_branch._instruct(
                                 ins_, **(explore_kwargs or {})
                             )
                             return InstructResponse(
@@ -386,7 +400,7 @@ async def brainstorm(
                                     f"\n-----Exploring Idea (sequential in chunk)-----\n{snippet}"
                                 )
 
-                            seq_resp = await local_branch.instruct(
+                            seq_resp = await local_branch._instruct(
                                 ins_, **(explore_kwargs or {})
                             )
                             chunk_results.append(

lionagi/operations/brainstorm/prompt.py

@@ -1,3 +1,7 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 PROMPT = """Perform a brainstorm session. Generate {num_instruct} concise and distinct instructions (Instruct), each representing a potential next step. We will run them in parallel under the same context. Ensure each idea:
 
 1. Adheres to project guidelines and standards.

lionagi/operations/plan/__init__.py

@@ -1,3 +1,7 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from .plan import plan
 
 __all__ = ["plan"]