lionagi 0.15.14__py3-none-any.whl → 0.16.1__py3-none-any.whl

lionagi/ln/fuzzy/_fuzzy_match.py

@@ -0,0 +1,172 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any, ClassVar, Literal
+
+from ..types import KeysDict, Params, Unset
+from ._string_similarity import (
+    SIMILARITY_ALGO_MAP,
+    SIMILARITY_TYPE,
+    SimilarityFunc,
+    string_similarity,
+)
+
+__all__ = (
+    "fuzzy_match_keys",
+    "FuzzyMatchKeysParams",
+)
+
+
+HandleUnmatched = Literal["ignore", "raise", "remove", "fill", "force"]
+
+
+def fuzzy_match_keys(
+    d_: dict[str, Any],
+    keys: Sequence[str] | KeysDict,
+    /,
+    *,
+    similarity_algo: SIMILARITY_TYPE | SimilarityFunc = "jaro_winkler",
+    similarity_threshold: float = 0.85,
+    fuzzy_match: bool = True,
+    handle_unmatched: HandleUnmatched = "ignore",
+    fill_value: Any = None,
+    fill_mapping: dict[str, Any] | None = None,
+    strict: bool = False,
+) -> dict[str, Any]:
+    """
+    Validate and correct dictionary keys based on expected keys using string similarity.
+
+    Args:
+        d_: The dictionary to validate and correct keys for.
+        keys: List of expected keys or dictionary mapping keys to types.
+        similarity_algo: String similarity algorithm to use or custom function.
+        similarity_threshold: Minimum similarity score for fuzzy matching.
+        fuzzy_match: If True, use fuzzy matching for key correction.
+        handle_unmatched: Specifies how to handle unmatched keys:
+            - "ignore": Keep unmatched keys in output.
+            - "raise": Raise ValueError if unmatched keys exist.
+            - "remove": Remove unmatched keys from output.
+            - "fill": Fill unmatched keys with default value/mapping.
+            - "force": Combine "fill" and "remove" behaviors.
+        fill_value: Default value for filling unmatched keys.
+        fill_mapping: Dictionary mapping unmatched keys to default values.
+        strict: If True, raise ValueError if any expected key is missing.
+
+    Returns:
+        A new dictionary with validated and corrected keys.
+
+    Raises:
+        ValueError: If validation fails based on specified parameters.
+        TypeError: If input types are invalid.
+        AttributeError: If key validation fails.
+    """
+    # Input validation
+    if not isinstance(d_, dict):
+        raise TypeError("First argument must be a dictionary")
+    if keys is None:
+        raise TypeError("Keys argument cannot be None")
+    if not 0.0 <= similarity_threshold <= 1.0:
+        raise ValueError("similarity_threshold must be between 0.0 and 1.0")
+
+    # Extract expected keys
+    fields_set = set(keys) if isinstance(keys, list) else set(keys.keys())
+    if not fields_set:
+        return d_.copy()  # Return copy of original if no expected keys
+
+    # Initialize output dictionary and tracking sets
+    corrected_out = {}
+    matched_expected = set()
+    matched_input = set()
+
+    # Get similarity function
+    if isinstance(similarity_algo, str):
+        if similarity_algo not in SIMILARITY_ALGO_MAP:
+            raise ValueError(
+                f"Unknown similarity algorithm: {similarity_algo}"
+            )
+        similarity_func = SIMILARITY_ALGO_MAP[similarity_algo]
+    else:
+        similarity_func = similarity_algo
+
+    # First pass: exact matches
+    for key in d_:
+        if key in fields_set:
+            corrected_out[key] = d_[key]
+            matched_expected.add(key)
+            matched_input.add(key)
+
+    # Second pass: fuzzy matching if enabled
+    if fuzzy_match:
+        remaining_input = set(d_.keys()) - matched_input
+        remaining_expected = fields_set - matched_expected
+
+        for key in remaining_input:
+            if not remaining_expected:
+                break
+
+            matches = string_similarity(
+                key,
+                list(remaining_expected),
+                algorithm=similarity_func,
+                threshold=similarity_threshold,
+                return_most_similar=True,
+            )
+
+            if matches:
+                match = matches
+                corrected_out[match] = d_[key]
+                matched_expected.add(match)
+                matched_input.add(key)
+                remaining_expected.remove(match)
+            elif handle_unmatched == "ignore":
+                corrected_out[key] = d_[key]
+
+    # Handle unmatched keys based on handle_unmatched parameter
+    unmatched_input = set(d_.keys()) - matched_input
+    unmatched_expected = fields_set - matched_expected
+
+    if handle_unmatched == "raise" and unmatched_input:
+        raise ValueError(f"Unmatched keys found: {unmatched_input}")
+
+    elif handle_unmatched == "ignore":
+        for key in unmatched_input:
+            corrected_out[key] = d_[key]
+
+    elif handle_unmatched in ("fill", "force"):
+        # Fill missing expected keys
+        for key in unmatched_expected:
+            if fill_mapping and key in fill_mapping:
+                corrected_out[key] = fill_mapping[key]
+            else:
+                corrected_out[key] = fill_value
+
+        # For "fill" mode, also keep unmatched original keys
+        if handle_unmatched == "fill":
+            for key in unmatched_input:
+                corrected_out[key] = d_[key]
+
+    # Check strict mode
+    if strict and unmatched_expected:
+        raise ValueError(f"Missing required keys: {unmatched_expected}")
+
+    return corrected_out
+
+
+@dataclass(slots=True, init=False, frozen=True)
+class FuzzyMatchKeysParams(Params):
+    _none_as_sentinel: ClassVar[bool] = False
+    _func: ClassVar[Any] = fuzzy_match_keys
+
+    similarity_algo: SIMILARITY_TYPE | SimilarityFunc = "jaro_winkler"
+    similarity_threshold: float = 0.85
+
+    fuzzy_match: bool = True
+    handle_unmatched: HandleUnmatched = "ignore"
+
+    fill_value: Any = Unset
+    fill_mapping: dict[str, Any] | Any = Unset
+    strict: bool = False
+
+    def __call__(
+        self, d_: dict[str, Any], keys: Sequence[str] | KeysDict
+    ) -> dict[str, Any]:
+        return fuzzy_match_keys(d_, keys, **self.default_kw())

lionagi/ln/fuzzy/_fuzzy_validate.py

@@ -0,0 +1,46 @@
+from pydantic import BaseModel
+
+from lionagi._errors import ValidationError
+
+from ._extract_json import extract_json
+from ._fuzzy_match import FuzzyMatchKeysParams, fuzzy_match_keys
+
+__all__ = ("fuzzy_validate_pydantic",)
+
+
+def fuzzy_validate_pydantic(
+    text,
+    /,
+    model_type: type[BaseModel],
+    fuzzy_parse: bool = True,
+    fuzzy_match: bool = False,
+    fuzzy_match_params: FuzzyMatchKeysParams | dict = None,
+):
+    try:
+        model_data = extract_json(text, fuzzy_parse=fuzzy_parse)
+    except Exception as e:
+        raise ValidationError(
+            f"Failed to extract valid JSON from model response: {e}"
+        ) from e
+
+    d = model_data
+    if fuzzy_match:
+        if fuzzy_match_params is None:
+            model_data = fuzzy_match_keys(
+                d, model_type.model_fields, handle_unmatched="remove"
+            )
+        elif isinstance(fuzzy_match_params, dict):
+            model_data = fuzzy_match_keys(
+                d, model_type.model_fields, **fuzzy_match_params
+            )
+        elif isinstance(fuzzy_match_params, FuzzyMatchKeysParams):
+            model_data = fuzzy_match_params(d, model_type.model_fields)
+        else:
+            raise TypeError(
+                "fuzzy_keys_params must be a dict or FuzzyMatchKeysParams instance"
+            )
+
+    try:
+        return model_type.model_validate(model_data)
+    except Exception as e:
+        raise ValidationError(f"Validation failed: {e}") from e

lionagi/ln/fuzzy/_string_similarity.py

@@ -0,0 +1,332 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+from itertools import product
+from typing import Literal
+
+__all__ = ("string_similarity",)
+
+
+def cosine_similarity(s1: str, s2: str) -> float:
+    """Calculate the cosine similarity between two strings.
+
+    Args:
+        s1: First input string
+        s2: Second input string
+
+    Returns:
+        float: Cosine similarity score between 0 and 1
+    """
+    if not s1 or not s2:
+        return 0.0
+
+    set1, set2 = set(s1), set(s2)
+    intersection = set1.intersection(set2)
+
+    if not set1 or not set2:
+        return 0.0
+
+    return len(intersection) / ((len(set1) * len(set2)) ** 0.5)
+
+
+def hamming_similarity(s1: str, s2: str) -> float:
+    """Calculate the Hamming similarity between two strings.
+
+    The strings must be of equal length. Returns the proportion of positions
+    at which corresponding symbols are the same.
+
+    Args:
+        s1: First input string
+        s2: Second input string
+
+    Returns:
+        float: Hamming similarity score between 0 and 1
+    """
+    if not s1 or not s2 or len(s1) != len(s2):
+        return 0.0
+
+    matches = sum(c1 == c2 for c1, c2 in zip(s1, s2))
+    return matches / len(s1)
+
+
+def jaro_distance(s: str, t: str) -> float:
+    """Calculate the Jaro distance between two strings.
+
+    Args:
+        s: First input string
+        t: Second input string
+
+    Returns:
+        float: Jaro distance score between 0 and 1
+    """
+    s_len = len(s)
+    t_len = len(t)
+
+    if s_len == 0 and t_len == 0:
+        return 1.0
+    elif s_len == 0 or t_len == 0:
+        return 0.0
+
+    match_distance = (max(s_len, t_len) // 2) - 1
+    match_distance = max(0, match_distance)  # Ensure non-negative
+
+    s_matches = [False] * s_len
+    t_matches = [False] * t_len
+
+    matches = 0
+    transpositions = 0
+
+    # Identify matches
+    for i in range(s_len):
+        start = max(0, i - match_distance)
+        end = min(i + match_distance + 1, t_len)
+
+        for j in range(start, end):
+            if t_matches[j] or s[i] != t[j]:
+                continue
+            s_matches[i] = t_matches[j] = True
+            matches += 1
+            break
+
+    if matches == 0:
+        return 0.0
+
+    # Count transpositions
+    k = 0
+    for i in range(s_len):
+        if not s_matches[i]:
+            continue
+        while not t_matches[k]:
+            k += 1
+        if s[i] != t[k]:
+            transpositions += 1
+        k += 1
+
+    transpositions //= 2
+
+    return (
+        matches / s_len
+        + matches / t_len
+        + (matches - transpositions) / matches
+    ) / 3.0
+
+
+def jaro_winkler_similarity(s: str, t: str, scaling: float = 0.1) -> float:
+    """Calculate the Jaro-Winkler similarity between two strings.
+
+    Args:
+        s: First input string
+        t: Second input string
+        scaling: Scaling factor for common prefix adjustment
+
+    Returns:
+        float: Jaro-Winkler similarity score between 0 and 1
+
+    Raises:
+        ValueError: If scaling factor is not between 0 and 0.25
+    """
+    if not 0 <= scaling <= 0.25:
+        raise ValueError("Scaling factor must be between 0 and 0.25")
+
+    jaro_sim = jaro_distance(s, t)
+
+    # Find length of common prefix (up to 4 chars)
+    prefix_len = 0
+    for s_char, t_char in zip(s, t):
+        if s_char != t_char:
+            break
+        prefix_len += 1
+        if prefix_len == 4:
+            break
+
+    return jaro_sim + (prefix_len * scaling * (1 - jaro_sim))
+
+
+def levenshtein_distance(a: str, b: str) -> int:
+    """Calculate the Levenshtein (edit) distance between two strings.
+
+    Args:
+        a: First input string
+        b: Second input string
+
+    Returns:
+        int: Minimum number of single-character edits needed to change one
+             string into the other
+    """
+    if not a:
+        return len(b)
+    if not b:
+        return len(a)
+
+    m, n = len(a), len(b)
+    d = [[0] * (n + 1) for _ in range(m + 1)]
+
+    for i in range(m + 1):
+        d[i][0] = i
+    for j in range(n + 1):
+        d[0][j] = j
+
+    for i, j in product(range(1, m + 1), range(1, n + 1)):
+        cost = 0 if a[i - 1] == b[j - 1] else 1
+        d[i][j] = min(
+            d[i - 1][j] + 1,  # deletion
+            d[i][j - 1] + 1,  # insertion
+            d[i - 1][j - 1] + cost,  # substitution
+        )
+
+    return d[m][n]
+
+
+def levenshtein_similarity(s1: str, s2: str) -> float:
+    """Calculate the Levenshtein similarity between two strings.
+
+    Converts Levenshtein distance to a similarity score between 0 and 1.
+
+    Args:
+        s1: First input string
+        s2: Second input string
+
+    Returns:
+        float: Levenshtein similarity score between 0 and 1
+    """
+    if not s1 and not s2:
+        return 1.0
+    if not s1 or not s2:
+        return 0.0
+
+    distance = levenshtein_distance(s1, s2)
+    max_len = max(len(s1), len(s2))
+    return 1 - (distance / max_len)
+
+
+def sequence_matcher_similarity(s1: str, s2: str) -> float:
+    """Calculate similarity using Python's SequenceMatcher.
+
+    Args:
+        s1: First input string
+        s2: Second input string
+
+    Returns:
+        float: Similarity score between 0 and 1
+    """
+    return SequenceMatcher(None, s1, s2).ratio()
+
+
+# Map of available similarity algorithms
+SIMILARITY_ALGO_MAP = {
+    "jaro_winkler": jaro_winkler_similarity,
+    "levenshtein": levenshtein_similarity,
+    "sequence_matcher": sequence_matcher_similarity,
+    "hamming": hamming_similarity,
+    "cosine": cosine_similarity,
+}
+
+
+SIMILARITY_TYPE = Literal[
+    "jaro_winkler",
+    "levenshtein",
+    "sequence_matcher",
+    "hamming",
+    "cosine",
+]
+
+# Type alias for similarity functions
+SimilarityFunc = Callable[[str, str], float]
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    """Represents a string matching result."""
+
+    word: str
+    score: float
+    index: int
+
+
+def string_similarity(
+    word: str,
+    correct_words: Sequence[str],
+    algorithm: SIMILARITY_TYPE | Callable[[str, str], float] = "jaro_winkler",
+    threshold: float = 0.0,
+    case_sensitive: bool = False,
+    return_most_similar: bool = False,
+) -> str | list[str] | None:
+    """Find similar strings using specified similarity algorithm.
+
+    Args:
+        word: The input string to find matches for
+        correct_words: List of strings to compare against
+        algorithm: Similarity algorithm to use
+        threshold: Minimum similarity score (0.0 to 1.0)
+        case_sensitive: Whether to consider case when matching
+        return_most_similar: Return only the most similar match
+
+    Returns:
+        Matching string(s) or None if no matches found
+
+    Raises:
+        ValueError: If correct_words is empty or threshold is invalid
+    """
+    if not correct_words:
+        raise ValueError("correct_words must not be empty")
+
+    if not 0.0 <= threshold <= 1.0:
+        raise ValueError("threshold must be between 0.0 and 1.0")
+
+    # Convert inputs to strings
+    compare_word = str(word)
+    original_words = [str(w) for w in correct_words]
+
+    # Handle case sensitivity
+    if not case_sensitive:
+        compare_word = compare_word.lower()
+        compare_words = [w.lower() for w in original_words]
+    else:
+        compare_words = original_words.copy()
+
+    # Get scoring function
+    if isinstance(algorithm, str):
+        score_func = SIMILARITY_ALGO_MAP.get(algorithm)
+        if score_func is None:
+            raise ValueError(f"Unsupported algorithm: {algorithm}")
+    elif callable(algorithm):
+        score_func = algorithm
+    else:
+        raise ValueError(
+            "algorithm must be a string specifying a built-in algorithm or a callable"
+        )
+
+    # Calculate similarities
+    results = []
+    for idx, (orig_word, comp_word) in enumerate(
+        zip(original_words, compare_words)
+    ):
+        # Skip different length strings for hamming similarity
+        if algorithm == "hamming" and len(comp_word) != len(compare_word):
+            continue
+
+        score = score_func(compare_word, comp_word)
+        if score >= threshold:
+            results.append(MatchResult(orig_word, score, idx))
+
+    # Return None if no matches
+    if not results:
+        return None
+
+    # Sort by score (descending) and index (ascending) for stable ordering
+    results.sort(key=lambda x: (-x.score, x.index))
+
+    # Filter exact matches for case sensitive comparisons
+    if case_sensitive:
+        max_score = results[0].score
+        results = [r for r in results if r.score == max_score]
+
+    # Return results
+    if return_most_similar:
+        return results[0].word
+
+    return [r.word for r in results]

lionagi/ln/{_models.py → types.py}

@@ -1,11 +1,160 @@
+from __future__ import annotations
+
 from dataclasses import dataclass, field
-from typing import Any, ClassVar
+from enum import Enum as _Enum
+from typing import Any, ClassVar, Final, Literal, TypeVar, Union
+
+from typing_extensions import TypedDict, override
+
+__all__ = (
+    "Undefined",
+    "Unset",
+    "MaybeUndefined",
+    "MaybeUnset",
+    "MaybeSentinel",
+    "SingletonType",
+    "UndefinedType",
+    "UnsetType",
+    "KeysDict",
+    "T",
+    "Enum",
+    "is_sentinel",
+    "not_sentinel",
+    "Params",
+    "DataClass",
+)
+
+T = TypeVar("T")
+
+
+class _SingletonMeta(type):
+    """Metaclass that guarantees exactly one instance per subclass.
+
+    This ensures that sentinel values maintain identity across the entire application,
+    allowing safe identity checks with 'is' operator.
+    """
+
+    _cache: dict[type, SingletonType] = {}
+
+    def __call__(cls, *a, **kw):
+        if cls not in cls._cache:
+            cls._cache[cls] = super().__call__(*a, **kw)
+        return cls._cache[cls]
+
+
+class SingletonType(metaclass=_SingletonMeta):
+    """Base class for singleton sentinel types.
+
+    Provides consistent interface for sentinel values with:
+    - Identity preservation across deepcopy
+    - Falsy boolean evaluation
+    - Clear string representation
+    """
+
+    __slots__: tuple[str, ...] = ()
+
+    def __deepcopy__(self, memo):  # copy & deepcopy both noop
+        return self
+
+    def __copy__(self):
+        return self
+
+    # concrete classes *must* override the two methods below
+    def __bool__(self) -> bool: ...
+    def __repr__(self) -> str: ...
+
+
+class UndefinedType(SingletonType):
+    """Sentinel for a key or field entirely missing from a namespace.
+
+    Use this when:
+    - A field has never been set
+    - A key doesn't exist in a mapping
+    - A value is conceptually undefined (not just unset)
+
+    Example:
+        >>> d = {"a": 1}
+        >>> d.get("b", Undefined) is Undefined
+        True
+    """
+
+    __slots__ = ()
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    def __repr__(self) -> Literal["Undefined"]:
+        return "Undefined"
+
+    def __str__(self) -> Literal["Undefined"]:
+        return "Undefined"
+
+    def __reduce__(self):
+        """Ensure pickle preservation of singleton identity."""
+        return "Undefined"
+
+
+class UnsetType(SingletonType):
+    """Sentinel for a key present but value not yet provided.
+
+    Use this when:
+    - A parameter exists but hasn't been given a value
+    - Distinguishing between None and "not provided"
+    - API parameters that are optional but need explicit handling
+
+    Example:
+        >>> def func(param=Unset):
+        ...     if param is not Unset:
+        ...         # param was explicitly provided
+        ...         process(param)
+    """
+
+    __slots__ = ()
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    def __repr__(self) -> Literal["Unset"]:
+        return "Unset"
+
+    def __str__(self) -> Literal["Unset"]:
+        return "Unset"
+
+    def __reduce__(self):
+        """Ensure pickle preservation of singleton identity."""
+        return "Unset"
+
+
+Undefined: Final = UndefinedType()
+"""A key or field entirely missing from a namespace"""
+Unset: Final = UnsetType()
+"""A key present but value not yet provided."""
+
+MaybeUndefined = Union[T, UndefinedType]
+MaybeUnset = Union[T, UnsetType]
+MaybeSentinel = Union[T, UndefinedType, UnsetType]
+
+
+def is_sentinel(value: Any) -> bool:
+    """Check if a value is any sentinel (Undefined or Unset)."""
+    return value is Undefined or value is Unset
+
+
+def not_sentinel(value: Any) -> bool:
+    """Check if a value is NOT a sentinel. Useful for filtering operations."""
+    return value is not Undefined and value is not Unset
+
+
+class Enum(_Enum):
+    @classmethod
+    def allowed(cls) -> tuple[str, ...]:
+        return tuple(e.value for e in cls)
 
-from typing_extensions import override
 
-from ._types import Undefined, Unset, is_sentinel
+class KeysDict(TypedDict, total=False):
+    """TypedDict for keys dictionary."""
 
-__all__ = ("Params", "DataClass")
+    key: Any  # Represents any key-type pair
 
 
 @dataclass(slots=True, frozen=True, init=False)