lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/libs/parse/validate/keys.py

@@ -0,0 +1,150 @@
+from collections.abc import Callable, Sequence
+from typing import Any, Literal, TypedDict
+
+from ...string_similarity import (
+    SIMILARITY_ALGO_MAP,
+    SIMILARITY_TYPE,
+    string_similarity,
+)
+
+
+class KeysDict(TypedDict):
+    """Dictionary mapping keys to their expected types."""
+
+    pass
+
+
+def validate_keys(
+    d_: dict[str, Any],
+    keys: Sequence[str] | KeysDict,
+    /,
+    *,
+    similarity_algo: (
+        SIMILARITY_TYPE | Callable[[str, str], float]
+    ) = "jaro_winkler",
+    similarity_threshold: float = 0.85,
+    fuzzy_match: bool = True,
+    handle_unmatched: Literal[
+        "ignore", "raise", "remove", "fill", "force"
+    ] = "ignore",
+    fill_value: Any = None,
+    fill_mapping: dict[str, Any] | None = None,
+    strict: bool = False,
+) -> dict[str, Any]:
+    """
+    Validate and correct dictionary keys based on expected keys using string similarity.
+
+    Args:
+        d_: The dictionary to validate and correct keys for.
+        keys: List of expected keys or dictionary mapping keys to types.
+        similarity_algo: String similarity algorithm to use or custom function.
+        similarity_threshold: Minimum similarity score for fuzzy matching.
+        fuzzy_match: If True, use fuzzy matching for key correction.
+        handle_unmatched: Specifies how to handle unmatched keys:
+            - "ignore": Keep unmatched keys in output.
+            - "raise": Raise ValueError if unmatched keys exist.
+            - "remove": Remove unmatched keys from output.
+            - "fill": Fill unmatched keys with default value/mapping.
+            - "force": Combine "fill" and "remove" behaviors.
+        fill_value: Default value for filling unmatched keys.
+        fill_mapping: Dictionary mapping unmatched keys to default values.
+        strict: If True, raise ValueError if any expected key is missing.
+
+    Returns:
+        A new dictionary with validated and corrected keys.
+
+    Raises:
+        ValueError: If validation fails based on specified parameters.
+        TypeError: If input types are invalid.
+        AttributeError: If key validation fails.
+    """
+    # Input validation
+    if not isinstance(d_, dict):
+        raise TypeError("First argument must be a dictionary")
+    if keys is None:
+        raise TypeError("Keys argument cannot be None")
+    if not 0.0 <= similarity_threshold <= 1.0:
+        raise ValueError("similarity_threshold must be between 0.0 and 1.0")
+
+    # Extract expected keys
+    fields_set = set(keys) if isinstance(keys, list) else set(keys.keys())
+    if not fields_set:
+        return d_.copy()  # Return copy of original if no expected keys
+
+    # Initialize output dictionary and tracking sets
+    corrected_out = {}
+    matched_expected = set()
+    matched_input = set()
+
+    # Get similarity function
+    if isinstance(similarity_algo, str):
+        if similarity_algo not in SIMILARITY_ALGO_MAP:
+            raise ValueError(
+                f"Unknown similarity algorithm: {similarity_algo}"
+            )
+        similarity_func = SIMILARITY_ALGO_MAP[similarity_algo]
+    else:
+        similarity_func = similarity_algo
+
+    # First pass: exact matches
+    for key in d_:
+        if key in fields_set:
+            corrected_out[key] = d_[key]
+            matched_expected.add(key)
+            matched_input.add(key)
+
+    # Second pass: fuzzy matching if enabled
+    if fuzzy_match:
+        remaining_input = set(d_.keys()) - matched_input
+        remaining_expected = fields_set - matched_expected
+
+        for key in remaining_input:
+            if not remaining_expected:
+                break
+
+            matches = string_similarity(
+                key,
+                list(remaining_expected),
+                algorithm=similarity_func,
+                threshold=similarity_threshold,
+                return_most_similar=True,
+            )
+
+            if matches:
+                match = matches
+                corrected_out[match] = d_[key]
+                matched_expected.add(match)
+                matched_input.add(key)
+                remaining_expected.remove(match)
+            elif handle_unmatched == "ignore":
+                corrected_out[key] = d_[key]
+
+    # Handle unmatched keys based on handle_unmatched parameter
+    unmatched_input = set(d_.keys()) - matched_input
+    unmatched_expected = fields_set - matched_expected
+
+    if handle_unmatched == "raise" and unmatched_input:
+        raise ValueError(f"Unmatched keys found: {unmatched_input}")
+
+    elif handle_unmatched == "ignore":
+        for key in unmatched_input:
+            corrected_out[key] = d_[key]
+
+    elif handle_unmatched in ("fill", "force"):
+        # Fill missing expected keys
+        for key in unmatched_expected:
+            if fill_mapping and key in fill_mapping:
+                corrected_out[key] = fill_mapping[key]
+            else:
+                corrected_out[key] = fill_value
+
+        # For "fill" mode, also keep unmatched original keys
+        if handle_unmatched == "fill":
+            for key in unmatched_input:
+                corrected_out[key] = d_[key]
+
+    # Check strict mode
+    if strict and unmatched_expected:
+        raise ValueError(f"Missing required keys: {unmatched_expected}")
+
+    return corrected_out

lionagi/libs/parse/validate/mapping.py

@@ -0,0 +1,109 @@
+from collections.abc import Callable, Sequence
+from typing import Any, Literal
+
+from ...string_similarity import SIMILARITY_TYPE
+from ..json.to_json import to_json
+from ..type_convert.to_dict import to_dict
+from .keys import KeysDict, validate_keys
+
+
+def validate_mapping(
+    d: Any,
+    keys: Sequence[str] | KeysDict,
+    /,
+    *,
+    similarity_algo: (
+        SIMILARITY_TYPE | Callable[[str, str], float]
+    ) = "jaro_winkler",
+    similarity_threshold: float = 0.85,
+    fuzzy_match: bool = True,
+    handle_unmatched: Literal[
+        "ignore", "raise", "remove", "fill", "force"
+    ] = "ignore",
+    fill_value: Any = None,
+    fill_mapping: dict[str, Any] | None = None,
+    strict: bool = False,
+    suppress_conversion_errors: bool = False,
+) -> dict[str, Any]:
+    """
+    Validate and correct any input into a dictionary with expected keys.
+
+    Args:
+        d: Input to validate. Can be:
+            - Dictionary
+            - JSON string or markdown code block
+            - XML string
+            - Object with to_dict/model_dump method
+            - Any type convertible to dictionary
+        keys: List of expected keys or dictionary mapping keys to types.
+        similarity_algo: String similarity algorithm or custom function.
+        similarity_threshold: Minimum similarity score for fuzzy matching.
+        fuzzy_match: If True, use fuzzy matching for key correction.
+        handle_unmatched: How to handle unmatched keys:
+            - "ignore": Keep unmatched keys
+            - "raise": Raise error for unmatched keys
+            - "remove": Remove unmatched keys
+            - "fill": Fill missing keys with default values
+            - "force": Combine "fill" and "remove" behaviors
+        fill_value: Default value for filling unmatched keys.
+        fill_mapping: Dictionary mapping keys to default values.
+        strict: Raise error if any expected key is missing.
+        suppress_conversion_errors: Return empty dict on conversion errors.
+
+    Returns:
+        Validated and corrected dictionary.
+
+    Raises:
+        ValueError: If input cannot be converted or validation fails.
+        TypeError: If input types are invalid.
+    """
+    if d is None:
+        raise TypeError("Input cannot be None")
+
+    # Try converting to dictionary
+    try:
+        if isinstance(d, str):
+            # First try to_json for JSON strings and code blocks
+            try:
+                json_result = to_json(d)
+                dict_input = (
+                    json_result[0]
+                    if isinstance(json_result, list)
+                    else json_result
+                )
+            except Exception:
+                # Fall back to to_dict for other string formats
+                dict_input = to_dict(
+                    d, str_type="json", fuzzy_parse=True, suppress=True
+                )
+        else:
+            dict_input = to_dict(
+                d, use_model_dump=True, fuzzy_parse=True, suppress=True
+            )
+
+        if not isinstance(dict_input, dict):
+            if suppress_conversion_errors:
+                dict_input = {}
+            else:
+                raise ValueError(
+                    f"Failed to convert input to dictionary: {type(dict_input)}"
+                )
+
+    except Exception as e:
+        if suppress_conversion_errors:
+            dict_input = {}
+        else:
+            raise ValueError(f"Failed to convert input to dictionary: {e}")
+
+    # Validate the dictionary
+    return validate_keys(
+        dict_input,
+        keys,
+        similarity_algo=similarity_algo,
+        similarity_threshold=similarity_threshold,
+        fuzzy_match=fuzzy_match,
+        handle_unmatched=handle_unmatched,
+        fill_value=fill_value,
+        fill_mapping=fill_mapping,
+        strict=strict,
+    )

lionagi/libs/parse/validate/params.py

@@ -0,0 +1,62 @@
+from collections.abc import Callable
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+from lionagi.libs.string_similarity import SIMILARITY_TYPE
+from lionagi.libs.string_similarity.algorithms import SIMILARITY_ALGO_MAP
+
+
+class ValidateKeysParams(BaseModel):
+    """Parameters for validate_keys function.
+
+    Attributes:
+        similarity_algo: String similarity algorithm to use or custom function
+        similarity_threshold: Minimum similarity score for fuzzy matching
+        fuzzy_match: If True, use fuzzy matching for key correction
+        handle_unmatched: How to handle unmatched keys
+        fill_value: Default value for filling unmatched keys
+        fill_mapping: Dictionary mapping unmatched keys to default values
+        strict: If True, raise ValueError if any expected key is missing
+    """
+
+    similarity_algo: SIMILARITY_TYPE | Callable[[str, str], float] = Field(
+        default="jaro_winkler",
+        description="String similarity algorithm to use or custom function",
+    )
+    similarity_threshold: float = Field(
+        default=0.85,
+        description="Minimum similarity score for fuzzy matching",
+        ge=0.0,
+        le=1.0,
+    )
+    fuzzy_match: bool = Field(
+        default=True,
+        description="If True, use fuzzy matching for key correction",
+    )
+    handle_unmatched: Literal["ignore", "raise", "remove", "fill", "force"] = (
+        Field(
+            default="ignore",
+            description="How to handle unmatched keys",
+        )
+    )
+    fill_value: Any = Field(
+        default=None,
+        description="Default value for filling unmatched keys",
+    )
+    fill_mapping: dict[str, Any] | None = Field(
+        default=None,
+        description="Dictionary mapping unmatched keys to default values",
+    )
+    strict: bool = Field(
+        default=False,
+        description="If True, raise ValueError if any expected key is missing",
+    )
+
+    @field_validator("similarity_algo")
+    def validate_similarity_algo(
+        cls, v: SIMILARITY_TYPE | Callable[[str, str], float]
+    ) -> SIMILARITY_TYPE | Callable[[str, str], float]:
+        if isinstance(v, str) and v not in SIMILARITY_ALGO_MAP:
+            raise ValueError(f"Unknown similarity algorithm: {v}")
+        return v

lionagi/libs/parse/xml/__init__.py

@@ -0,0 +1,10 @@
+"""XML parsing and conversion utilities."""
+
+from .convert import dict_to_xml, xml_to_dict
+from .parser import XMLParser
+
+__all__ = [
+    "dict_to_xml",
+    "xml_to_dict",
+    "XMLParser",
+]

lionagi/libs/parse/xml/convert.py

@@ -0,0 +1,56 @@
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from .parser import XMLParser
+
+
+def xml_to_dict(
+    xml_string: str,
+    /,
+    suppress=False,
+    remove_root: bool = True,
+    root_tag: str = None,
+) -> dict[str, Any]:
+    """
+    Parse an XML string into a nested dictionary structure.
+
+    This function converts an XML string into a dictionary where:
+    - Element tags become dictionary keys
+    - Text content is assigned directly to the tag key if there are no children
+    - Attributes are stored in a '@attributes' key
+    - Multiple child elements with the same tag are stored as lists
+
+    Args:
+        xml_string: The XML string to parse.
+
+    Returns:
+        A dictionary representation of the XML structure.
+
+    Raises:
+        ValueError: If the XML is malformed or parsing fails.
+    """
+    try:
+        a = XMLParser(xml_string).parse()
+        if remove_root and (root_tag or "root") in a:
+            a = a[root_tag or "root"]
+        return a
+    except ValueError as e:
+        if not suppress:
+            raise e
+
+
+def dict_to_xml(data: dict, /, root_tag: str = "root") -> str:
+
+    root = ET.Element(root_tag)
+
+    def convert(dict_obj: dict, parent: Any) -> None:
+        for key, val in dict_obj.items():
+            if isinstance(val, dict):
+                element = ET.SubElement(parent, key)
+                convert(dict_obj=val, parent=element)
+            else:
+                element = ET.SubElement(parent, key)
+                element.text = str(object=val)
+
+    convert(dict_obj=data, parent=root)
+    return ET.tostring(root, encoding="unicode")

lionagi/libs/parse/xml/parser.py

@@ -0,0 +1,93 @@
+import re
+from typing import Any
+
+
+class XMLParser:
+    def __init__(self, xml_string: str):
+        self.xml_string = xml_string.strip()
+        self.index = 0
+
+    def parse(self) -> dict[str, Any]:
+        """Parse the XML string and return the root element as a dictionary."""
+        return self._parse_element()
+
+    def _parse_element(self) -> dict[str, Any]:
+        """Parse a single XML element and its children."""
+        self._skip_whitespace()
+        if self.xml_string[self.index] != "<":
+            raise ValueError(
+                f"Expected '<', found '{self.xml_string[self.index]}'"
+            )
+
+        tag, attributes = self._parse_opening_tag()
+        children: dict[str, str | list | dict] = {}
+        text = ""
+
+        while self.index < len(self.xml_string):
+            self._skip_whitespace()
+            if self.xml_string.startswith("</", self.index):
+                closing_tag = self._parse_closing_tag()
+                if closing_tag != tag:
+                    raise ValueError(
+                        f"Mismatched tags: '{tag}' and '{closing_tag}'"
+                    )
+                break
+            elif self.xml_string.startswith("<", self.index):
+                child = self._parse_element()
+                child_tag, child_data = next(iter(child.items()))
+                if child_tag in children:
+                    if not isinstance(children[child_tag], list):
+                        children[child_tag] = [children[child_tag]]
+                    children[child_tag].append(child_data)
+                else:
+                    children[child_tag] = child_data
+            else:
+                text += self._parse_text()
+
+        result: dict[str, Any] = {}
+        if attributes:
+            result["@attributes"] = attributes
+        if children:
+            result.update(children)
+        elif text.strip():
+            result = text.strip()
+
+        return {tag: result}
+
+    def _parse_opening_tag(self) -> tuple[str, dict[str, str]]:
+        """Parse an opening XML tag and its attributes."""
+        match = re.match(
+            r'<(\w+)((?:\s+\w+="[^"]*")*)\s*/?>',
+            self.xml_string[self.index :],  # noqa
+        )
+        if not match:
+            raise ValueError("Invalid opening tag")
+        self.index += match.end()
+        tag = match.group(1)
+        attributes = dict(re.findall(r'(\w+)="([^"]*)"', match.group(2)))
+        return tag, attributes
+
+    def _parse_closing_tag(self) -> str:
+        """Parse a closing XML tag."""
+        match = re.match(r"</(\w+)>", self.xml_string[self.index :])  # noqa
+        if not match:
+            raise ValueError("Invalid closing tag")
+        self.index += match.end()
+        return match.group(1)
+
+    def _parse_text(self) -> str:
+        """Parse text content between XML tags."""
+        start = self.index
+        while (
+            self.index < len(self.xml_string)
+            and self.xml_string[self.index] != "<"
+        ):
+            self.index += 1
+        return self.xml_string[start : self.index]  # noqa
+
+    def _skip_whitespace(self) -> None:
+        """Skip any whitespace characters at the current parsing position."""
+        p_ = len(self.xml_string[self.index :])  # noqa
+        m_ = len(self.xml_string[self.index :].lstrip())  # noqa
+
+        self.index += p_ - m_

lionagi/libs/string_similarity/__init__.py

@@ -0,0 +1,32 @@
+"""String similarity calculation and matching functionality."""
+
+from .algorithms import (
+    SIMILARITY_ALGO_MAP,
+    cosine_similarity,
+    hamming_similarity,
+    jaro_distance,
+    jaro_winkler_similarity,
+    levenshtein_distance,
+    levenshtein_similarity,
+    sequence_matcher_similarity,
+)
+from .matcher import MatchResult, string_similarity
+from .utils import SIMILARITY_TYPE, SimilarityFunc
+
+__all__ = [
+    # Main functionality
+    "string_similarity",
+    "MatchResult",
+    # Algorithms
+    "cosine_similarity",
+    "hamming_similarity",
+    "jaro_distance",
+    "jaro_winkler_similarity",
+    "levenshtein_distance",
+    "levenshtein_similarity",
+    "sequence_matcher_similarity",
+    # Types and constants
+    "SIMILARITY_TYPE",
+    "SimilarityFunc",
+    "SIMILARITY_ALGO_MAP",
+]