lionagi 0.10.7__py3-none-any.whl → 0.12.0__py3-none-any.whl

lionagi/libs/parse/fuzzy_parse_json.py

@@ -0,0 +1,117 @@
+import contextlib
+import json
+import re
+from typing import Any
+
+
+def fuzzy_parse_json(
+    str_to_parse: str, /
+) -> dict[str, Any] | list[dict[str, Any]]:
+    """
+    Attempt to parse a JSON string, trying a few minimal "fuzzy" fixes if needed.
+
+    Steps:
+    1. Parse directly with json.loads.
+    2. Replace single quotes with double quotes, normalize spacing, and try again.
+    3. Attempt to fix unmatched brackets using fix_json_string.
+    4. If all fail, raise ValueError.
+
+    Args:
+        str_to_parse: The JSON string to parse
+
+    Returns:
+        Parsed JSON (dict or list of dicts)
+
+    Raises:
+        ValueError: If the string cannot be parsed as valid JSON
+        TypeError: If the input is not a string
+    """
+    _check_valid_str(str_to_parse)
+
+    # 1. Direct attempt
+    with contextlib.suppress(Exception):
+        return json.loads(str_to_parse)
+
+    # 2. Try cleaning: replace single quotes with double and normalize
+    cleaned = _clean_json_string(str_to_parse.replace("'", '"'))
+    with contextlib.suppress(Exception):
+        return json.loads(cleaned)
+
+    # 3. Try fixing brackets
+    fixed = fix_json_string(cleaned)
+    with contextlib.suppress(Exception):
+        return json.loads(fixed)
+
+    # If all attempts fail
+    raise ValueError("Invalid JSON string")
+
+
+def _check_valid_str(str_to_parse: str, /):
+    if not isinstance(str_to_parse, str):
+        raise TypeError("Input must be a string")
+    if not str_to_parse.strip():
+        raise ValueError("Input string is empty")
+
+
+def _clean_json_string(s: str) -> str:
+    """Basic normalization: replace unescaped single quotes, trim spaces, ensure keys are quoted."""
+    # Replace unescaped single quotes with double quotes
+    # '(?<!\\)'" means a single quote not preceded by a backslash
+    s = re.sub(r"(?<!\\)'", '"', s)
+    # Collapse multiple whitespaces
+    s = re.sub(r"\s+", " ", s)
+    # Ensure keys are quoted
+    # This attempts to find patterns like { key: value } and turn them into {"key": value}
+    s = re.sub(r'([{,])\s*([^"\s]+)\s*:', r'\1"\2":', s)
+    return s.strip()
+
+
+def fix_json_string(str_to_parse: str, /) -> str:
+    """Try to fix JSON string by ensuring brackets are matched properly."""
+    if not str_to_parse:
+        raise ValueError("Input string is empty")
+
+    brackets = {"{": "}", "[": "]"}
+    open_brackets = []
+    pos = 0
+    length = len(str_to_parse)
+
+    while pos < length:
+        char = str_to_parse[pos]
+
+        if char == "\\":
+            pos += 2  # Skip escaped chars
+            continue
+
+        if char == '"':
+            pos += 1
+            # skip string content
+            while pos < length:
+                if str_to_parse[pos] == "\\":
+                    pos += 2
+                    continue
+                if str_to_parse[pos] == '"':
+                    pos += 1
+                    break
+                pos += 1
+            continue
+
+        if char in brackets:
+            open_brackets.append(brackets[char])
+        elif char in brackets.values():
+            if not open_brackets:
+                # Extra closing bracket
+                # Better to raise error than guess
+                raise ValueError("Extra closing bracket found.")
+            if open_brackets[-1] != char:
+                # Mismatched bracket
+                raise ValueError("Mismatched brackets.")
+            open_brackets.pop()
+
+        pos += 1
+
+    # Add missing closing brackets if any
+    if open_brackets:
+        str_to_parse += "".join(reversed(open_brackets))
+
+    return str_to_parse

lionagi/libs/parse/to_dict.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from enum import Enum
+from functools import partial
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+from lionagi.utils import PydanticUndefinedType, UndefinedType
+
+
+def to_dict(
+    input_: Any,
+    /,
+    *,
+    use_model_dump: bool = True,
+    fuzzy_parse: bool = False,
+    suppress: bool = False,
+    str_type: Literal["json", "xml"] | None = "json",
+    parser: Callable[[str], Any] | None = None,
+    recursive: bool = False,
+    max_recursive_depth: int = None,
+    recursive_python_only: bool = True,
+    use_enum_values: bool = False,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """
+    Convert various input types to a dictionary, with optional recursive processing.
+
+    Args:
+        input_: The input to convert.
+        use_model_dump: Use model_dump() for Pydantic models if available.
+        fuzzy_parse: Use fuzzy parsing for string inputs.
+        suppress: Return empty dict on errors if True.
+        str_type: Input string type ("json" or "xml").
+        parser: Custom parser function for string inputs.
+        recursive: Enable recursive conversion of nested structures.
+        max_recursive_depth: Maximum recursion depth (default 5, max 10).
+        recursive_python_only: If False, attempts to convert custom types recursively.
+        use_enum_values: Use enum values instead of names.
+        **kwargs: Additional arguments for parsing functions.
+
+    Returns:
+        dict[str, Any]: A dictionary derived from the input.
+
+    Raises:
+        ValueError: If parsing fails and suppress is False.
+
+    Examples:
+        >>> to_dict({"a": 1, "b": [2, 3]})
+        {'a': 1, 'b': [2, 3]}
+        >>> to_dict('{"x": 10}', str_type="json")
+        {'x': 10}
+        >>> to_dict({"a": {"b": {"c": 1}}}, recursive=True, max_recursive_depth=2)
+        {'a': {'b': {'c': 1}}}
+    """
+
+    try:
+        if recursive:
+            input_ = recursive_to_dict(
+                input_,
+                use_model_dump=use_model_dump,
+                fuzzy_parse=fuzzy_parse,
+                str_type=str_type,
+                parser=parser,
+                max_recursive_depth=max_recursive_depth,
+                recursive_custom_types=not recursive_python_only,
+                use_enum_values=use_enum_values,
+                **kwargs,
+            )
+
+        return _to_dict(
+            input_,
+            fuzzy_parse=fuzzy_parse,
+            parser=parser,
+            str_type=str_type,
+            use_model_dump=use_model_dump,
+            use_enum_values=use_enum_values,
+            **kwargs,
+        )
+    except Exception as e:
+        if suppress or input_ == "":
+            return {}
+        raise e
+
+
+def recursive_to_dict(
+    input_: Any,
+    /,
+    *,
+    max_recursive_depth: int = None,
+    recursive_custom_types: bool = False,
+    **kwargs: Any,
+) -> Any:
+
+    if not isinstance(max_recursive_depth, int):
+        max_recursive_depth = 5
+    else:
+        if max_recursive_depth < 0:
+            raise ValueError(
+                "max_recursive_depth must be a non-negative integer"
+            )
+        if max_recursive_depth == 0:
+            return input_
+        if max_recursive_depth > 10:
+            raise ValueError(
+                "max_recursive_depth must be less than or equal to 10"
+            )
+
+    return _recur_to_dict(
+        input_,
+        max_recursive_depth=max_recursive_depth,
+        current_depth=0,
+        recursive_custom_types=recursive_custom_types,
+        **kwargs,
+    )
+
+
+def _recur_to_dict(
+    input_: Any,
+    /,
+    *,
+    max_recursive_depth: int,
+    current_depth: int = 0,
+    recursive_custom_types: bool = False,
+    **kwargs: Any,
+) -> Any:
+
+    if current_depth >= max_recursive_depth:
+        return input_
+
+    if isinstance(input_, str):
+        try:
+            # Attempt to parse the string
+            parsed = _to_dict(input_, **kwargs)
+            # Recursively process the parsed result
+            return _recur_to_dict(
+                parsed,
+                max_recursive_depth=max_recursive_depth,
+                current_depth=current_depth + 1,
+                recursive_custom_types=recursive_custom_types,
+                **kwargs,
+            )
+        except Exception:
+            # Return the original string if parsing fails
+            return input_
+
+    elif isinstance(input_, dict):
+        # Recursively process dictionary values
+        return {
+            key: _recur_to_dict(
+                value,
+                max_recursive_depth=max_recursive_depth,
+                current_depth=current_depth + 1,
+                recursive_custom_types=recursive_custom_types,
+                **kwargs,
+            )
+            for key, value in input_.items()
+        }
+
+    elif isinstance(input_, (list, tuple, set)):
+        # Recursively process list or tuple elements
+        processed = [
+            _recur_to_dict(
+                element,
+                max_recursive_depth=max_recursive_depth,
+                current_depth=current_depth + 1,
+                recursive_custom_types=recursive_custom_types,
+                **kwargs,
+            )
+            for element in input_
+        ]
+        return type(input_)(processed)
+
+    elif isinstance(input_, type) and issubclass(input_, Enum):
+        try:
+            obj_dict = _to_dict(input_, **kwargs)
+            return _recur_to_dict(
+                obj_dict,
+                max_recursive_depth=max_recursive_depth,
+                current_depth=current_depth + 1,
+                **kwargs,
+            )
+        except Exception:
+            return input_
+
+    elif recursive_custom_types:
+        # Process custom classes if enabled
+        try:
+            obj_dict = _to_dict(input_, **kwargs)
+            return _recur_to_dict(
+                obj_dict,
+                max_recursive_depth=max_recursive_depth,
+                current_depth=current_depth + 1,
+                recursive_custom_types=recursive_custom_types,
+                **kwargs,
+            )
+        except Exception:
+            return input_
+
+    else:
+        # Return the input as is for other data types
+        return input_
+
+
+def _enum_to_dict(input_, /, use_enum_values: bool = True):
+    dict_ = dict(input_.__members__).copy()
+    if use_enum_values:
+        return {key: value.value for key, value in dict_.items()}
+    return dict_
+
+
+def _str_to_dict(
+    input_: str,
+    /,
+    fuzzy_parse: bool = False,
+    str_type: Literal["json", "xml"] | None = "json",
+    parser: Callable[[str], Any] | None = None,
+    remove_root: bool = False,
+    root_tag: str = "root",
+    **kwargs: Any,
+):
+    """
+    kwargs for parser
+    """
+    if not parser:
+        if str_type == "xml" and not parser:
+            from lionagi.libs.parse.xml_parser import xml_to_dict
+
+            parser = partial(
+                xml_to_dict, remove_root=remove_root, root_tag=root_tag
+            )
+
+        elif fuzzy_parse:
+
+            from lionagi.libs.parse.fuzzy_parse_json import fuzzy_parse_json
+
+            parser = fuzzy_parse_json
+        else:
+            parser = json.loads
+
+    return parser(input_, **kwargs)
+
+
+def _na_to_dict(input_: type[None] | UndefinedType | PydanticUndefinedType, /):
+    return {}
+
+
+def _model_to_dict(input_: Any, /, use_model_dump=True, **kwargs):
+    """
+    kwargs: built-in serialization methods kwargs
+    accepted built-in serialization methods:
+        - mdoel_dump
+        - to_dict
+        - to_json
+        - dict
+        - json
+    """
+
+    if use_model_dump and hasattr(input_, "model_dump"):
+        return input_.model_dump(**kwargs)
+
+    methods = (
+        "to_dict",
+        "to_json",
+        "json",
+        "dict",
+    )
+    for method in methods:
+        if hasattr(input_, method):
+            result = getattr(input_, method)(**kwargs)
+            return json.loads(result) if isinstance(result, str) else result
+
+    if hasattr(input_, "__dict__"):
+        return input_.__dict__
+
+    try:
+        return dict(input_)
+    except Exception as e:
+        raise ValueError(f"Unable to convert input to dictionary: {e}")
+
+
+def _set_to_dict(input_: set, /) -> dict:
+    return {v: v for v in input_}
+
+
+def _iterable_to_dict(input_: Iterable, /) -> dict:
+    return {idx: v for idx, v in enumerate(input_)}
+
+
+def _to_dict(
+    input_: Any,
+    /,
+    *,
+    fuzzy_parse: bool = False,
+    str_type: Literal["json", "xml"] | None = "json",
+    parser: Callable[[str], Any] | None = None,
+    remove_root: bool = False,
+    root_tag: str = "root",
+    use_model_dump: bool = True,
+    use_enum_values: bool = True,
+    **kwargs: Any,
+) -> dict[str, Any]:
+
+    if isinstance(input_, set):
+        return _set_to_dict(input_)
+
+    if isinstance(input_, type) and issubclass(input_, Enum):
+        return _enum_to_dict(input_, use_enum_values=use_enum_values)
+
+    if isinstance(input_, Mapping):
+        return dict(input_)
+
+    if isinstance(input_, type(None) | UndefinedType | PydanticUndefinedType):
+        return _na_to_dict(input_)
+
+    if isinstance(input_, str):
+        return _str_to_dict(
+            input_,
+            fuzzy_parse=fuzzy_parse,
+            str_type=str_type,
+            parser=parser,
+            remove_root=remove_root,
+            root_tag=root_tag,
+            **kwargs,
+        )
+
+    if isinstance(input_, BaseModel) or not isinstance(input_, Sequence):
+        return _model_to_dict(input_, use_model_dump=use_model_dump, **kwargs)
+
+    if isinstance(input_, Iterable):
+        return _iterable_to_dict(input_)
+
+    return dict(input_)

lionagi/libs/parse/to_json.py

@@ -0,0 +1,61 @@
+import json
+import re
+from typing import Any
+
+from .fuzzy_parse_json import fuzzy_parse_json
+
+# Precompile the regex for extracting JSON code blocks
+_JSON_BLOCK_PATTERN = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL)
+
+
+def to_json(
+    input_data: str | list[str], /, *, fuzzy_parse: bool = False
+) -> dict[str, Any] | list[dict[str, Any]]:
+    """
+    Extract and parse JSON content from a string or markdown code blocks.
+
+    Attempts direct JSON parsing first. If that fails, looks for JSON content
+    within markdown code blocks denoted by ```json.
+
+    Args:
+        input_data (str | list[str]): The input string or list of strings to parse.
+        fuzzy_parse (bool): If True, attempts fuzzy JSON parsing on failed attempts.
+
+    Returns:
+        dict or list of dicts:
+            - If a single JSON object is found: returns a dict.
+            - If multiple JSON objects are found: returns a list of dicts.
+            - If no valid JSON found: returns an empty list.
+    """
+
+    # If input_data is a list, join into a single string
+    if isinstance(input_data, list):
+        input_str = "\n".join(input_data)
+    else:
+        input_str = input_data
+
+    # 1. Try direct parsing
+    try:
+        if fuzzy_parse:
+            return fuzzy_parse_json(input_str)
+        return json.loads(input_str)
+    except Exception:
+        pass
+
+    # 2. Attempt extracting JSON blocks from markdown
+    matches = _JSON_BLOCK_PATTERN.findall(input_str)
+    if not matches:
+        return []
+
+    # If only one match, return single dict; if multiple, return list of dicts
+    if len(matches) == 1:
+        data_str = matches[0]
+        return (
+            fuzzy_parse_json(data_str) if fuzzy_parse else json.loads(data_str)
+        )
+
+    # Multiple matches
+    if fuzzy_parse:
+        return [fuzzy_parse_json(m) for m in matches]
+    else:
+        return [json.loads(m) for m in matches]