lionagi 0.3.8__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/parse/flatten/flatten.py

@@ -0,0 +1,168 @@
+from collections import deque
+from collections.abc import Mapping, Sequence
+from typing import Any, Literal, TypeVar, overload
+
+T = TypeVar("T")
+
+
+@overload
+def flatten(
+    nested_structure: T,
+    /,
+    *,
+    parent_key: tuple = (),
+    sep: str = "|",
+    coerce_keys: Literal[True] = True,
+    dynamic: bool = True,
+    coerce_sequence: Literal["dict", None] = None,
+    max_depth: int | None = None,
+) -> dict[str, Any] | None: ...
+
+
+@overload
+def flatten(
+    nested_structure: T,
+    /,
+    *,
+    parent_key: tuple = (),
+    sep: str = "|",
+    coerce_keys: Literal[False],
+    dynamic: bool = True,
+    coerce_sequence: Literal["dict", "list", None] = None,
+    max_depth: int | None = None,
+) -> dict[tuple, Any] | None: ...
+
+
+def flatten(
+    nested_structure: Any,
+    /,
+    *,
+    parent_key: tuple = (),
+    sep: str = "|",
+    coerce_keys: bool = True,
+    dynamic: bool = True,
+    coerce_sequence: Literal["dict", "list"] | None = None,
+    max_depth: int | None = None,
+) -> dict[tuple | str, Any] | None:
+    """Flatten a nested structure into a single-level dictionary.
+
+    Recursively traverses the input, creating keys that represent the path
+    to each value in the flattened result.
+
+    Args:
+        nested_structure: The nested structure to flatten.
+        parent_key: Base key for the current recursion level. Default: ().
+        sep: Separator for joining keys. Default: "|".
+        coerce_keys: Join keys into strings if True, keep as tuples if False.
+            Default: True.
+        dynamic: Handle sequences (except strings) dynamically if True.
+            Default: True.
+        coerce_sequence: Force sequences to be treated as dicts or lists.
+            Options: "dict", "list", or None. Default: None.
+        max_depth: Maximum depth to flatten. None for complete flattening.
+            Default: None.
+
+    Returns:
+        A flattened dictionary with keys as tuples or strings (based on
+        coerce_keys) representing the path to each value.
+
+    Raises:
+        ValueError: If coerce_sequence is "list" and coerce_keys is True.
+
+    Example:
+        >>> nested = {"a": 1, "b": {"c": 2, "d": [3, 4]}}
+        >>> flatten(nested)
+        {'a': 1, 'b|c': 2, 'b|d|0': 3, 'b|d|1': 4}
+
+    Note:
+        - Preserves order of keys in dicts and indices in sequences.
+        - With dynamic=True, treats sequences (except strings) as nestable.
+        - coerce_sequence allows forcing sequence handling for homogeneity.
+    """
+
+    if coerce_keys and coerce_sequence == "list":
+        raise ValueError(
+            "coerce_sequence cannot be 'list' when coerce_keys is True"
+        )
+
+    coerce_sequence_to_list = None
+    coerce_sequence_to_dict = None
+
+    if dynamic and coerce_sequence:
+        if coerce_sequence == "dict":
+            coerce_sequence_to_dict = True
+        elif coerce_sequence == "list":
+            coerce_sequence_to_list = True
+
+    return _flatten_iterative(
+        obj=nested_structure,
+        parent_key=parent_key,
+        sep=sep,
+        coerce_keys=coerce_keys,
+        dynamic=dynamic,
+        coerce_sequence_to_list=coerce_sequence_to_list,
+        coerce_sequence_to_dict=coerce_sequence_to_dict,
+        max_depth=max_depth,
+    )
+
+
+def _flatten_iterative(
+    obj: Any,
+    parent_key: tuple,
+    sep: str,
+    coerce_keys: bool,
+    dynamic: bool,
+    coerce_sequence_to_list: bool = False,
+    coerce_sequence_to_dict: bool = False,
+    max_depth: int | None = None,
+) -> dict[tuple | str, Any]:
+    stack = deque([(obj, parent_key, 0)])
+    result = {}
+
+    while stack:
+        current_obj, current_key, depth = stack.pop()
+
+        if max_depth is not None and depth >= max_depth:
+            result[_format_key(current_key, sep, coerce_keys)] = current_obj
+            continue
+
+        if isinstance(current_obj, Mapping):
+            for k, v in current_obj.items():
+                new_key = current_key + (k,)
+                if (
+                    v
+                    and isinstance(v, (Mapping, Sequence))
+                    and not isinstance(v, (str, bytes, bytearray))
+                ):
+                    stack.appendleft((v, new_key, depth + 1))
+                else:
+                    result[_format_key(new_key, sep, coerce_keys)] = v
+
+        elif (
+            dynamic
+            and isinstance(current_obj, Sequence)
+            and not isinstance(current_obj, (str, bytes, bytearray))
+        ):
+            if coerce_sequence_to_dict:
+                dict_obj = {str(i): v for i, v in enumerate(current_obj)}
+                for k, v in dict_obj.items():
+                    new_key = current_key + (k,)
+                    stack.appendleft((v, new_key, depth + 1))
+            elif coerce_sequence_to_list:
+                for i, v in enumerate(current_obj):
+                    new_key = current_key + (i,)
+                    stack.appendleft((v, new_key, depth + 1))
+            else:
+                for i, v in enumerate(current_obj):
+                    new_key = current_key + (str(i),)
+                    stack.appendleft((v, new_key, depth + 1))
+        else:
+            result[_format_key(current_key, sep, coerce_keys)] = current_obj
+
+    return result
+
+
+def _format_key(key: tuple, sep: str, coerce_keys: bool, /) -> tuple | str:
+    if not key:
+        return key
+    return sep.join(map(str, key)) if coerce_keys else key

lionagi/libs/parse/flatten/params.py

@@ -0,0 +1,52 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class FlattenParams(BaseModel):
+    """Parameters for flatten function.
+
+    Attributes:
+        parent_key: Base key for the current recursion level
+        sep: Separator for joining keys
+        coerce_keys: Join keys into strings if True, keep as tuples if False
+        dynamic: Handle sequences (except strings) dynamically if True
+        coerce_sequence: Force sequences to be treated as dicts or lists
+        max_depth: Maximum depth to flatten
+    """
+
+    parent_key: tuple[Any, ...] = Field(
+        default=(),
+        description="Base key for the current recursion level",
+    )
+    sep: str = Field(
+        default="|",
+        description="Separator for joining keys",
+    )
+    coerce_keys: bool = Field(
+        default=True,
+        description="Join keys into strings if True, keep as tuples if False",
+    )
+    dynamic: bool = Field(
+        default=True,
+        description="Handle sequences (except strings) dynamically if True",
+    )
+    coerce_sequence: Literal["dict", "list"] | None = Field(
+        default=None,
+        description="Force sequences to be treated as dicts or lists",
+    )
+    max_depth: int | None = Field(
+        default=None,
+        description="Maximum depth to flatten",
+        ge=0,
+    )
+
+    @field_validator("coerce_sequence")
+    def validate_coerce_sequence(
+        cls, v: Literal["dict", "list"] | None, values: dict[str, Any]
+    ) -> Literal["dict", "list"] | None:
+        if v == "list" and values.get("coerce_keys", True):
+            raise ValueError(
+                "coerce_sequence cannot be 'list' when coerce_keys is True"
+            )
+        return v

lionagi/libs/parse/flatten/unflatten.py

@@ -0,0 +1,79 @@
+from typing import Any
+
+
+def unflatten(
+    flat_dict: dict[str, Any], sep: str = "|", inplace: bool = False
+) -> dict[str, Any] | list[Any]:
+    """
+    Unflatten a single-level dictionary into a nested dictionary or list.
+
+    Args:
+        flat_dict: The flattened dictionary to unflatten.
+        sep: The separator used for joining keys.
+        inplace: Whether to modify the input dictionary in place.
+
+    Returns:
+        The unflattened nested dictionary or list.
+
+    Examples:
+        >>> unflatten({"a|b|c": 1, "a|b|d": 2})
+        {'a': {'b': {'c': 1, 'd': 2}}}
+
+        >>> unflatten({"0": "a", "1": "b", "2": "c"})
+        ['a', 'b', 'c']
+    """
+
+    def _unflatten(data: dict) -> dict | list:
+        result = {}
+        for key, value in data.items():
+            parts = key.split(sep)
+            current = result
+            for part in parts[:-1]:
+                if part not in current:
+                    current[part] = {}
+                current = current[part]
+            if isinstance(value, dict):
+                current[parts[-1]] = _unflatten(value)
+            else:
+                current[parts[-1]] = value
+
+        # Convert dictionary to list if keys are consecutive integers
+        if result and all(
+            isinstance(key, str) and key.isdigit() for key in result
+        ):
+            return [result[str(i)] for i in range(len(result))]
+        return result
+
+    if inplace:
+        unflattened_dict = {}
+        for key, value in flat_dict.items():
+            parts = key.split(sep)
+            current = unflattened_dict
+            for part in parts[:-1]:
+                if part not in current:
+                    current[part] = {}
+                current = current[part]
+            current[parts[-1]] = value
+
+        unflattened_result = _unflatten(unflattened_dict)
+        flat_dict.clear()
+        if isinstance(unflattened_result, list):
+            flat_dict.update(
+                {str(i): v for i, v in enumerate(unflattened_result)}
+            )
+        else:
+            flat_dict.update(unflattened_result)
+        return flat_dict
+
+    else:
+        unflattened_dict = {}
+        for key, value in flat_dict.items():
+            parts = key.split(sep)
+            current = unflattened_dict
+            for part in parts[:-1]:
+                if part not in current:
+                    current[part] = {}
+                current = current[part]
+            current[parts[-1]] = value
+
+        return _unflatten(unflattened_dict)

lionagi/libs/parse/json/__init__.py

@@ -0,0 +1,27 @@
+"""JSON parsing and manipulation utilities."""
+
+from .as_readable import as_readable, as_readable_json
+from .extract import extract_block, extract_json_blocks
+from .parse import _clean_json_string, fix_json_string, fuzzy_parse_json
+from .schema import (
+    extract_json_schema,
+    json_schema_to_cfg,
+    json_schema_to_regex,
+    print_cfg,
+)
+from .to_json import to_json
+
+__all__ = [
+    "as_readable",
+    "as_readable_json",
+    "extract_block",
+    "extract_json_blocks",
+    "fuzzy_parse_json",
+    "fix_json_string",
+    "_clean_json_string",
+    "extract_json_schema",
+    "json_schema_to_cfg",
+    "json_schema_to_regex",
+    "print_cfg",
+    "to_json",
+]

lionagi/libs/parse/json/as_readable.py

@@ -0,0 +1,104 @@
+import json
+from typing import Any
+
+from ..type_convert.to_dict import to_dict
+
+
+def as_readable_json(input_: Any, /, **kwargs) -> str:
+    """Convert input to a human-readable JSON string.
+
+    Args:
+        input_: Object to convert to readable JSON
+        **kwargs: Additional arguments passed to json.dumps()
+
+    Returns:
+        A formatted, human-readable JSON string
+
+    Raises:
+        ValueError: If conversion to JSON fails
+    """
+    # Extract to_dict kwargs
+    to_dict_kwargs = {
+        "use_model_dump": True,
+        "fuzzy_parse": True,
+        "recursive": True,
+        "recursive_python_only": True,
+        "max_recursive_depth": 5,
+    }
+    to_dict_kwargs.update(kwargs)
+
+    # Handle empty input
+    if not input_:
+        if isinstance(input_, list):
+            return ""
+        return "{}"
+
+    try:
+        if isinstance(input_, list):
+            # For lists, convert and format each item separately
+            items = []
+            for item in input_:
+                dict_ = to_dict(item, **to_dict_kwargs)
+                items.append(
+                    json.dumps(
+                        dict_,
+                        indent=4,
+                        ensure_ascii=False,
+                        default=lambda o: to_dict(o),
+                    )
+                )
+            return "\n\n".join(items)
+
+        # Handle single items
+        dict_ = to_dict(input_, **to_dict_kwargs)
+
+        # Extract json.dumps kwargs from input kwargs
+        json_kwargs = {
+            "indent": 4,
+            "ensure_ascii": kwargs.get("ensure_ascii", False),
+            "default": lambda o: to_dict(o),
+        }
+
+        # Add any other JSON-specific kwargs
+        for k in ["indent", "separators", "cls"]:
+            if k in kwargs:
+                json_kwargs[k] = kwargs[k]
+
+        # Convert to JSON string
+        if kwargs.get("ensure_ascii", False):
+            # Force ASCII encoding for special characters
+            return json.dumps(
+                dict_,
+                ensure_ascii=True,
+                **{
+                    k: v for k, v in json_kwargs.items() if k != "ensure_ascii"
+                },
+            )
+
+        return json.dumps(dict_, **json_kwargs)
+
+    except Exception as e:
+        raise ValueError(
+            f"Failed to convert input to readable JSON: {e}"
+        ) from e
+
+
+def as_readable(input_: Any, /, *, md: bool = False, **kwargs) -> str:
+    """Convert input to readable string with optional markdown formatting.
+
+    Args:
+        input_: Object to convert
+        md: Whether to wrap in markdown block
+        **kwargs: Additional arguments for as_readable_json()
+
+    Returns:
+        Formatted string representation
+    """
+    try:
+        result = as_readable_json(input_, **kwargs)
+        if md:
+            return f"```json\n{result}\n```"
+        return result
+
+    except Exception:
+        return str(input_)

lionagi/libs/parse/json/extract.py

@@ -0,0 +1,102 @@
+import re
+from collections.abc import Callable
+from typing import Any
+
+from ..type_convert.to_dict import to_dict
+
+
+def extract_json_blocks(
+    str_to_parse: str,
+    suppress: bool = True,
+    fuzzy_parse: bool = True,
+    dropna: bool = True,
+) -> list[dict[str, Any]]:
+    """
+    Extract and parse JSON blocks from the given text.
+
+    This function searches for JSON blocks enclosed in triple backticks
+    within the input text, parses them, and returns a list of parsed
+    dictionaries.
+
+    Args:
+        text: The input text containing JSON blocks.
+        suppress: If True, suppress errors during parsing. Default is True.
+        fuzzy_parse: If True, use fuzzy parsing for JSON. Default is True.
+        dropna: If True, remove None values from the result. Default is True.
+
+    Returns:
+        A list of parsed JSON blocks as dictionaries.
+
+    Example:
+        >>> text = "```json\n{\"key\": \"value\"}\n```"
+        >>> extract_json_blocks(text)
+        [{'key': 'value'}]
+    """
+    pattern = r"```json\s*(.*?)\s*```"
+    matches = re.findall(pattern, str_to_parse, re.DOTALL)
+
+    json_blocks = [
+        to_dict(match, fuzzy_parse=fuzzy_parse, suppress=suppress)
+        for match in matches
+    ]
+
+    return [block for block in json_blocks if block] if dropna else json_blocks
+
+
+def extract_block(
+    str_to_parse: str,
+    language: str = "json",
+    regex_pattern: str | None = None,
+    *,
+    parser: Callable[[str], Any] | None = None,
+    suppress: bool = False,
+) -> dict[str, Any] | None:
+    """
+    Extract and parse a code block from the given string.
+
+    This function searches for a code block in the input string using a
+    regular expression pattern, extracts it, and parses it using the
+    provided parser function.
+
+    Args:
+        str_to_parse: The input string containing the code block.
+        language: The language of the code block. Default is "json".
+        regex_pattern: Custom regex pattern to find the code block.
+            If provided, overrides the default pattern.
+        parser: A function to parse the extracted code string.
+            If not provided, uses `to_dict` with fuzzy parsing.
+        suppress: If True, return None instead of raising an error
+            when no code block is found. Default is False.
+
+    Returns:
+        The parsed content of the code block as a dictionary,
+        or None if no block is found and suppress is True.
+
+    Raises:
+        ValueError: If no code block is found and suppress is False.
+
+    Example:
+        >>> text = "```json\n{\"key\": \"value\"}\n```"
+        >>> extract_block(text)
+        {'key': 'value'}
+    """
+    if not regex_pattern:
+        regex_pattern = rf"```{language}\n?(.*?)\n?```"
+    if not language:
+        regex_pattern = r"```\n?(.*?)\n?```"
+
+    match = re.search(regex_pattern, str_to_parse, re.DOTALL)
+
+    if match:
+        code_str = match.group(1).strip()
+    elif str_to_parse.startswith(f"```{language}\n") and str_to_parse.endswith(
+        "\n```"
+    ):
+        code_str = str_to_parse[4 + len(language) : -4].strip()
+    elif suppress:
+        return None
+    else:
+        raise ValueError("No code block found in the input string.")
+
+    parser = parser or (lambda x: to_dict(x, fuzzy_parse=True, suppress=True))
+    return parser(code_str)