lionagi 0.3.8__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/libs/parse/json/parse.py

@@ -0,0 +1,179 @@
+import contextlib
+import json
+import re
+from typing import Any
+
+
+def fuzzy_parse_json(
+    str_to_parse: str, /
+) -> dict[str, Any] | list[dict[str, Any]]:
+    """Parse a JSON string with automatic fixing of common formatting issues.
+
+    Args:
+        str_to_parse: The JSON string to parse
+
+    Returns:
+        The parsed JSON object as a dictionary
+
+    Raises:
+        ValueError: If the string cannot be parsed as valid JSON
+        TypeError: If the input is not a string or the result is not a dict
+    """
+    _check_valid_str(str_to_parse)
+
+    with contextlib.suppress(Exception):
+        return json.loads(str_to_parse)
+
+    with contextlib.suppress(Exception):
+        return json.loads(_fix_braces(str_to_parse))
+
+    with contextlib.suppress(Exception):
+        str_ = _fix_braces(str_to_parse.replace("'", '"'))
+        return json.loads(str_)
+
+    with contextlib.suppress(Exception):
+        return json.loads(fix_json_string(str_to_parse))
+
+    with contextlib.suppress(Exception):
+        return json.loads(_clean_json_string(str_to_parse))
+
+    with contextlib.suppress(Exception):
+        return json.loads(fix_json_string(_clean_json_string(str_to_parse)))
+
+    with contextlib.suppress(Exception):
+        assembled = _assemble(str_to_parse)
+        return json.loads(assembled)
+
+    with contextlib.suppress(Exception):
+        assembled = _assemble(str_to_parse.replace("'", '"'))
+        return json.loads(str_)
+
+    raise ValueError("Invalid JSON string")
+
+
+def _clean_json_string(s: str) -> str:
+    """Clean and standardize a JSON string."""
+    s = re.sub(r"(?<!\\)'", '"', s)
+    s = re.sub(r"\s+", " ", s)
+    s = re.sub(r'([{,])\s*([^"\s]+):', r'\1"\2":', s)
+    return s.strip()
+
+
+def fix_json_string(str_to_parse: str, /) -> str:
+    """Fix a JSON string by ensuring all brackets are properly closed.
+
+    Args:
+        str_to_parse: JSON string to fix
+
+    Returns:
+        Fixed JSON string with proper bracket closure
+
+    Raises:
+        ValueError: If mismatched or extra closing brackets are found
+    """
+    if not str_to_parse:
+        raise ValueError("Input string is empty")
+
+    brackets = {"{": "}", "[": "]"}
+    open_brackets = []
+    pos = 0
+    length = len(str_to_parse)
+
+    while pos < length:
+        char = str_to_parse[pos]
+
+        # Handle escape sequences
+        if char == "\\":
+            pos += 2  # Skip escape sequence
+            continue
+
+        # Handle string content
+        if char == '"':
+            pos += 1
+            # Skip until closing quote, accounting for escapes
+            while pos < length:
+                if str_to_parse[pos] == "\\":
+                    pos += 2  # Skip escape sequence
+                    continue
+                if str_to_parse[pos] == '"':
+                    break
+                pos += 1
+            pos += 1
+            continue
+
+        # Handle brackets
+        if char in brackets:
+            open_brackets.append(brackets[char])
+        elif char in brackets.values():
+            if not open_brackets:
+                raise ValueError(
+                    f"Extra closing bracket '{char}' at position {pos}"
+                )
+            if open_brackets[-1] != char:
+                raise ValueError(
+                    f"Mismatched bracket '{char}' at position {pos}"
+                )
+            open_brackets.pop()
+
+        pos += 1
+
+    # Add missing closing brackets
+    closing_brackets = "".join(reversed(open_brackets))
+    return str_to_parse + closing_brackets
+
+
+def _check_valid_str(str_to_parse, /):
+    if not isinstance(str_to_parse, str):
+        raise TypeError("Input must be a string")
+
+    if not str_to_parse.strip():
+        raise ValueError("Input string is empty")
+
+
+def _fix_braces(str_to_parse):
+    fixed = fix_json_string(str_to_parse)
+    str_ = fixed.strip()
+    if not str_.endswith("}"):
+        str_ += "}"
+    if not str_.startswith("{"):
+        str_ = "{" + str_
+    if str_.endswith(",}"):
+        str_ = str_[:-2] + "}"
+    return str_
+
+
+def _assemble(str_to_parse):
+    fixed = fix_json_string(str_to_parse)
+    if fixed.startswith("{"):
+        fixed = fixed[1:]
+    if fixed.endswith("}"):
+        fixed = fixed[:-1]
+    parts = []
+    for i in fixed.split(":"):
+        str_ = i.strip().replace("'", '"')
+        if str_.count('"') == 2 and ' "' in str_:
+            parts.append(str_.split(' "')[0])
+            parts.append('"' + str_.split(' "')[1])
+
+        elif str_.count('"') == 4 and ' "' in str_:
+            parts.append(str_.split(' "')[0])
+            parts.append('"' + str_.split(' "')[1])
+
+        else:
+            parts.append(i.strip())
+
+    assembled = ""
+    for i in range(len(parts)):
+        if i % 2 == 0:
+            assembled += parts[i] + ":"
+        else:
+            assembled += parts[i] + ","
+
+    if assembled.endswith(","):
+        assembled = assembled[:-1]
+    if not assembled.startswith("{"):
+        assembled = "{" + assembled
+    if not assembled.endswith("}"):
+        assembled += "}"
+
+    return assembled

lionagi/libs/parse/json/schema.py

@@ -0,0 +1,227 @@
+from typing import Any, Literal
+
+from ..flatten.flatten import flatten
+
+
+def extract_json_schema(
+    data: Any,
+    *,
+    sep: str = "|",
+    coerce_keys: bool = True,
+    dynamic: bool = True,
+    coerce_sequence: Literal["dict", "list"] | None = None,
+    max_depth: int | None = None,
+) -> dict[str, Any]:
+    """
+    Extract a JSON schema from JSON data.
+
+    This function uses the flatten function to create a flat representation
+    of the JSON data, then builds a schema based on the flattened structure.
+
+    Args:
+        data: The JSON data to extract the schema from.
+        sep: Separator used in flattened keys.
+        coerce_keys: Whether to coerce keys to strings.
+        dynamic: Whether to use dynamic flattening.
+        coerce_sequence: How to coerce sequences ("dict", "list", or None).
+        max_depth: Maximum depth to flatten.
+
+    Returns:
+        A dictionary representing the JSON schema.
+    """
+    flattened = flatten(
+        data,
+        sep=sep,
+        coerce_keys=coerce_keys,
+        dynamic=dynamic,
+        coerce_sequence=coerce_sequence,
+        max_depth=max_depth,
+    )
+
+    schema = {}
+    for key, value in flattened.items():
+        key_parts = key.split(sep) if isinstance(key, str) else key
+        current = schema
+        for part in key_parts[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        current[key_parts[-1]] = _get_type(value)
+
+    return {"type": "object", "properties": _consolidate_schema(schema)}
+
+
+def _get_type(value: Any) -> dict[str, Any]:
+    if isinstance(value, str):
+        return {"type": "string"}
+    elif isinstance(value, bool):
+        return {"type": "boolean"}
+    elif isinstance(value, int):
+        return {"type": "integer"}
+    elif isinstance(value, float):
+        return {"type": "number"}
+    elif isinstance(value, list):
+        if not value:
+            return {"type": "array", "items": {}}
+        item_types = [_get_type(item) for item in value]
+        if all(item_type == item_types[0] for item_type in item_types):
+            return {"type": "array", "items": item_types[0]}
+        else:
+            return {"type": "array", "items": {"oneOf": item_types}}
+    elif isinstance(value, dict):
+        return {
+            "type": "object",
+            "properties": _consolidate_schema(
+                {k: _get_type(v) for k, v in value.items()}
+            ),
+        }
+    elif value is None:
+        return {"type": "null"}
+    else:
+        return {"type": "any"}
+
+
+def _consolidate_schema(schema: dict) -> dict:
+    """
+    Consolidate the schema to handle lists and nested structures.
+    """
+    consolidated = {}
+    for key, value in schema.items():
+        if isinstance(value, dict) and all(k.isdigit() for k in value.keys()):
+            # This is likely a list
+            item_types = list(value.values())
+            if all(item_type == item_types[0] for item_type in item_types):
+                consolidated[key] = {"type": "array", "items": item_types[0]}
+            else:
+                consolidated[key] = {
+                    "type": "array",
+                    "items": {"oneOf": item_types},
+                }
+        elif isinstance(value, dict) and "type" in value:
+            consolidated[key] = value
+        else:
+            consolidated[key] = _consolidate_schema(value)
+    return consolidated
+
+
+def json_schema_to_cfg(
+    schema: dict[str, Any], start_symbol: str = "S"
+) -> list[tuple[str, list[str]]]:
+    productions = []
+    visited = set()
+    symbol_counter = 0
+
+    def generate_symbol(base: str) -> str:
+        nonlocal symbol_counter
+        symbol = f"{base}@{symbol_counter}"
+        symbol_counter += 1
+        return symbol
+
+    def generate_rules(s: dict[str, Any], symbol: str):
+        if symbol in visited:
+            return
+        visited.add(symbol)
+
+        if s.get("type") == "object":
+            properties = s.get("properties", {})
+            if properties:
+                props_symbol = generate_symbol("PROPS")
+                productions.append((symbol, ["{", props_symbol, "}"]))
+
+                productions.append((props_symbol, []))  # Empty object
+                for i, prop in enumerate(properties):
+                    prop_symbol = generate_symbol(prop)
+                    if i == 0:
+                        productions.append((props_symbol, [prop_symbol]))
+                    else:
+                        productions.append(
+                            (props_symbol, [props_symbol, ",", prop_symbol])
+                        )
+
+                for prop, prop_schema in properties.items():
+                    prop_symbol = generate_symbol(prop)
+                    value_symbol = generate_symbol("VALUE")
+                    productions.append(
+                        (prop_symbol, [f'"{prop}"', ":", value_symbol])
+                    )
+                    generate_rules(prop_schema, value_symbol)
+            else:
+                productions.append((symbol, ["{", "}"]))
+
+        elif s.get("type") == "array":
+            items = s.get("items", {})
+            items_symbol = generate_symbol("ITEMS")
+            value_symbol = generate_symbol("VALUE")
+            productions.append((symbol, ["[", "]"]))
+            productions.append((symbol, ["[", items_symbol, "]"]))
+            productions.append((items_symbol, [value_symbol]))
+            productions.append(
+                (items_symbol, [value_symbol, ",", items_symbol])
+            )
+            generate_rules(items, value_symbol)
+
+        elif s.get("type") == "string":
+            productions.append((symbol, ["STRING"]))
+
+        elif s.get("type") == "number":
+            productions.append((symbol, ["NUMBER"]))
+
+        elif s.get("type") == "integer":
+            productions.append((symbol, ["INTEGER"]))
+
+        elif s.get("type") == "boolean":
+            productions.append((symbol, ["BOOLEAN"]))
+
+        elif s.get("type") == "null":
+            productions.append((symbol, ["NULL"]))
+
+    generate_rules(schema, start_symbol)
+    return productions
+
+
+def json_schema_to_regex(schema: dict[str, Any]) -> str:
+    def schema_to_regex(s):
+        if s.get("type") == "object":
+            properties = s.get("properties", {})
+            prop_patterns = [
+                rf'"{prop}"\s*:\s*{schema_to_regex(prop_schema)}'
+                for prop, prop_schema in properties.items()
+            ]
+            return (
+                r"\{"
+                + r"\s*("
+                + r"|".join(prop_patterns)
+                + r")"
+                + r"(\s*,\s*("
+                + r"|".join(prop_patterns)
+                + r"))*\s*\}"
+            )
+        elif s.get("type") == "array":
+            items = s.get("items", {})
+            return (
+                r"\[\s*("
+                + schema_to_regex(items)
+                + r"(\s*,\s*"
+                + schema_to_regex(items)
+                + r")*)?\s*\]"
+            )
+        elif s.get("type") == "string":
+            return r'"[^"]*"'
+        elif s.get("type") == "integer":
+            return r"-?\d+"
+        elif s.get("type") == "number":
+            return r"-?\d+(\.\d+)?"
+        elif s.get("type") == "boolean":
+            return r"(true|false)"
+        elif s.get("type") == "null":
+            return r"null"
+        else:
+            return r".*"
+
+    return "^" + schema_to_regex(schema) + "$"
+
+
+def print_cfg(productions: list[tuple[str, list[str]]]):
+    for lhs, rhs in productions:
+        print(f"{lhs} -> {' '.join(rhs)}")

lionagi/libs/parse/json/to_json.py

@@ -0,0 +1,71 @@
+import json
+import re
+from typing import Any
+
+from .parse import fuzzy_parse_json
+
+
+def to_json(
+    string: str | list[str], /, fuzzy_parse: bool = False
+) -> list[dict[str, Any]] | dict:
+    """Extract and parse JSON content from a string or markdown code blocks.
+
+    This function attempts to parse JSON directly from the input string first.
+    If that fails, it looks for JSON content within markdown code blocks
+    (denoted by ```json).
+
+    Args:
+        string: Input string or list of strings to parse. If a list is provided,
+               it will be joined with newlines.
+
+    Returns:
+        - A dictionary if a single JSON object is found
+        - A list of dictionaries if multiple JSON objects are found
+        - An empty list if no valid JSON is found
+
+    Examples:
+        >>> to_json('{"key": "value"}')
+        {'key': 'value'}
+
+        >>> to_json('''
+        ... ```json
+        ... {"key": "value"}
+        ... ```
+        ... ''')
+        {'key': 'value'}
+
+        >>> to_json('''
+        ... ```json
+        ... {"key1": "value1"}
+        ... ```
+        ... ```json
+        ... {"key2": "value2"}
+        ... ```
+        ... ''')
+        [{'key1': 'value1'}, {'key2': 'value2'}]
+    """
+
+    if isinstance(string, list):
+        string = "\n".join(string)
+
+    # Try direct JSON parsing first
+    try:
+        if fuzzy_parse:
+            return fuzzy_parse_json(string)
+        return json.loads(string)
+    except Exception:
+        pass
+
+    # Look for JSON in markdown code blocks
+    pattern = r"```json\s*(.*?)\s*```"
+    matches = re.findall(pattern, string, re.DOTALL)
+
+    if not matches:
+        return []
+
+    if len(matches) == 1:
+        return json.loads(matches[0])
+
+    if fuzzy_parse:
+        return [fuzzy_parse_json(match) for match in matches]
+    return [json.loads(match) for match in matches]

lionagi/libs/parse/nested/__init__.py

@@ -0,0 +1,33 @@
+"""Nested data structure manipulation utilities."""
+
+from .nfilter import nfilter
+from .nget import nget
+from .ninsert import ninsert
+from .nmerge import nmerge
+from .npop import npop
+from .nset import nset
+from .to_flat_list import to_flat_list
+from .utils import (
+    deep_update,
+    ensure_list_index,
+    get_target_container,
+    is_homogeneous,
+    is_same_dtype,
+    is_structure_homogeneous,
+)
+
+__all__ = [
+    "nfilter",
+    "nget",
+    "ninsert",
+    "nmerge",
+    "npop",
+    "nset",
+    "to_flat_list",
+    "deep_update",
+    "ensure_list_index",
+    "get_target_container",
+    "is_homogeneous",
+    "is_same_dtype",
+    "is_structure_homogeneous",
+]

lionagi/libs/parse/nested/nfilter.py

@@ -0,0 +1,55 @@
+from collections.abc import Callable
+from typing import Any
+
+
+def nfilter(
+    nested_structure: dict[Any, Any] | list[Any],
+    /,
+    condition: Callable[[Any], bool],
+) -> dict[Any, Any] | list[Any]:
+    """Filter elements in a nested structure based on a condition.
+
+    Args:
+        nested_structure: The nested structure (dict or list) to filter.
+        condition: Function returning True for elements to keep, False to
+            discard.
+
+    Returns:
+        The filtered nested structure.
+
+    Raises:
+        TypeError: If nested_structure is not a dict or list.
+
+    Example:
+        >>> data = {"a": 1, "b": {"c": 2, "d": 3}, "e": [4, 5, 6]}
+        >>> nfilter(data, lambda x: isinstance(x, int) and x > 2)
+        {'b': {'d': 3}, 'e': [4, 5, 6]}
+    """
+    if isinstance(nested_structure, dict):
+        return _filter_dict(nested_structure, condition)
+    elif isinstance(nested_structure, list):
+        return _filter_list(nested_structure, condition)
+    else:
+        raise TypeError(
+            "The nested_structure must be either a dict or a list."
+        )
+
+
+def _filter_dict(
+    dictionary: dict[Any, Any], condition: Callable[[tuple[Any, Any]], bool]
+) -> dict[Any, Any]:
+    return {
+        k: nfilter(v, condition) if isinstance(v, dict | list) else v
+        for k, v in dictionary.items()
+        if condition(v) or isinstance(v, dict | list)
+    }
+
+
+def _filter_list(
+    lst: list[Any], condition: Callable[[Any], bool]
+) -> list[Any]:
+    return [
+        nfilter(item, condition) if isinstance(item, dict | list) else item
+        for item in lst
+        if condition(item) or isinstance(item, dict | list)
+    ]

lionagi/libs/parse/nested/nget.py

@@ -0,0 +1,40 @@
+from typing import Any
+
+from ...constants import UNDEFINED
+from .utils import get_target_container
+
+
+def nget(
+    nested_structure: dict[Any, Any] | list[Any],
+    /,
+    indices: list[int | str],
+    default: Any = UNDEFINED,
+) -> Any:
+    try:
+        target_container = get_target_container(nested_structure, indices[:-1])
+        last_index = indices[-1]
+
+        if (
+            isinstance(target_container, list)
+            and isinstance(last_index, int)
+            and last_index < len(target_container)
+        ):
+            return target_container[last_index]
+        elif (
+            isinstance(target_container, dict)
+            and last_index in target_container
+        ):
+            return target_container[last_index]
+        elif default is not UNDEFINED:
+            return default
+        else:
+            raise LookupError(
+                "Target not found and no default value provided."
+            )
+    except (IndexError, KeyError, TypeError):
+        if default is not UNDEFINED:
+            return default
+        else:
+            raise LookupError(
+                "Target not found and no default value provided."
+            )