lionagi 0.16.1__py3-none-any.whl → 0.16.3__py3-none-any.whl

lionagi/libs/nested/unflatten.py

@@ -1,83 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Any
-
-
-def unflatten(
-    flat_dict: dict[str, Any], sep: str = "|", inplace: bool = False
-) -> dict[str, Any] | list[Any]:
-    """
-    Unflatten a single-level dictionary into a nested dictionary or list.
-
-    Args:
-        flat_dict: The flattened dictionary to unflatten.
-        sep: The separator used for joining keys.
-        inplace: Whether to modify the input dictionary in place.
-
-    Returns:
-        The unflattened nested dictionary or list.
-
-    Examples:
-        >>> unflatten({"a|b|c": 1, "a|b|d": 2})
-        {'a': {'b': {'c': 1, 'd': 2}}}
-
-        >>> unflatten({"0": "a", "1": "b", "2": "c"})
-        ['a', 'b', 'c']
-    """
-
-    def _unflatten(data: dict) -> dict | list:
-        result = {}
-        for key, value in data.items():
-            parts = key.split(sep)
-            current = result
-            for part in parts[:-1]:
-                if part not in current:
-                    current[part] = {}
-                current = current[part]
-            if isinstance(value, dict):
-                current[parts[-1]] = _unflatten(value)
-            else:
-                current[parts[-1]] = value
-
-        # Convert dictionary to list if keys are consecutive integers
-        if result and all(
-            isinstance(key, str) and key.isdigit() for key in result
-        ):
-            return [result[str(i)] for i in range(len(result))]
-        return result
-
-    if inplace:
-        unflattened_dict = {}
-        for key, value in flat_dict.items():
-            parts = key.split(sep)
-            current = unflattened_dict
-            for part in parts[:-1]:
-                if part not in current:
-                    current[part] = {}
-                current = current[part]
-            current[parts[-1]] = value
-
-        unflattened_result = _unflatten(unflattened_dict)
-        flat_dict.clear()
-        if isinstance(unflattened_result, list):
-            flat_dict.update(
-                {str(i): v for i, v in enumerate(unflattened_result)}
-            )
-        else:
-            flat_dict.update(unflattened_result)
-        return flat_dict
-
-    else:
-        unflattened_dict = {}
-        for key, value in flat_dict.items():
-            parts = key.split(sep)
-            current = unflattened_dict
-            for part in parts[:-1]:
-                if part not in current:
-                    current[part] = {}
-                current = current[part]
-            current[parts[-1]] = value
-
-        return _unflatten(unflattened_dict)

lionagi/libs/nested/utils.py

@@ -1,189 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Any
-
-from lionagi.utils import UNDEFINED
-
-
-def is_homogeneous(
-    iterables: list[Any] | dict[Any, Any], type_check: type | tuple[type, ...]
-) -> bool:
-    """
-    Check if all elements in a list or all values in a dict are of same type.
-
-    Args:
-        iterables: The list or dictionary to check.
-        type_check: The type to check against.
-
-    Returns:
-        True if all elements/values are of the same type, False otherwise.
-    """
-    if isinstance(iterables, list):
-        return all(isinstance(it, type_check) for it in iterables)
-
-    elif isinstance(iterables, dict):
-        return all(isinstance(val, type_check) for val in iterables.values())
-
-    else:
-        return isinstance(iterables, type_check)
-
-
-def is_same_dtype(
-    input_: list[Any] | dict[Any, Any],
-    dtype: type | None = None,
-    return_dtype: bool = False,
-) -> bool | tuple[bool, type | None]:
-    """
-    Check if all elements in a list or dict values are of the same data type.
-
-    Args:
-        input_: The input list or dictionary to check.
-        dtype: The data type to check against. If None, uses the type of the
-            first element.
-        return_dtype: If True, return the data type with the check result.
-
-    Returns:
-        If return_dtype is False, returns True if all elements are of the
-        same type (or if the input is empty), False otherwise.
-        If return_dtype is True, returns a tuple (bool, type | None).
-    """
-    if not input_:
-        return True
-
-    iterable = input_.values() if isinstance(input_, dict) else input_
-    first_element_type = type(next(iter(iterable), None))
-
-    dtype = dtype or first_element_type
-
-    result = all(isinstance(element, dtype) for element in iterable)
-    return (result, dtype) if return_dtype else result
-
-
-def is_structure_homogeneous(
-    structure: Any, return_structure_type: bool = False
-) -> bool | tuple[bool, type | None]:
-    """
-    Check if a nested structure is homogeneous (no mix of lists and dicts).
-
-    Args:
-        structure: The nested structure to check.
-        return_structure_type: If True, return the type of the homogeneous
-            structure.
-
-    Returns:
-        If return_structure_type is False, returns True if the structure is
-        homogeneous, False otherwise.
-        If True, returns a tuple (bool, type | None).
-
-    Examples:
-        >>> is_structure_homogeneous({'a': {'b': 1}, 'c': {'d': 2}})
-        True
-        >>> is_structure_homogeneous({'a': {'b': 1}, 'c': [1, 2]})
-        False
-    """
-
-    def _check_structure(substructure):
-        structure_type = None
-        if isinstance(substructure, list):
-            structure_type = list
-            for item in substructure:
-                if not isinstance(item, structure_type) and isinstance(
-                    item, list | dict
-                ):
-                    return False, None
-                result, _ = _check_structure(item)
-                if not result:
-                    return False, None
-        elif isinstance(substructure, dict):
-            structure_type = dict
-            for item in substructure.values():
-                if not isinstance(item, structure_type) and isinstance(
-                    item, list | dict
-                ):
-                    return False, None
-                result, _ = _check_structure(item)
-                if not result:
-                    return False, None
-        return True, structure_type
-
-    is_homogeneous, structure_type = _check_structure(structure)
-    return (
-        (is_homogeneous, structure_type)
-        if return_structure_type
-        else is_homogeneous
-    )
-
-
-def deep_update(
-    original: dict[Any, Any], update: dict[Any, Any]
-) -> dict[Any, Any]:
-    """
-    Recursively merge two dicts, updating nested dicts instead of overwriting.
-
-    Args:
-        original: The dictionary to update.
-        update: The dictionary containing updates to apply to `original`.
-
-    Returns:
-        The `original` dictionary after applying updates from `update`.
-
-    Note:
-        This method modifies the `original` dictionary in place.
-    """
-    for key, value in update.items():
-        if isinstance(value, dict) and key in original:
-            original[key] = deep_update(original.get(key, {}), value)
-        else:
-            original[key] = value
-    return original
-
-
-def get_target_container(
-    nested: list[Any] | dict[Any, Any], indices: list[int | str]
-) -> list[Any] | dict[Any, Any]:
-    """
-    Retrieve the target container in a nested structure using indices.
-
-    Args:
-        nested: The nested structure to navigate.
-        indices: A list of indices to navigate through the nested structure.
-
-    Returns:
-        The target container at the specified path.
-
-    Raises:
-        IndexError: If a list index is out of range.
-        KeyError: If a dictionary key is not found.
-        TypeError: If the current element is neither a list nor a dictionary.
-    """
-    current_element = nested
-    for index in indices:
-        if isinstance(current_element, list):
-            if isinstance(index, str) and index.isdigit():
-                index = int(index)
-
-            if isinstance(index, int) and 0 <= index < len(current_element):
-                current_element = current_element[index]
-
-            else:
-                raise IndexError("List index is invalid or out of range")
-
-        elif isinstance(current_element, dict):
-            if index in current_element:
-                current_element = current_element.get(index, None)
-            else:
-                raise KeyError("Key not found in dictionary")
-        else:
-            raise TypeError(
-                "Current element is neither a list nor a dictionary"
-            )
-    return current_element
-
-
-def ensure_list_index(
-    lst: list[Any], index: int, default: Any = UNDEFINED
-) -> None:
-    while len(lst) <= index:
-        lst.append(default if default is not UNDEFINED else None)

lionagi/libs/parse.py

@@ -1,31 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from lionagi.libs.schema.as_readable import as_readable
-from lionagi.libs.schema.extract_code_block import extract_code_block
-from lionagi.libs.schema.function_to_schema import function_to_schema
-from lionagi.libs.validate.fuzzy_match_keys import fuzzy_match_keys
-from lionagi.libs.validate.fuzzy_validate_mapping import fuzzy_validate_mapping
-from lionagi.libs.validate.string_similarity import string_similarity
-from lionagi.libs.validate.to_num import to_num
-from lionagi.utils import fuzzy_parse_json, to_dict, to_json
-
-validate_keys = fuzzy_match_keys  # for backward compatibility
-validate_mapping = fuzzy_validate_mapping  # for backward compatibility
-
-
-__all__ = (
-    "as_readable",
-    "extract_code_block",
-    "function_to_schema",
-    "fuzzy_match_keys",
-    "fuzzy_validate_mapping",
-    "string_similarity",
-    "validate_keys",
-    "validate_mapping",
-    "to_dict",
-    "to_json",
-    "to_num",
-    "fuzzy_parse_json",
-)

lionagi/libs/schema/json_schema.py

@@ -1,231 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Any, Literal
-
-from ..nested.flatten import flatten
-
-
-def extract_json_schema(
-    data: Any,
-    *,
-    sep: str = "|",
-    coerce_keys: bool = True,
-    dynamic: bool = True,
-    coerce_sequence: Literal["dict", "list"] | None = None,
-    max_depth: int | None = None,
-) -> dict[str, Any]:
-    """
-    Extract a JSON schema from JSON data.
-
-    This function uses the flatten function to create a flat representation
-    of the JSON data, then builds a schema based on the flattened structure.
-
-    Args:
-        data: The JSON data to extract the schema from.
-        sep: Separator used in flattened keys.
-        coerce_keys: Whether to coerce keys to strings.
-        dynamic: Whether to use dynamic flattening.
-        coerce_sequence: How to coerce sequences ("dict", "list", or None).
-        max_depth: Maximum depth to flatten.
-
-    Returns:
-        A dictionary representing the JSON schema.
-    """
-    flattened = flatten(
-        data,
-        sep=sep,
-        coerce_keys=coerce_keys,
-        dynamic=dynamic,
-        coerce_sequence=coerce_sequence,
-        max_depth=max_depth,
-    )
-
-    schema = {}
-    for key, value in flattened.items():
-        key_parts = key.split(sep) if isinstance(key, str) else key
-        current = schema
-        for part in key_parts[:-1]:
-            if part not in current:
-                current[part] = {}
-            current = current[part]
-
-        current[key_parts[-1]] = _get_type(value)
-
-    return {"type": "object", "properties": _consolidate_schema(schema)}
-
-
-def _get_type(value: Any) -> dict[str, Any]:
-    if isinstance(value, str):
-        return {"type": "string"}
-    elif isinstance(value, bool):
-        return {"type": "boolean"}
-    elif isinstance(value, int):
-        return {"type": "integer"}
-    elif isinstance(value, float):
-        return {"type": "number"}
-    elif isinstance(value, list):
-        if not value:
-            return {"type": "array", "items": {}}
-        item_types = [_get_type(item) for item in value]
-        if all(item_type == item_types[0] for item_type in item_types):
-            return {"type": "array", "items": item_types[0]}
-        else:
-            return {"type": "array", "items": {"oneOf": item_types}}
-    elif isinstance(value, dict):
-        return {
-            "type": "object",
-            "properties": _consolidate_schema(
-                {k: _get_type(v) for k, v in value.items()}
-            ),
-        }
-    elif value is None:
-        return {"type": "null"}
-    else:
-        return {"type": "any"}
-
-
-def _consolidate_schema(schema: dict) -> dict:
-    """
-    Consolidate the schema to handle lists and nested structures.
-    """
-    consolidated = {}
-    for key, value in schema.items():
-        if isinstance(value, dict) and all(k.isdigit() for k in value.keys()):
-            # This is likely a list
-            item_types = list(value.values())
-            if all(item_type == item_types[0] for item_type in item_types):
-                consolidated[key] = {"type": "array", "items": item_types[0]}
-            else:
-                consolidated[key] = {
-                    "type": "array",
-                    "items": {"oneOf": item_types},
-                }
-        elif isinstance(value, dict) and "type" in value:
-            consolidated[key] = value
-        else:
-            consolidated[key] = _consolidate_schema(value)
-    return consolidated
-
-
-def json_schema_to_cfg(
-    schema: dict[str, Any], start_symbol: str = "S"
-) -> list[tuple[str, list[str]]]:
-    productions = []
-    visited = set()
-    symbol_counter = 0
-
-    def generate_symbol(base: str) -> str:
-        nonlocal symbol_counter
-        symbol = f"{base}@{symbol_counter}"
-        symbol_counter += 1
-        return symbol
-
-    def generate_rules(s: dict[str, Any], symbol: str):
-        if symbol in visited:
-            return
-        visited.add(symbol)
-
-        if s.get("type") == "object":
-            properties = s.get("properties", {})
-            if properties:
-                props_symbol = generate_symbol("PROPS")
-                productions.append((symbol, ["{", props_symbol, "}"]))
-
-                productions.append((props_symbol, []))  # Empty object
-                for i, prop in enumerate(properties):
-                    prop_symbol = generate_symbol(prop)
-                    if i == 0:
-                        productions.append((props_symbol, [prop_symbol]))
-                    else:
-                        productions.append(
-                            (props_symbol, [props_symbol, ",", prop_symbol])
-                        )
-
-                for prop, prop_schema in properties.items():
-                    prop_symbol = generate_symbol(prop)
-                    value_symbol = generate_symbol("VALUE")
-                    productions.append(
-                        (prop_symbol, [f'"{prop}"', ":", value_symbol])
-                    )
-                    generate_rules(prop_schema, value_symbol)
-            else:
-                productions.append((symbol, ["{", "}"]))
-
-        elif s.get("type") == "array":
-            items = s.get("items", {})
-            items_symbol = generate_symbol("ITEMS")
-            value_symbol = generate_symbol("VALUE")
-            productions.append((symbol, ["[", "]"]))
-            productions.append((symbol, ["[", items_symbol, "]"]))
-            productions.append((items_symbol, [value_symbol]))
-            productions.append(
-                (items_symbol, [value_symbol, ",", items_symbol])
-            )
-            generate_rules(items, value_symbol)
-
-        elif s.get("type") == "string":
-            productions.append((symbol, ["STRING"]))
-
-        elif s.get("type") == "number":
-            productions.append((symbol, ["NUMBER"]))
-
-        elif s.get("type") == "integer":
-            productions.append((symbol, ["INTEGER"]))
-
-        elif s.get("type") == "boolean":
-            productions.append((symbol, ["BOOLEAN"]))
-
-        elif s.get("type") == "null":
-            productions.append((symbol, ["NULL"]))
-
-    generate_rules(schema, start_symbol)
-    return productions
-
-
-def json_schema_to_regex(schema: dict[str, Any]) -> str:
-    def schema_to_regex(s):
-        if s.get("type") == "object":
-            properties = s.get("properties", {})
-            prop_patterns = [
-                rf'"{prop}"\s*:\s*{schema_to_regex(prop_schema)}'
-                for prop, prop_schema in properties.items()
-            ]
-            return (
-                r"\{"
-                + r"\s*("
-                + r"|".join(prop_patterns)
-                + r")"
-                + r"(\s*,\s*("
-                + r"|".join(prop_patterns)
-                + r"))*\s*\}"
-            )
-        elif s.get("type") == "array":
-            items = s.get("items", {})
-            return (
-                r"\[\s*("
-                + schema_to_regex(items)
-                + r"(\s*,\s*"
-                + schema_to_regex(items)
-                + r")*)?\s*\]"
-            )
-        elif s.get("type") == "string":
-            return r'"[^"]*"'
-        elif s.get("type") == "integer":
-            return r"-?\d+"
-        elif s.get("type") == "number":
-            return r"-?\d+(\.\d+)?"
-        elif s.get("type") == "boolean":
-            return r"(true|false)"
-        elif s.get("type") == "null":
-            return r"null"
-        else:
-            return r".*"
-
-    return "^" + schema_to_regex(schema) + "$"
-
-
-def print_cfg(productions: list[tuple[str, list[str]]]):
-    for lhs, rhs in productions:
-        print(f"{lhs} -> {' '.join(rhs)}")

lionagi/libs/token_transform/base.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from pathlib import Path
-
-from pydantic import Field
-
-from lionagi.tools.base import Resource, ResourceCategory
-
-here = Path(__file__).parent.resolve()
-MAPPING_PATH = "synthlang_/resources/mapping"
-
-
-class TokenMappingTemplate(str, Enum):
-    RUST_CHINESE = "rust_chinese"
-    LION_EMOJI = "lion_emoji"
-    PYTHON_MATH = "python_math"
-
-    @property
-    def fp(self) -> Path:
-        return here / MAPPING_PATH / f"{self.value}_mapping.toml"
-
-
-class TokenMapping(Resource):
-    category: ResourceCategory = Field(
-        default=ResourceCategory.UTILITY, frozen=True
-    )
-    content: dict
-
-    @classmethod
-    def load_from_template(
-        cls, template: TokenMappingTemplate | str
-    ) -> TokenMapping:
-        if isinstance(template, str):
-            template = template.lower().strip()
-            template = (
-                template.replace(".toml", "")
-                .replace(" ", "_")
-                .replace("-", "_")
-                .strip()
-            )
-            if template.endswith("_mapping"):
-                template = template[:-8]
-            if "/" in template:
-                template = template.split("/")[-1]
-            template = TokenMappingTemplate(template)
-
-        if isinstance(template, TokenMappingTemplate):
-            template = template.fp
-            return cls.adapt_from(template, ".toml", many=False)
-
-        raise ValueError(
-            f"Invalid template: {template}. Must be a TokenMappingTemplate or a valid path."
-        )

lionagi/libs/token_transform/llmlingua.py

@@ -1 +0,0 @@
-# TODO