lionagi 0.16.2__py3-none-any.whl → 0.17.0__py3-none-any.whl

lionagi/libs/parse.py

@@ -1,31 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from lionagi.libs.schema.as_readable import as_readable
-from lionagi.libs.schema.extract_code_block import extract_code_block
-from lionagi.libs.schema.function_to_schema import function_to_schema
-from lionagi.libs.validate.fuzzy_match_keys import fuzzy_match_keys
-from lionagi.libs.validate.fuzzy_validate_mapping import fuzzy_validate_mapping
-from lionagi.libs.validate.string_similarity import string_similarity
-from lionagi.libs.validate.to_num import to_num
-from lionagi.utils import fuzzy_parse_json, to_dict, to_json
-
-validate_keys = fuzzy_match_keys  # for backward compatibility
-validate_mapping = fuzzy_validate_mapping  # for backward compatibility
-
-
-__all__ = (
-    "as_readable",
-    "extract_code_block",
-    "function_to_schema",
-    "fuzzy_match_keys",
-    "fuzzy_validate_mapping",
-    "string_similarity",
-    "validate_keys",
-    "validate_mapping",
-    "to_dict",
-    "to_json",
-    "to_num",
-    "fuzzy_parse_json",
-)

lionagi/libs/schema/json_schema.py

@@ -1,231 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Any, Literal
-
-from ..nested.flatten import flatten
-
-
-def extract_json_schema(
-    data: Any,
-    *,
-    sep: str = "|",
-    coerce_keys: bool = True,
-    dynamic: bool = True,
-    coerce_sequence: Literal["dict", "list"] | None = None,
-    max_depth: int | None = None,
-) -> dict[str, Any]:
-    """
-    Extract a JSON schema from JSON data.
-
-    This function uses the flatten function to create a flat representation
-    of the JSON data, then builds a schema based on the flattened structure.
-
-    Args:
-        data: The JSON data to extract the schema from.
-        sep: Separator used in flattened keys.
-        coerce_keys: Whether to coerce keys to strings.
-        dynamic: Whether to use dynamic flattening.
-        coerce_sequence: How to coerce sequences ("dict", "list", or None).
-        max_depth: Maximum depth to flatten.
-
-    Returns:
-        A dictionary representing the JSON schema.
-    """
-    flattened = flatten(
-        data,
-        sep=sep,
-        coerce_keys=coerce_keys,
-        dynamic=dynamic,
-        coerce_sequence=coerce_sequence,
-        max_depth=max_depth,
-    )
-
-    schema = {}
-    for key, value in flattened.items():
-        key_parts = key.split(sep) if isinstance(key, str) else key
-        current = schema
-        for part in key_parts[:-1]:
-            if part not in current:
-                current[part] = {}
-            current = current[part]
-
-        current[key_parts[-1]] = _get_type(value)
-
-    return {"type": "object", "properties": _consolidate_schema(schema)}
-
-
-def _get_type(value: Any) -> dict[str, Any]:
-    if isinstance(value, str):
-        return {"type": "string"}
-    elif isinstance(value, bool):
-        return {"type": "boolean"}
-    elif isinstance(value, int):
-        return {"type": "integer"}
-    elif isinstance(value, float):
-        return {"type": "number"}
-    elif isinstance(value, list):
-        if not value:
-            return {"type": "array", "items": {}}
-        item_types = [_get_type(item) for item in value]
-        if all(item_type == item_types[0] for item_type in item_types):
-            return {"type": "array", "items": item_types[0]}
-        else:
-            return {"type": "array", "items": {"oneOf": item_types}}
-    elif isinstance(value, dict):
-        return {
-            "type": "object",
-            "properties": _consolidate_schema(
-                {k: _get_type(v) for k, v in value.items()}
-            ),
-        }
-    elif value is None:
-        return {"type": "null"}
-    else:
-        return {"type": "any"}
-
-
-def _consolidate_schema(schema: dict) -> dict:
-    """
-    Consolidate the schema to handle lists and nested structures.
-    """
-    consolidated = {}
-    for key, value in schema.items():
-        if isinstance(value, dict) and all(k.isdigit() for k in value.keys()):
-            # This is likely a list
-            item_types = list(value.values())
-            if all(item_type == item_types[0] for item_type in item_types):
-                consolidated[key] = {"type": "array", "items": item_types[0]}
-            else:
-                consolidated[key] = {
-                    "type": "array",
-                    "items": {"oneOf": item_types},
-                }
-        elif isinstance(value, dict) and "type" in value:
-            consolidated[key] = value
-        else:
-            consolidated[key] = _consolidate_schema(value)
-    return consolidated
-
-
-def json_schema_to_cfg(
-    schema: dict[str, Any], start_symbol: str = "S"
-) -> list[tuple[str, list[str]]]:
-    productions = []
-    visited = set()
-    symbol_counter = 0
-
-    def generate_symbol(base: str) -> str:
-        nonlocal symbol_counter
-        symbol = f"{base}@{symbol_counter}"
-        symbol_counter += 1
-        return symbol
-
-    def generate_rules(s: dict[str, Any], symbol: str):
-        if symbol in visited:
-            return
-        visited.add(symbol)
-
-        if s.get("type") == "object":
-            properties = s.get("properties", {})
-            if properties:
-                props_symbol = generate_symbol("PROPS")
-                productions.append((symbol, ["{", props_symbol, "}"]))
-
-                productions.append((props_symbol, []))  # Empty object
-                for i, prop in enumerate(properties):
-                    prop_symbol = generate_symbol(prop)
-                    if i == 0:
-                        productions.append((props_symbol, [prop_symbol]))
-                    else:
-                        productions.append(
-                            (props_symbol, [props_symbol, ",", prop_symbol])
-                        )
-
-                for prop, prop_schema in properties.items():
-                    prop_symbol = generate_symbol(prop)
-                    value_symbol = generate_symbol("VALUE")
-                    productions.append(
-                        (prop_symbol, [f'"{prop}"', ":", value_symbol])
-                    )
-                    generate_rules(prop_schema, value_symbol)
-            else:
-                productions.append((symbol, ["{", "}"]))
-
-        elif s.get("type") == "array":
-            items = s.get("items", {})
-            items_symbol = generate_symbol("ITEMS")
-            value_symbol = generate_symbol("VALUE")
-            productions.append((symbol, ["[", "]"]))
-            productions.append((symbol, ["[", items_symbol, "]"]))
-            productions.append((items_symbol, [value_symbol]))
-            productions.append(
-                (items_symbol, [value_symbol, ",", items_symbol])
-            )
-            generate_rules(items, value_symbol)
-
-        elif s.get("type") == "string":
-            productions.append((symbol, ["STRING"]))
-
-        elif s.get("type") == "number":
-            productions.append((symbol, ["NUMBER"]))
-
-        elif s.get("type") == "integer":
-            productions.append((symbol, ["INTEGER"]))
-
-        elif s.get("type") == "boolean":
-            productions.append((symbol, ["BOOLEAN"]))
-
-        elif s.get("type") == "null":
-            productions.append((symbol, ["NULL"]))
-
-    generate_rules(schema, start_symbol)
-    return productions
-
-
-def json_schema_to_regex(schema: dict[str, Any]) -> str:
-    def schema_to_regex(s):
-        if s.get("type") == "object":
-            properties = s.get("properties", {})
-            prop_patterns = [
-                rf'"{prop}"\s*:\s*{schema_to_regex(prop_schema)}'
-                for prop, prop_schema in properties.items()
-            ]
-            return (
-                r"\{"
-                + r"\s*("
-                + r"|".join(prop_patterns)
-                + r")"
-                + r"(\s*,\s*("
-                + r"|".join(prop_patterns)
-                + r"))*\s*\}"
-            )
-        elif s.get("type") == "array":
-            items = s.get("items", {})
-            return (
-                r"\[\s*("
-                + schema_to_regex(items)
-                + r"(\s*,\s*"
-                + schema_to_regex(items)
-                + r")*)?\s*\]"
-            )
-        elif s.get("type") == "string":
-            return r'"[^"]*"'
-        elif s.get("type") == "integer":
-            return r"-?\d+"
-        elif s.get("type") == "number":
-            return r"-?\d+(\.\d+)?"
-        elif s.get("type") == "boolean":
-            return r"(true|false)"
-        elif s.get("type") == "null":
-            return r"null"
-        else:
-            return r".*"
-
-    return "^" + schema_to_regex(schema) + "$"
-
-
-def print_cfg(productions: list[tuple[str, list[str]]]):
-    for lhs, rhs in productions:
-        print(f"{lhs} -> {' '.join(rhs)}")

lionagi/libs/unstructured/pdf_to_image.py

@@ -1,45 +0,0 @@
-from lionagi.utils import import_module, is_import_installed
-
-_HAS_PDF2IMAGE = is_import_installed("pdf2image")
-
-
-def pdf_to_images(
-    pdf_path: str, output_folder: str, dpi: int = 300, fmt: str = "jpeg"
-) -> list:
-    """
-    Convert a PDF file into images, one image per page.
-
-    Args:
-        pdf_path (str): Path to the input PDF file.
-        output_folder (str): Directory to save the output images.
-        dpi (int): Dots per inch (resolution) for conversion (default: 300).
-        fmt (str): Image format (default: 'jpeg'). Use 'png' if preferred.
-
-    Returns:
-        list: A list of file paths for the saved images.
-    """
-    if not _HAS_PDF2IMAGE:
-        raise ModuleNotFoundError(
-            "pdf2image is not installed, please install it with `pip install lionagi[unstructured]`"
-        )
-
-    import os
-
-    convert_from_path = import_module(
-        "pdf2image", import_name="convert_from_path"
-    )
-
-    # Ensure the output folder exists
-    os.makedirs(output_folder, exist_ok=True)
-
-    # Convert PDF to a list of PIL Image objects
-    images = convert_from_path(pdf_path, dpi=dpi)
-
-    saved_paths = []
-    for i, image in enumerate(images):
-        # Construct the output file name
-        image_file = os.path.join(output_folder, f"page_{i + 1}.{fmt}")
-        image.save(image_file, fmt.upper())
-        saved_paths.append(image_file)
-
-    return saved_paths

lionagi/libs/unstructured/read_image_to_base64.py

@@ -1,33 +0,0 @@
-from pathlib import Path
-
-from lionagi.utils import is_import_installed
-
-_HAS_OPENCV = is_import_installed("cv2")
-
-
-__all__ = ("read_image_to_base64",)
-
-
-def read_image_to_base64(image_path: str | Path) -> str:
-    if not _HAS_OPENCV:
-        raise ModuleNotFoundError(
-            "OpenCV is not installed, please install it with `pip install lionagi[unstructured]`"
-        )
-
-    import base64
-
-    import cv2
-
-    image_path = str(image_path)
-    image = cv2.imread(image_path, cv2.COLOR_BGR2RGB)
-
-    if image is None:
-        raise ValueError(f"Could not read image from path: {image_path}")
-
-    file_extension = "." + image_path.split(".")[-1]
-
-    success, buffer = cv2.imencode(file_extension, image)
-    if not success:
-        raise ValueError(f"Could not encode image to {file_extension} format.")
-    encoded_image = base64.b64encode(buffer).decode("utf-8")
-    return encoded_image

lionagi/libs/validate/fuzzy_match_keys.py

@@ -1,7 +0,0 @@
-from lionagi.ln.fuzzy._fuzzy_match import (
-    FuzzyMatchKeysParams,
-    HandleUnmatched,
-    fuzzy_match_keys,
-)
-
-__all__ = ("fuzzy_match_keys", "FuzzyMatchKeysParams", "HandleUnmatched")

lionagi/libs/validate/fuzzy_validate_mapping.py

@@ -1,144 +0,0 @@
-# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from collections.abc import Callable, Sequence
-from typing import Any, Literal
-
-from lionagi.utils import KeysDict, Params, to_dict, to_json
-
-from .fuzzy_match_keys import fuzzy_match_keys
-from .string_similarity import SIMILARITY_TYPE
-
-
-class FuzzyValidateMappingParams(Params):
-    similarity_algo: SIMILARITY_TYPE | Callable[[str, str], float] = (
-        "jaro_winkler"
-    )
-    similarity_threshold: float = 0.85
-    fuzzy_match: bool = True
-    handle_unmatched: Literal["ignore", "raise", "remove", "fill", "force"] = (
-        "ignore"
-    )
-    fill_value: Any = None
-    fill_mapping: dict[str, Any] | None = None
-    strict: bool = False
-    suppress_conversion_errors: bool = False
-
-    def __call__(
-        self, d_: dict[str, Any], keys: Sequence[str] | KeysDict
-    ) -> dict[str, Any]:
-        return fuzzy_validate_mapping(
-            d_,
-            keys,
-            similarity_algo=self.similarity_algo,
-            similarity_threshold=self.similarity_threshold,
-            fuzzy_match=self.fuzzy_match,
-            handle_unmatched=self.handle_unmatched,
-            fill_value=self.fill_value,
-            fill_mapping=self.fill_mapping,
-            strict=self.strict,
-            suppress_conversion_errors=self.suppress_conversion_errors,
-        )
-
-
-def fuzzy_validate_mapping(
-    d: Any,
-    keys: Sequence[str] | KeysDict,
-    /,
-    *,
-    similarity_algo: (
-        SIMILARITY_TYPE | Callable[[str, str], float]
-    ) = "jaro_winkler",
-    similarity_threshold: float = 0.85,
-    fuzzy_match: bool = True,
-    handle_unmatched: Literal[
-        "ignore", "raise", "remove", "fill", "force"
-    ] = "ignore",
-    fill_value: Any = None,
-    fill_mapping: dict[str, Any] | None = None,
-    strict: bool = False,
-    suppress_conversion_errors: bool = False,
-) -> dict[str, Any]:
-    """
-    Validate and correct any input into a dictionary with expected keys.
-
-    Args:
-        d: Input to validate. Can be:
-            - Dictionary
-            - JSON string or markdown code block
-            - XML string
-            - Object with to_dict/model_dump method
-            - Any type convertible to dictionary
-        keys: List of expected keys or dictionary mapping keys to types.
-        similarity_algo: String similarity algorithm or custom function.
-        similarity_threshold: Minimum similarity score for fuzzy matching.
-        fuzzy_match: If True, use fuzzy matching for key correction.
-        handle_unmatched: How to handle unmatched keys:
-            - "ignore": Keep unmatched keys
-            - "raise": Raise error for unmatched keys
-            - "remove": Remove unmatched keys
-            - "fill": Fill missing keys with default values
-            - "force": Combine "fill" and "remove" behaviors
-        fill_value: Default value for filling unmatched keys.
-        fill_mapping: Dictionary mapping keys to default values.
-        strict: Raise error if any expected key is missing.
-        suppress_conversion_errors: Return empty dict on conversion errors.
-
-    Returns:
-        Validated and corrected dictionary.
-
-    Raises:
-        ValueError: If input cannot be converted or validation fails.
-        TypeError: If input types are invalid.
-    """
-    if d is None:
-        raise TypeError("Input cannot be None")
-
-    # Try converting to dictionary
-    try:
-        if isinstance(d, str):
-            # First try to_json for JSON strings and code blocks
-            try:
-                json_result = to_json(d)
-                dict_input = (
-                    json_result[0]
-                    if isinstance(json_result, list)
-                    else json_result
-                )
-            except Exception:
-                # Fall back to to_dict for other string formats
-                dict_input = to_dict(
-                    d, str_type="json", fuzzy_parse=True, suppress=True
-                )
-        else:
-            dict_input = to_dict(
-                d, use_model_dump=True, fuzzy_parse=True, suppress=True
-            )
-
-        if not isinstance(dict_input, dict):
-            if suppress_conversion_errors:
-                dict_input = {}
-            else:
-                raise ValueError(
-                    f"Failed to convert input to dictionary: {type(dict_input)}"
-                )
-
-    except Exception as e:
-        if suppress_conversion_errors:
-            dict_input = {}
-        else:
-            raise ValueError(f"Failed to convert input to dictionary: {e}")
-
-    # Validate the dictionary
-    return fuzzy_match_keys(
-        dict_input,
-        keys,
-        similarity_algo=similarity_algo,
-        similarity_threshold=similarity_threshold,
-        fuzzy_match=fuzzy_match,
-        handle_unmatched=handle_unmatched,
-        fill_value=fill_value,
-        fill_mapping=fill_mapping,
-        strict=strict,
-    )

lionagi/libs/validate/string_similarity.py

@@ -1,7 +0,0 @@
-from lionagi.ln.fuzzy._string_similarity import (
-    SIMILARITY_TYPE,
-    SimilarityFunc,
-    string_similarity,
-)
-
-__all__ = ("string_similarity", "SIMILARITY_TYPE", "SimilarityFunc")