lionherd-core 1.0.0a3__py3-none-any.whl

lionherd_core/ln/_fuzzy_validate.py

@@ -0,0 +1,151 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal
+
+from lionherd_core.errors import ValidationError
+
+from ..libs.string_handlers._extract_json import extract_json
+from ..libs.string_handlers._string_similarity import SIMILARITY_TYPE
+from ..types import KeysLike
+from ._fuzzy_match import FuzzyMatchKeysParams, fuzzy_match_keys
+from ._to_dict import to_dict
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
+
+__all__ = ("fuzzy_validate_pydantic",)
+
+
+def fuzzy_validate_pydantic(
+    text,
+    /,
+    model_type: "type[BaseModel]",
+    fuzzy_parse: bool = True,
+    fuzzy_match: bool = False,
+    fuzzy_match_params: FuzzyMatchKeysParams | dict = None,
+):
+    """Validate and parse text/dict into Pydantic model with fuzzy parsing.
+
+    Args:
+        text: Input data (BaseModel instance, dict, or JSON string)
+        model_type: Target Pydantic model class
+        fuzzy_parse: Enable fuzzy JSON extraction from text
+        fuzzy_match: Enable fuzzy key matching for field names
+        fuzzy_match_params: Parameters for fuzzy matching (dict or FuzzyMatchKeysParams)
+
+    Returns:
+        Validated Pydantic model instance
+
+    Raises:
+        ValidationError: If JSON extraction or model validation fails
+        TypeError: If fuzzy_match_params is invalid type
+    """
+    # Handle already-valid model instances
+    if isinstance(text, model_type):
+        return text
+
+    # Handle dict inputs directly (skip JSON extraction)
+    if isinstance(text, dict):
+        model_data = text
+    else:
+        # Handle string inputs (JSON strings, markdown, etc.)
+        try:
+            model_data = extract_json(text, fuzzy_parse=fuzzy_parse)
+        except Exception as e:
+            raise ValidationError(f"Failed to extract valid JSON from model response: {e}") from e
+
+    d = model_data
+    if fuzzy_match:
+        if fuzzy_match_params is None:
+            model_data = fuzzy_match_keys(d, model_type.model_fields, handle_unmatched="remove")
+        elif isinstance(fuzzy_match_params, dict):
+            model_data = fuzzy_match_keys(d, model_type.model_fields, **fuzzy_match_params)
+        elif isinstance(fuzzy_match_params, FuzzyMatchKeysParams):
+            model_data = fuzzy_match_params(d, model_type.model_fields)
+        else:
+            raise TypeError("fuzzy_keys_params must be a dict or FuzzyMatchKeysParams instance")
+
+    try:
+        return model_type.model_validate(model_data)
+    except Exception as e:
+        raise ValidationError(f"Validation failed: {e}") from e
+
+
+def fuzzy_validate_mapping(
+    d: Any,
+    keys: KeysLike,
+    /,
+    *,
+    similarity_algo: SIMILARITY_TYPE | Callable[[str, str], float] = "jaro_winkler",
+    similarity_threshold: float = 0.85,
+    fuzzy_match: bool = True,
+    handle_unmatched: Literal["ignore", "raise", "remove", "fill", "force"] = "ignore",
+    fill_value: Any = None,
+    fill_mapping: dict[str, Any] | None = None,
+    strict: bool = False,
+    suppress_conversion_errors: bool = False,
+) -> dict[str, Any]:
+    """Validate any input into dict with expected keys and fuzzy matching.
+
+    Converts input (dict, JSON string, XML, object) to dict and validates keys.
+
+    Args:
+        d: Input to convert and validate
+        keys: Expected keys (list or dict-like)
+        similarity_algo: String similarity algorithm
+        similarity_threshold: Minimum similarity score (0.0-1.0)
+        fuzzy_match: Enable fuzzy key matching
+        handle_unmatched: How to handle unmatched keys
+        fill_value: Default value for missing keys
+        fill_mapping: Custom values for specific keys
+        strict: Raise if expected keys are missing
+        suppress_conversion_errors: Return empty dict on conversion failure
+
+    Returns:
+        Validated dictionary with corrected keys
+
+    Raises:
+        TypeError: If d is None
+        ValueError: If conversion fails and suppress_conversion_errors is False
+    """
+    if d is None:
+        raise TypeError("Input cannot be None")
+
+    # Try converting to dictionary
+    try:
+        if isinstance(d, str):
+            try:
+                json_result = extract_json(d, fuzzy_parse=True, return_one_if_single=True)
+                dict_input = json_result[0] if isinstance(json_result, list) else json_result
+            except Exception:
+                dict_input = to_dict(d, fuzzy_parse=True, suppress=True)
+        else:
+            dict_input = to_dict(d, prioritize_model_dump=True, fuzzy_parse=True, suppress=True)
+
+        if not isinstance(dict_input, dict):
+            if suppress_conversion_errors:
+                dict_input = {}
+            else:
+                raise ValueError(f"Failed to convert input to dictionary: {type(dict_input)}")
+
+    except Exception as e:
+        if suppress_conversion_errors:
+            dict_input = {}
+        else:
+            raise ValueError(f"Failed to convert input to dictionary: {e}")
+
+    # Validate the dictionary
+    return fuzzy_match_keys(
+        dict_input,
+        keys,
+        similarity_algo=similarity_algo,
+        similarity_threshold=similarity_threshold,
+        fuzzy_match=fuzzy_match,
+        handle_unmatched=handle_unmatched,
+        fill_value=fill_value,
+        fill_mapping=fill_mapping,
+        strict=strict,
+    )

lionherd_core/ln/_hash.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import contextlib
+import copy
+from typing import Any
+
+__all__ = ("hash_dict",)
+
+# Global initialization state
+_INITIALIZED = False
+PydanticBaseModel = None
+
+# --- Canonical Representation Generator ---
+_PRIMITIVE_TYPES = (str, int, float, bool, type(None))
+_TYPE_MARKER_DICT = 0
+_TYPE_MARKER_LIST = 1
+_TYPE_MARKER_TUPLE = 2
+_TYPE_MARKER_SET = 3
+_TYPE_MARKER_FROZENSET = 4
+_TYPE_MARKER_PYDANTIC = 5  # Distinguishes dumped Pydantic models
+
+
+def _generate_hashable_representation(item: Any) -> Any:
+    """Convert object to stable hashable representation recursively.
+
+    Ensures order-independent hashing for dicts/sets and consistent handling of collections.
+    """
+    if isinstance(item, _PRIMITIVE_TYPES):
+        return item
+
+    if PydanticBaseModel and isinstance(item, PydanticBaseModel):
+        # Process the Pydantic model by first dumping it to a dict, then processing that dict.
+        # The type marker distinguishes this from a regular dictionary.
+        return (
+            _TYPE_MARKER_PYDANTIC,
+            _generate_hashable_representation(item.model_dump()),
+        )
+
+    if isinstance(item, dict):
+        # Sort dictionary items by key (stringified) for order-insensitivity.
+        return (
+            _TYPE_MARKER_DICT,
+            tuple(
+                (str(k), _generate_hashable_representation(v))
+                for k, v in sorted(item.items(), key=lambda x: str(x[0]))
+            ),
+        )
+
+    if isinstance(item, list):
+        return (
+            _TYPE_MARKER_LIST,
+            tuple(_generate_hashable_representation(elem) for elem in item),
+        )
+
+    if isinstance(item, tuple):
+        return (
+            _TYPE_MARKER_TUPLE,
+            tuple(_generate_hashable_representation(elem) for elem in item),
+        )
+
+    # frozenset must be checked before set
+    if isinstance(item, frozenset):
+        try:  # Attempt direct sort for comparable elements
+            sorted_elements = sorted(list(item))
+        except TypeError:  # Fallback for unorderable mixed types
+
+            def sort_key(x):
+                # Deterministic ordering across mixed, unorderable types
+                # Sort strictly by textual type then textual value.
+                # This also naturally places bool before int because
+                # "<class 'bool'>" < "<class 'int'>" lexicographically.
+                return (str(type(x)), str(x))
+
+            sorted_elements = sorted(list(item), key=sort_key)
+        return (
+            _TYPE_MARKER_FROZENSET,
+            tuple(_generate_hashable_representation(elem) for elem in sorted_elements),
+        )
+
+    if isinstance(item, set):
+        try:
+            sorted_elements = sorted(list(item))
+        except TypeError:
+            # For mixed types, use a deterministic, portable sort key
+            def sort_key(x):
+                # Sort by textual type then textual value for stability.
+                return (str(type(x)), str(x))
+
+            sorted_elements = sorted(list(item), key=sort_key)
+        return (
+            _TYPE_MARKER_SET,
+            tuple(_generate_hashable_representation(elem) for elem in sorted_elements),
+        )
+
+    # Fallback for other types (e.g., custom objects not derived from the above)
+    with contextlib.suppress(Exception):
+        return str(item)
+    with contextlib.suppress(Exception):
+        return repr(item)
+
+    # If both str() and repr() fail, return a stable fallback based on type and id
+    return f"<unhashable:{type(item).__name__}:{id(item)}>"
+
+
+def hash_dict(data: Any, strict: bool = False) -> int:
+    """Generate stable hash for any data structure including dicts, lists, and Pydantic models.
+
+    Args:
+        data: Data to hash (dict, list, BaseModel, or any object)
+        strict: If True, deepcopy data before hashing to prevent mutation side effects
+
+    Returns:
+        Integer hash value (stable across equivalent structures)
+
+    Raises:
+        TypeError: If generated representation is not hashable
+    """
+    global _INITIALIZED, PydanticBaseModel
+    if _INITIALIZED is False:
+        from pydantic import BaseModel
+
+        PydanticBaseModel = BaseModel
+        _INITIALIZED = True
+
+    data_to_process = data
+    if strict:
+        data_to_process = copy.deepcopy(data)
+
+    hashable_repr = _generate_hashable_representation(data_to_process)
+
+    try:
+        return hash(hashable_repr)
+    except TypeError as e:
+        raise TypeError(
+            f"The generated representation for the input data was not hashable. "
+            f"Input type: {type(data).__name__}, Representation type: {type(hashable_repr).__name__}. "
+            f"Original error: {e}"
+        )

lionherd_core/ln/_json_dump.py

@@ -0,0 +1,347 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import contextlib
+import datetime as dt
+import decimal
+import re
+from collections.abc import Callable, Iterable, Mapping
+from enum import Enum
+from functools import lru_cache
+from pathlib import Path
+from textwrap import shorten
+from typing import Any
+from uuid import UUID
+
+import orjson
+
+__all__ = [
+    "get_orjson_default",
+    "json_dumpb",
+    "json_dumps",
+    "json_lines_iter",
+    "make_options",
+]
+
+# Types orjson already serializes natively at C/Rust speed.
+# (We only route them through default() when passthrough is requested.)
+_NATIVE = (dt.datetime, dt.date, dt.time, UUID)
+
+# --------- helpers ------------------------------------------------------------
+
+_ADDR_PAT = re.compile(r" at 0x[0-9A-Fa-f]+")
+
+
+def _clip(s: str, limit: int = 2048) -> str:
+    return shorten(s, width=limit, placeholder=f"...(+{len(s) - limit} chars)")  # type: ignore[arg-type]
+
+
+def _normalize_for_sorting(x: Any) -> str:
+    """Normalize repr/str to remove process-specific addresses."""
+    s = str(x)
+    return _ADDR_PAT.sub(" at 0x?", s)
+
+
+def _stable_sorted_iterable(o: Iterable[Any]) -> list[Any]:
+    """
+    Deterministic ordering for sets (including mixed types).
+    Key: (class name, normalized str) avoids comparisons across unlike types
+    and removes memory address variance in default reprs.
+    """
+    return sorted(o, key=lambda x: (x.__class__.__name__, _normalize_for_sorting(x)))
+
+
+def _safe_exception_payload(ex: Exception) -> dict[str, str]:
+    return {"type": ex.__class__.__name__, "message": str(ex)}
+
+
+def _default_serializers(
+    deterministic_sets: bool,
+    decimal_as_float: bool,
+    enum_as_name: bool,
+    passthrough_datetime: bool,
+) -> dict[type, Callable[[Any], Any]]:
+    ser: dict[type, Callable[[Any], Any]] = {
+        Path: str,
+        decimal.Decimal: (float if decimal_as_float else str),
+        set: (_stable_sorted_iterable if deterministic_sets else list),
+        frozenset: (_stable_sorted_iterable if deterministic_sets else list),
+    }
+    if enum_as_name:
+        ser[Enum] = lambda e: e.name
+    # Only needed if you also set OPT_PASSTHROUGH_DATETIME via options.
+    if passthrough_datetime:
+        ser[dt.datetime] = lambda o: o.isoformat()
+    return ser
+
+
+# --------- default() factory --------------------------------------------------
+
+
+def get_orjson_default(
+    *,
+    order: list[type] | None = None,
+    additional: Mapping[type, Callable[[Any], Any]] | None = None,
+    extend_default: bool = True,
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+) -> Callable[[Any], Any]:
+    """
+    Build a fast, extensible `default=` callable for orjson.dumps.
+
+    - deterministic_sets: sort set/frozenset deterministically (slower).
+    - decimal_as_float: serialize Decimal as float (faster/smaller; precision loss).
+    - enum_as_name: serialize Enum as .name (else orjson uses .value by default).
+    - passthrough_datetime: if True, also pass OPT_PASSTHROUGH_DATETIME in options.
+    - safe_fallback: if True, unknown objects never raise (for logs);
+      Exceptions become a tiny dict; all else becomes clipped repr(str).
+
+    'order' and 'additional' preserve your override semantics.
+    """
+    ser = _default_serializers(
+        deterministic_sets=deterministic_sets,
+        decimal_as_float=decimal_as_float,
+        enum_as_name=enum_as_name,
+        passthrough_datetime=passthrough_datetime,
+    )
+    if additional:
+        ser.update(additional)
+
+    base_order: list[type] = [Path, decimal.Decimal, set, frozenset]
+    if enum_as_name:
+        base_order.insert(0, Enum)
+    if passthrough_datetime:
+        base_order.insert(0, dt.datetime)
+
+    if order:
+        order_ = (
+            (base_order + [t for t in order if t not in base_order])
+            if extend_default
+            else list(order)
+        )
+    else:
+        order_ = base_order.copy()
+
+    if not passthrough_datetime:
+        # Avoid checks for types already on the orjson native fast path.
+        order_ = [t for t in order_ if t not in _NATIVE]
+
+    order_tuple = tuple(order_)
+    cache: dict[type, Callable[[Any], Any]] = {}
+
+    def default(obj: Any) -> Any:
+        typ = obj.__class__
+        func = cache.get(typ)
+        if func is None:
+            for T in order_tuple:
+                if issubclass(typ, T):
+                    f = ser.get(T)
+                    if f:
+                        cache[typ] = f
+                        func = f
+                        break
+            else:
+                # Duck-typed support for common data holders
+                md = getattr(obj, "model_dump", None)
+                if callable(md):
+                    with contextlib.suppress(Exception):
+                        return md()
+
+                dd = getattr(obj, "dict", None)
+                if callable(dd):
+                    with contextlib.suppress(Exception):
+                        return dd()
+                if safe_fallback:
+                    if isinstance(obj, Exception):
+                        return _safe_exception_payload(obj)
+                    return _clip(repr(obj), fallback_clip)
+                raise TypeError(f"Type is not JSON serializable: {typ.__name__}")
+        return func(obj)
+
+    return default
+
+
+@lru_cache(maxsize=128)
+def _cached_default(
+    deterministic_sets: bool,
+    decimal_as_float: bool,
+    enum_as_name: bool,
+    passthrough_datetime: bool,
+    safe_fallback: bool,
+    fallback_clip: int,
+):
+    return get_orjson_default(
+        deterministic_sets=deterministic_sets,
+        decimal_as_float=decimal_as_float,
+        enum_as_name=enum_as_name,
+        passthrough_datetime=passthrough_datetime,
+        safe_fallback=safe_fallback,
+        fallback_clip=fallback_clip,
+    )
+
+
+# --------- defaults & options -------------------------------------------------
+
+
+def make_options(
+    *,
+    pretty: bool = False,
+    sort_keys: bool = False,
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    append_newline: bool = False,
+    passthrough_datetime: bool = False,
+    allow_non_str_keys: bool = False,
+) -> int:
+    """
+    Compose orjson 'option' bit flags succinctly.
+    """
+    opt = 0
+    if append_newline:
+        opt |= orjson.OPT_APPEND_NEWLINE
+    if pretty:
+        opt |= orjson.OPT_INDENT_2
+    if sort_keys:
+        opt |= orjson.OPT_SORT_KEYS
+    if naive_utc:
+        opt |= orjson.OPT_NAIVE_UTC
+    if utc_z:
+        opt |= orjson.OPT_UTC_Z
+    if passthrough_datetime:
+        opt |= orjson.OPT_PASSTHROUGH_DATETIME
+    if allow_non_str_keys:
+        opt |= orjson.OPT_NON_STR_KEYS
+    return opt
+
+
+# --------- dump helpers -------------------------------------------------------
+
+
+def json_dumpb(
+    obj: Any,
+    *,
+    pretty: bool = False,
+    sort_keys: bool = False,
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    append_newline: bool = False,
+    allow_non_str_keys: bool = False,
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+    default: Callable[[Any], Any] | None = None,
+    options: int | None = None,
+) -> bytes:
+    """
+    Serialize to **bytes** (fast path). Prefer this in hot code.
+
+    Notes:
+      - If you set passthrough_datetime=True, you likely also want it in options.
+      - safe_fallback=True is recommended for LOGGING ONLY.
+    """
+    if default is None:
+        default = _cached_default(
+            deterministic_sets=deterministic_sets,
+            decimal_as_float=decimal_as_float,
+            enum_as_name=enum_as_name,
+            passthrough_datetime=passthrough_datetime,
+            safe_fallback=safe_fallback,
+            fallback_clip=fallback_clip,
+        )
+    opt = (
+        options
+        if options is not None
+        else make_options(
+            pretty=pretty,
+            sort_keys=sort_keys,
+            naive_utc=naive_utc,
+            utc_z=utc_z,
+            append_newline=append_newline,
+            passthrough_datetime=passthrough_datetime,
+            allow_non_str_keys=allow_non_str_keys,
+        )
+    )
+    return orjson.dumps(obj, default=default, option=opt)
+
+
+def json_dumps(
+    obj: Any,
+    /,
+    *,
+    decode: bool = True,
+    **kwargs: Any,
+) -> str | bytes:
+    """
+    Serialize to str by default (decode=True), or bytes if decode=False.
+    """
+    out = json_dumpb(obj, **kwargs)
+    return out.decode("utf-8") if decode else out
+
+
+# --------- streaming for very large outputs ----------------------------------
+
+
+def json_lines_iter(
+    it: Iterable[Any],
+    *,
+    # default() configuration for each line
+    deterministic_sets: bool = False,
+    decimal_as_float: bool = False,
+    enum_as_name: bool = False,
+    passthrough_datetime: bool = False,
+    safe_fallback: bool = False,
+    fallback_clip: int = 2048,
+    # options
+    naive_utc: bool = False,
+    utc_z: bool = False,
+    allow_non_str_keys: bool = False,
+    # advanced
+    default: Callable[[Any], Any] | None = None,
+    options: int | None = None,
+) -> Iterable[bytes]:
+    """
+    Stream an iterable as **NDJSON** (one JSON object per line) in **bytes**.
+
+    Always ensures a trailing newline per line (OPT_APPEND_NEWLINE).
+    """
+    if default is None:
+        default = _cached_default(
+            deterministic_sets=deterministic_sets,
+            decimal_as_float=decimal_as_float,
+            enum_as_name=enum_as_name,
+            passthrough_datetime=passthrough_datetime,
+            safe_fallback=safe_fallback,
+            fallback_clip=fallback_clip,
+        )
+    if options is None:
+        opt = make_options(
+            pretty=False,
+            sort_keys=False,
+            naive_utc=naive_utc,
+            utc_z=utc_z,
+            append_newline=True,  # enforce newline for NDJSON
+            passthrough_datetime=passthrough_datetime,
+            allow_non_str_keys=allow_non_str_keys,
+        )
+    else:
+        opt = options | orjson.OPT_APPEND_NEWLINE
+
+    for item in it:
+        yield orjson.dumps(item, default=default, option=opt)
+
+
+def json_dict(
+    obj: Any,
+    /,
+    **kwargs: Any,
+):
+    return orjson.loads(json_dumpb(obj, **kwargs))