lionherd-core 1.0.0a3__py3-none-any.whl

lionherd_core/libs/string_handlers/_to_num.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from decimal import Decimal
+from typing import Any
+
+
+def to_num(
+    input_: Any,
+    /,
+    *,
+    upper_bound: int | float | None = None,
+    lower_bound: int | float | None = None,
+    num_type: type[int] | type[float] = float,
+    precision: int | None = None,
+) -> int | float:
+    """Convert input to numeric type with validation and bounds checking.
+
+    Args:
+        input_: Value to convert to number
+        upper_bound: Maximum allowed value (inclusive)
+        lower_bound: Minimum allowed value (inclusive)
+        num_type: Target numeric type (int or float)
+        precision: Number of decimal places for rounding (float only)
+
+    Returns:
+        Converted number
+
+    Raises:
+        ValueError: For invalid input or out of bounds values
+        TypeError: For invalid input types
+    """
+    # Validate num_type
+    if num_type not in (int, float):
+        raise ValueError(f"Invalid number type: {num_type}")
+
+    # Handle boolean (special case - must check before int)
+    if isinstance(input_, (bool, int, float, Decimal)):
+        value = float(input_)
+    # Handle string input
+    elif isinstance(input_, str):
+        input_ = input_.strip()
+        if not input_:
+            raise ValueError("Empty string cannot be converted to number")
+        try:
+            value = float(input_)
+        except ValueError as e:
+            raise ValueError(f"Cannot convert '{input_}' to number") from e
+    else:
+        raise TypeError(f"Cannot convert {type(input_).__name__} to number")
+
+    # Apply bounds checking
+    if upper_bound is not None and value > upper_bound:
+        raise ValueError(f"Value {value} exceeds upper bound {upper_bound}")
+    if lower_bound is not None and value < lower_bound:
+        raise ValueError(f"Value {value} below lower bound {lower_bound}")
+
+    # Apply precision for float
+    if precision is not None and num_type is float:
+        value = round(value, precision)
+
+    # Convert to target type
+    return num_type(value)

lionherd_core/ln/__init__.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from ._async_call import AlcallParams, BcallParams, alcall, bcall
+from ._fuzzy_match import FuzzyMatchKeysParams, fuzzy_match_keys
+from ._fuzzy_validate import fuzzy_validate_mapping, fuzzy_validate_pydantic
+from ._hash import hash_dict
+from ._json_dump import (
+    get_orjson_default,
+    json_dict,
+    json_dumpb,
+    json_dumps,
+    json_lines_iter,
+    make_options,
+)
+from ._list_call import lcall
+from ._to_dict import to_dict
+from ._to_list import to_list
+from ._utils import acreate_path, get_bins, import_module, is_import_installed, now_utc
+
+__all__ = (
+    "AlcallParams",
+    "BcallParams",
+    "FuzzyMatchKeysParams",
+    "acreate_path",
+    "alcall",
+    "bcall",
+    "fuzzy_match_keys",
+    "fuzzy_validate_mapping",
+    "fuzzy_validate_pydantic",
+    "get_bins",
+    "get_orjson_default",
+    "hash_dict",
+    "import_module",
+    "is_import_installed",
+    "json_dict",
+    "json_dumpb",
+    "json_dumps",
+    "json_lines_iter",
+    "lcall",
+    "make_options",
+    "now_utc",
+    "to_dict",
+    "to_list",
+)

lionherd_core/ln/_async_call.py

@@ -0,0 +1,314 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+import threading
+from collections.abc import AsyncGenerator, Callable
+from dataclasses import dataclass
+from typing import Any, ClassVar, ParamSpec, TypeVar
+
+from lionherd_core.libs.concurrency import (
+    Semaphore,
+    create_task_group,
+    get_cancelled_exc_class,
+    is_coro_func,
+    move_on_after,
+    non_cancel_subgroup,
+    run_sync,
+    sleep,
+)
+from lionherd_core.types import ModelConfig, Params, Unset, not_sentinel
+
+from ._to_list import to_list
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+_INITIALIZED = False
+_MODEL_LIKE = None
+_INIT_LOCK = threading.RLock()
+
+
+__all__ = (
+    "AlcallParams",
+    "BcallParams",
+    "alcall",
+    "bcall",
+)
+
+
+async def alcall(
+    input_: list[Any],
+    func: Callable[..., T],
+    /,
+    *,
+    input_flatten: bool = False,
+    input_dropna: bool = False,
+    input_unique: bool = False,
+    input_flatten_tuple_set: bool = False,
+    output_flatten: bool = False,
+    output_dropna: bool = False,
+    output_unique: bool = False,
+    output_flatten_tuple_set: bool = False,
+    delay_before_start: float = 0,
+    retry_initial_delay: float = 0,
+    retry_backoff: float = 1,
+    retry_default: Any = Unset,
+    retry_timeout: float | None = None,
+    retry_attempts: int = 0,
+    max_concurrent: int | None = None,
+    throttle_period: float | None = None,
+    return_exceptions: bool = False,
+    **kwargs: Any,
+) -> list[T | BaseException]:
+    """Apply function to each list element asynchronously with retry and concurrency control.
+
+    Args:
+        input_: List of items to process (or iterable that will be converted)
+        func: Callable to apply (sync or async)
+        input_flatten: Flatten nested input structures
+        input_dropna: Remove None/undefined from input
+        input_unique: Remove duplicate inputs (requires flatten)
+        input_flatten_tuple_set: Include tuples/sets in flattening
+        output_flatten: Flatten nested output structures
+        output_dropna: Remove None/undefined from output
+        output_unique: Remove duplicate outputs (requires flatten)
+        output_flatten_tuple_set: Include tuples/sets in output flattening
+        delay_before_start: Initial delay before processing (seconds)
+        retry_initial_delay: Initial retry delay (seconds)
+        retry_backoff: Backoff multiplier for retry delays
+        retry_default: Default value on retry exhaustion (Unset = raise)
+        retry_timeout: Timeout per function call (seconds)
+        retry_attempts: Maximum retry attempts (0 = no retry)
+        max_concurrent: Max concurrent executions (None = unlimited)
+        throttle_period: Delay between starting tasks (seconds)
+        return_exceptions: Return exceptions instead of raising
+        **kwargs: Additional arguments passed to func
+
+    Returns:
+        List of results (preserves input order, may include exceptions if return_exceptions=True)
+
+    Raises:
+        ValueError: If func is not callable
+        TimeoutError: If retry_timeout exceeded
+        ExceptionGroup: If return_exceptions=False and tasks raise
+    """
+
+    global _INITIALIZED, _MODEL_LIKE
+    if _INITIALIZED is False:
+        with _INIT_LOCK:
+            # Double-checked locking pattern
+            if _INITIALIZED is False:
+                from pydantic import BaseModel
+
+                _MODEL_LIKE = (BaseModel,)
+                _INITIALIZED = True
+
+    # Validate func is a single callable
+    if not callable(func):
+        # If func is not callable, maybe it's an iterable. Extract one callable if possible.
+        try:
+            func_list = list(func)  # Convert iterable to list
+        except TypeError:
+            raise ValueError("func must be callable or an iterable containing one callable.")
+
+        # Ensure exactly one callable is present
+        if len(func_list) != 1 or not callable(func_list[0]):
+            raise ValueError("Only one callable function is allowed.")
+
+        func = func_list[0]
+
+    # Process input if requested
+    if any((input_flatten, input_dropna)):
+        input_ = to_list(
+            input_,
+            flatten=input_flatten,
+            dropna=input_dropna,
+            unique=input_unique,
+            flatten_tuple_set=input_flatten_tuple_set,
+        )
+    else:
+        if not isinstance(input_, list):
+            # Attempt to iterate
+            if isinstance(input_, _MODEL_LIKE):
+                # Pydantic model, convert to list
+                input_ = [input_]
+            else:
+                try:
+                    iter(input_)
+                    # It's iterable (tuple), convert to list of its contents
+                    input_ = list(input_)
+                except TypeError:
+                    # Not iterable, just wrap in a list
+                    input_ = [input_]
+
+    # Optional initial delay before processing
+    if delay_before_start:
+        await sleep(delay_before_start)
+
+    semaphore = Semaphore(max_concurrent) if max_concurrent else None
+    throttle_delay = throttle_period or 0
+    coro_func = is_coro_func(func)
+
+    async def call_func(item: Any) -> T:
+        if coro_func:
+            # Async function
+            if retry_timeout is not None:
+                with move_on_after(retry_timeout) as cancel_scope:
+                    result = await func(item, **kwargs)
+                if cancel_scope.cancelled_caught:
+                    raise TimeoutError(f"Function call timed out after {retry_timeout}s")
+                return result
+            else:
+                return await func(item, **kwargs)
+        else:
+            # Sync function
+            if retry_timeout is not None:
+                with move_on_after(retry_timeout) as cancel_scope:
+                    result = await run_sync(func, item, **kwargs)
+                if cancel_scope.cancelled_caught:
+                    raise TimeoutError(f"Function call timed out after {retry_timeout}s")
+                return result
+            else:
+                return await run_sync(func, item, **kwargs)
+
+    async def execute_task(i: Any, index: int) -> Any:
+        attempts = 0
+        current_delay = retry_initial_delay
+        while True:
+            try:
+                result = await call_func(i)
+                return index, result
+
+            # if cancelled, re-raise
+            except get_cancelled_exc_class():
+                raise
+
+            # handle other exceptions
+            except Exception:
+                attempts += 1
+                if attempts <= retry_attempts:
+                    if current_delay:
+                        await sleep(current_delay)
+                        current_delay *= retry_backoff
+                    # Retry loop continues
+                else:
+                    # Exhausted retries
+                    if not_sentinel(retry_default):
+                        return index, retry_default
+                    # No default, re-raise
+                    raise
+
+    # Preallocate result list and fill by index — preserves order with no lock/sort
+    n_items = len(input_)
+    out: list[Any] = [None] * n_items
+
+    async def task_wrapper(item: Any, idx: int) -> None:
+        try:
+            if semaphore:
+                async with semaphore:
+                    _, result = await execute_task(item, idx)
+            else:
+                _, result = await execute_task(item, idx)
+            out[idx] = result
+        except BaseException as exc:
+            out[idx] = exc
+            if not return_exceptions:
+                raise  # Propagate to TaskGroup
+
+    # Execute all tasks using task group
+    try:
+        async with create_task_group() as tg:
+            for idx, item in enumerate(input_):
+                tg.start_soon(task_wrapper, item, idx)
+                # Apply throttle delay between starting tasks
+                if throttle_delay and idx < n_items - 1:
+                    await sleep(throttle_delay)
+    except ExceptionGroup as eg:
+        if not return_exceptions:
+            # Surface only the non-cancellation subgroup to preserve structure & tracebacks
+            rest = non_cancel_subgroup(eg)
+            if rest is not None:
+                raise rest
+            raise
+
+    output_list = out  # already in original order
+    return to_list(
+        output_list,
+        flatten=output_flatten,
+        dropna=output_dropna,
+        unique=output_unique,
+        flatten_tuple_set=output_flatten_tuple_set,
+    )
+
+
+async def bcall(
+    input_: list[Any],
+    func: Callable[..., T],
+    /,
+    batch_size: int,
+    **kwargs: Any,
+) -> AsyncGenerator[list[T | BaseException], None]:
+    """Process input in batches using alcall. Yields results batch by batch.
+
+    Args:
+        input_: Items to process
+        func: Callable to apply
+        batch_size: Number of items per batch
+        **kwargs: Arguments passed to alcall (see alcall for details)
+
+    Yields:
+        List of results for each batch
+    """
+    input_ = to_list(input_, flatten=True, dropna=True)
+
+    for i in range(0, len(input_), batch_size):
+        batch = input_[i : i + batch_size]
+        yield await alcall(batch, func, **kwargs)
+
+
+@dataclass(slots=True, init=False, frozen=True)
+class AlcallParams(Params):
+    # ClassVar attributes
+    _config: ClassVar[ModelConfig] = ModelConfig(none_as_sentinel=True)
+    _func: ClassVar[Any] = alcall
+
+    # input processing
+    input_flatten: bool
+    input_dropna: bool
+    input_unique: bool
+    input_flatten_tuple_set: bool
+
+    # output processing
+    output_flatten: bool
+    output_dropna: bool
+    output_unique: bool
+    output_flatten_tuple_set: bool
+
+    # retry and timeout
+    delay_before_start: float
+    retry_initial_delay: float
+    retry_backoff: float
+    retry_default: Any
+    retry_timeout: float
+    retry_attempts: int
+
+    # concurrency and throttling
+    max_concurrent: int
+    throttle_period: float
+
+    kw: dict[str, Any] = Unset
+
+    async def __call__(self, input_: list[Any], func: Callable[..., T], **kw: Any) -> list[T]:
+        kwargs = {**self.default_kw(), **kw}
+        return await alcall(input_, func, **kwargs)
+
+
+@dataclass(slots=True, init=False, frozen=True)
+class BcallParams(AlcallParams):
+    _func: ClassVar[Any] = bcall
+
+    batch_size: int
+
+    async def __call__(self, input_: list[Any], func: Callable[..., T], **kw: Any) -> list[T]:
+        kwargs = {**self.default_kw(), **kw}
+        return await bcall(input_, func, self.batch_size, **kwargs)

lionherd_core/ln/_fuzzy_match.py

@@ -0,0 +1,166 @@
+# Copyright (c) 2025, HaiyangLi <quantocean.li at gmail dot com>
+# SPDX-License-Identifier: Apache-2.0
+
+from dataclasses import dataclass
+from typing import Any, ClassVar, Literal
+
+from ..libs.string_handlers._string_similarity import (
+    SIMILARITY_ALGO_MAP,
+    SIMILARITY_TYPE,
+    SimilarityAlgo,
+    SimilarityFunc,
+    string_similarity,
+)
+from ..types import KeysLike, ModelConfig, Params, Unset
+
+__all__ = (
+    "FuzzyMatchKeysParams",
+    "fuzzy_match_keys",
+)
+
+
+HandleUnmatched = Literal["ignore", "raise", "remove", "fill", "force"]
+
+
+def fuzzy_match_keys(
+    d_: dict[str, Any],
+    keys: KeysLike,
+    /,
+    *,
+    similarity_algo: SIMILARITY_TYPE | SimilarityAlgo | SimilarityFunc = "jaro_winkler",
+    similarity_threshold: float = 0.85,
+    fuzzy_match: bool = True,
+    handle_unmatched: HandleUnmatched = "ignore",
+    fill_value: Any = Unset,
+    fill_mapping: dict[str, Any] | None = None,
+    strict: bool = False,
+) -> dict[str, Any]:
+    """Validate and correct dict keys using fuzzy string matching.
+
+    Args:
+        d_: Input dictionary to validate
+        keys: Expected keys (list or dict-like with .keys())
+        similarity_algo: Algorithm for string similarity
+        similarity_threshold: Minimum similarity score (0.0-1.0)
+        fuzzy_match: Enable fuzzy matching for unmatched keys
+        handle_unmatched: How to handle unmatched keys ("ignore", "raise", "remove", "fill", "force")
+        fill_value: Default value for missing keys when filling
+        fill_mapping: Custom values for specific missing keys
+        strict: Raise if expected keys are missing
+
+    Returns:
+        Dictionary with corrected keys
+
+    Raises:
+        TypeError: If d_ is not a dict or keys is None
+        ValueError: If similarity_threshold out of range or unmatched keys found in raise mode
+    """
+    # Input validation
+    if not isinstance(d_, dict):
+        raise TypeError("First argument must be a dictionary")
+    if keys is None:
+        raise TypeError("Keys argument cannot be None")
+    if not 0.0 <= similarity_threshold <= 1.0:
+        raise ValueError("similarity_threshold must be between 0.0 and 1.0")
+
+    # Extract expected keys
+    fields_set = set(keys) if isinstance(keys, list) else set(keys.keys())
+    if not fields_set:
+        return d_.copy()  # Return copy of original if no expected keys
+
+    # Initialize output dictionary and tracking sets
+    corrected_out = {}
+    matched_expected = set()
+    matched_input = set()
+
+    # Get similarity function
+    if isinstance(similarity_algo, SimilarityAlgo):
+        similarity_func = SIMILARITY_ALGO_MAP[similarity_algo.value]
+    elif isinstance(similarity_algo, str):
+        if similarity_algo not in SIMILARITY_ALGO_MAP:
+            raise ValueError(f"Unknown similarity algorithm: {similarity_algo}")
+        similarity_func = SIMILARITY_ALGO_MAP[similarity_algo]
+    else:
+        similarity_func = similarity_algo
+
+    # First pass: exact matches
+    for key in d_:
+        if key in fields_set:
+            corrected_out[key] = d_[key]
+            matched_expected.add(key)
+            matched_input.add(key)
+
+    # Second pass: fuzzy matching if enabled
+    if fuzzy_match:
+        remaining_input = set(d_.keys()) - matched_input
+        remaining_expected = fields_set - matched_expected
+
+        for key in remaining_input:
+            if not remaining_expected:
+                break
+
+            matches = string_similarity(
+                key,
+                list(remaining_expected),
+                algorithm=similarity_func,
+                threshold=similarity_threshold,
+                return_most_similar=True,
+            )
+
+            if matches:
+                match = matches
+                corrected_out[match] = d_[key]
+                matched_expected.add(match)
+                matched_input.add(key)
+                remaining_expected.remove(match)
+            elif handle_unmatched == "ignore":
+                corrected_out[key] = d_[key]
+
+    # Handle unmatched keys based on handle_unmatched parameter
+    unmatched_input = set(d_.keys()) - matched_input
+    unmatched_expected = fields_set - matched_expected
+
+    if handle_unmatched == "raise" and unmatched_input:
+        raise ValueError(f"Unmatched keys found: {unmatched_input}")
+
+    elif handle_unmatched == "ignore":
+        for key in unmatched_input:
+            corrected_out[key] = d_[key]
+
+    elif handle_unmatched in ("fill", "force"):
+        # Fill missing expected keys
+        for key in unmatched_expected:
+            if fill_mapping and key in fill_mapping:
+                corrected_out[key] = fill_mapping[key]
+            else:
+                corrected_out[key] = fill_value
+
+        # For "fill" mode, also keep unmatched original keys
+        if handle_unmatched == "fill":
+            for key in unmatched_input:
+                corrected_out[key] = d_[key]
+
+    # Check strict mode
+    if strict and unmatched_expected:
+        raise ValueError(f"Missing required keys: {unmatched_expected}")
+
+    return corrected_out
+
+
+@dataclass(slots=True, init=False, frozen=True)
+class FuzzyMatchKeysParams(Params):
+    _config: ClassVar[ModelConfig] = ModelConfig(none_as_sentinel=False)
+    _func: ClassVar[Any] = fuzzy_match_keys
+
+    similarity_algo: SIMILARITY_TYPE | SimilarityAlgo | SimilarityFunc = "jaro_winkler"
+    similarity_threshold: float = 0.85
+
+    fuzzy_match: bool = True
+    handle_unmatched: HandleUnmatched = "ignore"
+
+    fill_value: Any = Unset
+    fill_mapping: dict[str, Any] | Any = Unset
+    strict: bool = False
+
+    def __call__(self, d_: dict[str, Any], keys: KeysLike) -> dict[str, Any]:
+        return fuzzy_match_keys(d_, keys, **self.default_kw())