judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/utils/version_check.py

@@ -0,0 +1,28 @@
+import importlib.metadata
+import httpx
+import threading
+from judgeval.logger import judgeval_logger
+from judgeval.utils.decorators.use_once import use_once
+
+
+@use_once
+def check_latest_version(package_name: str = "judgeval"):
+    def _check():
+        try:
+            current_version = importlib.metadata.version(package_name)
+            response = httpx.get(
+                f"https://pypi.org/pypi/{package_name}/json", timeout=2
+            )
+            latest_version = response.json()["info"]["version"]
+
+            if current_version != latest_version:
+                judgeval_logger.warning(
+                    f"UPDATE AVAILABLE: You are using '{package_name}=={current_version}', "
+                    f"but the latest version is '{latest_version}'. While this version is still supported, "
+                    f"we recommend upgrading to avoid potential issues or missing features: "
+                    f"`pip install --upgrade {package_name}`"
+                )
+        except Exception:
+            pass
+
+    threading.Thread(target=_check, daemon=True).start()

judgeval/utils/wrappers/README.md

@@ -0,0 +1,3 @@
+# Wrapper Utilities
+
+Ensure 100% test coverage for all files in this folder

judgeval/utils/wrappers/__init__.py

@@ -0,0 +1,15 @@
+from .immutable_wrap_sync import immutable_wrap_sync
+from .immutable_wrap_async import immutable_wrap_async
+from .immutable_wrap_sync_iterator import immutable_wrap_sync_iterator
+from .immutable_wrap_async_iterator import immutable_wrap_async_iterator
+from .mutable_wrap_sync import mutable_wrap_sync
+from .mutable_wrap_async import mutable_wrap_async
+
+__all__ = [
+    "immutable_wrap_sync",
+    "immutable_wrap_async",
+    "immutable_wrap_sync_iterator",
+    "immutable_wrap_async_iterator",
+    "mutable_wrap_sync",
+    "mutable_wrap_async",
+]

judgeval/utils/wrappers/immutable_wrap_async.py

@@ -0,0 +1,74 @@
+from functools import wraps
+from typing import (
+    Awaitable,
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    ParamSpec,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx, result: Any) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_async(
+    func: Callable[P, Awaitable[R]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    post_hook: Callable[[Ctx, R], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, Awaitable[R]]:
+    """
+    Wraps an async function with lifecycle hooks.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - post_hook: called after successful func execution with (ctx, result)
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The wrapped function's result is returned unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            result = await func(*args, **kwargs)
+            post_hook(ctx, result)
+            return result
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_async_iterator.py

@@ -0,0 +1,84 @@
+from functools import wraps
+from typing import (
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    Mapping,
+    ParamSpec,
+    AsyncIterator,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+Y = TypeVar("Y")
+Ctx = Dict[str, Any]
+ImmCtx = Mapping[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_yield_hook(ctx: Ctx, value: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_async_iterator(
+    func: Callable[P, AsyncIterator[Y]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    yield_hook: Callable[[Ctx, Y], None] = _void_yield_hook,
+    post_hook: Callable[[Ctx], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, AsyncIterator[Y]]:
+    """
+    Wraps an async iterator function with lifecycle hooks.
+
+    - pre_hook: called when iterator function is invoked with (ctx, *args, **kwargs) matching func's signature
+    - yield_hook: called after each yield with (ctx, yielded_value)
+    - post_hook: called when iterator completes successfully with (ctx)
+    - error_hook: called if iterator raises an exception with (ctx, error)
+    - finally_hook: called when iterator closes with (ctx)
+
+    The wrapped iterator yields values unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    yield_hook = dont_throw(yield_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> AsyncIterator[Y]:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            async for value in func(*args, **kwargs):
+                yield_hook(ctx, value)
+                yield value
+            post_hook(ctx)
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_sync.py

@@ -0,0 +1,66 @@
+from functools import wraps
+from typing import Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx, result: Any) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_sync(
+    func: Callable[P, R],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    post_hook: Callable[[Ctx, R], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, R]:
+    """
+    Wraps a function with lifecycle hooks.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - post_hook: called after successful func execution with (ctx, result)
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The wrapped function's result is returned unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            result = func(*args, **kwargs)
+            post_hook(ctx, result)
+            return result
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_sync_iterator.py

@@ -0,0 +1,84 @@
+from functools import wraps
+from typing import (
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    Mapping,
+    ParamSpec,
+    Iterator,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+Y = TypeVar("Y")
+Ctx = Dict[str, Any]
+ImmCtx = Mapping[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_yield_hook(ctx: Ctx, value: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_sync_iterator(
+    func: Callable[P, Iterator[Y]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    yield_hook: Callable[[Ctx, Y], None] = _void_yield_hook,
+    post_hook: Callable[[Ctx], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, Iterator[Y]]:
+    """
+    Wraps an iterator function with lifecycle hooks.
+
+    - pre_hook: called when iterator function is invoked with (ctx, *args, **kwargs) matching func's signature
+    - yield_hook: called after each yield with (ctx, yielded_value)
+    - post_hook: called when iterator completes successfully with (ctx)
+    - error_hook: called if iterator raises an exception with (ctx, error)
+    - finally_hook: called when iterator closes with (ctx)
+
+    The wrapped iterator yields values unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    yield_hook = dont_throw(yield_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> Iterator[Y]:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            for value in func(*args, **kwargs):
+                yield_hook(ctx, value)
+                yield value
+            post_hook(ctx)
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/mutable_wrap_async.py

@@ -0,0 +1,67 @@
+from functools import wraps
+from typing import Awaitable, Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+from judgeval.utils.wrappers.utils import identity_on_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def mutable_wrap_async(
+    func: Callable[P, Awaitable[R]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] | None = None,
+    mutate_args_hook: Callable[[Ctx, tuple[Any, ...]], tuple[Any, ...]] | None = None,
+    mutate_kwargs_hook: Callable[[Ctx, dict[str, Any]], dict[str, Any]] | None = None,
+    post_hook: Callable[[Ctx, R], None] | None = None,
+    mutate_hook: Callable[[Ctx, R], R] | None = None,
+    error_hook: Callable[[Ctx, Exception], None] | None = None,
+    finally_hook: Callable[[Ctx], None] | None = None,
+) -> Callable[P, Awaitable[R]]:
+    """
+    Wraps an async function with lifecycle hooks that can mutate args, kwargs, and result.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - mutate_args_hook: called after pre_hook with (ctx, args), returns potentially modified args
+    - mutate_kwargs_hook: called after pre_hook with (ctx, kwargs), returns potentially modified kwargs
+    - post_hook: called after successful func execution with (ctx, result)
+    - mutate_hook: called after post_hook with (ctx, result), returns potentially modified result
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The mutate hooks can transform args/kwargs/result. Exceptions are re-raised.
+    """
+
+    safe_pre_hook = dont_throw(pre_hook) if pre_hook else (lambda ctx, *a, **kw: None)
+    safe_post_hook = dont_throw(post_hook) if post_hook else (lambda ctx, r: None)
+    safe_error_hook = dont_throw(error_hook) if error_hook else (lambda ctx, e: None)
+    safe_finally_hook = dont_throw(finally_hook) if finally_hook else (lambda ctx: None)
+
+    safe_mutate_args = identity_on_throw(mutate_args_hook) if mutate_args_hook else None
+    safe_mutate_kwargs = (
+        identity_on_throw(mutate_kwargs_hook) if mutate_kwargs_hook else None
+    )
+    safe_mutate_hook = identity_on_throw(mutate_hook) if mutate_hook else None
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        safe_pre_hook(ctx, *args, **kwargs)
+
+        final_args = safe_mutate_args(ctx, args) if safe_mutate_args else args
+        final_kwargs = safe_mutate_kwargs(ctx, kwargs) if safe_mutate_kwargs else kwargs
+
+        try:
+            result = await func(*final_args, **final_kwargs)
+            safe_post_hook(ctx, result)
+            return safe_mutate_hook(ctx, result) if safe_mutate_hook else result
+        except Exception as e:
+            safe_error_hook(ctx, e)
+            raise
+        finally:
+            safe_finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/mutable_wrap_sync.py

@@ -0,0 +1,67 @@
+from functools import wraps
+from typing import Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+from judgeval.utils.wrappers.utils import identity_on_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def mutable_wrap_sync(
+    func: Callable[P, R],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] | None = None,
+    mutate_args_hook: Callable[[Ctx, tuple[Any, ...]], tuple[Any, ...]] | None = None,
+    mutate_kwargs_hook: Callable[[Ctx, dict[str, Any]], dict[str, Any]] | None = None,
+    post_hook: Callable[[Ctx, R], None] | None = None,
+    mutate_hook: Callable[[Ctx, R], R] | None = None,
+    error_hook: Callable[[Ctx, Exception], None] | None = None,
+    finally_hook: Callable[[Ctx], None] | None = None,
+) -> Callable[P, R]:
+    """
+    Wraps a function with lifecycle hooks that can mutate args, kwargs, and result.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - mutate_args_hook: called after pre_hook with (ctx, args), returns potentially modified args
+    - mutate_kwargs_hook: called after pre_hook with (ctx, kwargs), returns potentially modified kwargs
+    - post_hook: called after successful func execution with (ctx, result)
+    - mutate_hook: called after post_hook with (ctx, result), returns potentially modified result
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The mutate hooks can transform args/kwargs/result. Exceptions are re-raised.
+    """
+
+    safe_pre_hook = dont_throw(pre_hook) if pre_hook else (lambda ctx, *a, **kw: None)
+    safe_post_hook = dont_throw(post_hook) if post_hook else (lambda ctx, r: None)
+    safe_error_hook = dont_throw(error_hook) if error_hook else (lambda ctx, e: None)
+    safe_finally_hook = dont_throw(finally_hook) if finally_hook else (lambda ctx: None)
+
+    safe_mutate_args = identity_on_throw(mutate_args_hook) if mutate_args_hook else None
+    safe_mutate_kwargs = (
+        identity_on_throw(mutate_kwargs_hook) if mutate_kwargs_hook else None
+    )
+    safe_mutate_hook = identity_on_throw(mutate_hook) if mutate_hook else None
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        safe_pre_hook(ctx, *args, **kwargs)
+
+        final_args = safe_mutate_args(ctx, args) if safe_mutate_args else args
+        final_kwargs = safe_mutate_kwargs(ctx, kwargs) if safe_mutate_kwargs else kwargs
+
+        try:
+            result = func(*final_args, **final_kwargs)
+            safe_post_hook(ctx, result)
+            return safe_mutate_hook(ctx, result) if safe_mutate_hook else result
+        except Exception as e:
+            safe_error_hook(ctx, e)
+            raise
+        finally:
+            safe_finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/utils.py

@@ -0,0 +1,35 @@
+from typing import Callable, TypeVar, ParamSpec
+
+from judgeval.logger import judgeval_logger
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+def identity_on_throw(func: Callable[P, T]) -> Callable[P, T]:
+    """
+    Wraps a mutation function to preserve the last argument (identity) if it fails.
+
+    This is used for mutation hooks where we want to fall back to the original value
+    if the mutation fails, ensuring the wrapper is always safe and non-breaking.
+
+    Args:
+        func: A mutation function where the last positional argument is the value to mutate.
+              The function should return a potentially modified version of this value.
+
+    Returns:
+        A wrapped function that returns the last positional argument (original value) if mutation fails
+    """
+
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            judgeval_logger.debug(
+                f"[Caught] Mutation function {func.__name__} failed, using identity",
+                exc_info=e,
+            )
+            # The last positional argument is always the value to mutate
+            return args[-1]  # type: ignore[return-value]
+
+    return wrapper

judgeval/version.py

@@ -0,0 +1,5 @@
+__version__ = "0.22.2"
+
+
+def get_version() -> str:
+    return __version__

judgeval/warnings.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+
+class JudgmentWarning(Warning): ...