judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/utils/url.py

@@ -0,0 +1,10 @@
+from urllib.parse import urljoin
+
+from judgeval.env import JUDGMENT_API_URL
+
+
+def url_for(path: str, base: str = JUDGMENT_API_URL) -> str:
+    return urljoin(base, path)
+
+
+__all__ = ("url_for",)

judgeval/{version_check.py → utils/version_check.py}

@@ -1,14 +1,16 @@
 import importlib.metadata
-from judgeval.utils.requests import requests
+import httpx
 import threading
-from judgeval.common.logger import judgeval_logger
+from judgeval.logger import judgeval_logger
+from judgeval.utils.decorators.use_once import use_once
 
 
+@use_once
 def check_latest_version(package_name: str = "judgeval"):
     def _check():
         try:
             current_version = importlib.metadata.version(package_name)
-            response = requests.get(
+            response = httpx.get(
                 f"https://pypi.org/pypi/{package_name}/json", timeout=2
             )
             latest_version = response.json()["info"]["version"]

judgeval/utils/wrappers/README.md

@@ -0,0 +1,3 @@
+# Wrapper Utilities
+
+Ensure 100% test coverage for all files in this folder

judgeval/utils/wrappers/__init__.py

@@ -0,0 +1,15 @@
+from .immutable_wrap_sync import immutable_wrap_sync
+from .immutable_wrap_async import immutable_wrap_async
+from .immutable_wrap_sync_iterator import immutable_wrap_sync_iterator
+from .immutable_wrap_async_iterator import immutable_wrap_async_iterator
+from .mutable_wrap_sync import mutable_wrap_sync
+from .mutable_wrap_async import mutable_wrap_async
+
+__all__ = [
+    "immutable_wrap_sync",
+    "immutable_wrap_async",
+    "immutable_wrap_sync_iterator",
+    "immutable_wrap_async_iterator",
+    "mutable_wrap_sync",
+    "mutable_wrap_async",
+]

judgeval/utils/wrappers/immutable_wrap_async.py

@@ -0,0 +1,74 @@
+from functools import wraps
+from typing import (
+    Awaitable,
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    ParamSpec,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx, result: Any) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_async(
+    func: Callable[P, Awaitable[R]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    post_hook: Callable[[Ctx, R], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, Awaitable[R]]:
+    """
+    Wraps an async function with lifecycle hooks.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - post_hook: called after successful func execution with (ctx, result)
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The wrapped function's result is returned unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            result = await func(*args, **kwargs)
+            post_hook(ctx, result)
+            return result
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_async_iterator.py

@@ -0,0 +1,84 @@
+from functools import wraps
+from typing import (
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    Mapping,
+    ParamSpec,
+    AsyncIterator,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+Y = TypeVar("Y")
+Ctx = Dict[str, Any]
+ImmCtx = Mapping[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_yield_hook(ctx: Ctx, value: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_async_iterator(
+    func: Callable[P, AsyncIterator[Y]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    yield_hook: Callable[[Ctx, Y], None] = _void_yield_hook,
+    post_hook: Callable[[Ctx], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, AsyncIterator[Y]]:
+    """
+    Wraps an async iterator function with lifecycle hooks.
+
+    - pre_hook: called when iterator function is invoked with (ctx, *args, **kwargs) matching func's signature
+    - yield_hook: called after each yield with (ctx, yielded_value)
+    - post_hook: called when iterator completes successfully with (ctx)
+    - error_hook: called if iterator raises an exception with (ctx, error)
+    - finally_hook: called when iterator closes with (ctx)
+
+    The wrapped iterator yields values unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    yield_hook = dont_throw(yield_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> AsyncIterator[Y]:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            async for value in func(*args, **kwargs):
+                yield_hook(ctx, value)
+                yield value
+            post_hook(ctx)
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_sync.py

@@ -0,0 +1,66 @@
+from functools import wraps
+from typing import Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx, result: Any) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_sync(
+    func: Callable[P, R],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    post_hook: Callable[[Ctx, R], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, R]:
+    """
+    Wraps a function with lifecycle hooks.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - post_hook: called after successful func execution with (ctx, result)
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The wrapped function's result is returned unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            result = func(*args, **kwargs)
+            post_hook(ctx, result)
+            return result
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/immutable_wrap_sync_iterator.py

@@ -0,0 +1,84 @@
+from functools import wraps
+from typing import (
+    Callable,
+    TypeVar,
+    Any,
+    Dict,
+    Mapping,
+    ParamSpec,
+    Iterator,
+    Concatenate,
+)
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+
+P = ParamSpec("P")
+Y = TypeVar("Y")
+Ctx = Dict[str, Any]
+ImmCtx = Mapping[str, Any]
+
+
+def _void_pre_hook(ctx: Ctx, *args: Any, **kwargs: Any) -> None:
+    pass
+
+
+def _void_yield_hook(ctx: Ctx, value: Any) -> None:
+    pass
+
+
+def _void_post_hook(ctx: Ctx) -> None:
+    pass
+
+
+def _void_error_hook(ctx: Ctx, error: Exception) -> None:
+    pass
+
+
+def _void_finally_hook(ctx: Ctx) -> None:
+    pass
+
+
+def immutable_wrap_sync_iterator(
+    func: Callable[P, Iterator[Y]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] = _void_pre_hook,
+    yield_hook: Callable[[Ctx, Y], None] = _void_yield_hook,
+    post_hook: Callable[[Ctx], None] = _void_post_hook,
+    error_hook: Callable[[Ctx, Exception], None] = _void_error_hook,
+    finally_hook: Callable[[Ctx], None] = _void_finally_hook,
+) -> Callable[P, Iterator[Y]]:
+    """
+    Wraps an iterator function with lifecycle hooks.
+
+    - pre_hook: called when iterator function is invoked with (ctx, *args, **kwargs) matching func's signature
+    - yield_hook: called after each yield with (ctx, yielded_value)
+    - post_hook: called when iterator completes successfully with (ctx)
+    - error_hook: called if iterator raises an exception with (ctx, error)
+    - finally_hook: called when iterator closes with (ctx)
+
+    The wrapped iterator yields values unchanged, and exceptions are re-raised.
+    """
+
+    pre_hook = dont_throw(pre_hook)
+    yield_hook = dont_throw(yield_hook)
+    post_hook = dont_throw(post_hook)
+    error_hook = dont_throw(error_hook)
+    finally_hook = dont_throw(finally_hook)
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> Iterator[Y]:
+        ctx: Ctx = {}
+        pre_hook(ctx, *args, **kwargs)
+        try:
+            for value in func(*args, **kwargs):
+                yield_hook(ctx, value)
+                yield value
+            post_hook(ctx)
+        except Exception as e:
+            error_hook(ctx, e)
+            raise
+        finally:
+            finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/mutable_wrap_async.py

@@ -0,0 +1,67 @@
+from functools import wraps
+from typing import Awaitable, Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+from judgeval.utils.wrappers.utils import identity_on_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def mutable_wrap_async(
+    func: Callable[P, Awaitable[R]],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] | None = None,
+    mutate_args_hook: Callable[[Ctx, tuple[Any, ...]], tuple[Any, ...]] | None = None,
+    mutate_kwargs_hook: Callable[[Ctx, dict[str, Any]], dict[str, Any]] | None = None,
+    post_hook: Callable[[Ctx, R], None] | None = None,
+    mutate_hook: Callable[[Ctx, R], R] | None = None,
+    error_hook: Callable[[Ctx, Exception], None] | None = None,
+    finally_hook: Callable[[Ctx], None] | None = None,
+) -> Callable[P, Awaitable[R]]:
+    """
+    Wraps an async function with lifecycle hooks that can mutate args, kwargs, and result.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - mutate_args_hook: called after pre_hook with (ctx, args), returns potentially modified args
+    - mutate_kwargs_hook: called after pre_hook with (ctx, kwargs), returns potentially modified kwargs
+    - post_hook: called after successful func execution with (ctx, result)
+    - mutate_hook: called after post_hook with (ctx, result), returns potentially modified result
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The mutate hooks can transform args/kwargs/result. Exceptions are re-raised.
+    """
+
+    safe_pre_hook = dont_throw(pre_hook) if pre_hook else (lambda ctx, *a, **kw: None)
+    safe_post_hook = dont_throw(post_hook) if post_hook else (lambda ctx, r: None)
+    safe_error_hook = dont_throw(error_hook) if error_hook else (lambda ctx, e: None)
+    safe_finally_hook = dont_throw(finally_hook) if finally_hook else (lambda ctx: None)
+
+    safe_mutate_args = identity_on_throw(mutate_args_hook) if mutate_args_hook else None
+    safe_mutate_kwargs = (
+        identity_on_throw(mutate_kwargs_hook) if mutate_kwargs_hook else None
+    )
+    safe_mutate_hook = identity_on_throw(mutate_hook) if mutate_hook else None
+
+    @wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        safe_pre_hook(ctx, *args, **kwargs)
+
+        final_args = safe_mutate_args(ctx, args) if safe_mutate_args else args
+        final_kwargs = safe_mutate_kwargs(ctx, kwargs) if safe_mutate_kwargs else kwargs
+
+        try:
+            result = await func(*final_args, **final_kwargs)
+            safe_post_hook(ctx, result)
+            return safe_mutate_hook(ctx, result) if safe_mutate_hook else result
+        except Exception as e:
+            safe_error_hook(ctx, e)
+            raise
+        finally:
+            safe_finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/mutable_wrap_sync.py

@@ -0,0 +1,67 @@
+from functools import wraps
+from typing import Callable, TypeVar, Any, Dict, ParamSpec, Concatenate
+
+from judgeval.utils.decorators.dont_throw import dont_throw
+from judgeval.utils.wrappers.utils import identity_on_throw
+
+P = ParamSpec("P")
+R = TypeVar("R")
+Ctx = Dict[str, Any]
+
+
+def mutable_wrap_sync(
+    func: Callable[P, R],
+    /,
+    *,
+    pre_hook: Callable[Concatenate[Ctx, P], None] | None = None,
+    mutate_args_hook: Callable[[Ctx, tuple[Any, ...]], tuple[Any, ...]] | None = None,
+    mutate_kwargs_hook: Callable[[Ctx, dict[str, Any]], dict[str, Any]] | None = None,
+    post_hook: Callable[[Ctx, R], None] | None = None,
+    mutate_hook: Callable[[Ctx, R], R] | None = None,
+    error_hook: Callable[[Ctx, Exception], None] | None = None,
+    finally_hook: Callable[[Ctx], None] | None = None,
+) -> Callable[P, R]:
+    """
+    Wraps a function with lifecycle hooks that can mutate args, kwargs, and result.
+
+    - pre_hook: called before func with (ctx, *args, **kwargs) matching func's signature
+    - mutate_args_hook: called after pre_hook with (ctx, args), returns potentially modified args
+    - mutate_kwargs_hook: called after pre_hook with (ctx, kwargs), returns potentially modified kwargs
+    - post_hook: called after successful func execution with (ctx, result)
+    - mutate_hook: called after post_hook with (ctx, result), returns potentially modified result
+    - error_hook: called if func raises an exception with (ctx, error)
+    - finally_hook: called in finally block with (ctx)
+
+    The mutate hooks can transform args/kwargs/result. Exceptions are re-raised.
+    """
+
+    safe_pre_hook = dont_throw(pre_hook) if pre_hook else (lambda ctx, *a, **kw: None)
+    safe_post_hook = dont_throw(post_hook) if post_hook else (lambda ctx, r: None)
+    safe_error_hook = dont_throw(error_hook) if error_hook else (lambda ctx, e: None)
+    safe_finally_hook = dont_throw(finally_hook) if finally_hook else (lambda ctx: None)
+
+    safe_mutate_args = identity_on_throw(mutate_args_hook) if mutate_args_hook else None
+    safe_mutate_kwargs = (
+        identity_on_throw(mutate_kwargs_hook) if mutate_kwargs_hook else None
+    )
+    safe_mutate_hook = identity_on_throw(mutate_hook) if mutate_hook else None
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ctx: Ctx = {}
+        safe_pre_hook(ctx, *args, **kwargs)
+
+        final_args = safe_mutate_args(ctx, args) if safe_mutate_args else args
+        final_kwargs = safe_mutate_kwargs(ctx, kwargs) if safe_mutate_kwargs else kwargs
+
+        try:
+            result = func(*final_args, **final_kwargs)
+            safe_post_hook(ctx, result)
+            return safe_mutate_hook(ctx, result) if safe_mutate_hook else result
+        except Exception as e:
+            safe_error_hook(ctx, e)
+            raise
+        finally:
+            safe_finally_hook(ctx)
+
+    return wrapper

judgeval/utils/wrappers/utils.py

@@ -0,0 +1,35 @@
+from typing import Callable, TypeVar, ParamSpec
+
+from judgeval.logger import judgeval_logger
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+def identity_on_throw(func: Callable[P, T]) -> Callable[P, T]:
+    """
+    Wraps a mutation function to preserve the last argument (identity) if it fails.
+
+    This is used for mutation hooks where we want to fall back to the original value
+    if the mutation fails, ensuring the wrapper is always safe and non-breaking.
+
+    Args:
+        func: A mutation function where the last positional argument is the value to mutate.
+              The function should return a potentially modified version of this value.
+
+    Returns:
+        A wrapped function that returns the last positional argument (original value) if mutation fails
+    """
+
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            judgeval_logger.debug(
+                f"[Caught] Mutation function {func.__name__} failed, using identity",
+                exc_info=e,
+            )
+            # The last positional argument is always the value to mutate
+            return args[-1]  # type: ignore[return-value]
+
+    return wrapper

judgeval/v1/__init__.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_API_URL, JUDGMENT_ORG_ID
+
+
+class Judgeval:
+    __slots__ = ("_api_key", "_organization_id", "_api_url", "_internal_client")
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        organization_id: Optional[str] = None,
+        api_url: Optional[str] = None,
+    ):
+        api_key = api_key or JUDGMENT_API_KEY
+        organization_id = organization_id or JUDGMENT_ORG_ID
+        api_url = api_url or JUDGMENT_API_URL
+
+        if not api_key:
+            raise ValueError("api_key is required")
+        if not organization_id:
+            raise ValueError("organization_id is required")
+        if not api_url:
+            raise ValueError("api_url is required")
+
+        self._api_key = api_key
+        self._organization_id = organization_id
+        self._api_url = api_url
+
+        self._internal_client = JudgmentSyncClient(
+            self._api_url,
+            self._api_key,
+            self._organization_id,
+        )
+
+    @property
+    def tracer(self):
+        from judgeval.v1.tracer.tracer_factory import TracerFactory
+
+        return TracerFactory(
+            client=self._internal_client,
+        )
+
+    @property
+    def scorers(self):
+        from judgeval.v1.scorers.scorers_factory import ScorersFactory
+
+        return ScorersFactory(
+            client=self._internal_client,
+        )
+
+    @property
+    def evaluation(self):
+        from judgeval.v1.evaluation.evaluation_factory import EvaluationFactory
+
+        return EvaluationFactory(
+            client=self._internal_client,
+        )
+
+    @property
+    def trainers(self):
+        from judgeval.v1.trainers.trainers_factory import TrainersFactory
+
+        return TrainersFactory(
+            client=self._internal_client,
+        )
+
+    @property
+    def datasets(self):
+        from judgeval.v1.datasets.dataset_factory import DatasetFactory
+
+        return DatasetFactory(
+            client=self._internal_client,
+        )
+
+    @property
+    def prompts(self):
+        from judgeval.v1.prompts.prompt_factory import PromptFactory
+
+        return PromptFactory(
+            client=self._internal_client,
+        )
+
+
+__all__ = ["Judgeval"]

judgeval/v1/data/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from judgeval.v1.data.example import Example
+from judgeval.v1.data.scorer_data import ScorerData
+from judgeval.v1.data.scoring_result import ScoringResult
+
+__all__ = ["Example", "ScorerData", "ScoringResult"]

judgeval/v1/data/example.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from judgeval.v1.internal.api.api_types import Example as APIExample
+
+
+@dataclass(slots=True)
+class Example:
+    example_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    created_at: str = field(default_factory=lambda: datetime.utcnow().isoformat())
+    name: Optional[str] = None
+    _properties: Dict[str, Any] = field(default_factory=dict)
+
+    def set_property(self, key: str, value: Any) -> Example:
+        self._properties[key] = value
+        return self
+
+    def get_property(self, key: str) -> Any:
+        return self._properties.get(key)
+
+    @classmethod
+    def create(cls, **kwargs: Any) -> Example:
+        example = cls()
+        for key, value in kwargs.items():
+            example.set_property(key, value)
+        return example
+
+    def to_dict(self) -> APIExample:
+        result: APIExample = {
+            "example_id": self.example_id,
+            "created_at": self.created_at,
+            "name": self.name,
+        }
+        for key, value in self._properties.items():
+            result[key] = value  # type: ignore[literal-required]
+        return result
+
+    @property
+    def properties(self) -> Dict[str, Any]:
+        return self._properties.copy()