judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/trainer/trainer.py

@@ -0,0 +1,70 @@
+from typing import Optional
+from .config import TrainerConfig
+from .base_trainer import BaseTrainer
+from .fireworks_trainer import FireworksTrainer
+from .trainable_model import TrainableModel
+from judgeval.tracer import Tracer
+from judgeval.exceptions import JudgmentRuntimeError
+
+
+def JudgmentTrainer(
+    config: TrainerConfig,
+    trainable_model: TrainableModel,
+    tracer: Tracer,
+    project_name: Optional[str] = None,
+) -> BaseTrainer:
+    """
+    Factory function for creating reinforcement learning trainers.
+
+    This factory creates and returns provider-specific trainer implementations
+    (FireworksTrainer, VerifiersTrainer, etc.) based on the configured RFT provider.
+
+    The factory pattern allows for easy extension to support multiple training
+    providers without changing the client-facing API.
+
+    Example:
+        config = TrainerConfig(
+            deployment_id="my-deployment",
+            user_id="my-user",
+            model_id="my-model",
+            rft_provider="fireworks"  # or "verifiers" in the future
+        )
+
+        # User creates and configures the trainable model
+        trainable_model = TrainableModel(config)
+        tracer = Tracer()
+
+        # JudgmentTrainer automatically creates the appropriate provider-specific trainer
+        trainer = JudgmentTrainer(config, trainable_model, tracer)
+
+        # The returned trainer implements the BaseTrainer interface
+        model_config = await trainer.train(agent_function, scorers, prompts)
+
+        Args:
+        config: TrainerConfig instance with training parameters including rft_provider
+        trainable_model: Provider-specific trainable model instance (e.g., TrainableModel for Fireworks)
+        tracer: Tracer for observability
+        project_name: Project name for organizing training runs and evaluations
+
+        Returns:
+        Provider-specific trainer instance (FireworksTrainer, etc.) that implements
+        the BaseTrainer interface
+
+    Raises:
+        JudgmentRuntimeError: If the specified provider is not supported
+    """
+    provider = config.rft_provider.lower()
+
+    if provider == "fireworks":
+        return FireworksTrainer(config, trainable_model, tracer, project_name)
+    elif provider == "verifiers":
+        # Placeholder for future implementation
+        raise JudgmentRuntimeError(
+            "Verifiers provider is not yet implemented. "
+            "Currently supported providers: 'fireworks'"
+        )
+    else:
+        raise JudgmentRuntimeError(
+            f"Unsupported RFT provider: '{config.rft_provider}'. "
+            f"Currently supported providers: 'fireworks'"
+        )

judgeval/utils/async_utils.py

@@ -0,0 +1,39 @@
+"""Async utilities for judgeval."""
+
+import asyncio
+import concurrent.futures
+from typing import Awaitable, TypeVar, Coroutine
+
+
+T = TypeVar("T")
+
+
+def safe_run_async(coro: Awaitable[T]) -> T:
+    """Safely execute an async *coro* from synchronous code.
+
+    This helper handles two common situations:
+
+    1. **No running event loop** - Simply delegates to ``asyncio.run``.
+    2. **Existing running loop** - Executes the coroutine in a separate
+       thread so that we don't attempt to nest event loops (which would raise
+       ``RuntimeError``).
+
+    Args:
+        coro: The coroutine to execute.
+
+    Returns:
+        The result returned by *coro*.
+    """
+    if not isinstance(coro, Coroutine):
+        raise TypeError("The provided awaitable must be a coroutine.")
+
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future: concurrent.futures.Future[T] = executor.submit(
+            lambda: asyncio.run(coro)
+        )
+        return future.result()

judgeval/utils/decorators/dont_throw.py

@@ -0,0 +1,37 @@
+from functools import wraps
+from typing import Any, Callable, ParamSpec, TypeVar, overload
+
+from judgeval.logger import judgeval_logger
+
+T = TypeVar("T")
+D = TypeVar("D")
+P = ParamSpec("P")
+
+
+@overload
+def dont_throw(func: Callable[P, T], /) -> Callable[P, T | None]: ...
+
+
+@overload
+def dont_throw(
+    func: None = None, /, *, default: D
+) -> Callable[[Callable[P, T]], Callable[P, T | D]]: ...
+
+
+def dont_throw(func: Callable[P, T] | None = None, /, *, default: Any = None):
+    def decorator(f: Callable[P, T]) -> Callable[P, T | Any]:
+        @wraps(f)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T | Any:
+            try:
+                return f(*args, **kwargs)
+            except Exception as e:
+                judgeval_logger.debug(
+                    f"[Caught] An exception was raised in {f.__name__}", exc_info=e
+                )
+                return default
+
+        return wrapper
+
+    if func is None:
+        return decorator
+    return decorator(func)

judgeval/utils/decorators/use_once.py

@@ -0,0 +1,13 @@
+from functools import lru_cache, wraps
+from typing import Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def use_once(func: Callable[..., T]) -> Callable[..., T]:
+    @lru_cache(maxsize=1)
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper

judgeval/utils/file_utils.py

@@ -0,0 +1,97 @@
+import importlib.util
+import yaml
+import orjson
+from pathlib import Path
+from typing import List
+from judgeval.logger import judgeval_logger
+
+from judgeval.data.example import Example
+
+
+def get_examples_from_yaml(file_path: str) -> List[Example]:
+    """
+    Adds examples from a YAML file.
+
+    The YAML file is expected to have the following format:
+    - key_01: value_01
+        key_02: value_02
+    - key_11: value_11
+        key_12: value_12
+        key_13: value_13
+    ...
+    """
+    try:
+        with open(file_path, "r") as file:
+            payload = yaml.safe_load(file)
+            if payload is None:
+                raise ValueError("The YAML file is empty.")
+    except FileNotFoundError:
+        judgeval_logger.error(f"YAML file not found: {file_path}")
+        raise FileNotFoundError(f"The file {file_path} was not found.")
+    except yaml.YAMLError:
+        judgeval_logger.error(f"Invalid YAML file: {file_path}")
+        raise ValueError(f"The file {file_path} is not a valid YAML file.")
+
+    new_examples = [Example(**e) for e in payload]
+    return new_examples
+
+
+def get_examples_from_json(file_path: str) -> List[Example]:
+    """
+    Adds examples from a JSON file.
+
+    The JSON file is expected to have the following format:
+    [
+        {
+            "key_01": "value_01",
+            "key_02": "value_02"
+        },
+        {
+            "key_11": "value_11",
+            "key_12": "value_12",
+            "key_13": "value_13"
+        },
+        ...
+    ]
+    """
+    try:
+        with open(file_path, "rb") as file:
+            payload = orjson.loads(file.read())
+    except FileNotFoundError:
+        judgeval_logger.error(f"JSON file not found: {file_path}")
+        raise FileNotFoundError(f"The file {file_path} was not found.")
+    except orjson.JSONDecodeError:
+        judgeval_logger.error(f"Invalid JSON file: {file_path}")
+        raise ValueError(f"The file {file_path} is not a valid JSON file.")
+
+    new_examples = [Example(**e) for e in payload]
+    return new_examples
+
+
+def extract_scorer_name(scorer_file_path: str) -> str:
+    try:
+        spec = importlib.util.spec_from_file_location("scorer_module", scorer_file_path)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load spec from {scorer_file_path}")
+
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if (
+                isinstance(attr, type)
+                and any("Scorer" in str(base) for base in attr.__mro__)
+                and attr.__module__ == "scorer_module"
+            ):
+                try:
+                    scorer_instance = attr()
+                    if hasattr(scorer_instance, "name"):
+                        return scorer_instance.name
+                except Exception:
+                    continue
+
+        raise AttributeError("No scorer class found or could be instantiated")
+    except Exception as e:
+        judgeval_logger.warning(f"Could not extract scorer name: {e}")
+        return Path(scorer_file_path).stem

judgeval/utils/guards.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from judgeval.logger import judgeval_logger
+
+if TYPE_CHECKING:
+    from typing import TypeVar
+
+    T = TypeVar("T")
+
+
+def expect_exists(value: T | None, message: str, default: T) -> T:
+    if not value:
+        judgeval_logger.error(message)
+        return default
+
+    return value
+
+
+def expect_api_key(api_key: str | None) -> str | None:
+    return expect_exists(
+        api_key,
+        "API Key is not set, please set JUDGMENT_API_KEY in the environment variables or pass it as `api_key`",
+        default=None,
+    )
+
+
+def expect_organization_id(organization_id: str | None) -> str | None:
+    return expect_exists(
+        organization_id,
+        "Organization ID is not set, please set JUDGMENT_ORG_ID in the environment variables or pass it as `organization_id`",
+        default=None,
+    )
+
+
+__all__ = ("expect_exists", "expect_api_key", "expect_organization_id")

judgeval/utils/meta.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+from typing import TypeVar, Dict, cast, Type
+
+T = TypeVar("T")
+
+
+class SingletonMeta(type):
+    """
+    Metaclass for creating singleton classes.
+    """
+
+    _instances: Dict[type, object] = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in SingletonMeta._instances:
+            SingletonMeta._instances[cls] = super(SingletonMeta, cls).__call__(
+                *args, **kwargs
+            )
+        return SingletonMeta._instances[cls]
+
+    def get_instance(cls: Type[T]) -> T | None:
+        """Get the singleton instance if it exists, otherwise return None"""
+        instance = SingletonMeta._instances.get(cls, None)
+        return cast(T, instance) if instance is not None else None
+
+
+__all__ = ("SingletonMeta",)

judgeval/utils/project.py

@@ -0,0 +1,15 @@
+from judgeval.utils.decorators.dont_throw import dont_throw
+import functools
+from judgeval.api import JudgmentSyncClient
+
+
+@dont_throw
+@functools.lru_cache(maxsize=64)
+def _resolve_project_id(project_name: str, api_key: str, organization_id: str) -> str:
+    """Resolve project_id from project_name using the API."""
+    client = JudgmentSyncClient(
+        api_key=api_key,
+        organization_id=organization_id,
+    )
+    response = client.projects_resolve({"project_name": project_name})
+    return response["project_id"]

judgeval/utils/serialize.py

@@ -0,0 +1,253 @@
+"""
+
+This is a modified version of https://docs.powertools.aws.dev/lambda/python/2.35.1/api/event_handler/openapi/encoders.html
+
+"""
+
+import dataclasses
+import datetime
+from collections import defaultdict, deque
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path, PurePath
+from re import Pattern
+from types import GeneratorType
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic.types import SecretBytes, SecretStr
+import orjson
+
+from judgeval.logger import judgeval_logger
+
+
+"""
+This module contains the encoders used by jsonable_encoder to convert Python objects to JSON serializable data types.
+"""
+
+
+def _model_dump(
+    model: BaseModel, mode: Literal["json", "python"] = "json", **kwargs: Any
+) -> Any:
+    return model.model_dump(mode=mode, **kwargs)
+
+
+def json_encoder(
+    obj: Any,
+    custom_serializer: Optional[Callable[[Any], str]] = None,
+) -> Any:
+    """
+    JSON encodes an arbitrary Python object into JSON serializable data types.
+
+    This is a modified version of fastapi.encoders.jsonable_encoder that supports
+    encoding of pydantic.BaseModel objects.
+
+    Parameters
+    ----------
+    obj : Any
+        The object to encode
+    custom_serializer : Callable, optional
+        A custom serializer to use for encoding the object, when everything else fails.
+
+    Returns
+    -------
+    Any
+        The JSON serializable data types
+    """
+    # Pydantic models
+    if isinstance(obj, BaseModel):
+        return _dump_base_model(
+            obj=obj,
+        )
+
+    # Dataclasses
+    if dataclasses.is_dataclass(obj):
+        obj_dict = dataclasses.asdict(obj)  # type: ignore[arg-type]
+        return json_encoder(
+            obj_dict,
+        )
+
+    # Enums
+    if isinstance(obj, Enum):
+        return obj.value
+
+    # Paths
+    if isinstance(obj, PurePath):
+        return str(obj)
+
+    # Scalars
+    if isinstance(obj, (str, int, float, type(None))):
+        return obj
+
+    # Dictionaries
+    if isinstance(obj, dict):
+        return _dump_dict(
+            obj=obj,
+        )
+
+    # Sequences
+    if isinstance(obj, (list, set, frozenset, tuple, deque)):
+        return _dump_sequence(
+            obj=obj,
+        )
+
+    # Other types
+    if type(obj) in ENCODERS_BY_TYPE:
+        return ENCODERS_BY_TYPE[type(obj)](obj)
+
+    for encoder, classes_tuple in encoders_by_class_tuples.items():
+        if isinstance(obj, classes_tuple):
+            return encoder(obj)
+
+    # Use custom serializer if present
+    if custom_serializer:
+        return custom_serializer(obj)
+
+    # Default
+    return _dump_other(
+        obj=obj,
+    )
+
+
+def _dump_base_model(
+    *,
+    obj: Any,
+):
+    """
+    Dump a BaseModel object to a dict, using the same parameters as jsonable_encoder
+    """
+    obj_dict = _model_dump(
+        obj,
+        mode="json",
+    )
+    if "__root__" in obj_dict:
+        obj_dict = obj_dict["__root__"]
+
+    return json_encoder(
+        obj_dict,
+    )
+
+
+def _dump_dict(
+    *,
+    obj: Any,
+) -> Dict[str, Any]:
+    """
+    Dump a dict to a dict, using the same parameters as jsonable_encoder
+    """
+    encoded_dict = {}
+    allowed_keys = set(obj.keys())
+    for key, value in obj.items():
+        if key in allowed_keys:
+            encoded_key = json_encoder(
+                key,
+            )
+            encoded_value = json_encoder(
+                value,
+            )
+            encoded_dict[encoded_key] = encoded_value
+    return encoded_dict
+
+
+def _dump_sequence(
+    *,
+    obj: Any,
+) -> List[Any]:
+    """
+    Dump a sequence to a list, using the same parameters as jsonable_encoder
+    """
+    encoded_list = []
+    for item in obj:
+        encoded_list.append(
+            json_encoder(
+                item,
+            ),
+        )
+    return encoded_list
+
+
+def _dump_other(
+    *,
+    obj: Any,
+) -> Any:
+    """
+    Dump an object to a representation without iterating it.
+
+    Avoids calling dict(obj) which can consume iterators/generators or
+    invoke user-defined iteration protocols.
+    """
+    try:
+        return repr(obj)
+    except Exception:
+        return str(obj)
+
+
+def iso_format(o: Union[datetime.date, datetime.time]) -> str:
+    """
+    ISO format for date and time
+    """
+    return o.isoformat()
+
+
+def decimal_encoder(dec_value: Decimal) -> Union[int, float]:
+    """
+    Encodes a Decimal as int of there's no exponent, otherwise float
+
+    This is useful when we use ConstrainedDecimal to represent Numeric(x,0)
+    where an integer (but not int typed) is used. Encoding this as a float
+    results in failed round-tripping between encode and parse.
+
+    >>> decimal_encoder(Decimal("1.0"))
+    1.0
+
+    >>> decimal_encoder(Decimal("1"))
+    1
+    """
+    if dec_value.as_tuple().exponent >= 0:  # type: ignore[operator]
+        return int(dec_value)
+    else:
+        return float(dec_value)
+
+
+ENCODERS_BY_TYPE: Dict[Type[Any], Callable[[Any], Any]] = {
+    bytes: lambda o: o.decode(),
+    datetime.date: iso_format,
+    datetime.datetime: iso_format,
+    datetime.time: iso_format,
+    datetime.timedelta: lambda td: td.total_seconds(),
+    Decimal: decimal_encoder,
+    Enum: lambda o: o.value,
+    frozenset: list,
+    deque: list,
+    GeneratorType: repr,
+    Path: str,
+    Pattern: lambda o: o.pattern,
+    SecretBytes: str,
+    SecretStr: str,
+    set: list,
+    UUID: str,
+}
+
+
+# Generates a mapping of encoders to a tuple of classes that they can encode
+def generate_encoders_by_class_tuples(
+    type_encoder_map: Dict[Any, Callable[[Any], Any]],
+) -> Dict[Callable[[Any], Any], Tuple[Any, ...]]:
+    encoders: Dict[Callable[[Any], Any], Tuple[Any, ...]] = defaultdict(tuple)
+    for type_, encoder in type_encoder_map.items():
+        encoders[encoder] += (type_,)
+    return encoders
+
+
+# Mapping of encoders to a tuple of classes that they can encode
+encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)
+
+
+# Seralize arbitrary object to a json string
+def safe_serialize(obj: Any) -> str:
+    try:
+        return orjson.dumps(json_encoder(obj), option=orjson.OPT_NON_STR_KEYS).decode()
+    except Exception as e:
+        judgeval_logger.warning(f"Error serializing object: {e}")
+        return repr(obj)

judgeval/utils/testing.py

@@ -0,0 +1,70 @@
+from rich import print as rprint
+
+from typing import List
+from judgeval.evaluation import ScoringResult
+from judgeval.data import ScorerData
+from judgeval.exceptions import JudgmentTestError
+
+
+def assert_test_results(scoring_results: List[ScoringResult]) -> None:
+    failed_cases: List[List[ScorerData]] = []
+    for result in scoring_results:
+        if not result.success:
+            test_case = []
+            if result.scorers_data:
+                for scorer_data in result.scorers_data:
+                    if not scorer_data.success:
+                        test_case.append(scorer_data)
+            failed_cases.append(test_case)
+
+    if failed_cases:
+        error_msg = "The following test cases failed: \n"
+        for fail_case in failed_cases:
+            for fail_scorer in fail_case:
+                error_msg += (
+                    f"\nScorer Name: {fail_scorer.name}\n"
+                    f"Threshold: {fail_scorer.threshold}\n"
+                    f"Success: {fail_scorer.success}\n"
+                    f"Score: {fail_scorer.score}\n"
+                    f"Reason: {fail_scorer.reason}\n"
+                    f"Strict Mode: {fail_scorer.strict_mode}\n"
+                    f"Evaluation Model: {fail_scorer.evaluation_model}\n"
+                    f"Error: {fail_scorer.error}\n"
+                    f"Additional Metadata: {fail_scorer.additional_metadata}\n"
+                )
+            error_msg += "-" * 100
+
+        total_tests = len(scoring_results)
+        failed_tests = len(failed_cases)
+        passed_tests = total_tests - failed_tests
+
+        rprint("\n" + "=" * 80)
+        if failed_tests == 0:
+            rprint(
+                f"[bold green]🎉 ALL TESTS PASSED! {passed_tests}/{total_tests} tests successful[/bold green]"
+            )
+        else:
+            rprint(
+                f"[bold red]⚠️  TEST RESULTS: {passed_tests}/{total_tests} passed ({failed_tests} failed)[/bold red]"
+            )
+        rprint("=" * 80 + "\n")
+
+        for i, result in enumerate(scoring_results):
+            test_num = i + 1
+            if result.success:
+                rprint(f"[green]✓ Test {test_num}: PASSED[/green]")
+            else:
+                rprint(f"[red]✗ Test {test_num}: FAILED[/red]")
+                if result.scorers_data:
+                    for scorer_data in result.scorers_data:
+                        if not scorer_data.success:
+                            rprint(f"  [yellow]Scorer: {scorer_data.name}[/yellow]")
+                            rprint(f"  [red]  Score: {scorer_data.score}[/red]")
+                            rprint(f"  [red]  Reason: {scorer_data.reason}[/red]")
+                            if scorer_data.error:
+                                rprint(f"  [red]  Error: {scorer_data.error}[/red]")
+                rprint("  " + "-" * 40)
+
+        rprint("\n" + "=" * 80)
+        if failed_tests > 0:
+            raise JudgmentTestError(failed_cases)

judgeval/utils/url.py

@@ -0,0 +1,10 @@
+from urllib.parse import urljoin
+
+from judgeval.env import JUDGMENT_API_URL
+
+
+def url_for(path: str, base: str = JUDGMENT_API_URL) -> str:
+    return urljoin(base, path)
+
+
+__all__ = ("url_for",)