judgeval 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

judgeval/{common/trainer → trainer}/trainable_model.py

@@ -2,7 +2,7 @@ from fireworks import LLM
 from .config import TrainerConfig, ModelConfig
 from typing import Optional, Dict, Any, Callable
 from .console import _model_spinner_progress, _print_model_progress
-from judgeval.common.exceptions import JudgmentAPIError
+from judgeval.exceptions import JudgmentRuntimeError
 
 
 class TrainableModel:
@@ -14,6 +14,12 @@ class TrainableModel:
     abstracting away manual snapshot management from users.
     """
 
+    config: TrainerConfig
+    current_step: int
+    _current_model: LLM
+    _tracer_wrapper_func: Optional[Callable]
+    _base_model: LLM
+
     def __init__(self, config: TrainerConfig):
         """
         Initialize the TrainableModel.
@@ -24,13 +30,12 @@ class TrainableModel:
         try:
             self.config = config
             self.current_step = 0
-            self._current_model = None
             self._tracer_wrapper_func = None
 
             self._base_model = self._create_base_model()
             self._current_model = self._base_model
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to initialize TrainableModel: {str(e)}"
             ) from e
 
@@ -80,7 +85,7 @@ class TrainableModel:
             _print_model_progress("Base model deployment ready")
             return base_model
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to create and deploy base model '{self.config.base_model_name}': {str(e)}"
             ) from e
 
@@ -103,7 +108,7 @@ class TrainableModel:
             if self._tracer_wrapper_func:
                 self._tracer_wrapper_func(self._current_model)
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to load and deploy trained model '{model_name}': {str(e)}"
             ) from e
 
@@ -150,7 +155,7 @@ class TrainableModel:
                 if self._tracer_wrapper_func:
                     self._tracer_wrapper_func(self._current_model)
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to advance to training step {step}: {str(e)}"
             ) from e
 
@@ -176,7 +181,7 @@ class TrainableModel:
                 accelerator_type=self.config.accelerator_type,
             )
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to start reinforcement learning step {step + 1}: {str(e)}"
             ) from e
 

judgeval/{common/trainer → trainer}/trainer.py

@@ -1,15 +1,19 @@
 import asyncio
+import json
 import time
-from typing import Optional, Callable, Any, List, Union
+from typing import Optional, Callable, Any, List, Union, Dict
 from fireworks import Dataset
 from .config import TrainerConfig, ModelConfig
 from .trainable_model import TrainableModel
 from judgeval.tracer import Tracer
-from judgeval.judgment_client import JudgmentClient
+from judgeval.tracer.exporters.store import SpanStore
+from judgeval.tracer.exporters import InMemorySpanExporter
+from judgeval.tracer.keys import AttributeKeys
+from judgeval import JudgmentClient
 from judgeval.scorers import BaseScorer, APIScorerConfig
 from judgeval.data import Example
 from .console import _spinner_progress, _print_progress, _print_progress_update
-from judgeval.common.exceptions import JudgmentAPIError
+from judgeval.exceptions import JudgmentRuntimeError
 
 
 class JudgmentTrainer:
@@ -39,20 +43,114 @@ class JudgmentTrainer:
         try:
             self.config = config
             self.tracer = tracer
-            self.tracer.show_trace_urls = False
             self.project_name = project_name or "judgment_training"
-
-            if trainable_model is None:
-                self.trainable_model = TrainableModel(self.config)
-            else:
-                self.trainable_model = trainable_model
+            self.trainable_model = trainable_model
 
             self.judgment_client = JudgmentClient()
+            self.span_store = SpanStore()
+            self.span_exporter = InMemorySpanExporter(self.span_store)
         except Exception as e:
-            raise JudgmentAPIError(
+            raise JudgmentRuntimeError(
                 f"Failed to initialize JudgmentTrainer: {str(e)}"
             ) from e
 
+    def _extract_message_history_from_spans(self) -> List[Dict[str, str]]:
+        """
+        Extract message history from spans in the span store for training purposes.
+
+        This method processes trace spans to reconstruct the conversation flow,
+        extracting messages in chronological order from LLM, user, and tool spans.
+
+        Returns:
+            List of message dictionaries with 'role' and 'content' keys
+        """
+        spans = self.span_store.get_all()
+        if not spans:
+            return []
+
+        messages = []
+        first_found = False
+
+        for span in sorted(spans, key=lambda s: getattr(s, "start_time", 0)):
+            span_attributes = span.attributes or {}
+            span_type = span_attributes.get(AttributeKeys.JUDGMENT_SPAN_KIND, "span")
+
+            if (
+                not span_attributes.get(AttributeKeys.JUDGMENT_OUTPUT)
+                and span_type != "llm"
+            ):
+                continue
+
+            if span_type == "llm":
+                if not first_found and span_attributes.get(
+                    AttributeKeys.JUDGMENT_INPUT
+                ):
+                    input_data = span_attributes.get(AttributeKeys.JUDGMENT_INPUT, {})
+                    if isinstance(input_data, dict) and "messages" in input_data:
+                        input_messages = input_data["messages"]
+                        if input_messages:
+                            first_found = True
+                            for msg in input_messages:
+                                if (
+                                    isinstance(msg, dict)
+                                    and "role" in msg
+                                    and "content" in msg
+                                ):
+                                    messages.append(
+                                        {"role": msg["role"], "content": msg["content"]}
+                                    )
+
+                # Add assistant response from span output
+                output = span_attributes.get(AttributeKeys.JUDGMENT_OUTPUT)
+                if output is not None:
+                    content = str(output)
+                    try:
+                        parsed = json.loads(content)
+                        if isinstance(parsed, dict) and "messages" in parsed:
+                            # Extract the actual assistant message content
+                            for msg in parsed["messages"]:
+                                if (
+                                    isinstance(msg, dict)
+                                    and msg.get("role") == "assistant"
+                                ):
+                                    content = msg.get("content", content)
+                                    break
+                    except (json.JSONDecodeError, KeyError):
+                        pass
+                    messages.append({"role": "assistant", "content": content})
+
+            elif span_type == "user":
+                output = span_attributes.get(AttributeKeys.JUDGMENT_OUTPUT)
+                if output is not None:
+                    content = str(output)
+                    try:
+                        parsed = json.loads(content)
+                        if isinstance(parsed, dict) and "messages" in parsed:
+                            for msg in parsed["messages"]:
+                                if isinstance(msg, dict) and msg.get("role") == "user":
+                                    content = msg.get("content", content)
+                                    break
+                    except (json.JSONDecodeError, KeyError):
+                        pass
+                    messages.append({"role": "user", "content": content})
+
+            elif span_type == "tool":
+                output = span_attributes.get(AttributeKeys.JUDGMENT_OUTPUT)
+                if output is not None:
+                    content = str(output)
+                    try:
+                        parsed = json.loads(content)
+                        if isinstance(parsed, dict) and "messages" in parsed:
+                            for msg in parsed["messages"]:
+                                if isinstance(msg, dict) and msg.get("role") == "user":
+                                    content = msg.get("content", content)
+                                    break
+                    except (json.JSONDecodeError, KeyError):
+                        pass
+                    messages.append({"role": "user", "content": content})
+
+        return messages
+
     async def generate_rollouts_and_rewards(
         self,
         agent_function: Callable[[Any], Any],
@@ -95,13 +193,16 @@ class JudgmentTrainer:
                 messages = response_data.get("messages", [])
 
                 try:
-                    traced_messages = self.tracer.get_current_message_history()
+                    traced_messages = self._extract_message_history_from_spans()
                     if traced_messages:
                         messages = traced_messages
                 except Exception as e:
                     print(f"Warning: Failed to get message history from trace: {e}")
                     pass
 
+                finally:
+                    self.span_store.spans = []
+
                 example = Example(
                     input=prompt_input,
                     messages=messages,
@@ -113,14 +214,15 @@ class JudgmentTrainer:
                     scorers=scorers,
                     project_name=self.project_name,
                     eval_run_name=f"training_step_{self.trainable_model.current_step}_prompt_{prompt_id}_gen_{generation_id}",
-                    show_url=False,
                 )
 
                 if scoring_results and scoring_results[0].scorers_data:
-                    reward = sum(
+                    scores = [
                         scorer_data.score
                         for scorer_data in scoring_results[0].scorers_data
-                    ) / len(scoring_results[0].scorers_data)
+                        if scorer_data.score is not None
+                    ]
+                    reward = sum(scores) / len(scores) if scores else 0.0
                 else:
                     reward = 0.0
 
@@ -246,7 +348,7 @@ class JudgmentTrainer:
                     time.sleep(10)
                     job = job.get()
                     if job is None:
-                        raise JudgmentAPIError(
+                        raise JudgmentRuntimeError(
                             "Training job was deleted while waiting for completion"
                         )
 
@@ -294,8 +396,8 @@ class JudgmentTrainer:
             return await self.run_reinforcement_learning(
                 agent_function, scorers, prompts
             )
-        except JudgmentAPIError:
+        except JudgmentRuntimeError:
             # Re-raise JudgmentAPIError as-is
             raise
         except Exception as e:
-            raise JudgmentAPIError(f"Training process failed: {str(e)}") from e
+            raise JudgmentRuntimeError(f"Training process failed: {str(e)}") from e

judgeval/utils/async_utils.py

@@ -5,7 +5,6 @@ import concurrent.futures
 from typing import Awaitable, TypeVar
 
 
-# Generic type variable for coroutine return type
 T = TypeVar("T")
 
 
@@ -14,8 +13,8 @@ def safe_run_async(coro: Awaitable[T]) -> T:  # type: ignore[type-var]
 
     This helper handles two common situations:
 
-    1. **No running event loop** – Simply delegates to ``asyncio.run``.
-    2. **Existing running loop** – Executes the coroutine in a separate
+    1. **No running event loop** - Simply delegates to ``asyncio.run``.
+    2. **Existing running loop** - Executes the coroutine in a separate
        thread so that we don't attempt to nest event loops (which would raise
        ``RuntimeError``).
 

judgeval/utils/decorators.py

@@ -0,0 +1,24 @@
+from functools import lru_cache, wraps
+from typing import Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def use_once(func: Callable[..., T]) -> Callable[..., T]:
+    @lru_cache(maxsize=1)
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def dont_throw(func: Callable[..., T]) -> Callable[..., T | None]:
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            pass
+
+    return wrapper

judgeval/utils/file_utils.py

@@ -1,12 +1,14 @@
+import importlib.util
 import yaml
 import orjson
+from pathlib import Path
 from typing import List
-from judgeval.common.logger import judgeval_logger
+from judgeval.logger import judgeval_logger
 
-from judgeval.data import Example
+from judgeval.data.example import Example
 
 
-def get_examples_from_yaml(file_path: str) -> List[Example] | None:
+def get_examples_from_yaml(file_path: str) -> List[Example]:
     """
     Adds examples from a YAML file.
 
@@ -34,7 +36,7 @@ def get_examples_from_yaml(file_path: str) -> List[Example] | None:
     return new_examples
 
 
-def get_examples_from_json(file_path: str) -> List[Example] | None:
+def get_examples_from_json(file_path: str) -> List[Example]:
     """
     Adds examples from a JSON file.
 
@@ -64,3 +66,34 @@ def get_examples_from_json(file_path: str) -> List[Example] | None:
 
     new_examples = [Example(**e) for e in payload]
     return new_examples
+
+
+def extract_scorer_name(scorer_file_path: str) -> str:
+    try:
+        spec = importlib.util.spec_from_file_location("scorer_module", scorer_file_path)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load spec from {scorer_file_path}")
+
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if (
+                isinstance(attr, type)
+                and any("Scorer" in str(base) for base in attr.__mro__)
+                and attr.__module__ == "scorer_module"
+            ):
+                try:
+                    # Instantiate the scorer and get its name
+                    scorer_instance = attr()
+                    if hasattr(scorer_instance, "name"):
+                        return scorer_instance.name
+                except Exception:
+                    # Skip if instantiation fails
+                    continue
+
+        raise AttributeError("No scorer class found or could be instantiated")
+    except Exception as e:
+        judgeval_logger.warning(f"Could not extract scorer name: {e}")
+        return Path(scorer_file_path).stem

judgeval/utils/guards.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import TypeVar
+
+    T = TypeVar("T")
+
+
+def expect_exists(value: T | None, message: str) -> T:
+    if value is None:
+        raise ValueError(message)
+
+    return value
+
+
+def expect_api_key(api_key: str | None) -> str:
+    return expect_exists(
+        api_key,
+        "API Key is not set, please set JUDGMENT_API_KEY in the environment variables or pass it as `api_key`",
+    )
+
+
+def expect_organization_id(organization_id: str | None) -> str:
+    return expect_exists(
+        organization_id,
+        "Organization ID is not set, please set JUDGMENT_ORG_ID in the environment variables or pass it as `organization_id`",
+    )
+
+
+__all__ = ("expect_exists", "expect_api_key", "expect_organization_id")

judgeval/utils/meta.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+
+class SingletonMeta(type):
+    """
+    Metaclass for creating singleton classes.
+    """
+
+    _instances: dict[type, object] = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]

judgeval/{common/api/json_encoder.py → utils/serialize.py}

@@ -17,6 +17,7 @@ from uuid import UUID
 
 from pydantic import BaseModel
 from pydantic.types import SecretBytes, SecretStr
+import orjson
 
 
 """
@@ -60,7 +61,7 @@ def json_encoder(
 
     # Dataclasses
     if dataclasses.is_dataclass(obj):
-        obj_dict = dataclasses.asdict(obj)
+        obj_dict = dataclasses.asdict(obj)  # type: ignore[arg-type]
         return json_encoder(
             obj_dict,
         )
@@ -239,3 +240,8 @@ def generate_encoders_by_class_tuples(
 
 # Mapping of encoders to a tuple of classes that they can encode
 encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)
+
+
+# Seralize arbitrary object to a json string
+def safe_serialize(obj: Any) -> str:
+    return orjson.dumps(json_encoder(obj)).decode()

judgeval/utils/testing.py

@@ -0,0 +1,88 @@
+from rich import print as rprint
+
+from typing import List
+from judgeval.evaluation import ScoringResult
+from judgeval.data import ScorerData
+from judgeval.exceptions import JudgmentTestError
+
+
+def assert_test_results(scoring_results: List[ScoringResult]) -> None:
+    """
+    Collects all failed scorers from the scoring results.
+
+    Args:
+        ScoringResults (List[ScoringResult]): List of scoring results to check
+
+    Returns:
+        None. Raises exceptions for any failed test cases.
+    """
+    failed_cases: List[List[ScorerData]] = []
+
+    for result in scoring_results:
+        if not result.success:
+            # Create a test case context with all relevant fields
+            test_case = []
+            if result.scorers_data:
+                # If the result was not successful, check each scorer_data
+                for scorer_data in result.scorers_data:
+                    if not scorer_data.success:
+                        if scorer_data.name == "Tool Order":
+                            # Remove threshold, evaluation model for Tool Order scorer
+                            scorer_data.threshold = None
+                            scorer_data.evaluation_model = None
+                        test_case.append(scorer_data)
+            failed_cases.append(test_case)
+
+    if failed_cases:
+        error_msg = "The following test cases failed: \n"
+        for fail_case in failed_cases:
+            for fail_scorer in fail_case:
+                error_msg += (
+                    f"\nScorer Name: {fail_scorer.name}\n"
+                    f"Threshold: {fail_scorer.threshold}\n"
+                    f"Success: {fail_scorer.success}\n"
+                    f"Score: {fail_scorer.score}\n"
+                    f"Reason: {fail_scorer.reason}\n"
+                    f"Strict Mode: {fail_scorer.strict_mode}\n"
+                    f"Evaluation Model: {fail_scorer.evaluation_model}\n"
+                    f"Error: {fail_scorer.error}\n"
+                    f"Additional Metadata: {fail_scorer.additional_metadata}\n"
+                )
+            error_msg += "-" * 100
+
+        total_tests = len(scoring_results)
+        failed_tests = len(failed_cases)
+        passed_tests = total_tests - failed_tests
+
+        # Print summary with colors
+        rprint("\n" + "=" * 80)
+        if failed_tests == 0:
+            rprint(
+                f"[bold green]🎉 ALL TESTS PASSED! {passed_tests}/{total_tests} tests successful[/bold green]"
+            )
+        else:
+            rprint(
+                f"[bold red]⚠️  TEST RESULTS: {passed_tests}/{total_tests} passed ({failed_tests} failed)[/bold red]"
+            )
+        rprint("=" * 80 + "\n")
+
+        # Print individual test cases
+        for i, result in enumerate(scoring_results):
+            test_num = i + 1
+            if result.success:
+                rprint(f"[green]✓ Test {test_num}: PASSED[/green]")
+            else:
+                rprint(f"[red]✗ Test {test_num}: FAILED[/red]")
+                if result.scorers_data:
+                    for scorer_data in result.scorers_data:
+                        if not scorer_data.success:
+                            rprint(f"  [yellow]Scorer: {scorer_data.name}[/yellow]")
+                            rprint(f"  [red]  Score: {scorer_data.score}[/red]")
+                            rprint(f"  [red]  Reason: {scorer_data.reason}[/red]")
+                            if scorer_data.error:
+                                rprint(f"  [red]  Error: {scorer_data.error}[/red]")
+                rprint("  " + "-" * 40)
+
+        rprint("\n" + "=" * 80)
+        if failed_tests > 0:
+            raise JudgmentTestError(failed_cases)

judgeval/utils/url.py

@@ -0,0 +1,10 @@
+from urllib.parse import urljoin
+
+from judgeval.env import JUDGMENT_API_URL
+
+
+def url_for(path: str, base: str = JUDGMENT_API_URL) -> str:
+    return urljoin(base, path)
+
+
+__all__ = ("url_for",)

judgeval/{version_check.py → utils/version_check.py}

@@ -1,14 +1,14 @@
 import importlib.metadata
-from judgeval.utils.requests import requests
+import httpx
 import threading
-from judgeval.common.logger import judgeval_logger
+from judgeval.logger import judgeval_logger
 
 
 def check_latest_version(package_name: str = "judgeval"):
     def _check():
         try:
             current_version = importlib.metadata.version(package_name)
-            response = requests.get(
+            response = httpx.get(
                 f"https://pypi.org/pypi/{package_name}/json", timeout=2
             )
             latest_version = response.json()["info"]["version"]

judgeval/version.py

@@ -0,0 +1,5 @@
+__version__ = "0.0.0"
+
+
+def get_version() -> str:
+    return __version__

judgeval/warnings.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+
+class JudgmentWarning(Warning): ...

{judgeval-0.8.0.dist-info → judgeval-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.8.0
+Version: 0.9.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -10,27 +10,25 @@ License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
-Requires-Dist: boto3
+Requires-Dist: boto3>=1.40.11
 Requires-Dist: click<8.2.0
-Requires-Dist: fireworks-ai>=0.19.18
-Requires-Dist: langchain-anthropic
-Requires-Dist: langchain-core
-Requires-Dist: langchain-huggingface
-Requires-Dist: langchain-openai
-Requires-Dist: litellm>=1.61.15
-Requires-Dist: nest-asyncio>=1.6.0
-Requires-Dist: opentelemetry-api>=1.34.1
-Requires-Dist: opentelemetry-sdk>=1.34.1
+Requires-Dist: dotenv
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: litellm<1.75.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.36.0
+Requires-Dist: opentelemetry-sdk>=1.36.0
+Requires-Dist: opentelemetry-semantic-conventions>=0.57b0
 Requires-Dist: orjson>=3.9.0
-Requires-Dist: python-dotenv
-Requires-Dist: requests
-Requires-Dist: rich
 Requires-Dist: typer>=0.9.0
 Provides-Extra: langchain
 Requires-Dist: langchain-anthropic; extra == 'langchain'
 Requires-Dist: langchain-core; extra == 'langchain'
 Requires-Dist: langchain-huggingface; extra == 'langchain'
 Requires-Dist: langchain-openai; extra == 'langchain'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.40.11; extra == 's3'
+Provides-Extra: trainer
+Requires-Dist: fireworks-ai>=0.19.18; extra == 'trainer'
 Description-Content-Type: text/markdown
 
 <div align="center">