judgeval 0.3.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

judgeval/local_eval_queue.py

@@ -0,0 +1,192 @@
+"""Local evaluation queue for batching custom scorer evaluations.
+
+This module provides a simple in-memory queue for EvaluationRun objects that contain
+only local (BaseScorer) scorers. Useful for batching evaluations and processing them
+either synchronously or in a background thread.
+"""
+
+import queue
+import threading
+from typing import Callable, List, Optional
+import time
+
+from judgeval.common.logger import judgeval_logger
+from judgeval.constants import MAX_CONCURRENT_EVALUATIONS
+from judgeval.data import ScoringResult
+from judgeval.evaluation_run import EvaluationRun
+from judgeval.utils.async_utils import safe_run_async
+from judgeval.scorers import BaseScorer
+from judgeval.scorers.score import a_execute_scoring
+
+
+class LocalEvaluationQueue:
+    """Lightweight in-memory queue for local evaluation runs.
+
+    Only supports EvaluationRuns with local scorers (BaseScorer instances).
+    API scorers (APIScorerConfig) are not supported as they have their own queue.
+    """
+
+    def __init__(
+        self, max_concurrent: int = MAX_CONCURRENT_EVALUATIONS, num_workers: int = 4
+    ):
+        if num_workers <= 0:
+            raise ValueError("num_workers must be a positive integer.")
+        self._queue: queue.Queue[Optional[EvaluationRun]] = queue.Queue()
+        self._max_concurrent = max_concurrent
+        self._num_workers = num_workers  # Number of worker threads
+        self._worker_threads: List[threading.Thread] = []
+        self._shutdown_event = threading.Event()
+
+    def enqueue(self, evaluation_run: EvaluationRun) -> None:
+        """Add evaluation run to the queue."""
+        self._queue.put(evaluation_run)
+
+    def _process_run(self, evaluation_run: EvaluationRun) -> List[ScoringResult]:
+        """Execute evaluation run locally and return results."""
+        local_scorers = [s for s in evaluation_run.scorers if isinstance(s, BaseScorer)]
+
+        if not local_scorers:
+            raise ValueError(
+                "LocalEvaluationQueue only supports runs with local scorers (BaseScorer). "
+                "Found only APIScorerConfig instances."
+            )
+
+        return safe_run_async(
+            a_execute_scoring(
+                evaluation_run.examples,
+                local_scorers,
+                model=evaluation_run.model,
+                throttle_value=0,
+                max_concurrent=self._max_concurrent // self._num_workers,
+                show_progress=False,
+            )
+        )
+
+    def run_all(
+        self,
+        callback: Optional[Callable[[EvaluationRun, List[ScoringResult]], None]] = None,
+    ) -> None:
+        """Process all queued runs synchronously.
+
+        Args:
+            callback: Optional function called after each run with (run, results).
+        """
+        while not self._queue.empty():
+            run = self._queue.get()
+            if run is None:  # Sentinel for worker shutdown
+                self._queue.put(None)
+                break
+            results = self._process_run(run)
+            if callback:
+                callback(run, results)
+            self._queue.task_done()
+
+    def start_workers(
+        self,
+        callback: Optional[Callable[[EvaluationRun, List[ScoringResult]], None]] = None,
+    ) -> List[threading.Thread]:
+        """Start multiple background threads to process runs in parallel.
+
+        Args:
+            callback: Optional function called after each run with (run, results).
+
+        Returns:
+            List of started worker threads.
+        """
+
+        def _worker(worker_id: int) -> None:
+            while not self._shutdown_event.is_set():
+                try:
+                    # Use timeout so workers can check shutdown event periodically
+                    run = self._queue.get(timeout=1.0)
+                    if run is None:  # Sentinel to stop worker
+                        # Put sentinel back for other workers
+                        self._queue.put(None)
+                        self._queue.task_done()
+                        break
+
+                    try:
+                        results = self._process_run(run)
+                        if callback:
+                            callback(run, results)
+                    except Exception as exc:
+                        judgeval_logger.error(
+                            f"Worker {worker_id} error processing {run.eval_name}: {exc}"
+                        )
+                        # Continue processing other runs instead of shutting down all workers
+                    finally:
+                        self._queue.task_done()
+
+                except queue.Empty:
+                    # Timeout - check shutdown event and continue
+                    continue
+
+        # Start worker threads
+        for i in range(self._num_workers):
+            thread = threading.Thread(target=_worker, args=(i,), daemon=True)
+            thread.start()
+            self._worker_threads.append(thread)
+
+        return self._worker_threads
+
+    def start_worker(
+        self,
+        callback: Optional[Callable[[EvaluationRun, List[ScoringResult]], None]] = None,
+    ) -> Optional[threading.Thread]:
+        """Start a single background thread to process runs (backward compatibility).
+
+        Args:
+            callback: Optional function called after each run with (run, results).
+
+        Returns:
+            The started thread, or None if no threads were started.
+        """
+        threads = self.start_workers(callback)
+        return threads[0] if threads else None
+
+    def wait_for_completion(self, timeout: Optional[float] = None) -> bool:
+        """Wait for all queued tasks to complete.
+
+        Args:
+            timeout: Maximum time to wait in seconds. None means wait indefinitely.
+
+        Returns:
+            True if all tasks completed, False if timeout occurred.
+        """
+        try:
+            if timeout is None:
+                self._queue.join()
+                return True
+            else:
+                start_time = time.time()
+                while not self._queue.empty() or self._queue.unfinished_tasks > 0:
+                    if time.time() - start_time > timeout:
+                        return False
+                    time.sleep(0.1)
+                return True
+        except Exception:
+            return False
+
+    def stop_workers(self) -> None:
+        """Signal all background workers to stop after current tasks complete."""
+        if not self._worker_threads:
+            return
+
+        # Signal shutdown
+        self._shutdown_event.set()
+
+        # Send sentinel to wake up any blocking workers
+        for _ in range(self._num_workers):
+            self._queue.put(None)
+
+        # Wait for all workers to finish with timeout
+        for thread in self._worker_threads:
+            if thread.is_alive():
+                thread.join(timeout=5.0)
+                if thread.is_alive():
+                    judgeval_logger.warning(
+                        f"Worker thread {thread.name} did not shut down gracefully"
+                    )
+
+        self._worker_threads.clear()
+        self._shutdown_event.clear()

judgeval/run_evaluation.py

@@ -1,10 +1,12 @@
+from __future__ import annotations
+
 import asyncio
 import concurrent.futures
 import time
 import orjson
 import sys
 import threading
-from typing import List, Dict, Union, Optional, Callable, Tuple, Any
+from typing import List, Dict, Union, Optional, Callable, Tuple, Any, TYPE_CHECKING
 from rich import print as rprint
 
 from judgeval.data import ScorerData, ScoringResult, Example, Trace
@@ -17,10 +19,13 @@ from judgeval.constants import (
 from judgeval.common.exceptions import JudgmentAPIError
 from judgeval.common.api.api import JudgmentAPIException
 from judgeval.common.logger import judgeval_logger
-from judgeval.evaluation_run import EvaluationRun
-from judgeval.data.trace_run import TraceRun
-from judgeval.common.tracer import Tracer
-from judgeval.integrations.langgraph import JudgevalCallbackHandler
+
+
+if TYPE_CHECKING:
+    from judgeval.common.tracer import Tracer
+    from judgeval.data.trace_run import TraceRun
+    from judgeval.evaluation_run import EvaluationRun
+    from judgeval.integrations.langgraph import JudgevalCallbackHandler
 
 
 def safe_run_async(coro):
@@ -282,7 +287,7 @@ def run_trace_eval(
     judgment_api_key: str,
     override: bool = False,
     function: Optional[Callable] = None,
-    tracer: Optional[Union[Tracer, JudgevalCallbackHandler]] = None,
+    tracer: Optional[Union[Tracer, "JudgevalCallbackHandler"]] = None,
     examples: Optional[List[Example]] = None,
 ) -> List[ScoringResult]:
     # Call endpoint to check to see if eval run name exists (if we DON'T want to override and DO want to log results)

judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -1,27 +1,29 @@
 from judgeval.scorers.api_scorer import APIScorerConfig
 from judgeval.constants import APIScorerType
-from typing import Mapping, Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.common.api import JudgmentApiClient, JudgmentAPIException
 import os
 from judgeval.common.exceptions import JudgmentAPIError
+from copy import copy
+from judgeval.common.logger import judgeval_logger
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
-    options: Mapping[str, float],
+    options: Optional[Dict[str, float]] = None,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
 ) -> str:
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        r = client.save_scorer(name, prompt, dict(options))
+        r = client.save_scorer(name, prompt, options)
     except JudgmentAPIException as e:
         if e.status_code == 500:
             raise JudgmentAPIError(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
-        raise JudgmentAPIError(f"Failed to save classifier scorer: {e.error_detail}")
+        raise JudgmentAPIError(f"Failed to save prompt scorer: {e.error_detail}")
     return r["name"]
 
 
@@ -32,7 +34,7 @@ def fetch_prompt_scorer(
 ):
     client = JudgmentApiClient(judgment_api_key, organization_id)
     try:
-        scorer_config = client.fetch_scorer(name)
+        scorer_config = client.fetch_scorer(name)["scorer"]
         scorer_config.pop("created_at")
         scorer_config.pop("updated_at")
         return scorer_config
@@ -42,7 +44,7 @@ def fetch_prompt_scorer(
                 f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}"
             )
         raise JudgmentAPIError(
-            f"Failed to fetch classifier scorer '{name}': {e.error_detail}"
+            f"Failed to fetch prompt scorer '{name}': {e.error_detail}"
         )
 
 
@@ -72,7 +74,7 @@ class PromptScorer(APIScorerConfig):
     """
 
     prompt: str
-    options: Mapping[str, float]
+    options: Optional[Dict[str, float]] = None
     score_type: APIScorerType = APIScorerType.PROMPT_SCORER
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
@@ -88,7 +90,7 @@ class PromptScorer(APIScorerConfig):
         return cls(
             name=name,
             prompt=scorer_config["prompt"],
-            options=scorer_config["options"],
+            options=scorer_config.get("options"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
         )
@@ -98,12 +100,13 @@ class PromptScorer(APIScorerConfig):
         cls,
         name: str,
         prompt: str,
-        options: Mapping[str, float],
+        options: Optional[Dict[str, float]] = None,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
     ):
         if not scorer_exists(name, judgment_api_key, organization_id):
             push_prompt_scorer(name, prompt, options, judgment_api_key, organization_id)
+            judgeval_logger.info(f"Successfully created PromptScorer: {name}")
             return cls(
                 name=name,
                 prompt=prompt,
@@ -117,13 +120,6 @@ class PromptScorer(APIScorerConfig):
             )
 
     # Setter functions. Each setter function pushes the scorer to the DB.
-    def set_name(self, name: str):
-        """
-        Updates the name of the scorer.
-        """
-        self.name = name
-        self.push_prompt_scorer()
-
     def set_threshold(self, threshold: float):
         """
         Updates the threshold of the scorer.
@@ -140,8 +136,9 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt = prompt
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated prompt for {self.name}")
 
-    def set_options(self, options: Mapping[str, float]):
+    def set_options(self, options: Dict[str, float]):
         """
         Updates the options with the new options.
 
@@ -150,6 +147,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.options = options
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
 
     def append_to_prompt(self, prompt_addition: str):
         """
@@ -157,6 +155,7 @@ class PromptScorer(APIScorerConfig):
         """
         self.prompt += prompt_addition
         self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully appended to prompt for {self.name}")
 
     # Getters
     def get_prompt(self) -> str | None:
@@ -165,11 +164,11 @@ class PromptScorer(APIScorerConfig):
         """
         return self.prompt
 
-    def get_options(self) -> Mapping[str, float] | None:
+    def get_options(self) -> Dict[str, float] | None:
         """
         Returns the options of the scorer.
         """
-        return self.options
+        return copy(self.options) if self.options is not None else None
 
     def get_name(self) -> str | None:
         """

judgeval/scorers/score.py

@@ -17,6 +17,7 @@ from judgeval.scorers import BaseScorer
 from judgeval.scorers.utils import clone_scorers
 from judgeval.common.logger import judgeval_logger
 from judgeval.judges import JudgevalJudge
+from judgeval.constants import DEFAULT_GPT_MODEL
 
 
 async def safe_a_score_example(
@@ -48,16 +49,18 @@ async def safe_a_score_example(
         judgeval_logger.error(f"Error during scoring: {str(e)}")
         scorer.error = str(e)
         scorer.success = False
+        scorer.score = 0
         return
 
 
 async def a_execute_scoring(
     examples: List[Example],
     scorers: List[BaseScorer],
-    model: Optional[Union[str, List[str], JudgevalJudge]] = "gpt-4.1",
+    model: Optional[Union[str, List[str], JudgevalJudge]] = DEFAULT_GPT_MODEL,
     ignore_errors: bool = False,
     throttle_value: int = 0,
     max_concurrent: int = 100,
+    show_progress: bool = True,
 ) -> List[ScoringResult]:
     """
     Executes evaluations of `Example`s asynchronously using one or more `BaseScorer`s.
@@ -70,8 +73,7 @@ async def a_execute_scoring(
         ignore_errors (bool): Whether to ignore errors during evaluation.
         throttle_value (int): The amount of time to wait between starting each task.
         max_concurrent (int): The maximum number of concurrent tasks.
-
-        _use_bar_indicator (bool): Whether to use a progress bar indicator.
+        show_progress (bool): Whether to show the progress bar indicator.
 
     Returns:
         List[ScoringResult]: A list of `ScoringResult` objects containing the evaluation results.
@@ -100,16 +102,37 @@ async def a_execute_scoring(
     tasks = []
     cloned_scorers: List[BaseScorer]
 
-    with tqdm_asyncio(
-        desc=f"Evaluating {len(examples)} example(s) in parallel",
-        unit="Example",
-        total=len(examples),
-        bar_format="{desc}: |{bar}|{percentage:3.0f}% ({n_fmt}/{total_fmt}) [Time Taken: {elapsed}, {rate_fmt}{postfix}]",
-    ) as pbar:
+    if show_progress:
+        with tqdm_asyncio(
+            desc=f"Evaluating {len(examples)} example(s) in parallel",
+            unit="Example",
+            total=len(examples),
+            bar_format="{desc}: |{bar}|{percentage:3.0f}% ({n_fmt}/{total_fmt}) [Time Taken: {elapsed}, {rate_fmt}{postfix}]",
+        ) as pbar:
+            for i, ex in enumerate(examples):
+                if isinstance(ex, Example):
+                    if len(scorers) == 0:
+                        pbar.update(1)
+                        continue
+
+                    cloned_scorers = clone_scorers(scorers)
+                    task = execute_with_semaphore(
+                        func=a_eval_examples_helper,
+                        scorers=cloned_scorers,
+                        example=ex,
+                        scoring_results=scoring_results,
+                        score_index=i,
+                        ignore_errors=ignore_errors,
+                        pbar=pbar,
+                    )
+                    tasks.append(asyncio.create_task(task))
+
+                await asyncio.sleep(throttle_value)
+            await asyncio.gather(*tasks)
+    else:
         for i, ex in enumerate(examples):
             if isinstance(ex, Example):
                 if len(scorers) == 0:
-                    pbar.update(1)
                     continue
 
                 cloned_scorers = clone_scorers(scorers)
@@ -120,7 +143,7 @@ async def a_execute_scoring(
                     scoring_results=scoring_results,
                     score_index=i,
                     ignore_errors=ignore_errors,
-                    pbar=pbar,
+                    pbar=None,
                 )
                 tasks.append(asyncio.create_task(task))
 

judgeval/utils/async_utils.py

@@ -0,0 +1,36 @@
+"""Async utilities for judgeval."""
+
+import asyncio
+import concurrent.futures
+from typing import Awaitable, TypeVar
+
+
+# Generic type variable for coroutine return type
+T = TypeVar("T")
+
+
+def safe_run_async(coro: Awaitable[T]) -> T:  # type: ignore[type-var]
+    """Safely execute an async *coro* from synchronous code.
+
+    This helper handles two common situations:
+
+    1. **No running event loop** – Simply delegates to ``asyncio.run``.
+    2. **Existing running loop** – Executes the coroutine in a separate
+       thread so that we don't attempt to nest event loops (which would raise
+       ``RuntimeError``).
+
+    Args:
+        coro: The coroutine to execute.
+
+    Returns:
+        The result returned by *coro*.
+    """
+
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future = executor.submit(lambda: asyncio.run(coro))
+        return future.result()

{judgeval-0.3.2.dist-info → judgeval-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.3.2
+Version: 0.5.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -10,27 +10,24 @@ License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
-Requires-Dist: anthropic
 Requires-Dist: boto3
-Requires-Dist: datamodel-code-generator>=0.31.1
-Requires-Dist: google-genai
-Requires-Dist: groq>=0.30.0
 Requires-Dist: langchain-anthropic
 Requires-Dist: langchain-core
 Requires-Dist: langchain-huggingface
 Requires-Dist: langchain-openai
 Requires-Dist: litellm>=1.61.15
-Requires-Dist: matplotlib>=3.10.3
-Requires-Dist: nest-asyncio
-Requires-Dist: openai
+Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: opentelemetry-api>=1.34.1
 Requires-Dist: opentelemetry-sdk>=1.34.1
 Requires-Dist: orjson>=3.9.0
-Requires-Dist: pandas
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: python-dotenv
 Requires-Dist: requests
-Requires-Dist: together
+Requires-Dist: rich
+Provides-Extra: langchain
+Requires-Dist: langchain-anthropic; extra == 'langchain'
+Requires-Dist: langchain-core; extra == 'langchain'
+Requires-Dist: langchain-huggingface; extra == 'langchain'
+Requires-Dist: langchain-openai; extra == 'langchain'
 Description-Content-Type: text/markdown
 
 <div align="center">

{judgeval-0.3.2.dist-info → judgeval-0.5.0.dist-info}/RECORD

@@ -1,53 +1,56 @@
-judgeval/__init__.py,sha256=HM1M8hmqRum6G554QKkXhB4DF4f5eh_xtYo0Kf-t3kw,332
-judgeval/clients.py,sha256=JnB8n90GyXiYaGmSEYaA67mdJSnr3SIrzArao7NGebw,980
-judgeval/constants.py,sha256=hWed25HwGUJy-tePbtoUZ0_Zg0X_MkAH84KiH-OHHFI,4150
-judgeval/dataset.py,sha256=rjV54XNTslNNtf-Uu2ndDIh602ZwSCFhPg2NuckDJ-w,6081
-judgeval/evaluation_run.py,sha256=edNpO444Fwt2ykWsflIzlYdDJUlUfbpXHHQSKfFS4y0,2876
-judgeval/judgment_client.py,sha256=vPoxbmxAlhbG5rXXqxWjMbyEqOI044BaQanr1fev2CE,11723
+judgeval/__init__.py,sha256=5Lm1JMYFREJGN_8X-Wpruu_ovwGLJ08gCzNAt-u-pQE,419
+judgeval/clients.py,sha256=HHul68PV1om0dxsVZZu90TtCiy5zaqAwph16jXTQzQo,989
+judgeval/constants.py,sha256=UNoTLHgbpZHRInPM2ZaI3m0XokPkee5ILlg20reqhzo,4180
+judgeval/dataset.py,sha256=vOrDKam2I-K1WcVF5IBkQruCDvXTc8PRaFm4-dV0lXs,6220
+judgeval/evaluation_run.py,sha256=FJpnc1sGncmAOAnEUO0n2vNXjlycljGqBdV99qPT5og,3087
+judgeval/judgment_client.py,sha256=tGhENRb2YVIe2WUlcssC8DuEijeUC7Ajj_rh_Dh7bzA,11878
+judgeval/local_eval_queue.py,sha256=fAI0_OlvCr-WOCQWw18C4JIRJHKYzlyGzsGUm8LcsYE,7076
 judgeval/rules.py,sha256=CoQjqmP8daEXewMkplmA-7urubDtweOr5O6z8klVwLI,20031
-judgeval/run_evaluation.py,sha256=7J6FHhWhB-IDPMSOcWkrjTpSNm2v3s_KBq8Np3y2pys,27652
+judgeval/run_evaluation.py,sha256=4kcaw3R_akhxqutGFGTaBS2pqD-3d0ET7zMDL1_7HK4,27741
 judgeval/version_check.py,sha256=FoLEtpCjDw2HuDQdpw5yT29UtwumSc6ZZN6AV_c9Mnw,1057
 judgeval/common/__init__.py,sha256=KH-QJyWtQ60R6yFIBDYS3WGRiNpEu1guynpxivZvpBQ,309
 judgeval/common/exceptions.py,sha256=OkgDznu2wpBQZMXiZarLJYNk1HIcC8qYW7VypDC3Ook,556
 judgeval/common/logger.py,sha256=514eFLYWS_UL8VY-zAR2ePUlpQe4rbYlleLASFllLE4,1511
 judgeval/common/utils.py,sha256=oxGDRVWOICKWeyGgsoc36_yAyHSYF4XtH842Mkznwis,34739
 judgeval/common/api/__init__.py,sha256=-E7lpZz1fG8puR_aYUMfPmQ-Vyhd0bgzoaU5EhIuFjQ,114
-judgeval/common/api/api.py,sha256=wty02HYANeOYlM8fHOLc33ux5bu9Ieq7iRqCr-UP0ng,14157
-judgeval/common/api/constants.py,sha256=vAW94pbyTS6rv1TKpt7z6xxMJvTaAxFiy1D4kzuLHeg,4567
+judgeval/common/api/api.py,sha256=uuLH6veC0LewfZ1IFiiUi5_OV7zTa7xTIK9LRlLoufc,13743
+judgeval/common/api/constants.py,sha256=DXej0m8HEhb871SdiR8t_o4fzeMoQjHYqb_X0Plj8wY,4577
+judgeval/common/api/json_encoder.py,sha256=XsScZe9hZP56yuxQ-3Ox6K8DcbjWxc2Yq7FcLF9qkUE,5852
 judgeval/common/storage/__init__.py,sha256=a-PI7OL-ydyzugGUKmJKRBASnK-Q-gs82L9K9rSyJP8,90
 judgeval/common/storage/s3_storage.py,sha256=0-bNKheqJJyBZ92KGrzQtd1zocIRWBlfn_58L4a-Ay0,3719
 judgeval/common/tracer/__init__.py,sha256=tJCJsmVmrL89Phv88gNCJ-j0ITPez6lh8vhMAAlLNSc,795
 judgeval/common/tracer/constants.py,sha256=yu5y8gMe5yb1AaBkPtAH-BNwIaAR3NwYCRoSf45wp5U,621
-judgeval/common/tracer/core.py,sha256=blHEh61CE5kZLYCgyRF4kU6dVzi_Ko6DrnBpw2-jByI,73973
+judgeval/common/tracer/core.py,sha256=rI7P0CaarP5FLQZmOGWpOJkjdf6WUgSds6i_QF04J3M,85071
 judgeval/common/tracer/otel_exporter.py,sha256=kZLlOQ6afQE4dmb9H1wgU4P3H5PG1D_zKyvnpWcT5Ak,3899
 judgeval/common/tracer/otel_span_processor.py,sha256=W7SM62KnxJ48vC9WllIHRKaLlvxkCwqYoT4KqZLfGNs,6497
+judgeval/common/tracer/providers.py,sha256=3c3YOtKuoBjlTL0rc2HAGnUpppqvsyzrN5H6EKCqEi0,2733
 judgeval/common/tracer/span_processor.py,sha256=eFjTgSWSkM6BWE94CrvgafDg_WkxLsFL_MafwBG-p9M,1145
-judgeval/common/tracer/span_transformer.py,sha256=nCnwRC52OKfYRFnsOwGdPaqb_U17yn5S_9jfhv1GaLM,7803
+judgeval/common/tracer/span_transformer.py,sha256=mUmfUYjEekUEOXAZMmH0WEF94ge05EBi5ftSc-T91zQ,7314
 judgeval/common/tracer/trace_manager.py,sha256=ltiXcWC-68DRc8uSa28qHiWRSIBf6NpYOPkZYooR8tg,3086
 judgeval/data/__init__.py,sha256=1QagDcSQtfnJ632t9Dnq8d7XjAqhmY4mInOWt8qH9tM,455
 judgeval/data/example.py,sha256=kRskIgsjwcvv2Y8jaPwV-PND7zlmMbFsvRVQ_b7SZY0,914
-judgeval/data/judgment_types.py,sha256=KE1HrFLfSxiu1zutaiZ7B7La9PGXIAsoWpo_5iy645c,8336
+judgeval/data/judgment_types.py,sha256=1DTpCnIdDM93Rozu9Dr812Q5K3lZfawMcWbPG2ofbxM,8407
 judgeval/data/result.py,sha256=OtSnBUrdQpjyAqxXRLTW3wC9v9lOm_GqzL14ccRQxrg,2124
 judgeval/data/scorer_data.py,sha256=5QBHtvOIWOq0Rn9_uPJzAMRYMlWxMB-rXnG_6kV4Z4Y,2955
 judgeval/data/tool.py,sha256=iWQSdy5uNbIeACu3gQy1DC2oGYxRVYNfkkczWdQMAiA,99
-judgeval/data/trace.py,sha256=tDOuYFPUssQInjsmwyxcXq-W3IB29Vq340VzqafuKJc,6942
-judgeval/data/trace_run.py,sha256=c6pRSv09Vj016hxM49I3kMftCwWg8hhkfT_1kBXluSI,1600
+judgeval/data/trace.py,sha256=LG-IZksynC1VgfUBuBfIIfR1DT9Bn-sY4vIj6Rc9K6Q,2791
+judgeval/data/trace_run.py,sha256=ZCAzktgOSUPD0p1XQj8qGcF-DdsdQFNZM2dtY0aKGbE,1657
 judgeval/data/scripts/fix_default_factory.py,sha256=lvp2JwYZqz-XpD9LZNa3mANZVP-jJSZoNzolI6JWERM,591
 judgeval/data/scripts/openapi_transform.py,sha256=Sm04JClzyP1ga8KA3gkIdsae8Hlx-XU7-x0gHCQYOhg,3877
 judgeval/integrations/langgraph.py,sha256=kJXLsgBY7DgsUTZyVQ47deDgHm887brFHfyIbuyerGw,29986
 judgeval/judges/__init__.py,sha256=6X7VSwrwsdxGBNxCyapVRWGghhKOy3MVxFNMQ62kCXM,308
 judgeval/judges/base_judge.py,sha256=_dz0qWsKRxzXxpRY9l6mrxTRYPSF2FE4ZXkrzhZ4gbY,986
-judgeval/judges/litellm_judge.py,sha256=yt6QvwKMmxZcrUtjbn3EiO5aVg7CHM2YZkBCSQLS8jk,2509
-judgeval/judges/mixture_of_judges.py,sha256=cecQ8mRmz2-dDoZl2MGsrhZICkpIvRovGPK3su0kc8s,14889
+judgeval/judges/litellm_judge.py,sha256=K9yCGOmozt7sYO0u8CHWyZNi8mXnSR3pPkP8yVsvuRc,2561
+judgeval/judges/mixture_of_judges.py,sha256=iTNjTX4Le1nCwGRm9qfMCv1lQjgqoIw3OE0teiLubwo,14946
 judgeval/judges/together_judge.py,sha256=5FADUhs6-FN1ZVV_1D3-8_gu9mPbZiG0PYTpme41SfM,2336
-judgeval/judges/utils.py,sha256=0CF9qtIUQUL3-W-qTGpmTjZbkUUBAM6TslDsrCHnTBU,2725
+judgeval/judges/utils.py,sha256=_t6oYN9q63wyP7D4jI8X0bNmvVw7OfaE7uMTYDVS14E,2782
 judgeval/scorers/__init__.py,sha256=4H_cinTQ4EogZv59YEV-3U9EOTLppNwgAPTi1-jI9Fw,746
 judgeval/scorers/agent_scorer.py,sha256=TjwD_YglSywr3EowEojiCyg5qDgCRa5LRGc5nFdmIBc,703
 judgeval/scorers/api_scorer.py,sha256=xlhqkeMUBFxl8daSXOTWOYwZjBAz7o6b4sVD5f8cIHw,2523
 judgeval/scorers/base_scorer.py,sha256=eDfQk8N8TQfM1ayJDWr0NTdSQxcbk9-VZHd0Igb9EbI,2878
 judgeval/scorers/example_scorer.py,sha256=2n45y3LMV1Q-ARyXLHqvVWETlnY1DqS7OLzPu9IBGz8,716
 judgeval/scorers/exceptions.py,sha256=ACDHK5-TWiF3NTk-wycaedpbrdobm-CvvC1JA_iP-Mk,179
-judgeval/scorers/score.py,sha256=2-M_AmOjIQR2c0qvuB4WIIQD-7zSNdzsWC8ttqltw2g,6601
+judgeval/scorers/score.py,sha256=SWyoqOOvyLpLy39tLyb_Q94sdh9r_IuDv6YNREw52lg,7546
 judgeval/scorers/utils.py,sha256=HQOYTJtNnsi_aPfMssePAaBbXpAv7LXgwUlWlDFuN2g,3965
 judgeval/scorers/judgeval_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=GX4KkwPR2p-c0Y5mZingJa8EUfjAbMGhrmRBDBunOGw,1484
@@ -58,14 +61,15 @@ judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py,sha256=NABO_iBd
 judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ps51bTgQsD9xGYsk1v9bx0WxQMqywSllCE9_xlJkLd8,531
 judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py,sha256=SnFLvU4FGsMeUVUp0SGHSy_6wgfwr_vHPGnZx5YJl_Q,691
 judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=aQzu-TiGqG74JDQ927evv5yGmnZw2AOolyHvlIhiUbI,683
-judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=TS3uZ6YQfMs2yGCwzlz-yxZ3Rid79MGxEQESZkSX_Vo,7038
+judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=nx73DeoVkSqJTP1hYxMsJobG9HVWgMDN5-xFOXt_8Ts,7348
 judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py,sha256=Mcp1CjMNyOax9UkvoRdSyUYdO2Os1-Nko43y89m2Luo,594
 judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py,sha256=Z2FLGBC7m_CLx-CMgXVuTvYvN0vY5yOcWA0ImBkeBfY,787
 judgeval/tracer/__init__.py,sha256=wkuXtOGDCrwgPPXlh_sSJmvGuWaAMHyNzk1TzB5f9aI,148
 judgeval/utils/alerts.py,sha256=3w_AjQrgfmOZvfqCridW8WAnHVxHHXokX9jNzVFyGjA,3297
+judgeval/utils/async_utils.py,sha256=uNx1SopEc0quSjc8GBQqyba0SmCMAzv2NKIq6xYwttc,989
 judgeval/utils/file_utils.py,sha256=PWHRs8dUr8iDwpglSSk4Yjd7C6ZhDzUaO-jV3m7riHM,1987
 judgeval/utils/requests.py,sha256=K3gUKrwL6TvwYKVYO5OeLWdUHn9NiUPmnIXhZEiEaHU,1534
-judgeval-0.3.2.dist-info/METADATA,sha256=3VDQzNPY0wKnaIpzmrP2OLsk6eEGJZH_dSl4QcJRVwE,10348
-judgeval-0.3.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.3.2.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.3.2.dist-info/RECORD,,
+judgeval-0.5.0.dist-info/METADATA,sha256=wwnunL-UcNKbB7D5t-UnOM_x3DVghU2BBPAVxa0tNfo,10348
+judgeval-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+judgeval-0.5.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
+judgeval-0.5.0.dist-info/RECORD,,