judgeval 0.0.20__py3-none-any.whl → 0.0.21__py3-none-any.whl

judgeval/common/tracer.py

@@ -10,6 +10,7 @@ import os
 import time
 import uuid
 import warnings
+from contextvars import ContextVar
 from contextlib import contextmanager
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -37,6 +38,7 @@ from judgeval.constants import (
     RABBITMQ_PORT,
     RABBITMQ_QUEUE,
     JUDGMENT_TRACES_DELETE_API_URL,
+    JUDGMENT_PROJECT_DELETE_API_URL,
     JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL
 )
 from judgeval.judgment_client import JudgmentClient
@@ -54,7 +56,7 @@ from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain_core.callbacks import CallbackManager, BaseCallbackHandler
 from langchain_core.agents import AgentAction, AgentFinish
 from langchain_core.outputs import LLMResult
-
+from langchain_core.tracers.context import register_configure_hook
 from langchain_core.messages.ai import AIMessage
 from langchain_core.messages.tool import ToolMessage
 from langchain_core.messages.base import BaseMessage
@@ -251,7 +253,8 @@ class TraceManagerClient:
             raise ValueError(f"Failed to save trace data: {response.text}")
         
         if not empty_save and "ui_results_url" in response.json():
-            rprint(f"\n🔍 You can view your trace data here: [rgb(106,0,255)]{response.json()['ui_results_url']}[/]\n")
+            pretty_str = f"\n🔍 You can view your trace data here: [rgb(106,0,255)][link={response.json()['ui_results_url']}]View Trace[/link]\n"
+            rprint(pretty_str)
 
     def delete_trace(self, trace_id: str):
         """
@@ -294,6 +297,27 @@ class TraceManagerClient:
             raise ValueError(f"Failed to delete trace: {response.text}")
         
         return response.json()
+    
+    def delete_project(self, project_name: str):
+        """
+        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
+        """
+        response = requests.delete(
+            JUDGMENT_PROJECT_DELETE_API_URL,
+            json={
+                "project_name": project_name,
+            },
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.judgment_api_key}",
+                "X-Organization-Id": self.organization_id
+            }
+        )
+
+        if response.status_code != HTTPStatus.OK:
+            raise ValueError(f"Failed to delete traces: {response.text}")
+            
+        return response.json()
 
 
 class TraceClient:
@@ -1152,3 +1176,18 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
             'args': str(messages),
             'kwargs': kwargs
         })
+
+judgeval_callback_handler_var: ContextVar[Optional[JudgevalCallbackHandler]] = ContextVar(
+    "judgeval_callback_handler", default=None
+)
+
+def set_global_handler(handler: JudgevalCallbackHandler):
+    judgeval_callback_handler_var.set(handler)
+
+def clear_global_handler():
+    judgeval_callback_handler_var.set(None)
+
+register_configure_hook(
+    context_var=judgeval_callback_handler_var,
+    inheritable=True,
+)

judgeval/constants.py

@@ -48,6 +48,7 @@ JUDGMENT_EVAL_LOG_API_URL = f"{ROOT_API}/log_eval_results/"
 JUDGMENT_EVAL_FETCH_API_URL = f"{ROOT_API}/fetch_eval_results/"
 JUDGMENT_EVAL_DELETE_API_URL = f"{ROOT_API}/delete_eval_results_by_project_and_run_name/"
 JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
+JUDGMENT_PROJECT_DELETE_API_URL = f"{ROOT_API}/projects/delete/"
 JUDGMENT_TRACES_FETCH_API_URL = f"{ROOT_API}/traces/fetch/"
 JUDGMENT_TRACES_SAVE_API_URL = f"{ROOT_API}/traces/save/"
 JUDGMENT_TRACES_DELETE_API_URL = f"{ROOT_API}/traces/delete/"

judgeval/judgment_client.py

@@ -27,7 +27,8 @@ from judgeval.judges import JudgevalJudge
 from judgeval.constants import (
     JUDGMENT_EVAL_FETCH_API_URL, 
     JUDGMENT_EVAL_DELETE_API_URL, 
-    JUDGMENT_EVAL_DELETE_PROJECT_API_URL
+    JUDGMENT_EVAL_DELETE_PROJECT_API_URL,
+    JUDGMENT_PROJECT_DELETE_API_URL
 )
 from judgeval.common.exceptions import JudgmentAPIError
 from pydantic import BaseModel
@@ -156,7 +157,7 @@ class JudgmentClient:
         metadata: Optional[Dict[str, Any]] = None,
         project_name: str = "",
         eval_run_name: str = "",
-        log_results: bool = False,
+        log_results: bool = True,
         use_judgment: bool = True,
         rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
@@ -362,7 +363,6 @@ class JudgmentClient:
         response = requests.delete(JUDGMENT_EVAL_DELETE_PROJECT_API_URL, 
                         json={
                             "project_name": project_name,
-                            "judgment_api_key": self.judgment_api_key,
                         },
                         headers={
                             "Content-Type": "application/json",
@@ -372,6 +372,23 @@ class JudgmentClient:
         if response.status_code != requests.codes.ok:
             raise ValueError(f"Error deleting eval results: {response.json()}")
         return response.json()
+    
+    def delete_project(self, project_name: str) -> bool:
+        """
+        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
+        """
+        response = requests.delete(JUDGMENT_PROJECT_DELETE_API_URL, 
+                        json={
+                            "project_name": project_name,
+                        },
+                        headers={
+                            "Content-Type": "application/json",
+                            "Authorization": f"Bearer {self.judgment_api_key}",
+                            "X-Organization-Id": self.organization_id
+                        })
+        if response.status_code != requests.codes.ok:
+            raise ValueError(f"Error deleting project: {response.json()}")
+        return response.json()
         
     def _validate_api_key(self):
         """

judgeval/run_evaluation.py

@@ -1,12 +1,17 @@
 import asyncio
 import requests
-from typing import List, Dict
+import time
+import sys
+import itertools
+import threading
+from typing import List, Dict, Any
 from datetime import datetime
 from rich import print as rprint
 
 from judgeval.data import (
     ScorerData, 
-    ScoringResult
+    ScoringResult,
+    Example
 )
 from judgeval.scorers import (
     JudgevalScorer, 
@@ -14,7 +19,6 @@ from judgeval.scorers import (
     ClassifierScorer
 )
 from judgeval.scorers.score import a_execute_scoring
-
 from judgeval.constants import (
     ROOT_API,
     JUDGMENT_EVAL_API_URL,
@@ -185,7 +189,7 @@ def check_eval_run_name_exists(eval_name: str, project_name: str, judgment_api_k
         raise JudgmentAPIError(f"Failed to check if eval run name exists: {str(e)}")
 
 
-def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run: EvaluationRun) -> None:
+def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run: EvaluationRun) -> str:
     """
     Logs evaluation results to the Judgment API database.
 
@@ -220,7 +224,9 @@ def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run:
             raise JudgmentAPIError(error_message)
         
         if "ui_results_url" in res.json():
-            rprint(f"\n🔍 You can view your evaluation results here: [rgb(106,0,255)]{res.json()['ui_results_url']}[/]\n")
+            url = res.json()['ui_results_url']
+            pretty_str = f"\n🔍 You can view your evaluation results here: [rgb(106,0,255)][link={url}]View Results[/link]\n"
+            return pretty_str
             
     except requests.exceptions.RequestException as e:
         error(f"Request failed while saving evaluation results to DB: {str(e)}")
@@ -229,6 +235,51 @@ def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run:
         error(f"Failed to save evaluation results to DB: {str(e)}")
         raise ValueError(f"Failed to save evaluation results to DB: {str(e)}")
 
+def run_with_spinner(message: str, func, *args, **kwargs) -> Any:
+        """Run a function with a spinner in the terminal."""
+        spinner = itertools.cycle(['|', '/', '-', '\\'])
+
+        def display_spinner():
+            while not stop_spinner_event.is_set():
+                sys.stdout.write(f'\r{message}{next(spinner)}')
+                sys.stdout.flush() 
+                time.sleep(0.1)
+
+        stop_spinner_event = threading.Event()
+        spinner_thread = threading.Thread(target=display_spinner)
+        spinner_thread.start()
+
+        try:
+            result = func(*args, **kwargs)
+        except Exception as e:
+            error(f"An error occurred: {str(e)}")
+            stop_spinner_event.set()
+            spinner_thread.join()
+            raise e
+        finally:
+            stop_spinner_event.set()
+            spinner_thread.join()
+
+            sys.stdout.write('\r' + ' ' * (len(message) + 1) + '\r')
+            sys.stdout.flush()
+
+        return result
+
+def check_examples(examples: List[Example], scorers: List[APIJudgmentScorer]) -> None:
+    """
+    Checks if the example contains the necessary parameters for the scorer.
+    """
+    for scorer in scorers:
+        if isinstance(scorer, APIJudgmentScorer):
+            for example in examples:
+                missing_params = []
+                for param in scorer.required_params:
+                    if getattr(example, param.value) is None:
+                        missing_params.append(f"'{param.value}'")
+                if missing_params:
+                    # We do this because we want to inform users that an example is missing parameters for a scorer
+                    # Example ID (usually random UUID) does not provide any helpful information for the user but printing the entire example is overdoing it
+                    print(f"WARNING: Example {example.example_id} is missing the following parameters: {missing_params} for scorer {scorer.score_type.value}")
 
 
 def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[ScoringResult]:
@@ -253,7 +304,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     Returns:
         List[ScoringResult]: The results of the evaluation. Each result is a dictionary containing the fields of a `ScoringResult` object.
     """
-    
+
     # Call endpoint to check to see if eval run name exists (if we DON'T want to override and DO want to log results)
     if not override and evaluation_run.log_results:
         check_eval_run_name_exists(
@@ -306,6 +357,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     
     # Execute evaluation using Judgment API
     if judgment_scorers:
+        check_examples(evaluation_run.examples, evaluation_run.scorers)
         info("Starting API evaluation")
         debug(f"Creating API evaluation run with {len(judgment_scorers)} scorers")
         try:  # execute an EvaluationRun with just JudgmentScorers
@@ -323,7 +375,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
                 rules=evaluation_run.rules
             )
             debug("Sending request to Judgment API")    
-            response_data: List[Dict] = execute_api_eval(api_evaluation_run)  # Dicts are `ScoringResult` objs
+            response_data: List[Dict] = run_with_spinner("Running Evaluation: ", execute_api_eval, api_evaluation_run)
             info(f"Received {len(response_data['results'])} results from API")
         except JudgmentAPIError as e:
             error(f"An error occurred while executing the Judgment API request: {str(e)}")
@@ -352,6 +404,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
                 api_results.append(ScoringResult(**filtered_result))
     # Run local evals
     if local_scorers:  # List[JudgevalScorer]
+        # We should be removing local scorers soon
         info("Starting local evaluation")
         for example in evaluation_run.examples:
             with example_logging_context(example.timestamp, example.example_id):
@@ -389,7 +442,8 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     #     )
     
     if evaluation_run.log_results:
-        log_evaluation_results(merged_results, evaluation_run)
+        pretty_str = run_with_spinner("Logging Results: ", log_evaluation_results, merged_results, evaluation_run)
+        rprint(pretty_str)
 
     for i, result in enumerate(merged_results):
         if not result.scorers_data:  # none of the scorers could be executed on this example

judgeval/scorers/api_scorer.py

@@ -5,8 +5,9 @@ Scores `Example`s using ready-made Judgment evaluators.
 """
 
 from pydantic import BaseModel, field_validator
+from typing import List
 from judgeval.common.logger import debug, info, warning, error
-
+from judgeval.data import ExampleParams
 from judgeval.constants import APIScorer, UNBOUNDED_SCORERS
 
 
@@ -20,6 +21,7 @@ class APIJudgmentScorer(BaseModel):
     """
     score_type: APIScorer
     threshold: float
+    required_params: List[ExampleParams] = [] # List of the required parameters on examples for the scorer
 
     @field_validator('threshold')
     def validate_threshold(cls, v, info):

judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -8,11 +8,19 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class AnswerCorrectnessScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_CORRECTNESS)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.ANSWER_CORRECTNESS,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -8,11 +8,18 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class AnswerRelevancyScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_RELEVANCY)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.ANSWER_RELEVANCY,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/comparison.py

@@ -9,12 +9,20 @@ TODO add link to docs page for this scorer
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
 from typing import Optional, Dict
-
+from judgeval.data import ExampleParams
 class ComparisonScorer(APIJudgmentScorer):
     kwargs: Optional[Dict] = None
 
     def __init__(self, threshold: float, criteria: str, description: str):
-        super().__init__(threshold=threshold, score_type=APIScorer.COMPARISON)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.COMPARISON,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
         self.kwargs = {"criteria": criteria, "description": description}
 
     @property

judgeval/scorers/judgeval_scorers/api_scorers/contextual_precision.py

@@ -8,11 +8,20 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class ContextualPrecisionScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_PRECISION)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.CONTEXTUAL_PRECISION,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/contextual_recall.py

@@ -8,12 +8,21 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
+from judgeval.data import ExampleParams
 
 
 class ContextualRecallScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_RECALL)
-
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.CONTEXTUAL_RECALL,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+            ]
+        )
     @property
     def __name__(self):
         return "Contextual Recall"

judgeval/scorers/judgeval_scorers/api_scorers/contextual_relevancy.py

@@ -8,15 +8,22 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class ContextualRelevancyScorer(APIJudgmentScorer):
     """
     Scorer that checks if the output of a model is relevant to the retrieval context
     """
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.CONTEXTUAL_RELEVANCY)
-
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.CONTEXTUAL_RELEVANCY,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+            ]
+        )
     @property
     def __name__(self):
         return "Contextual Relevancy"

judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py

@@ -8,13 +8,21 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-from typing import Optional, Dict
+from typing import Optional, Dict, List
+from judgeval.data import ExampleParams
 
 class ExecutionOrderScorer(APIJudgmentScorer):
     kwargs: Optional[Dict] = None
 
     def __init__(self, threshold: float, should_exact_match: bool = False, should_consider_ordering: bool = False):
-        super().__init__(threshold=threshold, score_type=APIScorer.EXECUTION_ORDER)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.EXECUTION_ORDER,
+            required_params=[
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
         self.kwargs = {"should_exact_match": should_exact_match, "should_consider_ordering": should_consider_ordering}
 
     @property

judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py

@@ -8,11 +8,19 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class FaithfulnessScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.FAITHFULNESS)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.FAITHFULNESS,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/groundedness.py

@@ -8,11 +8,19 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class GroundednessScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.GROUNDEDNESS)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.GROUNDEDNESS,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py

@@ -8,11 +8,19 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class HallucinationScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.HALLUCINATION)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.HALLUCINATION,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.CONTEXT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py

@@ -8,11 +8,18 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class InstructionAdherenceScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.INSTRUCTION_ADHERENCE)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.INSTRUCTION_ADHERENCE,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

judgeval/scorers/judgeval_scorers/api_scorers/json_correctness.py

@@ -11,13 +11,20 @@ from pydantic import BaseModel, Field
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class JSONCorrectnessScorer(APIJudgmentScorer):
     json_schema: BaseModel = Field(None, exclude=True)
     
     def __init__(self, threshold: float, json_schema: BaseModel):
-        super().__init__(threshold=threshold, score_type=APIScorer.JSON_CORRECTNESS)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.JSON_CORRECTNESS,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ]
+        )
         object.__setattr__(self, 'json_schema', json_schema)
     
     def to_dict(self):

judgeval/scorers/judgeval_scorers/api_scorers/summarization.py

@@ -7,12 +7,19 @@ TODO add link to docs page for this scorer
 
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.constants import APIScorer
-
+from judgeval.constants import APIScorer    
+from judgeval.data import ExampleParams
 
 class SummarizationScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.SUMMARIZATION)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.SUMMARIZATION,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

{judgeval-0.0.20.dist-info → judgeval-0.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.20
+Version: 0.0.21
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -12,9 +12,15 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Requires-Dist: anthropic
 Requires-Dist: fastapi
+Requires-Dist: langchain
+Requires-Dist: langchain-anthropic
+Requires-Dist: langchain-core
+Requires-Dist: langchain-huggingface
+Requires-Dist: langchain-openai
 Requires-Dist: litellm
 Requires-Dist: nest-asyncio
 Requires-Dist: openai
+Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pika
 Requires-Dist: python-dotenv==1.0.1
@@ -23,8 +29,6 @@ Requires-Dist: supabase
 Requires-Dist: together
 Requires-Dist: uvicorn
 Provides-Extra: dev
-Requires-Dist: langfuse==2.50.3; extra == 'dev'
-Requires-Dist: patronus; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.25.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.14.0; extra == 'dev'
 Requires-Dist: pytest>=8.3.4; extra == 'dev'

{judgeval-0.0.20.dist-info → judgeval-0.0.21.dist-info}/RECORD

@@ -1,14 +1,14 @@
 judgeval/__init__.py,sha256=dtXxsCmI4eEsZdGSUMy8P_pA0bc2-OSGAgb2C__yJoA,252
 judgeval/clients.py,sha256=6VQmEqmfCngUdS2MuPBIpHvtDFqOENm8-_BmMvjLyRQ,944
-judgeval/constants.py,sha256=i8JIDUyo38Vt0R1n0GRA4FaakkBC5F2o4hQa0ncSF2E,5008
+judgeval/constants.py,sha256=VhJppAECTUDQwzC_FpzJw2wPlkYoogsadHxaJIY_J8U,5073
 judgeval/evaluation_run.py,sha256=RgJD60lJsunNQzObjo7iXnAzXWgubCLOAAuuamAAuoI,6354
-judgeval/judgment_client.py,sha256=evlvcrYO9pF-oCgcvlGE59iODN0C6GJtn7bySFU_88k,23384
+judgeval/judgment_client.py,sha256=5lqp9X67qPzBUu7kQYETslsc3L5JjxrDVgVLslF07A0,24173
 judgeval/rules.py,sha256=ebsiDEBVAnYTQxwVNvh_RpmKeWBnjQXgHs8KofTjcAs,15526
-judgeval/run_evaluation.py,sha256=yLW24kFcw0xzXHvnDclYqtujTww6SDwvut6HM1x7SXk,21505
+judgeval/run_evaluation.py,sha256=YOzkyeWl-r3vaz0jB5nM-1VULi7ALmJ9_f58ENqexXk,23827
 judgeval/common/__init__.py,sha256=7d24BRxtncpMj3AAJCj8RS7TqgjXmW777HVZH6-3sBs,289
 judgeval/common/exceptions.py,sha256=U-TxHLn7oVMezsMuoYouNDb2XuS8RCggfntYf5_6u4E,565
 judgeval/common/logger.py,sha256=KO75wWXCxhUHUMvLaTU31ZzOk6tkZBa7heQ7y0f-zFE,6062
-judgeval/common/tracer.py,sha256=FYrAuav6OiiawHLQ2e154MLvCBMdh-z_ucU2h7XK08M,45295
+judgeval/common/tracer.py,sha256=WFjFNf3NZ2BN8UAu2MG0F3Om9LgJNma3m_GrxyXgJqE,46655
 judgeval/common/utils.py,sha256=LUQV5JfDr6wj7xHAJoNq-gofNZ6mjXbeKrGKzBME1KM,33533
 judgeval/data/__init__.py,sha256=QykVE22Qf-I2f1g-jC9-iQyLNXgDmX1-vHbCgZg8Ra8,558
 judgeval/data/api_example.py,sha256=dzkrQ0xno08y6qNfqL2djXbapUyc2B2aQ5iANn0o4CY,3667
@@ -27,7 +27,7 @@ judgeval/judges/mixture_of_judges.py,sha256=IJoi4Twk8ze1CJWVEp69k6TSqTCTGrmVYQ0q
 judgeval/judges/together_judge.py,sha256=l00hhPerAZXg3oYBd8cyMtWsOTNt_0FIqoxhKJKQe3k,2302
 judgeval/judges/utils.py,sha256=9lvUxziGV86ISvVFxYBWc09TWFyAQgUTyPf_a9mD5Rs,2686
 judgeval/scorers/__init__.py,sha256=gkeKJvjXhswCnkEyjijrVvGVM3Om86egrZ-PUOGvNvI,1158
-judgeval/scorers/api_scorer.py,sha256=wGqTQCbUE7uE-PzaKcCmexAqutdTunjFR0zVA6bUxdE,2518
+judgeval/scorers/api_scorer.py,sha256=NQ_CrrUPhSUk1k2Q8rKpCG_TU2FT32sFEqvb-Yi54B0,2688
 judgeval/scorers/base_scorer.py,sha256=xdUlY3CnLdCQ1Z5iUeY22Bim5v-OQruZmaVF_4Y1mC0,2183
 judgeval/scorers/exceptions.py,sha256=eGW5CuJgZ5YJBFrE4FHDSF651PO1dKAZ379mJ8gOsfo,178
 judgeval/scorers/judgeval_scorer.py,sha256=oIkfoGXA09wL_vcK1DRibzQSA-MFNa-hmw1IhGBErf8,6592
@@ -36,19 +36,19 @@ judgeval/scorers/score.py,sha256=GALVmeApP1Cyih2vY93zRaU6RShtW4jJDG47Pm6yfnw,186
 judgeval/scorers/utils.py,sha256=iHQVTlIANbmCTXz9kTeSdOytgUZ_T74Re61ajqsk_WQ,6827
 judgeval/scorers/judgeval_scorers/__init__.py,sha256=xFRb62sp4JmBUSeuAB_pC_7kEGp-lGdqCRIu9--Bbdg,5992
 judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=mZ6b_5Dl04k3PaG24ICBajB_j43ody1II1OJhO1DkXo,1648
-judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=690G5askjE8dcbKPGvCF6JxAEM9QJUqb-3K-D6lI6oM,463
-judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=CqvvjV7AZqPlXh-PZaPKYPILHr15u4bIYiKBFjlk5i0,457
-judgeval/scorers/judgeval_scorers/api_scorers/comparison.py,sha256=6Q1qbsANOoZ3PM8n_gtZLIMbTBB9879L3acRelNJ6Uk,1001
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_precision.py,sha256=2zBrm_EEc143bmPA4HVcf8XtQeuc_BexczGx-SHlwRY,473
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_recall.py,sha256=NyojBWy_lRYx8diREulSK8s9dfYdZav4eZjg3TwUm0M,461
-judgeval/scorers/judgeval_scorers/api_scorers/contextual_relevancy.py,sha256=wROMWOliCnB39ftX9TdeZmG9y0vrnxIGVby65tLOQRU,574
-judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py,sha256=qxnvEDeKRlyzxX3EX53sW4oXxAM8Fj_q6ibdTxJNTAc,1076
-judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=gNf_i5c0jjpz2zCGhe7TtDMLKxc1PdOExJMFB5X7hSg,442
-judgeval/scorers/judgeval_scorers/api_scorers/groundedness.py,sha256=esO76hEp0NzeBUdoSICPLdx5AeA5zWSt_2zpcSgvGis,442
-judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py,sha256=ffYwH3CexPkKgo1rCALMivypROQjG5WWEsKXEFZxe2k,446
-judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=t1lWYOF0Pxvw5-NrI1Dt9FojaOncOCRlZc4a2SA20h4,477
-judgeval/scorers/judgeval_scorers/api_scorers/json_correctness.py,sha256=CAZBQKwNSqpqAoOgStYfr-yP1Brug_6VRimRIQY-zdg,894
-judgeval/scorers/judgeval_scorers/api_scorers/summarization.py,sha256=-E3oxYbI0D_0q-_fGWh2jQHW9O4Pu7I7xvLWsHU6cn8,450
+judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=Fnd9CVIOZ73sWEWymsU5eBrrZqPFjMZ0BKpeW-PDyTg,711
+judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=oETeN9K0HSIRdL2SDqn82Vskpwh5SlKnZvs5VDm2OBU,658
+judgeval/scorers/judgeval_scorers/api_scorers/comparison.py,sha256=kuzf9OWvpY38yYSwlBgneLkUZwJNM4FQqvbS66keA90,1249
+judgeval/scorers/judgeval_scorers/api_scorers/contextual_precision.py,sha256=tpSuzFAaW8X9xqA0aLLKwh7qmBK0Pc_bJZMIe_q412U,770
+judgeval/scorers/judgeval_scorers/api_scorers/contextual_recall.py,sha256=pFVhk4pLtQ-FnNlbI-dFF-SIh69Jza7erHqiPkFWoBo,758
+judgeval/scorers/judgeval_scorers/api_scorers/contextual_relevancy.py,sha256=RQ6DZwEhChfecd89Ey-T7ke--7qTaXZlRsNxwH8gaME,823
+judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py,sha256=Pb3CiNF2Ca826B92wJCVAi_68lJjLhqqCKwQKaflSUg,1294
+judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=-BwOapqjryYNKNydtdkUiKIij76dY0O1jBmdc6dKazQ,692
+judgeval/scorers/judgeval_scorers/api_scorers/groundedness.py,sha256=ntEEeTANEOsGlcbiTAF_3r6BeSJEaVDns8po8T0L6Vg,692
+judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py,sha256=k5gDOki-8KXrZXydvdSqDt3NZqQ28hXoOCHQf6jNxr4,686
+judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=XnSGEkQfwVqaqnHEGMCsxNiHVzrsrej48uDbLoWc8CQ,678
+judgeval/scorers/judgeval_scorers/api_scorers/json_correctness.py,sha256=mMKEuR87_yanEuZJ5YSGFMHDD_oLVZ6-rQuciFaDOMA,1095
+judgeval/scorers/judgeval_scorers/api_scorers/summarization.py,sha256=QmWB8bVbDYHY5FcF0rYZE_3c2XXgMLRmR6aXJWfdMC4,655
 judgeval/scorers/judgeval_scorers/classifiers/__init__.py,sha256=Qt81W5ZCwMvBAne0LfQDb8xvg5iOG1vEYP7WizgwAZo,67
 judgeval/scorers/judgeval_scorers/classifiers/text2sql/__init__.py,sha256=8iTzMvou1Dr8pybul6lZHKjc9Ye2-0_racRGYkhEdTY,74
 judgeval/scorers/judgeval_scorers/classifiers/text2sql/text2sql_scorer.py,sha256=ly72Z7s_c8NID6-nQnuW8qEGEW2MqdvpJ-5WfXzbAQg,2579
@@ -88,7 +88,7 @@ judgeval/scorers/judgeval_scorers/local_implementations/summarization/prompts.py
 judgeval/scorers/judgeval_scorers/local_implementations/summarization/summarization_scorer.py,sha256=Qk7lwHgRPYeGoxTOyclAh1VfGItfvHJ6l1t7Nk3SWFM,20927
 judgeval/tracer/__init__.py,sha256=wy3DYpH8U_z0GO_K_gOSkK0tTTD-u5eLDo0T5xIBoAc,147
 judgeval/utils/alerts.py,sha256=RgW5R9Dn3Jtim0OyAYDbNzjoX2s6SA4Mw16GyyaikjI,1424
-judgeval-0.0.20.dist-info/METADATA,sha256=cz7uKUuHAc1rdANc8IJ5klQhlmrqOu_K1y6wwEIAdFU,1283
-judgeval-0.0.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-judgeval-0.0.20.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
-judgeval-0.0.20.dist-info/RECORD,,
+judgeval-0.0.21.dist-info/METADATA,sha256=jQW4w6jGNaHvPWTcqX3ZGr_SKeCpNl7DsNr-cwrYHsA,1378
+judgeval-0.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+judgeval-0.0.21.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
+judgeval-0.0.21.dist-info/RECORD,,