judgeval 0.0.20__tar.gz → 0.0.21__tar.gz

{judgeval-0.0.20 → judgeval-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.20
+Version: 0.0.21
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -12,9 +12,15 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Requires-Dist: anthropic
 Requires-Dist: fastapi
+Requires-Dist: langchain
+Requires-Dist: langchain-anthropic
+Requires-Dist: langchain-core
+Requires-Dist: langchain-huggingface
+Requires-Dist: langchain-openai
 Requires-Dist: litellm
 Requires-Dist: nest-asyncio
 Requires-Dist: openai
+Requires-Dist: openpyxl
 Requires-Dist: pandas
 Requires-Dist: pika
 Requires-Dist: python-dotenv==1.0.1
@@ -23,8 +29,6 @@ Requires-Dist: supabase
 Requires-Dist: together
 Requires-Dist: uvicorn
 Provides-Extra: dev
-Requires-Dist: langfuse==2.50.3; extra == 'dev'
-Requires-Dist: patronus; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.25.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.14.0; extra == 'dev'
 Requires-Dist: pytest>=8.3.4; extra == 'dev'

{judgeval-0.0.20 → judgeval-0.0.21}/Pipfile

@@ -29,8 +29,6 @@ pytest = "*"
 pytest-asyncio = "*"
 pytest-mock = "*"
 tavily-python = "*"
-patronus = "*"
-langfuse = "==2.50.3"
 
 [requires]
 python_version = "3.11"

{judgeval-0.0.20 → judgeval-0.0.21}/docs/integration/langgraph.mdx

@@ -14,10 +14,11 @@ graph_builder = StateGraph(State)
 # YOUR LANGGRAPH WORKFLOW
 
 handler = JudgevalCallbackHandler(judgment.get_current_trace())
+set_global_handler(handler)
 
 result = graph.invoke({
     "messages": [HumanMessage(content=prompt)]
-}, config=dict(callbacks=[handler]))
+})
 
 ```
 

{judgeval-0.0.20 → judgeval-0.0.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.20"
+version = "0.0.21"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },
@@ -28,6 +28,12 @@ dependencies = [
     "anthropic",
     "nest-asyncio",
     "pika",
+    "openpyxl",
+    "langchain",
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
 ]
 
 [project.optional-dependencies]
@@ -35,8 +41,6 @@ dev = [
     "pytest>=8.3.4",
     "pytest-asyncio>=0.25.0",
     "pytest-mock>=3.14.0",
-    "langfuse==2.50.3",
-    "patronus",
     "tavily-python"
 ]
 

judgeval-0.0.21/src/demo/cookbooks/test.py

@@ -0,0 +1,152 @@
+from judgeval.data import Example
+from judgeval.data.datasets import EvalDataset
+from judgeval.scorers import AnswerRelevancyScorer
+from judgeval import JudgmentClient
+
+
+def create_sample_dataset():
+    # Define sample inputs
+    inputs = [
+        # Highly relevant Q/A pairs
+        "Who founded Microsoft?",
+        "What is the capital of France?",
+        "How does photosynthesis work?",
+        "What are the benefits of exercise?",
+        "Explain quantum computing in simple terms.",
+        
+        # Somewhat relevant Q/A pairs
+        "What is machine learning?",
+        "How do electric cars work?",
+        "What causes climate change?",
+        "How does the human digestive system function?",
+        "What is blockchain technology?",
+        
+        # Minimally relevant Q/A pairs
+        "What are the main programming languages?",
+        "How do I bake a chocolate cake?",
+        "What is the history of the Roman Empire?",
+        "How do vaccines work?",
+        "What are black holes?",
+        
+        # Not relevant Q/A pairs
+        "What is the best smartphone to buy?",
+        "How tall is Mount Everest?",
+        "Who wrote Romeo and Juliet?",
+        "What is the population of Tokyo?",
+        "How do I change a flat tire?"
+    ]
+    
+    # Define corresponding outputs
+    actual_outputs = [
+        # Highly relevant answers
+        "Bill Gates and Paul Allen founded Microsoft in 1975.",
+        "The capital of France is Paris, known for the Eiffel Tower and Louvre Museum.",
+        "Photosynthesis is the process where plants convert sunlight, water, and carbon dioxide into glucose and oxygen.",
+        "Regular exercise improves cardiovascular health, builds muscle strength, reduces stress, and helps maintain a healthy weight.",
+        "Quantum computing uses quantum bits or qubits that can exist in multiple states simultaneously, allowing for potentially faster computation of certain problems compared to classical computers.",
+        
+        # Somewhat relevant answers (partial or tangential information)
+        "Machine learning involves statistical techniques, but it's primarily about natural language processing and computer vision applications in modern businesses.",
+        "Electric cars use batteries, though the most important aspect is their impact on reducing traffic congestion in urban areas.",
+        "Climate change is related to weather patterns, but it's mainly caused by volcanic eruptions and natural planetary cycles.",
+        "The digestive system breaks down food, but the most interesting part is how it connects to brain function and mental health.",
+        "Blockchain is a distributed ledger technology, though its primary purpose is to replace traditional banking systems entirely.",
+        
+        # Minimally relevant answers (mostly off-topic but with slight connection)
+        "Programming languages include Python and JavaScript, but the real question is whether AI will replace programmers in the next decade.",
+        "Chocolate cakes require flour and sugar, but I'd recommend focusing on gluten-free alternatives since they're healthier.",
+        "The Roman Empire lasted for centuries, but modern Italy's political system is more relevant to understand today's European politics.",
+        "Vaccines stimulate immune responses, but the pharmaceutical industry's profit motives are what you should really be concerned about.",
+        "Black holes are regions of spacetime, but the conspiracy theories about what NASA isn't telling us are far more interesting.",
+        
+        # Not relevant answers (completely off-topic)
+        "The migration patterns of monarch butterflies are fascinating examples of evolutionary adaptation.",
+        "The Great Wall of China was built over multiple dynasties and stretches over 13,000 miles.",
+        "Photosynthesis is how plants convert sunlight into energy, producing oxygen as a byproduct.",
+        "The human genome contains approximately 3 billion base pairs of DNA.",
+        "The Pythagorean theorem states that in a right-angled triangle, the square of the hypotenuse equals the sum of squares of the other two sides."
+    ]
+
+    # Create Example objects from inputs and outputs
+    examples = []
+    for i in range(len(inputs)):
+        examples.append(Example(
+            input=inputs[i],
+            actual_output=actual_outputs[i]
+        ))
+
+    return EvalDataset(examples=examples)
+
+
+def save_dataset(client, dataset, alias):
+    """Save the dataset to Judgment API with the given alias"""
+    client.push_dataset(alias=alias, dataset=dataset)
+    print(f"Dataset saved with alias: {alias}")
+
+
+def run_evaluation(client, dataset_alias, model="gpt-4o", project_name="jnpr_mist_demo_project", eval_run_name="jnpr_demo_eval_run"):
+    """Pull a dataset and run an evaluation on it"""
+    # Pull the dataset from Judgment API
+    eval_dataset = client.pull_dataset(alias=dataset_alias)
+    
+    # Run the evaluation
+    results = client.evaluate_dataset(
+        dataset=eval_dataset,
+        scorers=[AnswerRelevancyScorer(threshold=0.8)],
+        model=model,
+        eval_run_name=eval_run_name,
+        project_name=project_name,
+    )
+    
+    return results
+
+
+def run_assertion_test(client, dataset_alias, model="gpt-4o", project_name="jnpr_mist_demo_project", eval_run_name="jnpr_demo_assertion_run"):
+    """Pull a dataset and run assertion tests on its examples"""
+    # Pull the dataset from Judgment API
+    eval_dataset = client.pull_dataset(alias=dataset_alias)
+    
+    # Extract examples from the dataset
+    examples = eval_dataset.examples
+    
+    # Run assertion tests on each example
+    # Run assertion test on all examples at once
+    client.assert_test(
+        examples=examples,
+        scorers=[AnswerRelevancyScorer(threshold=0.8)],
+        model=model,
+        project_name=project_name,
+        eval_run_name=eval_run_name
+    )
+    
+
+def main():
+    client = JudgmentClient()
+    
+    # Uncomment to create and save a new dataset
+    # dataset = create_sample_dataset()
+    # save_dataset(client, dataset, "jnpr_demo_dataset")
+    
+    # # Run evaluation on the saved dataset
+    # results = run_evaluation(
+    #     client, 
+    #     dataset_alias="jnpr_demo_dataset",
+    #     model="gpt-4o",
+    #     project_name="jnpr_mist_demo_project",
+    #     eval_run_name="jnpr_demo_eval"
+    # )
+    
+    # Run assertion test on the saved dataset
+    results = run_assertion_test(
+        client, 
+        dataset_alias="jnpr_demo_dataset",
+        model="gpt-4o",
+        project_name="jnpr_mist_demo_project",
+        eval_run_name="jnpr_demo_assertion"
+    )
+    return results
+
+
+if __name__ == "__main__":
+    results = main()
+    print(results)

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/common/tracer.py

@@ -10,6 +10,7 @@ import os
 import time
 import uuid
 import warnings
+from contextvars import ContextVar
 from contextlib import contextmanager
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -37,6 +38,7 @@ from judgeval.constants import (
     RABBITMQ_PORT,
     RABBITMQ_QUEUE,
     JUDGMENT_TRACES_DELETE_API_URL,
+    JUDGMENT_PROJECT_DELETE_API_URL,
     JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL
 )
 from judgeval.judgment_client import JudgmentClient
@@ -54,7 +56,7 @@ from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain_core.callbacks import CallbackManager, BaseCallbackHandler
 from langchain_core.agents import AgentAction, AgentFinish
 from langchain_core.outputs import LLMResult
-
+from langchain_core.tracers.context import register_configure_hook
 from langchain_core.messages.ai import AIMessage
 from langchain_core.messages.tool import ToolMessage
 from langchain_core.messages.base import BaseMessage
@@ -251,7 +253,8 @@ class TraceManagerClient:
             raise ValueError(f"Failed to save trace data: {response.text}")
         
         if not empty_save and "ui_results_url" in response.json():
-            rprint(f"\n🔍 You can view your trace data here: [rgb(106,0,255)]{response.json()['ui_results_url']}[/]\n")
+            pretty_str = f"\n🔍 You can view your trace data here: [rgb(106,0,255)][link={response.json()['ui_results_url']}]View Trace[/link]\n"
+            rprint(pretty_str)
 
     def delete_trace(self, trace_id: str):
         """
@@ -294,6 +297,27 @@ class TraceManagerClient:
             raise ValueError(f"Failed to delete trace: {response.text}")
         
         return response.json()
+    
+    def delete_project(self, project_name: str):
+        """
+        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
+        """
+        response = requests.delete(
+            JUDGMENT_PROJECT_DELETE_API_URL,
+            json={
+                "project_name": project_name,
+            },
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.judgment_api_key}",
+                "X-Organization-Id": self.organization_id
+            }
+        )
+
+        if response.status_code != HTTPStatus.OK:
+            raise ValueError(f"Failed to delete traces: {response.text}")
+            
+        return response.json()
 
 
 class TraceClient:
@@ -1152,3 +1176,18 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
             'args': str(messages),
             'kwargs': kwargs
         })
+
+judgeval_callback_handler_var: ContextVar[Optional[JudgevalCallbackHandler]] = ContextVar(
+    "judgeval_callback_handler", default=None
+)
+
+def set_global_handler(handler: JudgevalCallbackHandler):
+    judgeval_callback_handler_var.set(handler)
+
+def clear_global_handler():
+    judgeval_callback_handler_var.set(None)
+
+register_configure_hook(
+    context_var=judgeval_callback_handler_var,
+    inheritable=True,
+)

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/constants.py

@@ -48,6 +48,7 @@ JUDGMENT_EVAL_LOG_API_URL = f"{ROOT_API}/log_eval_results/"
 JUDGMENT_EVAL_FETCH_API_URL = f"{ROOT_API}/fetch_eval_results/"
 JUDGMENT_EVAL_DELETE_API_URL = f"{ROOT_API}/delete_eval_results_by_project_and_run_name/"
 JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
+JUDGMENT_PROJECT_DELETE_API_URL = f"{ROOT_API}/projects/delete/"
 JUDGMENT_TRACES_FETCH_API_URL = f"{ROOT_API}/traces/fetch/"
 JUDGMENT_TRACES_SAVE_API_URL = f"{ROOT_API}/traces/save/"
 JUDGMENT_TRACES_DELETE_API_URL = f"{ROOT_API}/traces/delete/"

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/judgment_client.py

@@ -27,7 +27,8 @@ from judgeval.judges import JudgevalJudge
 from judgeval.constants import (
     JUDGMENT_EVAL_FETCH_API_URL, 
     JUDGMENT_EVAL_DELETE_API_URL, 
-    JUDGMENT_EVAL_DELETE_PROJECT_API_URL
+    JUDGMENT_EVAL_DELETE_PROJECT_API_URL,
+    JUDGMENT_PROJECT_DELETE_API_URL
 )
 from judgeval.common.exceptions import JudgmentAPIError
 from pydantic import BaseModel
@@ -156,7 +157,7 @@ class JudgmentClient:
         metadata: Optional[Dict[str, Any]] = None,
         project_name: str = "",
         eval_run_name: str = "",
-        log_results: bool = False,
+        log_results: bool = True,
         use_judgment: bool = True,
         rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
@@ -362,7 +363,6 @@ class JudgmentClient:
         response = requests.delete(JUDGMENT_EVAL_DELETE_PROJECT_API_URL, 
                         json={
                             "project_name": project_name,
-                            "judgment_api_key": self.judgment_api_key,
                         },
                         headers={
                             "Content-Type": "application/json",
@@ -372,6 +372,23 @@ class JudgmentClient:
         if response.status_code != requests.codes.ok:
             raise ValueError(f"Error deleting eval results: {response.json()}")
         return response.json()
+    
+    def delete_project(self, project_name: str) -> bool:
+        """
+        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
+        """
+        response = requests.delete(JUDGMENT_PROJECT_DELETE_API_URL, 
+                        json={
+                            "project_name": project_name,
+                        },
+                        headers={
+                            "Content-Type": "application/json",
+                            "Authorization": f"Bearer {self.judgment_api_key}",
+                            "X-Organization-Id": self.organization_id
+                        })
+        if response.status_code != requests.codes.ok:
+            raise ValueError(f"Error deleting project: {response.json()}")
+        return response.json()
         
     def _validate_api_key(self):
         """

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/run_evaluation.py

@@ -1,12 +1,17 @@
 import asyncio
 import requests
-from typing import List, Dict
+import time
+import sys
+import itertools
+import threading
+from typing import List, Dict, Any
 from datetime import datetime
 from rich import print as rprint
 
 from judgeval.data import (
     ScorerData, 
-    ScoringResult
+    ScoringResult,
+    Example
 )
 from judgeval.scorers import (
     JudgevalScorer, 
@@ -14,7 +19,6 @@ from judgeval.scorers import (
     ClassifierScorer
 )
 from judgeval.scorers.score import a_execute_scoring
-
 from judgeval.constants import (
     ROOT_API,
     JUDGMENT_EVAL_API_URL,
@@ -185,7 +189,7 @@ def check_eval_run_name_exists(eval_name: str, project_name: str, judgment_api_k
         raise JudgmentAPIError(f"Failed to check if eval run name exists: {str(e)}")
 
 
-def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run: EvaluationRun) -> None:
+def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run: EvaluationRun) -> str:
     """
     Logs evaluation results to the Judgment API database.
 
@@ -220,7 +224,9 @@ def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run:
             raise JudgmentAPIError(error_message)
         
         if "ui_results_url" in res.json():
-            rprint(f"\n🔍 You can view your evaluation results here: [rgb(106,0,255)]{res.json()['ui_results_url']}[/]\n")
+            url = res.json()['ui_results_url']
+            pretty_str = f"\n🔍 You can view your evaluation results here: [rgb(106,0,255)][link={url}]View Results[/link]\n"
+            return pretty_str
             
     except requests.exceptions.RequestException as e:
         error(f"Request failed while saving evaluation results to DB: {str(e)}")
@@ -229,6 +235,51 @@ def log_evaluation_results(merged_results: List[ScoringResult], evaluation_run:
         error(f"Failed to save evaluation results to DB: {str(e)}")
         raise ValueError(f"Failed to save evaluation results to DB: {str(e)}")
 
+def run_with_spinner(message: str, func, *args, **kwargs) -> Any:
+        """Run a function with a spinner in the terminal."""
+        spinner = itertools.cycle(['|', '/', '-', '\\'])
+
+        def display_spinner():
+            while not stop_spinner_event.is_set():
+                sys.stdout.write(f'\r{message}{next(spinner)}')
+                sys.stdout.flush() 
+                time.sleep(0.1)
+
+        stop_spinner_event = threading.Event()
+        spinner_thread = threading.Thread(target=display_spinner)
+        spinner_thread.start()
+
+        try:
+            result = func(*args, **kwargs)
+        except Exception as e:
+            error(f"An error occurred: {str(e)}")
+            stop_spinner_event.set()
+            spinner_thread.join()
+            raise e
+        finally:
+            stop_spinner_event.set()
+            spinner_thread.join()
+
+            sys.stdout.write('\r' + ' ' * (len(message) + 1) + '\r')
+            sys.stdout.flush()
+
+        return result
+
+def check_examples(examples: List[Example], scorers: List[APIJudgmentScorer]) -> None:
+    """
+    Checks if the example contains the necessary parameters for the scorer.
+    """
+    for scorer in scorers:
+        if isinstance(scorer, APIJudgmentScorer):
+            for example in examples:
+                missing_params = []
+                for param in scorer.required_params:
+                    if getattr(example, param.value) is None:
+                        missing_params.append(f"'{param.value}'")
+                if missing_params:
+                    # We do this because we want to inform users that an example is missing parameters for a scorer
+                    # Example ID (usually random UUID) does not provide any helpful information for the user but printing the entire example is overdoing it
+                    print(f"WARNING: Example {example.example_id} is missing the following parameters: {missing_params} for scorer {scorer.score_type.value}")
 
 
 def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[ScoringResult]:
@@ -253,7 +304,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     Returns:
         List[ScoringResult]: The results of the evaluation. Each result is a dictionary containing the fields of a `ScoringResult` object.
     """
-    
+
     # Call endpoint to check to see if eval run name exists (if we DON'T want to override and DO want to log results)
     if not override and evaluation_run.log_results:
         check_eval_run_name_exists(
@@ -306,6 +357,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     
     # Execute evaluation using Judgment API
     if judgment_scorers:
+        check_examples(evaluation_run.examples, evaluation_run.scorers)
         info("Starting API evaluation")
         debug(f"Creating API evaluation run with {len(judgment_scorers)} scorers")
         try:  # execute an EvaluationRun with just JudgmentScorers
@@ -323,7 +375,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
                 rules=evaluation_run.rules
             )
             debug("Sending request to Judgment API")    
-            response_data: List[Dict] = execute_api_eval(api_evaluation_run)  # Dicts are `ScoringResult` objs
+            response_data: List[Dict] = run_with_spinner("Running Evaluation: ", execute_api_eval, api_evaluation_run)
             info(f"Received {len(response_data['results'])} results from API")
         except JudgmentAPIError as e:
             error(f"An error occurred while executing the Judgment API request: {str(e)}")
@@ -352,6 +404,7 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
                 api_results.append(ScoringResult(**filtered_result))
     # Run local evals
     if local_scorers:  # List[JudgevalScorer]
+        # We should be removing local scorers soon
         info("Starting local evaluation")
         for example in evaluation_run.examples:
             with example_logging_context(example.timestamp, example.example_id):
@@ -389,7 +442,8 @@ def run_eval(evaluation_run: EvaluationRun, override: bool = False) -> List[Scor
     #     )
     
     if evaluation_run.log_results:
-        log_evaluation_results(merged_results, evaluation_run)
+        pretty_str = run_with_spinner("Logging Results: ", log_evaluation_results, merged_results, evaluation_run)
+        rprint(pretty_str)
 
     for i, result in enumerate(merged_results):
         if not result.scorers_data:  # none of the scorers could be executed on this example

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/scorers/api_scorer.py

@@ -5,8 +5,9 @@ Scores `Example`s using ready-made Judgment evaluators.
 """
 
 from pydantic import BaseModel, field_validator
+from typing import List
 from judgeval.common.logger import debug, info, warning, error
-
+from judgeval.data import ExampleParams
 from judgeval.constants import APIScorer, UNBOUNDED_SCORERS
 
 
@@ -20,6 +21,7 @@ class APIJudgmentScorer(BaseModel):
     """
     score_type: APIScorer
     threshold: float
+    required_params: List[ExampleParams] = [] # List of the required parameters on examples for the scorer
 
     @field_validator('threshold')
     def validate_threshold(cls, v, info):

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -8,11 +8,19 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class AnswerCorrectnessScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_CORRECTNESS)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.ANSWER_CORRECTNESS,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -8,11 +8,18 @@ TODO add link to docs page for this scorer
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from judgeval.data import ExampleParams
 
 class AnswerRelevancyScorer(APIJudgmentScorer):
     def __init__(self, threshold: float):
-        super().__init__(threshold=threshold, score_type=APIScorer.ANSWER_RELEVANCY)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.ANSWER_RELEVANCY,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+            ]
+        )
 
     @property
     def __name__(self):

{judgeval-0.0.20 → judgeval-0.0.21}/src/judgeval/scorers/judgeval_scorers/api_scorers/comparison.py

@@ -9,12 +9,20 @@ TODO add link to docs page for this scorer
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
 from typing import Optional, Dict
-
+from judgeval.data import ExampleParams
 class ComparisonScorer(APIJudgmentScorer):
     kwargs: Optional[Dict] = None
 
     def __init__(self, threshold: float, criteria: str, description: str):
-        super().__init__(threshold=threshold, score_type=APIScorer.COMPARISON)
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.COMPARISON,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
         self.kwargs = {"criteria": criteria, "description": description}
 
     @property

judgeval-0.0.21/src/judgeval/scorers/judgeval_scorers/api_scorers/contextual_precision.py

@@ -0,0 +1,28 @@
+"""
+`judgeval` contextual precision scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.api_scorer import APIJudgmentScorer
+from judgeval.constants import APIScorer
+from judgeval.data import ExampleParams
+
+class ContextualPrecisionScorer(APIJudgmentScorer):
+    def __init__(self, threshold: float):
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.CONTEXTUAL_PRECISION,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+                ExampleParams.EXPECTED_OUTPUT,
+            ]
+        )
+
+    @property
+    def __name__(self):
+        return "Contextual Precision"

judgeval-0.0.21/src/judgeval/scorers/judgeval_scorers/api_scorers/contextual_recall.py

@@ -0,0 +1,28 @@
+"""
+`judgeval` contextual recall scorer
+
+TODO add link to docs page for this scorer
+
+"""
+
+# Internal imports
+from judgeval.scorers.api_scorer import APIJudgmentScorer
+from judgeval.constants import APIScorer
+from judgeval.data import ExampleParams
+
+
+class ContextualRecallScorer(APIJudgmentScorer):
+    def __init__(self, threshold: float):
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.CONTEXTUAL_RECALL,
+            required_params=[
+                ExampleParams.INPUT,
+                ExampleParams.ACTUAL_OUTPUT,
+                ExampleParams.EXPECTED_OUTPUT,
+                ExampleParams.RETRIEVAL_CONTEXT,
+            ]
+        )
+    @property
+    def __name__(self):
+        return "Contextual Recall"