judgeval 0.0.37__py3-none-any.whl → 0.0.38__py3-none-any.whl

judgeval/judges/litellm_judge.py

@@ -12,7 +12,7 @@ BASE_CONVERSATION = [
 
 
 class LiteLLMJudge(JudgevalJudge):
-    def __init__(self, model: str = "gpt-4o-mini", **kwargs):
+    def __init__(self, model: str = "gpt-4.1-mini", **kwargs):
         debug(f"Initializing LiteLLMJudge with model={model}")
         self.model = model
         self.kwargs = kwargs

judgeval/judges/mixture_of_judges.py

@@ -136,7 +136,7 @@ class MixtureOfJudges(JudgevalJudge):
     """
     def __init__(self, 
                  models: List[str] = ['QWEN', 'LLAMA3_70B_INSTRUCT_TURBO', 'MISTRAL_8x22B_INSTRUCT'],
-                 aggregator: str = 'gpt-4o', 
+                 aggregator: str = 'gpt-4.1', 
                  **kwargs):
         """
         `models` are the individual judge models to be used for generating responses.

judgeval/judges/utils.py

@@ -23,7 +23,7 @@ def create_judge(
     If no model is provided, uses GPT4o as the default judge.
     """
     if model is None:  # default option
-        return LiteLLMJudge(model="gpt-4o"), True
+        return LiteLLMJudge(model="gpt-4.1"), True
     if not isinstance(model, (str, list, JudgevalJudge)):
         raise InvalidJudgeModelError(f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead.")
     # If model is already a valid judge type, return it and mark native

judgeval/judgment_client.py

@@ -12,7 +12,7 @@ from judgeval.data import (
     ScoringResult, 
     Example,
     CustomExample,
-    Sequence,
+    Trace,
 )
 from judgeval.scorers import (
     APIJudgmentScorer, 
@@ -23,9 +23,9 @@ from judgeval.evaluation_run import EvaluationRun
 from judgeval.run_evaluation import (
     run_eval, 
     assert_test,
-    run_sequence_eval
+    run_trace_eval
 )
-from judgeval.data.sequence_run import SequenceRun
+from judgeval.data.trace_run import TraceRun
 from judgeval.judges import JudgevalJudge
 from judgeval.constants import (
     JUDGMENT_EVAL_FETCH_API_URL, 
@@ -105,16 +105,16 @@ class JudgmentClient(metaclass=SingletonMeta):
             rules=rules
         )
 
-    def run_sequence_evaluation(
+    def run_trace_evaluation(
         self,
         scorers: List[Union[APIJudgmentScorer, JudgevalScorer]],
         model: Optional[Union[str, List[str], JudgevalJudge]] = "gpt-4.1",
-        sequences: Optional[List[Sequence]] = None,
+        traces: Optional[List[Trace]] = None,
         examples: Optional[List[Example]] = None,
         test_file: Optional[str] = None,
         aggregator: Optional[str] = None,
         project_name: str = "default_project",
-        eval_run_name: str = "default_eval_sequence",
+        eval_run_name: str = "default_eval_trace",
         log_results: bool = True,
         append: bool = False,
         override: bool = False,
@@ -134,16 +134,16 @@ class JudgmentClient(metaclass=SingletonMeta):
             if examples and not function:
                 raise ValueError("Cannot pass in examples without a function")
             
-            if sequences and function:
-                raise ValueError("Cannot pass in sequences and function")
+            if traces and function:
+                raise ValueError("Cannot pass in traces and function")
             
-            if examples and sequences:
-                raise ValueError("Cannot pass in both examples and sequences")
+            if examples and traces:
+                raise ValueError("Cannot pass in both examples and traces")
             
-            sequence_run = SequenceRun(
+            trace_run = TraceRun(
                 project_name=project_name,
                 eval_name=eval_run_name,
-                sequences=sequences,
+                traces=traces,
                 scorers=scorers,
                 model=model,
                 aggregator=aggregator,
@@ -152,9 +152,9 @@ class JudgmentClient(metaclass=SingletonMeta):
                 judgment_api_key=self.judgment_api_key,
                 organization_id=self.organization_id,
             )
-            return run_sequence_eval(sequence_run, override, ignore_errors, function, tracer, examples)
+            return run_trace_eval(trace_run, override, ignore_errors, function, tracer, examples)
         except ValueError as e:
-            raise ValueError(f"Please check your SequenceRun object, one or more fields are invalid: \n{str(e)}")
+            raise ValueError(f"Please check your TraceRun object, one or more fields are invalid: \n{str(e)}")
         except Exception as e:
             raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
 
@@ -245,12 +245,6 @@ class JudgmentClient(metaclass=SingletonMeta):
         """
         return self.eval_dataset_client.append_examples(alias, examples, project_name)
     
-    def append_sequence_dataset(self, alias: str, sequences: List[Sequence], project_name: str) -> bool:
-        """
-        Appends a `Sequence` to the Judgment platform for storage.
-        """
-        return self.eval_dataset_client.append_sequences(alias, sequences, project_name)
-    
     def pull_dataset(self, alias: str, project_name: str) -> EvalDataset:
         """
         Retrieves a saved `EvalDataset` from the Judgment platform.
@@ -523,7 +517,7 @@ class JudgmentClient(metaclass=SingletonMeta):
             raise ValueError("Exactly one of 'examples' or 'test_file' must be provided, but not both")
 
         if function:
-            results = self.run_sequence_evaluation(
+            results = self.run_trace_evaluation(
                 examples=examples,
                 scorers=scorers,
                 model=model,

judgeval/run_evaluation.py

@@ -13,7 +13,6 @@ from judgeval.data import (
     ScoringResult,
     Example,
     CustomExample,
-    Sequence,
     Trace
 )
 from judgeval.scorers import (
@@ -25,11 +24,10 @@ from judgeval.scorers.score import a_execute_scoring
 from judgeval.constants import (
     ROOT_API,
     JUDGMENT_EVAL_API_URL,
-    JUDGMENT_SEQUENCE_EVAL_API_URL,
+    JUDGMENT_TRACE_EVAL_API_URL,
     JUDGMENT_EVAL_LOG_API_URL,
     MAX_CONCURRENT_EVALUATIONS,
     JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL,
-    JUDGMENT_RETRIEVE_SEQUENCE_FROM_TRACE_API_URL
 )
 from judgeval.common.exceptions import JudgmentAPIError
 from judgeval.common.logger import (
@@ -39,7 +37,7 @@ from judgeval.common.logger import (
     example_logging_context
 )
 from judgeval.evaluation_run import EvaluationRun
-from judgeval.data.sequence_run import SequenceRun
+from judgeval.data.trace_run import TraceRun
 from judgeval.common.tracer import Tracer
 from langchain_core.callbacks import BaseCallbackHandler
 
@@ -98,20 +96,20 @@ def execute_api_eval(evaluation_run: EvaluationRun) -> List[Dict]:
         raise JudgmentAPIError(error_message)
     return response_data
 
-def execute_api_sequence_eval(sequence_run: SequenceRun) -> List[Dict]:
+def execute_api_trace_eval(trace_run: TraceRun) -> List[Dict]:
     """
     Executes an evaluation of a list of `Example`s using one or more `JudgmentScorer`s via the Judgment API.
     """
         
     try:
         # submit API request to execute evals
-        payload = sequence_run.model_dump(warnings=False)
+        payload = trace_run.model_dump(warnings=False)
         response = requests.post(
-            JUDGMENT_SEQUENCE_EVAL_API_URL, 
+            JUDGMENT_TRACE_EVAL_API_URL, 
             headers={
                 "Content-Type": "application/json",
-                "Authorization": f"Bearer {sequence_run.judgment_api_key}",
-                "X-Organization-Id": sequence_run.organization_id
+                "Authorization": f"Bearer {trace_run.judgment_api_key}",
+                "X-Organization-Id": trace_run.organization_id
             }, 
             json=payload,
             verify=True
@@ -282,7 +280,7 @@ def check_eval_run_name_exists(eval_name: str, project_name: str, judgment_api_k
         raise JudgmentAPIError(f"Failed to check if eval run name exists: {str(e)}")
 
 
-def log_evaluation_results(scoring_results: List[ScoringResult], run: Union[EvaluationRun, SequenceRun]) -> str:
+def log_evaluation_results(scoring_results: List[ScoringResult], run: Union[EvaluationRun, TraceRun]) -> str:
     """
     Logs evaluation results to the Judgment API database.
 
@@ -327,51 +325,6 @@ def log_evaluation_results(scoring_results: List[ScoringResult], run: Union[Eval
         error(f"Failed to save evaluation results to DB: {str(e)}")
         raise ValueError(f"Failed to save evaluation results to DB: {str(e)}")
 
-def retrieve_sequence_from_trace(trace_id: str, parent_span: str, judgment_api_key: str, organization_id: str) -> Sequence:
-    """
-    Retrieves a sequence from a trace ID.
-    """
-    """
-    Logs evaluation results to the Judgment API database.
-
-    Args:
-        merged_results (List[ScoringResult]): The results to log
-        evaluation_run (EvaluationRun): The evaluation run containing project info and API key
-
-    Raises:
-        JudgmentAPIError: If there's an API error during logging
-        ValueError: If there's a validation error with the results
-    """
-    try:
-        res = requests.post(
-            JUDGMENT_RETRIEVE_SEQUENCE_FROM_TRACE_API_URL,
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {judgment_api_key}",
-                "X-Organization-Id": organization_id
-            },
-            json={
-                "trace_id": trace_id,
-                "trace_span_id": parent_span,
-            },
-            verify=True
-        )
-        
-        if not res.ok:
-            response_data = res.json()
-            error_message = response_data.get('detail', 'An unknown error occurred.')
-            error(f"Error {res.status_code}: {error_message}")
-            raise JudgmentAPIError(error_message)
-        
-        return Sequence(**res.json())
-    except requests.exceptions.RequestException as e:
-        error(f"Request failed while saving evaluation results to DB: {str(e)}")
-        raise JudgmentAPIError(f"Request failed while saving evaluation results to DB: {str(e)}")
-    except Exception as e:
-        error(f"Failed to save evaluation results to DB: {str(e)}")
-        raise ValueError(f"Failed to save evaluation results to DB: {str(e)}")
-
-
 def run_with_spinner(message: str, func, *args, **kwargs) -> Any:
         """Run a function with a spinner in the terminal."""
         spinner = itertools.cycle(['|', '/', '-', '\\'])
@@ -415,62 +368,59 @@ def check_examples(examples: List[Example], scorers: List[Union[APIJudgmentScore
             if missing_params:
                 print(f"WARNING: Example {example.example_id} is missing the following parameters: {missing_params} for scorer {scorer.score_type.value}")
 
-def run_sequence_eval(sequence_run: SequenceRun, override: bool = False, ignore_errors: bool = True, function: Optional[Callable] = None, tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None, examples: Optional[List[Example]] = None) -> List[ScoringResult]:
+def run_trace_eval(trace_run: TraceRun, override: bool = False, ignore_errors: bool = True, function: Optional[Callable] = None, tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None, examples: Optional[List[Example]] = None) -> List[ScoringResult]:
     # Call endpoint to check to see if eval run name exists (if we DON'T want to override and DO want to log results)
-    if not override and sequence_run.log_results and not sequence_run.append:
+    if not override and trace_run.log_results and not trace_run.append:
         check_eval_run_name_exists(
-            sequence_run.eval_name,
-            sequence_run.project_name,
-            sequence_run.judgment_api_key,
-            sequence_run.organization_id
+            trace_run.eval_name,
+            trace_run.project_name,
+            trace_run.judgment_api_key,
+            trace_run.organization_id
         )
 
-    if sequence_run.append:
+    if trace_run.append:
         # Check that the current experiment, if one exists, has the same type (examples of sequences)
         check_experiment_type(
-            sequence_run.eval_name,
-            sequence_run.project_name,
-            sequence_run.judgment_api_key,
-            sequence_run.organization_id,
+            trace_run.eval_name,
+            trace_run.project_name,
+            trace_run.judgment_api_key,
+            trace_run.organization_id,
             True
         )
 
     if function and tracer:
-        new_sequences: List[Sequence] = []
+        new_traces: List[Trace] = []
+        tracer.offline_mode = True
         for example in examples:
             if example.input:
                 result = run_with_spinner("Running agent function: ", function, **example.input)
             else:
                 result = run_with_spinner("Running agent function: ", function)
         for i, trace in enumerate(tracer.traces):
-            trace_id = trace['trace_id']
-            parent_span = trace['entries'][0]['span_id']
-            new_sequence = retrieve_sequence_from_trace(trace_id, parent_span, sequence_run.judgment_api_key, sequence_run.organization_id)
-            new_sequence.expected_tools = examples[i].expected_tools
-            new_sequences.append(new_sequence)
-        sequence_run.sequences = new_sequences
-        
-    for sequence in sequence_run.sequences:
-        sequence.scorers = sequence_run.scorers
+            # We set the root-level trace span with the expected tools of the Trace
+            trace = Trace(**trace)
+            trace.entries[0].expected_tools = examples[i].expected_tools
+            new_traces.append(trace)
+        trace_run.traces = new_traces
         
     # Execute evaluation using Judgment API
     info("Starting API evaluation")
     try:  # execute an EvaluationRun with just JudgmentScorers
         debug("Sending request to Judgment API")    
-        response_data: List[Dict] = run_with_spinner("Running Sequence Evaluation: ", execute_api_sequence_eval, sequence_run)
+        response_data: List[Dict] = run_with_spinner("Running Trace Evaluation: ", execute_api_trace_eval, trace_run)
         scoring_results = [ScoringResult(**result) for result in response_data["results"]]
         info(f"Received {len(scoring_results)} results from API")
     except JudgmentAPIError as e:
         error(f"An error occurred while executing the Judgment API request: {str(e)}")
         raise JudgmentAPIError(f"An error occurred while executing the Judgment API request: {str(e)}")
     except ValueError as e:
-        raise ValueError(f"Please check your SequenceRun object, one or more fields are invalid: {str(e)}")
+        raise ValueError(f"Please check your TraceRun object, one or more fields are invalid: {str(e)}")
     
     # Convert the response data to `ScoringResult` objects
     debug("Processing API results")
-    # TODO: allow for custom scorer on sequences
-    if sequence_run.log_results:
-        pretty_str = run_with_spinner("Logging Results: ", log_evaluation_results, response_data["results"], sequence_run)
+    # TODO: allow for custom scorer on traces
+    if trace_run.log_results:
+        pretty_str = run_with_spinner("Logging Results: ", log_evaluation_results, response_data["results"], trace_run)
         rprint(pretty_str)
 
     return scoring_results

judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py

@@ -5,13 +5,15 @@
 # Internal imports
 from judgeval.scorers.api_scorer import APIJudgmentScorer
 from judgeval.constants import APIScorer
-
+from typing import Optional, Dict
 class ToolOrderScorer(APIJudgmentScorer):
-    def __init__(self, threshold: float=1.0):
+    kwargs: Optional[Dict] = None
+    def __init__(self, threshold: float=1.0, exact_match: bool=False):
         super().__init__(
             threshold=threshold, 
             score_type=APIScorer.TOOL_ORDER,
         )
+        self.kwargs = {"exact_match": exact_match}
 
     @property
     def __name__(self):

judgeval-0.0.38.dist-info/METADATA

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: judgeval
+Version: 0.0.38
+Summary: Judgeval Package
+Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
+Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
+Author-email: Andrew Li <andrew@judgmentlabs.ai>, Alex Shan <alex@judgmentlabs.ai>, Joseph Camyre <joseph@judgmentlabs.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.md
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: anthropic
+Requires-Dist: boto3
+Requires-Dist: google-genai
+Requires-Dist: langchain-anthropic
+Requires-Dist: langchain-core
+Requires-Dist: langchain-huggingface
+Requires-Dist: langchain-openai
+Requires-Dist: litellm==1.38.12
+Requires-Dist: nest-asyncio
+Requires-Dist: openai
+Requires-Dist: pandas
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: requests
+Requires-Dist: together
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<img src="assets/logo-light.svg#gh-light-mode-only" alt="Judgment Logo" width="400" />
+<img src="assets/logo-dark.svg#gh-dark-mode-only" alt="Judgment Logo" width="400" />
+
+**Build monitoring & evaluation pipelines for complex agents**
+
+<img src="assets/experiments_page.png" alt="Judgment Platform Experiments Page" width="800" />
+
+<br>
+
+## [🌐 Landing Page](https://www.judgmentlabs.ai/) • [Twitter/X](https://x.com/JudgmentLabs) • [💼 LinkedIn](https://www.linkedin.com/company/judgmentlabs) • [📚 Docs](https://judgment.mintlify.app/getting_started) • [🚀 Demos](https://www.youtube.com/@AlexShan-j3o) • [🎮 Discord](https://discord.gg/taAufyhf)
+</div>
+
+## Judgeval: open-source testing, monitoring, and optimization for AI agents
+
+Judgeval offers robust tooling for evaluating and tracing LLM agent systems. It is dev-friendly and open-source (licensed under Apache 2.0). 
+
+Judgeval gets you started in five minutes, after which you'll be ready to use all of its features as your agent becomes more complex. Judgeval is natively connected to the [Judgment Platform](https://www.judgmentlabs.ai/) for free and you can export your data and self-host at any time.
+
+We support tracing agents built with LangGraph, OpenAI SDK, Anthropic, ... and allow custom eval integrations for any use case. Check out our quickstarts below or our [setup guide](https://judgment.mintlify.app/getting_started) to get started.
+
+Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
+
+## 📋 Table of Contents
+* [✨ Features](#-features)
+    * [🔍 Tracing](#-tracing)
+    * [🧪 Evals](#-evals)
+    * [📡 Monitoring](#-monitoring)
+    * [📊 Datasets](#-datasets)
+    * [💡 Insights](#-insights)
+* [🛠️ Installation](#️-installation)
+* [🏁 Get Started](#-get-started)
+* [🏢 Self-Hosting](#-self-hosting)
+* [📚 Cookbooks](#-cookbooks)
+* [⭐ Star Us on GitHub](#-star-us-on-github)
+* [❤️ Contributors](#️-contributors)
+
+<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+
+
+## ✨ Features
+
+|  |  |
+|:---|:---:|
+| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic): **tracking inputs/outputs, latency, and cost** at every step.<br><br>Online evals can be applied to traces to measure quality on production data in real-time.<br><br>Export trace data to the Judgment Platform or your own S3 buckets, {Parquet, JSON, YAML} files, or data warehouse.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 👤 Tracking user activity <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
+| <h3>🧪 Evals</h3>15+ research-backed metrics including tool call accuracy, hallucinations, instruction adherence, and retrieval context recall.<br><br>Build custom evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 Experimental prompt testing<br>• 🛡️ Online guardrails <br><br> | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
+| <h3>📡 Monitoring</h3>Real-time performance tracking of your agents in production environments. **Track all your metrics in one place.**<br><br>Set up **Slack/email alerts** for critical metrics and receive notifications when thresholds are exceeded.<br><br> **Useful for:** <br>•📉 Identifying degradation early <br>•📈 Visualizing performance trends across versions and time | <p align="center"><img src="assets/monitoring_screenshot.png" alt="Monitoring Dashboard" width="1200"/></p> |
+| <h3>📊 Datasets</h3>Export trace data or import external testcases to datasets hosted on Judgment's Platform. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations. <br><br> **Useful for:**<br>• 🔄 Scaled analysis for A/B tests <br>• 🗃️ Filtered collections of agent runtime data| <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
+| <h3>💡 Insights</h3>Cluster on your data to reveal common use cases and failure modes.<br><br>Trace failures to their exact source with Judgment's Osiris agent, which localizes errors to specific components for precise fixes.<br><br> **Useful for:**<br>•🔮 Surfacing common inputs that lead to error<br>•🤖 Investigating agent/user behavior for optimization <br>| <p align="center"><img src="assets/dataset_clustering_screenshot_dm.png" alt="Insights dashboard" width="1200"/></p> |
+
+## 🛠️ Installation
+
+Get started with Judgeval by installing our SDK using pip:
+
+```bash
+pip install judgeval
+```
+
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment platform](https://app.judgmentlabs.ai/). 
+
+**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
+
+## 🏁 Get Started
+
+Here's how you can quickly start using Judgeval:
+
+### 🛰️ Tracing
+
+Track your agent execution with full observability with just a few lines of code.
+Create a file named `traces.py` with the following code:
+
+```python
+from judgeval.common.tracer import Tracer, wrap
+from openai import OpenAI
+
+client = wrap(OpenAI())
+judgment = Tracer(project_name="my_project")
+
+@judgment.observe(span_type="tool")
+def my_tool():
+    return "What's the capital of the U.S.?"
+
+@judgment.observe(span_type="function")
+def main():
+    task_input = my_tool()
+    res = client.chat.completions.create(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": f"{task_input}"}]
+    )
+    return res.choices[0].message.content
+
+main()
+```
+
+[Click here](https://judgment.mintlify.app/getting_started#create-your-first-trace) for a more detailed explanation.
+
+### 📝 Offline Evaluations
+
+You can evaluate your agent's execution to measure quality metrics such as hallucination.
+Create a file named `evaluate.py` with the following code:
+
+```python evaluate.py
+from judgeval import JudgmentClient
+from judgeval.data import Example
+from judgeval.scorers import FaithfulnessScorer
+
+client = JudgmentClient()
+
+example = Example(
+    input="What if these shoes don't fit?",
+    actual_output="We offer a 30-day full refund at no extra cost.",
+    retrieval_context=["All customers are eligible for a 30 day full refund at no extra cost."],
+)
+
+scorer = FaithfulnessScorer(threshold=0.5)
+results = client.run_evaluation(
+    examples=[example],
+    scorers=[scorer],
+    model="gpt-4.1",
+)
+print(results)
+```
+
+[Click here](https://judgment.mintlify.app/getting_started#create-your-first-experiment) for a more detailed explanation.
+
+### 📡 Online Evaluations
+
+Attach performance monitoring on traces to measure the quality of your systems in production.
+
+Using the same `traces.py` file we created earlier, modify `main` function:
+
+```python
+from judgeval.common.tracer import Tracer, wrap
+from judgeval.scorers import AnswerRelevancyScorer
+from openai import OpenAI
+
+client = wrap(OpenAI())
+judgment = Tracer(project_name="my_project")
+
+@judgment.observe(span_type="tool")
+def my_tool():
+    return "Hello world!"
+
+@judgment.observe(span_type="function")
+def main():
+    task_input = my_tool()
+    res = client.chat.completions.create(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": f"{task_input}"}]
+    ).choices[0].message.content
+
+    judgment.async_evaluate(
+        scorers=[AnswerRelevancyScorer(threshold=0.5)],
+        input=task_input,
+        actual_output=res,
+        model="gpt-4.1"
+    )
+    print("Online evaluation submitted.")
+    return res
+
+main()
+```
+
+[Click here](https://judgment.mintlify.app/getting_started#create-your-first-online-evaluation) for a more detailed explanation.
+
+## 🏢 Self-Hosting
+
+Run Judgment on your own infrastructure: we provide comprehensive self-hosting capabilities that give you full control over the backend and data plane that Judgeval interfaces with.
+
+### Key Features
+* Deploy Judgment on your own AWS account
+* Store data in your own Supabase instance
+* Access Judgment through your own custom domain
+
+### Getting Started
+1. Check out our [self-hosting documentation](https://judgment.mintlify.app/self_hosting/get_started) for detailed setup instructions, along with how your self-hosted instance can be accessed
+2. Use the [Judgment CLI](https://github.com/JudgmentLabs/judgment-cli) to deploy your self-hosted environment
+3. After your self-hosted instance is setup, make sure the `JUDGMENT_API_URL` environmental variable is set to your self-hosted backend endpoint
+
+## 📚 Cookbooks
+
+Have your own? We're happy to feature it if you create a PR or message us on [Discord](https://discord.gg/taAufyhf).
+
+You can access our repo of cookbooks [here](https://github.com/JudgmentLabs/judgment-cookbook). Here are some highlights:
+
+### Sample Agents
+
+#### 💰 [LangGraph Financial QA Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/financial_agent/demo.py)
+A LangGraph-based agent for financial queries, featuring RAG capabilities with a vector database for contextual data retrieval and evaluation of its reasoning and data accuracy.
+
+#### ✈️ [OpenAI Travel Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/openai_travel_agent/agent.py)
+A travel planning agent using OpenAI API calls, custom tool functions, and RAG with a vector database for up-to-date and contextual travel information. Evaluated for itinerary quality and information relevance.
+
+### Custom Evaluators
+
+#### 🔍 [PII Detection](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/classifier_scorer/pii_checker.py)
+Detecting and evaluating Personal Identifiable Information (PII) leakage.
+
+#### 📧 [Cold Email Generation](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/custom_scorers/cold_email_scorer.py)
+
+Evaluates if a cold email generator properly utilizes all relevant information about the target recipient.
+
+## ⭐ Star Us on GitHub
+
+If you find Judgeval useful, please consider giving us a star on GitHub! Your support helps us grow our community and continue improving the product.
+
+
+## ❤️ Contributors
+
+There are many ways to contribute to Judgeval:
+
+- Submit [bug reports](https://github.com/JudgmentLabs/judgeval/issues) and [feature requests](https://github.com/JudgmentLabs/judgeval/issues)
+- Review the documentation and submit [Pull Requests](https://github.com/JudgmentLabs/judgeval/pulls) to improve it
+- Speaking or writing about Judgment and letting us know!
+
+<!-- Contributors collage -->
+[![Contributors](https://contributors-img.web.app/image?repo=JudgmentLabs/judgeval)](https://github.com/JudgmentLabs/judgeval/graphs/contributors)
+