judgeval 0.0.41__tar.gz → 0.0.43__tar.gz

judgeval-0.0.43/.github/workflows/ci.yaml

@@ -0,0 +1,163 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions: read-all
+
+jobs:
+  validate-branch:
+    uses: ./.github/workflows/merge-branch-check.yaml
+
+  run-tests:
+    needs: [validate-branch]
+    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.11"
+    name: Unit Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      JUDGMENT_DEV: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Run tests
+        run: |
+          cd src
+          uv run pytest tests
+
+  run-e2e-tests-staging:
+    needs: [validate-branch]
+    if: "github.base_ref == 'staging' && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
+    name: Staging E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Wait for turn
+        uses: softprops/turnstyle@v2
+        with:
+          poll-interval-seconds: 10
+          same-branch-only: false
+          job-to-wait-for: "Staging E2E Tests"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+          
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+         
+      - name: Install judgeval dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s https://staging.api.judgmentlabs.ai/health > /dev/null; then
+            echo "Staging Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
+            exit 1
+          else
+            echo "Staging server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |      
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-stg-judgeval/api-keys/judgeval --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report (staging)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html-staging
+          path: src/htmlcov
+
+  run-e2e-tests-main:
+    needs: [validate-branch]
+    if: "github.base_ref == 'main' && !contains(github.actor, '[bot]') && needs.validate-branch.result == 'success'"
+    name: Production E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Wait for turn
+        uses: softprops/turnstyle@v2
+        with:
+          poll-interval-seconds: 10
+          same-branch-only: false
+          job-to-wait-for: "Production E2E Tests"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+          
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+         
+      - name: Install judgeval dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s https://api.judgmentlabs.ai/health > /dev/null; then
+            echo "Production Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
+            exit 1
+          else
+            echo "Production server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |      
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-judgeval/api-keys/judgeval --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report (production)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html-production
+          path: src/htmlcov

judgeval-0.0.41/.github/workflows/merge-to-main.yaml → judgeval-0.0.43/.github/workflows/merge-branch-check.yaml

@@ -1,8 +1,6 @@
-name: Enforce Main Branch Protection
+name: Branch Protection Check
 
-on:
-  pull_request:
-    types: [opened, synchronize, reopened, edited]
+on: workflow_call
 
 jobs:
   validate-branch:
@@ -10,20 +8,17 @@ jobs:
     steps:
       - name: Check branch name
         run: |
-          # Get the base and source branch names
           BASE_BRANCH="${{ github.base_ref }}"
           SOURCE_BRANCH="${{ github.head_ref }}"
 
           echo "BASE_BRANCH: $BASE_BRANCH"
           echo "SOURCE_BRANCH: $SOURCE_BRANCH"
           
-          # Only run validation if the base branch is main
           if [[ "$BASE_BRANCH" != "main" ]]; then
             echo "Skipping branch validation - not targeting main branch"
             exit 0
           fi
           
-          # Check if the source branch is staging or starts with hotfix/
           if [[ "$SOURCE_BRANCH" != "staging" && ! "$SOURCE_BRANCH" =~ ^hotfix/ ]]; then
             echo "::error::Pull requests to main can only be created from 'staging' or 'hotfix/*' branches. Current branch: $SOURCE_BRANCH"
             exit 1

judgeval-0.0.43/.github/workflows/validate-branch.yaml

@@ -0,0 +1,9 @@
+name: Branch Protection
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+
+jobs:
+  branch-protection:
+    uses: ./.github/workflows/merge-branch-check.yaml

{judgeval-0.0.41 → judgeval-0.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.41
+Version: 0.0.43
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -18,6 +18,7 @@ Requires-Dist: langchain-core
 Requires-Dist: langchain-huggingface
 Requires-Dist: langchain-openai
 Requires-Dist: litellm==1.61.15
+Requires-Dist: matplotlib>=3.10.3
 Requires-Dist: nest-asyncio
 Requires-Dist: openai
 Requires-Dist: pandas
@@ -31,37 +32,37 @@ Description-Content-Type: text/markdown
 <img src="assets/new_lightmode.svg#gh-light-mode-only" alt="Judgment Logo" width="400" />
 <img src="assets/new_darkmode.svg#gh-dark-mode-only" alt="Judgment Logo" width="400" />
 
-**Build monitoring & evaluation pipelines for complex agents**
+<br>
+<div style="font-size: 1.5em;">
+  Open source tracing, evals, and metrics to debug, test, and monitor LLM agents.
+</div>
 
-<img src="assets/experiments_pagev2.png" alt="Judgment Platform Experiments Page" width="800" />
+## [Judgment Cloud](https://app.judgmentlabs.ai/register)  • [Self-Host](https://docs.judgmentlabs.ai/self-hosting/get_started) • [Demo](https://www.youtube.com/watch?v=1S4LixpVbcc) 
 
-<br>
+[Docs](https://docs.judgmentlabs.ai/introduction) • [Bug Reports](https://github.com/JudgmentLabs/judgeval/issues) • [Changelog](https://docs.judgmentlabs.ai/changelog/2025-04-21)
 
-## [🌐 Landing Page](https://www.judgmentlabs.ai/)  • [📚 Docs](https://docs.judgmentlabs.ai/introduction) • [🚀 Demos](https://www.youtube.com/@AlexShan-j3o) 
+We're hiring! Join us in our mission to unleash optimized agents.
 
 [![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
 [![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
 [![Discord](https://img.shields.io/badge/-Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/ZCnSXYug)
 
-</div>
+<img src="assets/experiments_pagev2.png" alt="Judgment Platform Experiments Page" width="800" />
 
-## Judgeval: open-source testing, monitoring, and optimization for AI agents
+</div>
 
-Judgeval offers robust tooling for evaluating and tracing LLM agent systems. It is dev-friendly and open-source (licensed under Apache 2.0). 
 
-Judgeval gets you started in five minutes, after which you'll be ready to use all of its features as your agent becomes more complex. Judgeval is natively connected to the [Judgment Platform](https://www.judgmentlabs.ai/) for free and you can export your data and self-host at any time.
+Judgeval offers **robust open-source tooling** for tracing, evaluating, and monitoring LLM agents. It helps AI teams effectively **test and monitor** agents in development and production, **closing the agent feedback loop**. 
 
-We support tracing agents built with LangGraph, OpenAI SDK, Anthropic, ... and allow custom eval integrations for any use case. Check out our quickstarts below or our [setup guide](https://docs.judgmentlabs.ai/getting-started) to get started.
+Judgeval can be set up **(cloud-hosted or self-hosted) in 5 minutes**!
+> 🎁 Generous monthly [free tier](https://judgmentlabs.ai/pricing) (10k traces, 1k evals) - No credit card required!
 
 Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
 
 ## 📋 Table of Contents
-- [🌐 Landing Page  • 📚 Docs • 🚀 Demos](#-landing-page----docs---demos)
-- [Judgeval: open-source testing, monitoring, and optimization for AI agents](#judgeval-open-source-testing-monitoring-and-optimization-for-ai-agents)
-- [📋 Table of Contents](#-table-of-contents)
 - [✨ Features](#-features)
 - [🛠️ Installation](#️-installation)
-- [🏁 Get Started](#-get-started)
+- [🏁 Quickstarts](#-quickstarts)
   - [🛰️ Tracing](#️-tracing)
   - [📝 Offline Evaluations](#-offline-evaluations)
   - [📡 Online Evaluations](#-online-evaluations)
@@ -69,12 +70,6 @@ Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
   - [Key Features](#key-features)
   - [Getting Started](#getting-started)
 - [📚 Cookbooks](#-cookbooks)
-  - [Sample Agents](#sample-agents)
-    - [💰 LangGraph Financial QA Agent](#-langgraph-financial-qa-agent)
-    - [✈️ OpenAI Travel Agent](#️-openai-travel-agent)
-  - [Custom Evaluators](#custom-evaluators)
-    - [🔍 PII Detection](#-pii-detection)
-    - [📧 Cold Email Generation](#-cold-email-generation)
 - [💻 Development with Cursor](#-development-with-cursor)
 - [⭐ Star Us on GitHub](#-star-us-on-github)
 - [❤️ Contributors](#️-contributors)
@@ -86,11 +81,10 @@ Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
 
 |  |  |
 |:---|:---:|
-| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic): **tracking inputs/outputs, latency, and cost** at every step.<br><br>Online evals can be applied to traces to measure quality on production data in real-time.<br><br>Export trace data to the Judgment Platform or your own S3 buckets, {Parquet, JSON, YAML} files, or data warehouse.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 👤 Tracking user activity <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
-| <h3>🧪 Evals</h3>15+ research-backed metrics including tool call accuracy, hallucinations, instruction adherence, and retrieval context recall.<br><br>Build custom evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 Experimental prompt testing<br>• 🛡️ Online guardrails <br><br> | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
-| <h3>📡 Monitoring</h3>Real-time performance tracking of your agents in production environments. **Track all your metrics in one place.**<br><br>Set up **Slack/email alerts** for critical metrics and receive notifications when thresholds are exceeded.<br><br> **Useful for:** <br>•📉 Identifying degradation early <br>•📈 Visualizing performance trends across versions and time | <p align="center"><img src="assets/monitoring_screenshot.png" alt="Monitoring Dashboard" width="1200"/></p> |
-| <h3>📊 Datasets</h3>Export trace data or import external testcases to datasets hosted on Judgment's Platform. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations. <br><br> **Useful for:**<br>• 🔄 Scaled analysis for A/B tests <br>• 🗃️ Filtered collections of agent runtime data| <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
-| <h3>💡 Insights</h3>Cluster on your data to reveal common use cases and failure modes.<br><br>Trace failures to their exact source with Judgment's Osiris agent, which localizes errors to specific components for precise fixes.<br><br> **Useful for:**<br>•🔮 Surfacing common inputs that lead to error<br>•🤖 Investigating agent/user behavior for optimization <br>| <p align="center"><img src="assets/dataset_clustering_screenshot_dm.png" alt="Insights dashboard" width="1200"/></p> |
+| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic): **tracking inputs/outputs, agent tool calls, latency, and cost** at every step.<br><br>Online evals can be applied to traces to measure quality on production data in real-time.<br><br>Export trace data to the Judgment Platform or your own S3 buckets, {Parquet, JSON, YAML} files, or data warehouse.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 👤 Tracking user activity <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
+| <h3>🧪 Evals</h3>Evals are the key to regression testing for agents. Judgeval provides 15+ research-backed metrics including tool call accuracy, hallucinations, instruction adherence, and retrieval context recall.<br><br>Judgeval supports LLM-as-a-judge, manual labeling, and custom evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 Experimental prompt testing<br>• 🛡️ Online guardrails | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
+| <h3>📡 Monitoring</h3>Track all your agent metrics in production. **Catch production regressions early.**<br><br>Configure alerts to trigger automated actions when metric thresholds are exceeded (add agent trace to review queue/dataset, Slack notification, etc.).<br><br> **Useful for:** <br>• 📉 Identifying degradation early <br>• 📈 Visualizing performance trends across agent versions and time | <p align="center"><img src="assets/monitoring_screenshot.png" alt="Monitoring Dashboard" width="1200"/></p> |
+| <h3>📊 Datasets</h3>Export trace data or import external testcases to datasets for scaled unit testing and structured experiments. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations. <br><br> **Useful for:**<br>• 🗃️ Filtered agent runtime data for fine tuning<br>• 🔄 Scaled analysis for A/B tests | <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
 
 ## 🛠️ Installation
 
@@ -100,17 +94,19 @@ Get started with Judgeval by installing our SDK using pip:
 pip install judgeval
 ```
 
-Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment platform](https://app.judgmentlabs.ai/). 
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment platform](https://app.judgmentlabs.ai/).
 
-**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
 
-## 🏁 Get Started
+**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
 
-Here's how you can quickly start using Judgeval:
+## 🏁 Quickstarts
 
 ### 🛰️ Tracing
 
-Track your agent execution with full observability with just a few lines of code.
 Create a file named `traces.py` with the following code:
 
 ```python
@@ -135,12 +131,15 @@ def main():
 
 main()
 ```
+You'll see your trace exported to the Judgment Platform:
+
+<p align="center"><img src="assets/trace_demo.png" alt="Judgment Platform Trace Example" width="800" /></p>
+
 
 [Click here](https://docs.judgmentlabs.ai/getting-started#create-your-first-trace) for a more detailed explanation.
 
 ### 📝 Offline Evaluations
 
-You can evaluate your agent's execution to measure quality metrics such as hallucination.
 Create a file named `evaluate.py` with the following code:
 
 ```python evaluate.py
@@ -156,7 +155,7 @@ example = Example(
     retrieval_context=["All customers are eligible for a 30 day full refund at no extra cost."],
 )
 
-scorer = FaithfulnessScorer(threshold=0.5)
+scorer = FaithfulnessScorer(threshold=0.5)  # Hallucination detector
 results = client.run_evaluation(
     examples=[example],
     scorers=[scorer],
@@ -205,6 +204,8 @@ def main():
 main()
 ```
 
+You should see an evaluation attached to your trace on the Judgment Platform.
+
 [Click here](https://docs.judgmentlabs.ai/getting-started#create-your-first-online-evaluation) for a more detailed explanation.
 
 ## 🏢 Self-Hosting
@@ -229,20 +230,8 @@ You can access our repo of cookbooks [here](https://github.com/JudgmentLabs/judg
 
 ### Sample Agents
 
-#### 💰 [LangGraph Financial QA Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/financial_agent/demo.py)
-A LangGraph-based agent for financial queries, featuring RAG capabilities with a vector database for contextual data retrieval and evaluation of its reasoning and data accuracy.
-
-#### ✈️ [OpenAI Travel Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/openai_travel_agent/agent.py)
-A travel planning agent using OpenAI API calls, custom tool functions, and RAG with a vector database for up-to-date and contextual travel information. Evaluated for itinerary quality and information relevance.
-
-### Custom Evaluators
-
-#### 🔍 [PII Detection](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/classifier_scorer/pii_checker.py)
-Detecting and evaluating Personal Identifiable Information (PII) leakage.
-
-#### 📧 [Cold Email Generation](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/custom_scorers/cold_email_scorer.py)
-
-Evaluates if a cold email generator properly utilizes all relevant information about the target recipient.
+#### [Multi-Agent System](https://github.com/JudgmentLabs/judgment-cookbook/tree/main/cookbooks/agents/multi-agent)
+A multi-agent system augmented with tool calls designed for general purpose tasks like financial research and math. Traced and evaluated on Faithfulness (factual adherence to retrieval context).
 
 ## 💻 Development with Cursor
 When building agents and LLM workflows in Cursor, providing proper context to your coding assistant helps ensure seamless integration with Judgment. This rule file supplies the essential context your coding assistant needs for successful implementation.

{judgeval-0.0.41 → judgeval-0.0.43}/README.md

@@ -3,37 +3,37 @@
 <img src="assets/new_lightmode.svg#gh-light-mode-only" alt="Judgment Logo" width="400" />
 <img src="assets/new_darkmode.svg#gh-dark-mode-only" alt="Judgment Logo" width="400" />
 
-**Build monitoring & evaluation pipelines for complex agents**
+<br>
+<div style="font-size: 1.5em;">
+  Open source tracing, evals, and metrics to debug, test, and monitor LLM agents.
+</div>
 
-<img src="assets/experiments_pagev2.png" alt="Judgment Platform Experiments Page" width="800" />
+## [Judgment Cloud](https://app.judgmentlabs.ai/register)  • [Self-Host](https://docs.judgmentlabs.ai/self-hosting/get_started) • [Demo](https://www.youtube.com/watch?v=1S4LixpVbcc) 
 
-<br>
+[Docs](https://docs.judgmentlabs.ai/introduction) • [Bug Reports](https://github.com/JudgmentLabs/judgeval/issues) • [Changelog](https://docs.judgmentlabs.ai/changelog/2025-04-21)
 
-## [🌐 Landing Page](https://www.judgmentlabs.ai/)  • [📚 Docs](https://docs.judgmentlabs.ai/introduction) • [🚀 Demos](https://www.youtube.com/@AlexShan-j3o) 
+We're hiring! Join us in our mission to unleash optimized agents.
 
 [![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
 [![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
 [![Discord](https://img.shields.io/badge/-Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/ZCnSXYug)
 
-</div>
+<img src="assets/experiments_pagev2.png" alt="Judgment Platform Experiments Page" width="800" />
 
-## Judgeval: open-source testing, monitoring, and optimization for AI agents
+</div>
 
-Judgeval offers robust tooling for evaluating and tracing LLM agent systems. It is dev-friendly and open-source (licensed under Apache 2.0). 
 
-Judgeval gets you started in five minutes, after which you'll be ready to use all of its features as your agent becomes more complex. Judgeval is natively connected to the [Judgment Platform](https://www.judgmentlabs.ai/) for free and you can export your data and self-host at any time.
+Judgeval offers **robust open-source tooling** for tracing, evaluating, and monitoring LLM agents. It helps AI teams effectively **test and monitor** agents in development and production, **closing the agent feedback loop**. 
 
-We support tracing agents built with LangGraph, OpenAI SDK, Anthropic, ... and allow custom eval integrations for any use case. Check out our quickstarts below or our [setup guide](https://docs.judgmentlabs.ai/getting-started) to get started.
+Judgeval can be set up **(cloud-hosted or self-hosted) in 5 minutes**!
+> 🎁 Generous monthly [free tier](https://judgmentlabs.ai/pricing) (10k traces, 1k evals) - No credit card required!
 
 Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
 
 ## 📋 Table of Contents
-- [🌐 Landing Page  • 📚 Docs • 🚀 Demos](#-landing-page----docs---demos)
-- [Judgeval: open-source testing, monitoring, and optimization for AI agents](#judgeval-open-source-testing-monitoring-and-optimization-for-ai-agents)
-- [📋 Table of Contents](#-table-of-contents)
 - [✨ Features](#-features)
 - [🛠️ Installation](#️-installation)
-- [🏁 Get Started](#-get-started)
+- [🏁 Quickstarts](#-quickstarts)
   - [🛰️ Tracing](#️-tracing)
   - [📝 Offline Evaluations](#-offline-evaluations)
   - [📡 Online Evaluations](#-online-evaluations)
@@ -41,12 +41,6 @@ Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
   - [Key Features](#key-features)
   - [Getting Started](#getting-started)
 - [📚 Cookbooks](#-cookbooks)
-  - [Sample Agents](#sample-agents)
-    - [💰 LangGraph Financial QA Agent](#-langgraph-financial-qa-agent)
-    - [✈️ OpenAI Travel Agent](#️-openai-travel-agent)
-  - [Custom Evaluators](#custom-evaluators)
-    - [🔍 PII Detection](#-pii-detection)
-    - [📧 Cold Email Generation](#-cold-email-generation)
 - [💻 Development with Cursor](#-development-with-cursor)
 - [⭐ Star Us on GitHub](#-star-us-on-github)
 - [❤️ Contributors](#️-contributors)
@@ -58,11 +52,10 @@ Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).
 
 |  |  |
 |:---|:---:|
-| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic): **tracking inputs/outputs, latency, and cost** at every step.<br><br>Online evals can be applied to traces to measure quality on production data in real-time.<br><br>Export trace data to the Judgment Platform or your own S3 buckets, {Parquet, JSON, YAML} files, or data warehouse.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 👤 Tracking user activity <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
-| <h3>🧪 Evals</h3>15+ research-backed metrics including tool call accuracy, hallucinations, instruction adherence, and retrieval context recall.<br><br>Build custom evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 Experimental prompt testing<br>• 🛡️ Online guardrails <br><br> | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
-| <h3>📡 Monitoring</h3>Real-time performance tracking of your agents in production environments. **Track all your metrics in one place.**<br><br>Set up **Slack/email alerts** for critical metrics and receive notifications when thresholds are exceeded.<br><br> **Useful for:** <br>•📉 Identifying degradation early <br>•📈 Visualizing performance trends across versions and time | <p align="center"><img src="assets/monitoring_screenshot.png" alt="Monitoring Dashboard" width="1200"/></p> |
-| <h3>📊 Datasets</h3>Export trace data or import external testcases to datasets hosted on Judgment's Platform. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations. <br><br> **Useful for:**<br>• 🔄 Scaled analysis for A/B tests <br>• 🗃️ Filtered collections of agent runtime data| <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
-| <h3>💡 Insights</h3>Cluster on your data to reveal common use cases and failure modes.<br><br>Trace failures to their exact source with Judgment's Osiris agent, which localizes errors to specific components for precise fixes.<br><br> **Useful for:**<br>•🔮 Surfacing common inputs that lead to error<br>•🤖 Investigating agent/user behavior for optimization <br>| <p align="center"><img src="assets/dataset_clustering_screenshot_dm.png" alt="Insights dashboard" width="1200"/></p> |
+| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic): **tracking inputs/outputs, agent tool calls, latency, and cost** at every step.<br><br>Online evals can be applied to traces to measure quality on production data in real-time.<br><br>Export trace data to the Judgment Platform or your own S3 buckets, {Parquet, JSON, YAML} files, or data warehouse.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 👤 Tracking user activity <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
+| <h3>🧪 Evals</h3>Evals are the key to regression testing for agents. Judgeval provides 15+ research-backed metrics including tool call accuracy, hallucinations, instruction adherence, and retrieval context recall.<br><br>Judgeval supports LLM-as-a-judge, manual labeling, and custom evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 Experimental prompt testing<br>• 🛡️ Online guardrails | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
+| <h3>📡 Monitoring</h3>Track all your agent metrics in production. **Catch production regressions early.**<br><br>Configure alerts to trigger automated actions when metric thresholds are exceeded (add agent trace to review queue/dataset, Slack notification, etc.).<br><br> **Useful for:** <br>• 📉 Identifying degradation early <br>• 📈 Visualizing performance trends across agent versions and time | <p align="center"><img src="assets/monitoring_screenshot.png" alt="Monitoring Dashboard" width="1200"/></p> |
+| <h3>📊 Datasets</h3>Export trace data or import external testcases to datasets for scaled unit testing and structured experiments. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations. <br><br> **Useful for:**<br>• 🗃️ Filtered agent runtime data for fine tuning<br>• 🔄 Scaled analysis for A/B tests | <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
 
 ## 🛠️ Installation
 
@@ -72,17 +65,19 @@ Get started with Judgeval by installing our SDK using pip:
 pip install judgeval
 ```
 
-Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment platform](https://app.judgmentlabs.ai/). 
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment platform](https://app.judgmentlabs.ai/).
 
-**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
 
-## 🏁 Get Started
+**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
 
-Here's how you can quickly start using Judgeval:
+## 🏁 Quickstarts
 
 ### 🛰️ Tracing
 
-Track your agent execution with full observability with just a few lines of code.
 Create a file named `traces.py` with the following code:
 
 ```python
@@ -107,12 +102,15 @@ def main():
 
 main()
 ```
+You'll see your trace exported to the Judgment Platform:
+
+<p align="center"><img src="assets/trace_demo.png" alt="Judgment Platform Trace Example" width="800" /></p>
+
 
 [Click here](https://docs.judgmentlabs.ai/getting-started#create-your-first-trace) for a more detailed explanation.
 
 ### 📝 Offline Evaluations
 
-You can evaluate your agent's execution to measure quality metrics such as hallucination.
 Create a file named `evaluate.py` with the following code:
 
 ```python evaluate.py
@@ -128,7 +126,7 @@ example = Example(
     retrieval_context=["All customers are eligible for a 30 day full refund at no extra cost."],
 )
 
-scorer = FaithfulnessScorer(threshold=0.5)
+scorer = FaithfulnessScorer(threshold=0.5)  # Hallucination detector
 results = client.run_evaluation(
     examples=[example],
     scorers=[scorer],
@@ -177,6 +175,8 @@ def main():
 main()
 ```
 
+You should see an evaluation attached to your trace on the Judgment Platform.
+
 [Click here](https://docs.judgmentlabs.ai/getting-started#create-your-first-online-evaluation) for a more detailed explanation.
 
 ## 🏢 Self-Hosting
@@ -201,20 +201,8 @@ You can access our repo of cookbooks [here](https://github.com/JudgmentLabs/judg
 
 ### Sample Agents
 
-#### 💰 [LangGraph Financial QA Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/financial_agent/demo.py)
-A LangGraph-based agent for financial queries, featuring RAG capabilities with a vector database for contextual data retrieval and evaluation of its reasoning and data accuracy.
-
-#### ✈️ [OpenAI Travel Agent](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/openai_travel_agent/agent.py)
-A travel planning agent using OpenAI API calls, custom tool functions, and RAG with a vector database for up-to-date and contextual travel information. Evaluated for itinerary quality and information relevance.
-
-### Custom Evaluators
-
-#### 🔍 [PII Detection](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/classifier_scorer/pii_checker.py)
-Detecting and evaluating Personal Identifiable Information (PII) leakage.
-
-#### 📧 [Cold Email Generation](https://github.com/JudgmentLabs/judgment-cookbook/blob/main/cookbooks/custom_scorers/cold_email_scorer.py)
-
-Evaluates if a cold email generator properly utilizes all relevant information about the target recipient.
+#### [Multi-Agent System](https://github.com/JudgmentLabs/judgment-cookbook/tree/main/cookbooks/agents/multi-agent)
+A multi-agent system augmented with tool calls designed for general purpose tasks like financial research and math. Traced and evaluated on Faithfulness (factual adherence to retrieval context).
 
 ## 💻 Development with Cursor
 When building agents and LLM workflows in Cursor, providing proper context to your coding assistant helps ensure seamless integration with Judgment. This rule file supplies the essential context your coding assistant needs for successful implementation.

{judgeval-0.0.41 → judgeval-0.0.43}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.41"
+version = "0.0.43"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },
@@ -29,7 +29,8 @@ dependencies = [
     "langchain-anthropic",
     "langchain-core",
     "google-genai",
-    "boto3"
+    "boto3",
+    "matplotlib>=3.10.3",
 ]
 
 [project.urls]
@@ -47,6 +48,18 @@ include = [
     "/src/judgeval/**/*.py",
 ]
 
+[dependency-groups]
+dev = [
+    "chromadb>=1.0.12",
+    "langchain-community>=0.3.24",
+    "pytest>=8.4.0",
+    "pytest-asyncio>=1.0.0",
+    "pytest-cov>=6.1.1",
+    "pytest-mock>=3.14.1",
+    "tavily-python>=0.7.5",
+    "langgraph>=0.4.3",
+]
+
 [tool.hatch.build]
 directory = "dist"
 artifacts = [

{judgeval-0.0.41 → judgeval-0.0.43}/src/judgeval/common/s3_storage.py

@@ -46,7 +46,9 @@ class S3Storage:
                         Bucket=self.bucket_name,
                         CreateBucketConfiguration={
                             'LocationConstraint': self.s3_client.meta.region_name
-                        }
+                        }               
+                    ) if self.s3_client.meta.region_name != "us-east-1" else self.s3_client.create_bucket(
+                        Bucket=self.bucket_name
                     )
                     info(f"Created S3 bucket: {self.bucket_name}")
                 except ClientError as create_error: