judgeval 0.9.0__tar.gz → 0.22.3__tar.gz

judgeval-0.22.3/.github/workflows/ci.yaml

@@ -0,0 +1,148 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions: read-all
+
+jobs:
+  validate-branch:
+    uses: ./.github/workflows/merge-branch-check.yaml
+
+  run-tests:
+    needs: [validate-branch]
+    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+    name: Unit Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      JUDGMENT_DEV: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Install Claude Code CLI
+        run: |
+          npm install -g @anthropic-ai/claude-code
+
+      - name: Run tests
+        run: |
+          cd src
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          uv run pytest tests -n auto
+
+  run-e2e-tests:
+    needs: [validate-branch]
+    if: "(github.base_ref == 'staging' || github.base_ref == 'main') && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    name: E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set env based on branch
+        run: |
+          if [ "${{ github.base_ref }}" = "main" ]; then
+            echo "TARGET_ENV=main" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=prod/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-production-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          else
+            echo "TARGET_ENV=staging" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://staging.api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=stg/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-staging-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          fi
+
+      - name: Restore uv cache
+        uses: actions/cache/restore@v4
+        id: restore-uv-cache
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-judgment-
+            ${{ runner.os }}-uv-
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install judgeval dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s "$BASE_URL/health" > /dev/null; then
+            echo "Judgment server ($BASE_URL) is not running properly. Check CloudWatch logs."
+            exit 1
+          else
+            echo "Server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id "$SECRETS_PATH" --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          export JUDGMENT_API_URL="$BASE_URL"
+          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.COVERAGE_ARTIFACT }}
+          path: src/htmlcov
+
+      - name: Save uv cache
+        uses: actions/cache/save@v4
+        if: always() && steps.restore-uv-cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}

{judgeval-0.9.0 → judgeval-0.22.3}/.github/workflows/release.yaml

@@ -91,7 +91,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       
-      - name: Bump pyproject.toml version
+      - name: Bump pyproject.toml and version.py version
         run: |
           python update_version.py ${{ steps.bump_tag.outputs.new_version }}
 

judgeval-0.22.3/.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.9.7
+    hooks:
+      - id: uv-lock
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.3
+    hooks:
+      - id: ruff
+        name: ruff (linter)
+        args: [--fix]
+      - id: ruff-format
+        name: ruff (formatter)
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.18.2
+    hooks:
+      - id: mypy
+        language: system
+        # These next two lines allow commits even if mypy fails, REMOVE once we fix all mypy errors
+        verbose: true
+        entry: bash -c 'mypy src/judgeval/ || true'

judgeval-0.22.3/CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contribute to Judgeval
+
+There are many ways to contribute to Judgeval:
+
+- Submit [bug reports](https://github.com/JudgmentLabs/judgeval/issues) and [feature requests](https://github.com/JudgmentLabs/judgeval/issues)
+- Review the documentation and submit [Pull Requests](https://github.com/JudgmentLabs/judgeval/pulls) to improve it
+- Speaking or writing about Judgment and letting us know!
+
+<!-- Contributors collage -->
+[![Contributors](https://contributors-img.web.app/image?repo=JudgmentLabs/judgeval)](https://github.com/JudgmentLabs/judgeval/graphs/contributors)

judgeval-0.22.3/PKG-INFO

@@ -0,0 +1,266 @@
+Metadata-Version: 2.4
+Name: judgeval
+Version: 0.22.3
+Summary: The open source post-building layer for Agent Behavior Monitoring.
+Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
+Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
+Author-email: Andrew Li <andrew@judgmentlabs.ai>, Alex Shan <alex@judgmentlabs.ai>, Joseph Camyre <joseph@judgmentlabs.ai>
+Maintainer-email: Judgment Labs <contact@judgmentlabs.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.md
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: boto3>=1.40.11
+Requires-Dist: click<8.2.0
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: litellm>=1.75.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.36.0
+Requires-Dist: opentelemetry-sdk>=1.36.0
+Requires-Dist: orjson>=3.9.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: s3
+Requires-Dist: boto3>=1.40.11; extra == 's3'
+Provides-Extra: trainer
+Requires-Dist: fireworks-ai>=0.19.18; extra == 'trainer'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<a href="https://judgmentlabs.ai/">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="assets/logo_darkmode.svg">
+    <img src="assets/logo_lightmode.svg" alt="Judgment Logo" width="400" />
+  </picture>
+</a>
+
+<br>
+
+## Agent Behavior Monitoring (ABM)
+
+Track and judge any agent behavior in online and offline setups. Set up Sentry-style alerts and analyze agent behaviors / topic patterns at scale! 
+
+[![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.judgmentlabs.ai/documentation)
+[![Judgment Cloud](https://img.shields.io/badge/Judgment%20Cloud-brightgreen)](https://app.judgmentlabs.ai/register)
+[![Self-Host](https://img.shields.io/badge/Self--Host-orange)](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started)
+
+
+[![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
+[![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
+
+</div>
+
+
+</table>
+
+## [NEW] 🎆 Agent Reinforcement Learning
+
+Train your agents with multi-turn reinforcement learning using judgeval and [Fireworks AI](https://fireworks.ai/)! Judgeval's ABM now integrates with Fireworks' Reinforcement Fine-Tuning (RFT) endpoint, supporting gpt-oss, qwen3, Kimi2, DeepSeek, and more.
+
+Judgeval's agent monitoring infra provides a simple harness for integrating GRPO into any Python agent, giving builders a quick method to **try RL with minimal code changes** to their existing agents!
+
+```python
+await trainer.train(
+    agent_function=your_agent_function,  # entry point to your agent
+    scorers=[RewardScorer()],  # Custom scorer you define based on task criteria, acts as reward
+    prompts=training_prompts  # Tasks
+)
+```
+
+**That's it!** Judgeval automatically manages trajectory collection and reward tagging - your agent can learn from production data with minimal code changes. 
+
+👉 Check out the [Wikipedia Racer notebook](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/rl/WikiRacingAgent_RL.ipynb), where an agent learns to navigate Wikipedia using RL, to see Judgeval in action.
+
+
+You can view and monitor training progress for free via the [Judgment Dashboard](https://app.judgmentlabs.ai/).
+
+
+## Judgeval Overview
+
+Judgeval is an open-source framework for agent behavior monitoring. Judgeval offers a toolkit to track and judge agent behavior in online and offline setups, enabling you to convert interaction data from production/test environments into improved agents. To get started, try running one of the notebooks below or dive deeper in our [docs](https://docs.judgmentlabs.ai/documentation).
+
+Our mission is to unlock the power of production data for agent development, enabling teams to improve their apps by catching real-time failures and optimizing over their users' preferences.
+
+## 📚 Cookbooks
+
+| Try Out | Notebook | Description |
+|:---------|:-----|:------------|
+| RL | [Wikipedia Racer](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/rl/WikiRacingAgent_RL.ipynb) | Train agents with reinforcement learning |
+| Online ABM | [Research Agent](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/monitoring/Research_Agent_Online_Monitoring.ipynb) | Monitor agent behavior in production |
+| Custom Scorers | [HumanEval](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/custom_scorers/HumanEval_Custom_Scorer.ipynb) | Build custom evaluators for your agents |
+| Offline Testing | [Get Started For Free] | Compare how different prompts, models, or agent configs affect performance across ANY metric |
+
+You can access our [repo of cookbooks](https://github.com/JudgmentLabs/judgment-cookbook).
+
+You can find a list of [video tutorials for Judgeval use cases](https://www.youtube.com/@Alexshander-JL).
+
+## Why Judgeval?
+
+🤖 **Simple to run multi-turn RL**: Optimize your agents with multi-turn RL without managing compute infrastructure or data pipelines. Just add a few lines of code to your existing agent code and train!
+
+⚙️ **Custom Evaluators**: No restriction to only monitoring with prefab scorers. Judgeval provides simple abstractions for custom Python scorers, supporting any LLM-as-a-judge rubrics/models and code-based scorers that integrate to our live agent-tracking infrastructure. [Learn more](https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers)
+
+🚨 **Production Monitoring**: Run any custom scorer in a hosted, virtualized secure container to flag agent behaviors online in production. Get Slack alerts for failures and add custom hooks to address regressions before they impact users. [Learn more](https://docs.judgmentlabs.ai/documentation/performance/online-evals)
+
+📊 **Behavior/Topic Grouping**: Group agent runs by behavior type or topic for deeper analysis. Drill down into subsets of users, agents, or use cases to reveal patterns of agent behavior.
+<!-- Add link to Bucketing docs once we have it -->
+<!-- 
+TODO: Once we have trainer code docs, plug in here
+-->
+
+🧪 **Run experiments on your agents**: Compare test different prompts, models, or agent configs across customer segments. Measure which changes improve agent performance and decrease bad agent behaviors.
+
+<!-- 
+Use this once we have AI PM features:
+
+**Run experiments on your agents**: A/B test different prompts, models, or agent configs across customer segments. Measure which changes improve agent performance and decrease bad agent behaviors. [Learn more]
+
+-->
+
+## 🛠️ Quickstart
+
+Get started with Judgeval by installing our SDK using pip:
+
+```bash
+pip install judgeval
+```
+
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment Platform](https://app.judgmentlabs.ai/).
+
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
+
+**If you don't have keys, [create an account for free](https://app.judgmentlabs.ai/register) on the platform!**
+
+### Start monitoring with Judgeval
+
+```python
+from judgeval.tracer import Tracer, wrap
+from judgeval.data import Example
+from judgeval.scorers import AnswerRelevancyScorer
+from openai import OpenAI
+
+
+judgment = Tracer(project_name="default_project")
+client = wrap(OpenAI())  # tracks all LLM calls
+
+@judgment.observe(span_type="tool")
+def format_question(question: str) -> str:
+    # dummy tool
+    return f"Question : {question}"
+
+@judgment.observe(span_type="function")
+def run_agent(prompt: str) -> str:
+    task = format_question(prompt)
+    response = client.chat.completions.create(
+        model="gpt-5-mini",
+        messages=[{"role": "user", "content": task}]
+    )
+
+    judgment.async_evaluate(  # trigger online monitoring
+        scorer=AnswerRelevancyScorer(threshold=0.5),  # swap with any scorer
+        example=Example(input=task, actual_output=response),  # customize to your data
+        model="gpt-5",
+    )
+    return response.choices[0].message.content
+
+run_agent("What is the capital of the United States?")
+```
+
+Running this code will deliver monitoring results to your [free platform account](https://app.judgmentlabs.ai/register) and should look like this:
+
+![Judgment Platform Trajectory View](assets/quickstart_trajectory_ss.png)
+
+
+### Customizable Scorers Over Agent Behavior
+
+Judgeval's strongest suit is the full customization over the types of scorers you can run online monitoring with. No restrictions to only single-prompt LLM judges or prefab scorers - if you can express your scorer
+in python code, judgeval can monitor it! Under the hood, judgeval hosts your scorer in a virtualized secure container, enabling online monitoring for any scorer.
+
+
+First, create a behavior scorer in a file called `helpfulness_scorer.py`:
+
+```python
+from judgeval.data import Example
+from judgeval.scorers.example_scorer import ExampleScorer
+
+# Define custom example class
+class QuestionAnswer(Example):
+    question: str
+    answer: str
+
+# Define a server-hosted custom scorer
+class HelpfulnessScorer(ExampleScorer):
+    name: str = "Helpfulness Scorer"
+    server_hosted: bool = True  # Enable server hosting
+    async def a_score_example(self, example: QuestionAnswer):
+        # Custom scoring logic for agent behavior
+        # Can be an arbitrary combination of code and LLM calls
+        if len(example.answer) > 10 and "?" not in example.answer:
+            self.reason = "Answer is detailed and provides helpful information"
+            return 1.0
+        else:
+            self.reason = "Answer is too brief or unclear"
+            return 0.0
+```
+
+Then deploy your scorer to Judgment's infrastructure:
+
+```bash
+echo "pydantic" > requirements.txt
+uv run judgeval upload_scorer helpfulness_scorer.py requirements.txt
+```
+
+Now you can instrument your agent with monitoring and online evaluation:
+
+```python
+from judgeval.tracer import Tracer, wrap
+from helpfulness_scorer import HelpfulnessScorer, QuestionAnswer
+from openai import OpenAI
+
+judgment = Tracer(project_name="default_project")
+client = wrap(OpenAI())  # tracks all LLM calls
+
+@judgment.observe(span_type="tool")
+def format_task(question: str) -> str:  # replace with your prompt engineering
+    return f"Please answer the following question: {question}"
+
+@judgment.observe(span_type="tool")
+def answer_question(prompt: str) -> str:  # replace with your LLM system calls
+    response = client.chat.completions.create(
+        model="gpt-5-mini",
+        messages=[{"role": "user", "content": prompt}]
+    )
+    return response.choices[0].message.content
+
+@judgment.observe(span_type="function")
+def run_agent(question: str) -> str:
+    task = format_task(question)
+    answer = answer_question(task)
+
+    # Add online evaluation with server-hosted scorer
+    judgment.async_evaluate(
+        scorer=HelpfulnessScorer(),
+        example=QuestionAnswer(question=question, answer=answer),
+        sampling_rate=0.9  # Evaluate 90% of agent runs
+    )
+
+    return answer
+
+if __name__ == "__main__":
+    result = run_agent("What is the capital of the United States?")
+    print(result)
+```
+
+Congratulations! Your online eval result should look like this:
+
+![Custom Scorer Online ABM](assets/custom_scorer_online_abm.png)
+
+You can now run any online scorer in a secure Firecracker microVMs with no latency impact on your applications.
+
+---
+
+Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).