judgeval 0.0.54__tar.gz → 0.1.0__tar.gz

judgeval-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🤔 Questions or discussions
+    url: https://github.com/orgs/JudgmentLabs/discussions
+    about: Please use discussions for questions

{judgeval-0.0.54 → judgeval-0.1.0}/.github/workflows/release.yaml

@@ -28,6 +28,18 @@ jobs:
           version=$(curl -s https://pypi.org/pypi/judgeval/json | jq -r .info.version)
           echo "latest_version=$version" >> $GITHUB_OUTPUT
 
+      - name: Determine bump type (minor if commit message starts with [Bump Minor Version], else patch)
+        id: bump_type
+        run: |
+          # Get the latest commit message
+          commit_message=$(git log -1 --pretty=%B)
+          # Default bump type
+          bump_type=patch
+          if [[ "$commit_message" == "[Bump Minor Version]"* ]]; then
+            bump_type=minor
+          fi
+          echo "bump_type=$bump_type" >> $GITHUB_OUTPUT
+
       - name: Bump version and create new tag
         id: bump_tag
         run: |
@@ -37,9 +49,29 @@ jobs:
           # Extract version numbers
           IFS='.' read -r major minor patch <<< "$latest_version"
 
-          # Bump patch version
-          patch=$((patch + 1))
-          new_version="$major.$minor.$patch"
+          # Set major version (manually modify this if you want to change the major version)
+          new_major_version=0
+
+          # Validate that new major version is greater than current major version
+          if [ "$new_major_version" -lt "$major" ]; then
+            echo "Error: New major version ($new_major_version) must be greater than or equal to current major version ($major)"
+            exit 1
+          fi
+
+          if [ "$new_major_version" -ne "$major" ]; then  # If major version changed, set minor and patch to 0
+            echo "Major version bumped, setting minor and patch to 0"
+            minor=0
+            patch=0
+          elif [ "${{ steps.bump_type.outputs.bump_type }}" = "minor" ]; then
+            echo "Minor version bumped, setting patch to 0"
+            minor=$((minor + 1))
+            patch=0
+          else
+            echo "Patch version bumped"
+            patch=$((patch + 1))
+          fi
+
+          new_version="$new_major_version.$minor.$patch"
 
           echo "New version: $new_version"
           echo "new_version=$new_version" >> $GITHUB_OUTPUT

judgeval-0.1.0/PKG-INFO

@@ -0,0 +1,202 @@
+Metadata-Version: 2.4
+Name: judgeval
+Version: 0.1.0
+Summary: Judgeval Package
+Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
+Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
+Author-email: Andrew Li <andrew@judgmentlabs.ai>, Alex Shan <alex@judgmentlabs.ai>, Joseph Camyre <joseph@judgmentlabs.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.md
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: anthropic
+Requires-Dist: boto3
+Requires-Dist: datamodel-code-generator>=0.31.1
+Requires-Dist: google-genai
+Requires-Dist: langchain-anthropic
+Requires-Dist: langchain-core
+Requires-Dist: langchain-huggingface
+Requires-Dist: langchain-openai
+Requires-Dist: litellm>=1.61.15
+Requires-Dist: matplotlib>=3.10.3
+Requires-Dist: nest-asyncio
+Requires-Dist: openai
+Requires-Dist: pandas
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: requests
+Requires-Dist: together
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<img src="assets/new_lightmode.svg#gh-light-mode-only" alt="Judgment Logo" width="400" />
+<img src="assets/new_darkmode.svg#gh-dark-mode-only" alt="Judgment Logo" width="400" />
+
+<br>
+<div style="font-size: 1.5em;">
+    Enable self-learning agents with traces, evals, and environment data.
+</div>
+
+## [Docs](https://docs.judgmentlabs.ai/)  •  [Judgment Cloud](https://app.judgmentlabs.ai/register)  • [Self-Host](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started)
+
+ [Demo](https://www.youtube.com/watch?v=1S4LixpVbcc) • [Bug Reports](https://github.com/JudgmentLabs/judgeval/issues) • [Changelog](https://docs.judgmentlabs.ai/changelog/2025-04-21)
+
+We're hiring! Join us in our mission to enable self-learning agents by providing the data and signals needed for monitoring and post-training.
+
+[![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
+[![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
+[![Discord](https://img.shields.io/badge/-Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/tGVFf8UBUY)
+
+<img src="assets/product_shot.png" alt="Judgment Platform" width="800" />
+
+</div>
+
+Judgeval offers **open-source tooling** for tracing and evaluating autonomous, stateful agents. It **provides runtime data from agent-environment interactions** for continuous learning and self-improvement.
+
+## 🎬 See Judgeval in Action
+
+**[Multi-Agent System](https://github.com/JudgmentLabs/judgment-cookbook/tree/main/cookbooks/agents/multi-agent) with complete observability:** (1) A multi-agent system spawns agents to research topics on the internet. (2) With just **3 lines of code**, Judgeval traces every input/output + environment response across all agent tool calls for debugging. (3) After completion, (4) export all interaction data to enable further environment-specific learning and optimization.
+
+<table style="width: 100%; max-width: 800px; table-layout: fixed;">
+<tr>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/agent.gif" alt="Agent Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>🤖 Agents Running</strong>
+</td>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/trace.gif" alt="Trace Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>📊 Real-time Tracing</strong>
+</td>
+</tr>
+<tr>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/document.gif" alt="Agent Completed Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>✅ Agents Completed Running</strong>
+</td>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/data.gif" alt="Data Export Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>📤 Exporting Agent Environment Data</strong>
+</td>
+</tr>
+
+</table>
+
+## 📋 Table of Contents
+- [🛠️ Installation](#️-installation)
+- [🏁 Quickstarts](#-quickstarts)
+- [✨ Features](#-features)
+- [🏢 Self-Hosting](#-self-hosting)
+- [📚 Cookbooks](#-cookbooks)
+- [💻 Development with Cursor](#-development-with-cursor)
+
+## 🛠️ Installation
+
+Get started with Judgeval by installing our SDK using pip:
+
+```bash
+pip install judgeval
+```
+
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment Platform](https://app.judgmentlabs.ai/).
+
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
+
+**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
+
+## 🏁 Quickstarts
+
+### 🛰️ Tracing
+
+Create a file named `agent.py` with the following code:
+
+```python
+from judgeval.tracer import Tracer, wrap
+from openai import OpenAI
+
+client = wrap(OpenAI())  # tracks all LLM calls
+judgment = Tracer(project_name="my_project")
+
+@judgment.observe(span_type="tool")
+def format_question(question: str) -> str:
+    # dummy tool
+    return f"Question : {question}"
+
+@judgment.observe(span_type="function")
+def run_agent(prompt: str) -> str:
+    task = format_question(prompt)
+    response = client.chat.completions.create(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": task}]
+    )
+    return response.choices[0].message.content
+    
+run_agent("What is the capital of the United States?")
+```
+You'll see your trace exported to the Judgment Platform:
+
+<p align="center"><img src="assets/trace_demo.png" alt="Judgment Platform Trace Example" width="800" /></p>
+
+
+[Click here](https://docs.judgmentlabs.ai/documentation/tracing/introduction) for a more detailed explanation.
+
+
+<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+
+
+## ✨ Features
+
+|  |  |
+|:---|:---:|
+| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic). **Tracks inputs/outputs, agent tool calls, latency, cost, and custom metadata** at every step.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 📋 Collecting agent environment data <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
+| <h3>🧪 Evals</h3>Build custom evaluators on top of your agents. Judgeval supports LLM-as-a-judge, manual labeling, and code-based evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 A/B testing <br>• 🛡️ Online guardrails | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
+| <h3>📡 Monitoring</h3>Get Slack alerts for agent failures in production. Add custom hooks to address production regressions.<br><br> **Useful for:** <br>• 📉 Identifying degradation early <br>• 📈 Visualizing performance trends across agent versions and time | <p align="center"><img src="assets/error_analysis_dashboard.png" alt="Monitoring Dashboard" width="1200"/></p> |
+| <h3>📊 Datasets</h3>Export traces and test cases to datasets for scaled analysis and optimization. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations, enabling continuous learning from production interactions. <br><br> **Useful for:**<br>• 🗃️ Agent environment interaction data for optimization<br>• 🔄 Scaled analysis for A/B tests | <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
+
+## 🏢 Self-Hosting
+
+Run Judgment on your own infrastructure: we provide comprehensive self-hosting capabilities that give you full control over the backend and data plane that Judgeval interfaces with.
+
+### Key Features
+* Deploy Judgment on your own AWS account
+* Store data in your own Supabase instance
+* Access Judgment through your own custom domain
+
+### Getting Started
+1. Check out our [self-hosting documentation](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started) for detailed setup instructions, along with how your self-hosted instance can be accessed
+2. Use the [Judgment CLI](https://docs.judgmentlabs.ai/documentation/developer-tools/judgment-cli/installation) to deploy your self-hosted environment
+3. After your self-hosted instance is setup, make sure the `JUDGMENT_API_URL` environmental variable is set to your self-hosted backend endpoint
+
+## 📚 Cookbooks
+
+Have your own? We're happy to feature it if you create a PR or message us on [Discord](https://discord.gg/tGVFf8UBUY).
+
+You can access our repo of cookbooks [here](https://github.com/JudgmentLabs/judgment-cookbook).
+
+## 💻 Development with Cursor
+Building agents and LLM workflows in Cursor works best when your coding assistant has the proper context about Judgment integration. The Cursor rules file contains the key information needed for your assistant to implement Judgment features effectively.
+
+Refer to the official [documentation](https://docs.judgmentlabs.ai/documentation/developer-tools/cursor/cursor-rules) for access to the rules file and more information on integrating this rules file with your codebase.
+
+## ⭐ Star Us on GitHub
+
+If you find Judgeval useful, please consider giving us a star on GitHub! Your support helps us grow our community and continue improving the repository.
+
+## ❤️ Contributors
+
+There are many ways to contribute to Judgeval:
+
+- Submit [bug reports](https://github.com/JudgmentLabs/judgeval/issues) and [feature requests](https://github.com/JudgmentLabs/judgeval/issues)
+- Review the documentation and submit [Pull Requests](https://github.com/JudgmentLabs/judgeval/pulls) to improve it
+- Speaking or writing about Judgment and letting us know!
+
+<!-- Contributors collage -->
+[![Contributors](https://contributors-img.web.app/image?repo=JudgmentLabs/judgeval)](https://github.com/JudgmentLabs/judgeval/graphs/contributors)
+
+---
+
+Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).

judgeval-0.1.0/README.md

@@ -0,0 +1,171 @@
+<div align="center">
+
+<img src="assets/new_lightmode.svg#gh-light-mode-only" alt="Judgment Logo" width="400" />
+<img src="assets/new_darkmode.svg#gh-dark-mode-only" alt="Judgment Logo" width="400" />
+
+<br>
+<div style="font-size: 1.5em;">
+    Enable self-learning agents with traces, evals, and environment data.
+</div>
+
+## [Docs](https://docs.judgmentlabs.ai/)  •  [Judgment Cloud](https://app.judgmentlabs.ai/register)  • [Self-Host](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started)
+
+ [Demo](https://www.youtube.com/watch?v=1S4LixpVbcc) • [Bug Reports](https://github.com/JudgmentLabs/judgeval/issues) • [Changelog](https://docs.judgmentlabs.ai/changelog/2025-04-21)
+
+We're hiring! Join us in our mission to enable self-learning agents by providing the data and signals needed for monitoring and post-training.
+
+[![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
+[![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
+[![Discord](https://img.shields.io/badge/-Discord-5865F2?logo=discord&logoColor=white)](https://discord.gg/tGVFf8UBUY)
+
+<img src="assets/product_shot.png" alt="Judgment Platform" width="800" />
+
+</div>
+
+Judgeval offers **open-source tooling** for tracing and evaluating autonomous, stateful agents. It **provides runtime data from agent-environment interactions** for continuous learning and self-improvement.
+
+## 🎬 See Judgeval in Action
+
+**[Multi-Agent System](https://github.com/JudgmentLabs/judgment-cookbook/tree/main/cookbooks/agents/multi-agent) with complete observability:** (1) A multi-agent system spawns agents to research topics on the internet. (2) With just **3 lines of code**, Judgeval traces every input/output + environment response across all agent tool calls for debugging. (3) After completion, (4) export all interaction data to enable further environment-specific learning and optimization.
+
+<table style="width: 100%; max-width: 800px; table-layout: fixed;">
+<tr>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/agent.gif" alt="Agent Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>🤖 Agents Running</strong>
+</td>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/trace.gif" alt="Trace Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>📊 Real-time Tracing</strong>
+</td>
+</tr>
+<tr>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/document.gif" alt="Agent Completed Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>✅ Agents Completed Running</strong>
+</td>
+<td align="center" style="padding: 8px; width: 50%;">
+  <img src="assets/data.gif" alt="Data Export Demo" style="width: 100%; max-width: 350px; height: auto;" />
+  <br><strong>📤 Exporting Agent Environment Data</strong>
+</td>
+</tr>
+
+</table>
+
+## 📋 Table of Contents
+- [🛠️ Installation](#️-installation)
+- [🏁 Quickstarts](#-quickstarts)
+- [✨ Features](#-features)
+- [🏢 Self-Hosting](#-self-hosting)
+- [📚 Cookbooks](#-cookbooks)
+- [💻 Development with Cursor](#-development-with-cursor)
+
+## 🛠️ Installation
+
+Get started with Judgeval by installing our SDK using pip:
+
+```bash
+pip install judgeval
+```
+
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment Platform](https://app.judgmentlabs.ai/).
+
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
+
+**If you don't have keys, [create an account](https://app.judgmentlabs.ai/register) on the platform!**
+
+## 🏁 Quickstarts
+
+### 🛰️ Tracing
+
+Create a file named `agent.py` with the following code:
+
+```python
+from judgeval.tracer import Tracer, wrap
+from openai import OpenAI
+
+client = wrap(OpenAI())  # tracks all LLM calls
+judgment = Tracer(project_name="my_project")
+
+@judgment.observe(span_type="tool")
+def format_question(question: str) -> str:
+    # dummy tool
+    return f"Question : {question}"
+
+@judgment.observe(span_type="function")
+def run_agent(prompt: str) -> str:
+    task = format_question(prompt)
+    response = client.chat.completions.create(
+        model="gpt-4.1",
+        messages=[{"role": "user", "content": task}]
+    )
+    return response.choices[0].message.content
+    
+run_agent("What is the capital of the United States?")
+```
+You'll see your trace exported to the Judgment Platform:
+
+<p align="center"><img src="assets/trace_demo.png" alt="Judgment Platform Trace Example" width="800" /></p>
+
+
+[Click here](https://docs.judgmentlabs.ai/documentation/tracing/introduction) for a more detailed explanation.
+
+
+<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+
+
+## ✨ Features
+
+|  |  |
+|:---|:---:|
+| <h3>🔍 Tracing</h3>Automatic agent tracing integrated with common frameworks (LangGraph, OpenAI, Anthropic). **Tracks inputs/outputs, agent tool calls, latency, cost, and custom metadata** at every step.<br><br>**Useful for:**<br>• 🐛 Debugging agent runs <br>• 📋 Collecting agent environment data <br>• 🔬 Pinpointing performance bottlenecks| <p align="center"><img src="assets/trace_screenshot.png" alt="Tracing visualization" width="1200"/></p> |
+| <h3>🧪 Evals</h3>Build custom evaluators on top of your agents. Judgeval supports LLM-as-a-judge, manual labeling, and code-based evaluators that connect with our metric-tracking infrastructure. <br><br>**Useful for:**<br>• ⚠️ Unit-testing <br>• 🔬 A/B testing <br>• 🛡️ Online guardrails | <p align="center"><img src="assets/experiments_page.png" alt="Evaluation metrics" width="800"/></p> |
+| <h3>📡 Monitoring</h3>Get Slack alerts for agent failures in production. Add custom hooks to address production regressions.<br><br> **Useful for:** <br>• 📉 Identifying degradation early <br>• 📈 Visualizing performance trends across agent versions and time | <p align="center"><img src="assets/error_analysis_dashboard.png" alt="Monitoring Dashboard" width="1200"/></p> |
+| <h3>📊 Datasets</h3>Export traces and test cases to datasets for scaled analysis and optimization. Move datasets to/from Parquet, S3, etc. <br><br>Run evals on datasets as unit tests or to A/B test different agent configurations, enabling continuous learning from production interactions. <br><br> **Useful for:**<br>• 🗃️ Agent environment interaction data for optimization<br>• 🔄 Scaled analysis for A/B tests | <p align="center"><img src="assets/datasets_preview_screenshot.png" alt="Dataset management" width="1200"/></p> |
+
+## 🏢 Self-Hosting
+
+Run Judgment on your own infrastructure: we provide comprehensive self-hosting capabilities that give you full control over the backend and data plane that Judgeval interfaces with.
+
+### Key Features
+* Deploy Judgment on your own AWS account
+* Store data in your own Supabase instance
+* Access Judgment through your own custom domain
+
+### Getting Started
+1. Check out our [self-hosting documentation](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started) for detailed setup instructions, along with how your self-hosted instance can be accessed
+2. Use the [Judgment CLI](https://docs.judgmentlabs.ai/documentation/developer-tools/judgment-cli/installation) to deploy your self-hosted environment
+3. After your self-hosted instance is setup, make sure the `JUDGMENT_API_URL` environmental variable is set to your self-hosted backend endpoint
+
+## 📚 Cookbooks
+
+Have your own? We're happy to feature it if you create a PR or message us on [Discord](https://discord.gg/tGVFf8UBUY).
+
+You can access our repo of cookbooks [here](https://github.com/JudgmentLabs/judgment-cookbook).
+
+## 💻 Development with Cursor
+Building agents and LLM workflows in Cursor works best when your coding assistant has the proper context about Judgment integration. The Cursor rules file contains the key information needed for your assistant to implement Judgment features effectively.
+
+Refer to the official [documentation](https://docs.judgmentlabs.ai/documentation/developer-tools/cursor/cursor-rules) for access to the rules file and more information on integrating this rules file with your codebase.
+
+## ⭐ Star Us on GitHub
+
+If you find Judgeval useful, please consider giving us a star on GitHub! Your support helps us grow our community and continue improving the repository.
+
+## ❤️ Contributors
+
+There are many ways to contribute to Judgeval:
+
+- Submit [bug reports](https://github.com/JudgmentLabs/judgeval/issues) and [feature requests](https://github.com/JudgmentLabs/judgeval/issues)
+- Review the documentation and submit [Pull Requests](https://github.com/JudgmentLabs/judgeval/pulls) to improve it
+- Speaking or writing about Judgment and letting us know!
+
+<!-- Contributors collage -->
+[![Contributors](https://contributors-img.web.app/image?repo=JudgmentLabs/judgeval)](https://github.com/JudgmentLabs/judgeval/graphs/contributors)
+
+---
+
+Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).

{judgeval-0.0.54 → judgeval-0.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.54"
+version = "0.1.0"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },
@@ -16,7 +16,7 @@ classifiers = [
 license = "Apache-2.0"
 license-files = ["LICENSE.md"]
 dependencies = [
-    "litellm==1.61.15",
+    "litellm>=1.61.15",
     "python-dotenv==1.0.1",
     "requests",
     "pandas",
@@ -31,6 +31,7 @@ dependencies = [
     "google-genai",
     "boto3",
     "matplotlib>=3.10.3",
+    "python-slugify>=8.0.4",
     "datamodel-code-generator>=0.31.1",
 ]
 
@@ -60,6 +61,7 @@ dev = [
     "tavily-python>=0.7.5",
     "langgraph>=0.4.3",
     "pre-commit>=4.2.0",
+    "types-requests>=2.32.4.20250611",
 ]
 
 [tool.hatch.build]

judgeval-0.1.0/src/judgeval/common/api/__init__.py

@@ -0,0 +1,3 @@
+from .api import JudgmentApiClient, JudgmentAPIException
+
+__all__ = ["JudgmentApiClient", "JudgmentAPIException"]