judgeval 0.0.55__tar.gz → 0.22.3__tar.gz

judgeval-0.22.3/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🤔 Questions or discussions
+    url: https://github.com/orgs/JudgmentLabs/discussions
+    about: Please use discussions for questions

{judgeval-0.0.55 → judgeval-0.22.3}/.github/pull_request_template.md

@@ -10,14 +10,7 @@
 -->
 - [ ] 1. ...
 
-## 🎥 Demo of Changes
-
-<!-- Add a short 1-3 minute video describing/demoing the changes -->
-
 ## ✅ Checklist
 
-- [ ] Tagged Linear ticket in PR title. Ie. PR Title (JUD-XXXX)
-- [ ] Video demo of changes
-- [ ] Reviewers assigned
 - [ ] Docs updated ([if necessary](https://github.com/JudgmentLabs/docs))
-- [ ] Cookbooks updated ([if necessary](https://github.com/JudgmentLabs/judgment-cookbook))
+- [ ] Changelogs are updated ([if necessary](https://github.com/JudgmentLabs/docs/tree/main/content/docs/changelog/%28weekly%29))

judgeval-0.22.3/.github/workflows/ci.yaml

@@ -0,0 +1,148 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions: read-all
+
+jobs:
+  validate-branch:
+    uses: ./.github/workflows/merge-branch-check.yaml
+
+  run-tests:
+    needs: [validate-branch]
+    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+    name: Unit Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      JUDGMENT_DEV: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Install Claude Code CLI
+        run: |
+          npm install -g @anthropic-ai/claude-code
+
+      - name: Run tests
+        run: |
+          cd src
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          uv run pytest tests -n auto
+
+  run-e2e-tests:
+    needs: [validate-branch]
+    if: "(github.base_ref == 'staging' || github.base_ref == 'main') && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    name: E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set env based on branch
+        run: |
+          if [ "${{ github.base_ref }}" = "main" ]; then
+            echo "TARGET_ENV=main" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=prod/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-production-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          else
+            echo "TARGET_ENV=staging" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://staging.api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=stg/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-staging-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          fi
+
+      - name: Restore uv cache
+        uses: actions/cache/restore@v4
+        id: restore-uv-cache
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-judgment-
+            ${{ runner.os }}-uv-
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install judgeval dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s "$BASE_URL/health" > /dev/null; then
+            echo "Judgment server ($BASE_URL) is not running properly. Check CloudWatch logs."
+            exit 1
+          else
+            echo "Server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id "$SECRETS_PATH" --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          export JUDGMENT_API_URL="$BASE_URL"
+          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.COVERAGE_ARTIFACT }}
+          path: src/htmlcov
+
+      - name: Save uv cache
+        uses: actions/cache/save@v4
+        if: always() && steps.restore-uv-cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}

judgeval-0.22.3/.github/workflows/claude-code-review.yml

@@ -0,0 +1,35 @@
+name: Claude Code Review
+
+on:
+  issue_comment:
+    types: [created]
+jobs:
+  claude-review:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/claude review')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        id: claude-review
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          direct_prompt: |
+            Please review this pull request and provide feedback on:
+            - Code quality and best practices
+            - Potential bugs or issues
+            - Performance considerations
+            - Security concerns
+            - Test coverage
+            
+            Be constructive and helpful in your feedback.

judgeval-0.22.3/.github/workflows/claude.yml

@@ -0,0 +1,40 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          additional_permissions: |
+            actions: read

{judgeval-0.0.55 → judgeval-0.22.3}/.github/workflows/lint.yaml

@@ -10,20 +10,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
       - name: Install ruff
         uses: astral-sh/ruff-action@v3
         with:
           args: "--version"
 
-      - name: Install mypy and dependencies
-        run: |
-          pip install mypy types-requests types-PyYAML
-
       - name: Run ruff formatter
         if: always()
         run: ruff format --check .
@@ -31,7 +22,3 @@ jobs:
       - name: Run ruff linter
         if: always()
         run: ruff check .
-
-      - name: Run mypy
-        if: always()
-        run: mypy --explicit-package-bases --ignore-missing-imports .

judgeval-0.22.3/.github/workflows/mypy.yaml

@@ -0,0 +1,25 @@
+name: MyPy Check
+
+on:
+  pull_request:
+    branches: [ main, staging ]
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Run mypy
+        if: always()
+        run: uv run mypy ./src/judgeval/

judgeval-0.22.3/.github/workflows/pre-commit-autoupdate.yaml

@@ -0,0 +1,38 @@
+name: Pre-commit auto-update
+on:
+  schedule:
+    - cron: '0 0 * * 1'  # Weekly on Monday at midnight UTC
+  workflow_dispatch:
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: staging
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install and update pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit autoupdate
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: 'chore: update pre-commit hooks'
+          title: 'chore: update pre-commit hooks'
+          body: |
+            Auto-generated PR to update pre-commit hook versions.
+            
+            Please review the changes and merge if everything looks good.
+            
+            Updated by GitHub Actions on {{ date }}.
+          branch: update-pre-commit-hooks
+          base: staging

{judgeval-0.0.55 → judgeval-0.22.3}/.github/workflows/release.yaml

@@ -28,6 +28,18 @@ jobs:
           version=$(curl -s https://pypi.org/pypi/judgeval/json | jq -r .info.version)
           echo "latest_version=$version" >> $GITHUB_OUTPUT
 
+      - name: Determine bump type (minor if commit message starts with [Bump Minor Version], else patch)
+        id: bump_type
+        run: |
+          # Get the latest commit message
+          commit_message=$(git log -1 --pretty=%B)
+          # Default bump type
+          bump_type=patch
+          if [[ "$commit_message" == "[Bump Minor Version]"* ]]; then
+            bump_type=minor
+          fi
+          echo "bump_type=$bump_type" >> $GITHUB_OUTPUT
+
       - name: Bump version and create new tag
         id: bump_tag
         run: |
@@ -37,9 +49,29 @@ jobs:
           # Extract version numbers
           IFS='.' read -r major minor patch <<< "$latest_version"
 
-          # Bump patch version
-          patch=$((patch + 1))
-          new_version="$major.$minor.$patch"
+          # Set major version (manually modify this if you want to change the major version)
+          new_major_version=0
+
+          # Validate that new major version is greater than current major version
+          if [ "$new_major_version" -lt "$major" ]; then
+            echo "Error: New major version ($new_major_version) must be greater than or equal to current major version ($major)"
+            exit 1
+          fi
+
+          if [ "$new_major_version" -ne "$major" ]; then  # If major version changed, set minor and patch to 0
+            echo "Major version bumped, setting minor and patch to 0"
+            minor=0
+            patch=0
+          elif [ "${{ steps.bump_type.outputs.bump_type }}" = "minor" ]; then
+            echo "Minor version bumped, setting patch to 0"
+            minor=$((minor + 1))
+            patch=0
+          else
+            echo "Patch version bumped"
+            patch=$((patch + 1))
+          fi
+
+          new_version="$new_major_version.$minor.$patch"
 
           echo "New version: $new_version"
           echo "new_version=$new_version" >> $GITHUB_OUTPUT
@@ -59,7 +91,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       
-      - name: Bump pyproject.toml version
+      - name: Bump pyproject.toml and version.py version
         run: |
           python update_version.py ${{ steps.bump_tag.outputs.new_version }}
 

{judgeval-0.0.55 → judgeval-0.22.3}/.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.7.14
+    rev: 0.9.7
     hooks:
       - id: uv-lock
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.0
+    rev: v0.14.3
     hooks:
       - id: ruff
         name: ruff (linter)
@@ -14,8 +14,10 @@ repos:
         name: ruff (formatter)
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.16.1
+    rev: v1.18.2
     hooks:
       - id: mypy
-        args: [--explicit-package-bases, --ignore-missing-imports]
-        additional_dependencies: [types-requests, types-PyYAML]
+        language: system
+        # These next two lines allow commits even if mypy fails, REMOVE once we fix all mypy errors
+        verbose: true
+        entry: bash -c 'mypy src/judgeval/ || true'

judgeval-0.22.3/CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contribute to Judgeval
+
+There are many ways to contribute to Judgeval:
+
+- Submit [bug reports](https://github.com/JudgmentLabs/judgeval/issues) and [feature requests](https://github.com/JudgmentLabs/judgeval/issues)
+- Review the documentation and submit [Pull Requests](https://github.com/JudgmentLabs/judgeval/pulls) to improve it
+- Speaking or writing about Judgment and letting us know!
+
+<!-- Contributors collage -->
+[![Contributors](https://contributors-img.web.app/image?repo=JudgmentLabs/judgeval)](https://github.com/JudgmentLabs/judgeval/graphs/contributors)