judgeval 0.0.39__tar.gz → 0.0.41__tar.gz

judgeval-0.0.41/.github/workflows/blocked-pr.yaml

@@ -0,0 +1,19 @@
+name: Check Blocked PR
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+
+jobs:
+  fail-for-blocked:
+    if: contains(github.event.pull_request.labels.*.name, 'Blocked')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if PR is blocked
+        run: |
+          echo "This PR is currently blocked. Please unblock it before merging."
+          exit 1

judgeval-0.0.41/.github/workflows/ci-staging.yaml

@@ -0,0 +1,103 @@
+
+name: Staging CI Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - staging
+
+permissions: read-all
+
+jobs:
+  run-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.11"
+    name: Test
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      JUDGMENT_DEV: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install pipenv
+          pipenv install --dev
+          
+
+      - name: Run tests
+        run: |
+          cd src
+          pipenv run pytest tests
+
+  run-e2e-tests-staging:
+    if: "!contains(github.actor, '[bot]')"  # Exclude if the actor is a bot
+    name: Staging E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Wait for turn
+        uses: softprops/turnstyle@v2
+        with:
+          poll-interval-seconds: 10
+          same-branch-only: false
+          job-to-wait-for: "Staging E2E Tests"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+          
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+         
+      - name: Install judgeval dependencies
+        run: |
+          pip install pipenv
+          pipenv install --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s https://staging.api.judgmentlabs.ai/health > /dev/null; then
+            echo "Staging Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
+            exit 1
+          else
+            echo "Staging server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |      
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-stg-judgeval/api-keys/judgeval --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          timeout ${TEST_TIMEOUT_SECONDS}s pipenv run pytest --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html
+          path: src/htmlcov

{judgeval-0.0.39 → judgeval-0.0.41}/.github/workflows/ci.yaml

@@ -1,3 +1,4 @@
+
 name: CI Tests
 
 on:
@@ -48,6 +49,8 @@ jobs:
     if: "!contains(github.actor, '[bot]')"  # Exclude if the actor is a bot
     name: E2E Tests
     runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
     steps:
       - name: Wait for turn
         uses: softprops/turnstyle@v2
@@ -78,7 +81,7 @@ jobs:
 
       - name: Check if server is running
         run: |
-          if ! curl -s http://api.judgmentlabs.ai/health > /dev/null; then
+          if ! curl -s https://api.judgmentlabs.ai/health > /dev/null; then
             echo "Production Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
             exit 1
           else
@@ -88,6 +91,13 @@ jobs:
       - name: Run E2E tests
         working-directory: src
         run: |      
-          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions/api-keys/judgeval --query SecretString --output text)
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-judgeval/api-keys/judgeval --query SecretString --output text)
           export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
-          pipenv run pytest --durations=0 ./e2etests
+          timeout ${TEST_TIMEOUT_SECONDS}s pipenv run pytest --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html
+          path: src/htmlcov

judgeval-0.0.41/.github/workflows/merge-to-main.yaml

@@ -0,0 +1,32 @@
+name: Enforce Main Branch Protection
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+
+jobs:
+  validate-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check branch name
+        run: |
+          # Get the base and source branch names
+          BASE_BRANCH="${{ github.base_ref }}"
+          SOURCE_BRANCH="${{ github.head_ref }}"
+
+          echo "BASE_BRANCH: $BASE_BRANCH"
+          echo "SOURCE_BRANCH: $SOURCE_BRANCH"
+          
+          # Only run validation if the base branch is main
+          if [[ "$BASE_BRANCH" != "main" ]]; then
+            echo "Skipping branch validation - not targeting main branch"
+            exit 0
+          fi
+          
+          # Check if the source branch is staging or starts with hotfix/
+          if [[ "$SOURCE_BRANCH" != "staging" && ! "$SOURCE_BRANCH" =~ ^hotfix/ ]]; then
+            echo "::error::Pull requests to main can only be created from 'staging' or 'hotfix/*' branches. Current branch: $SOURCE_BRANCH"
+            exit 1
+          fi
+          
+          echo "Branch validation passed. Source branch: $SOURCE_BRANCH"

judgeval-0.0.41/.github/workflows/release.yaml

@@ -0,0 +1,92 @@
+name: Release on Main Merge
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    outputs:
+      new_version: ${{ steps.bump_tag.outputs.new_version }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Get latest version
+        id: get_version
+        run: |
+          version=$(curl -s https://pypi.org/pypi/judgeval/json | jq -r .info.version)
+          echo "latest_version=$version" >> $GITHUB_OUTPUT
+
+      - name: Bump version and create new tag
+        id: bump_tag
+        run: |
+          latest_version=${{ steps.get_version.outputs.latest_version }}
+          echo "Latest version: $latest_version"
+
+          # Extract version numbers
+          IFS='.' read -r major minor patch <<< "$latest_version"
+
+          # Bump patch version
+          patch=$((patch + 1))
+          new_version="$major.$minor.$patch"
+
+          echo "New version: $new_version"
+          echo "new_version=$new_version" >> $GITHUB_OUTPUT
+
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          git tag v$new_version
+          git push origin v$new_version
+
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.bump_tag.outputs.new_version }}
+          generate_release_notes: true
+          body: |
+            You can find this package release on PyPI: https://pypi.org/project/judgeval/${{ steps.bump_tag.outputs.new_version }}/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Bump pyproject.toml version
+        run: |
+          python update_version.py ${{ steps.bump_tag.outputs.new_version }}
+
+      - name: Build PyPI package
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+
+      - name: Create PyPI release
+        run: |
+          python -m pip install --upgrade twine
+          python -m twine upload --repository pypi -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/*
+
+  cleanup:
+    needs: release
+    if: failure()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Authenticate GitHub CLI
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
+
+      - name: Delete tag and release
+        run: |
+          gh release delete v${{ needs.release.outputs.new_version }} --yes
+          git push --delete origin v${{ needs.release.outputs.new_version }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}