judgeval 0.13.1__tar.gz → 0.14.1__tar.gz

judgeval-0.14.1/.github/workflows/ci.yaml

@@ -0,0 +1,141 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions: read-all
+
+jobs:
+  validate-branch:
+    uses: ./.github/workflows/merge-branch-check.yaml
+
+  run-tests:
+    needs: [validate-branch]
+    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
+    name: Unit Tests
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      JUDGMENT_DEV: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Run tests
+        run: |
+          cd src
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          uv run pytest tests
+
+  run-e2e-tests:
+    needs: [validate-branch]
+    if: "(github.base_ref == 'staging' || github.base_ref == 'main') && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    name: E2E Tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set env based on branch
+        run: |
+          if [ "${{ github.base_ref }}" = "main" ]; then
+            echo "TARGET_ENV=main" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=prod/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-production-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          else
+            echo "TARGET_ENV=staging" >> "$GITHUB_ENV"
+            echo "BASE_URL=https://staging.api.judgmentlabs.ai" >> "$GITHUB_ENV"
+            echo "SECRETS_PATH=stg/api-keys/e2e-tests" >> "$GITHUB_ENV"
+            echo "COVERAGE_ARTIFACT=coverage-html-staging-${{ matrix.python-version }}" >> "$GITHUB_ENV"
+          fi
+
+      - name: Restore uv cache
+        uses: actions/cache/restore@v4
+        id: restore-uv-cache
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-judgment-
+            ${{ runner.os }}-uv-
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install judgeval dependencies
+        run: |
+          pip install uv
+          uv sync --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s "$BASE_URL/health" > /dev/null; then
+            echo "Judgment server ($BASE_URL) is not running properly. Check CloudWatch logs."
+            exit 1
+          else
+            echo "Server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id "$SECRETS_PATH" --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          export JUDGMENT_API_URL="$BASE_URL"
+          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
+
+      - name: Upload coverage HTML report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.COVERAGE_ARTIFACT }}
+          path: src/htmlcov
+
+      - name: Save uv cache
+        uses: actions/cache/save@v4
+        if: always() && steps.restore-uv-cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/.cache/uv/
+          key: ${{ runner.os }}-uv-judgment-${{ hashFiles('./**/uv.lock') }}

{judgeval-0.13.1 → judgeval-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.13.1
+Version: 0.14.1
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.13.1 → judgeval-0.14.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.13.1"
+version = "0.14.1"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.13.1 → judgeval-0.14.1}/src/judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:18+00:00
+#   timestamp: 2025-09-29T19:54:47+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -54,6 +54,8 @@ class SavePromptScorerRequest(TypedDict):
     threshold: float
     model: NotRequired[str]
     is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
 
 
 class SavePromptScorerResponse(TypedDict):
@@ -143,6 +145,8 @@ class PromptScorer(TypedDict):
     prompt: str
     threshold: float
     model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
     created_at: NotRequired[Optional[str]]
     updated_at: NotRequired[Optional[str]]
     is_trace: NotRequired[Optional[bool]]

{judgeval-0.13.1 → judgeval-0.14.1}/src/judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-09-24T18:25:17+00:00
+#   timestamp: 2025-09-29T19:54:46+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -56,6 +56,8 @@ class SavePromptScorerRequest(BaseModel):
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
 
 
 class SavePromptScorerResponse(BaseModel):
@@ -156,6 +158,8 @@ class PromptScorer(BaseModel):
     prompt: Annotated[str, Field(title="Prompt")]
     threshold: Annotated[float, Field(title="Threshold")]
     model: Annotated[Optional[str], Field(title="Model")] = "gpt-5"
+    options: Annotated[Optional[Dict[str, float]], Field(title="Options")] = None
+    description: Annotated[Optional[str], Field(title="Description")] = None
     created_at: Annotated[Optional[AwareDatetime], Field(title="Created At")] = None
     updated_at: Annotated[Optional[AwareDatetime], Field(title="Updated At")] = None
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False

{judgeval-0.13.1 → judgeval-0.14.1}/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -4,19 +4,21 @@ from judgeval.scorers.api_scorer import (
     TraceAPIScorerConfig,
 )
 from judgeval.constants import APIScorerType
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 from judgeval.api import JudgmentSyncClient
 from judgeval.exceptions import JudgmentAPIError
 import os
 from judgeval.logger import judgeval_logger
 from abc import ABC
 from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from copy import copy
 
 
 def push_prompt_scorer(
     name: str,
     prompt: str,
     threshold: float,
+    options: Optional[Dict[str, float]] = None,
     model: str = JUDGMENT_DEFAULT_GPT_MODEL,
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -29,6 +31,7 @@ def push_prompt_scorer(
                 "name": name,
                 "prompt": prompt,
                 "threshold": threshold,
+                "options": options,
                 "model": model,
                 "is_trace": is_trace,
             }
@@ -98,6 +101,7 @@ def scorer_exists(
 class BasePromptScorer(ABC, APIScorerConfig):
     score_type: APIScorerType
     prompt: str
+    options: Optional[Dict[str, float]] = None
     judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or ""
     organization_id: str = os.getenv("JUDGMENT_ORG_ID") or ""
 
@@ -124,6 +128,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             name=name,
             prompt=scorer_config["prompt"],
             threshold=scorer_config["threshold"],
+            options=scorer_config.get("options"),
             model=scorer_config.get("model"),
             judgment_api_key=judgment_api_key,
             organization_id=organization_id,
@@ -135,6 +140,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         name: str,
         prompt: str,
         threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
         judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "",
         organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "",
@@ -150,6 +156,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name,
                 prompt,
                 threshold,
+                options,
                 model,
                 judgment_api_key,
                 organization_id,
@@ -161,6 +168,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
                 name=name,
                 prompt=prompt,
                 threshold=threshold,
+                options=options,
                 model=model,
                 judgment_api_key=judgment_api_key,
                 organization_id=organization_id,
@@ -199,6 +207,14 @@ class BasePromptScorer(ABC, APIScorerConfig):
         self.push_prompt_scorer()
         judgeval_logger.info(f"Successfully updated model for {self.name}")
 
+    def set_options(self, options: Optional[Dict[str, float]]):
+        """
+        Updates the options of the scorer.
+        """
+        self.options = options
+        self.push_prompt_scorer()
+        judgeval_logger.info(f"Successfully updated options for {self.name}")
+
     def append_to_prompt(self, prompt_addition: str):
         """
         Appends a string to the prompt.
@@ -226,6 +242,12 @@ class BasePromptScorer(ABC, APIScorerConfig):
         """
         return self.model
 
+    def get_options(self) -> Dict[str, float] | None:
+        """
+        Returns the options of the scorer.
+        """
+        return copy(self.options) if self.options is not None else None
+
     def get_name(self) -> str | None:
         """
         Returns the name of the scorer.
@@ -241,6 +263,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             "model": self.model,
             "prompt": self.prompt,
             "threshold": self.threshold,
+            "options": self.options,
         }
 
     def push_prompt_scorer(self):
@@ -251,6 +274,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
             self.name,
             self.prompt,
             self.threshold,
+            self.options,
             self.model,
             self.judgment_api_key,
             self.organization_id,
@@ -258,7 +282,7 @@ class BasePromptScorer(ABC, APIScorerConfig):
         )
 
     def __str__(self):
-        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold})"
+        return f"PromptScorer(name={self.name}, model={self.model}, prompt={self.prompt}, threshold={self.threshold}, options={self.options})"
 
     def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
         base = super().model_dump(*args, **kwargs)

{judgeval-0.13.1 → judgeval-0.14.1}/src/judgeval/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.13.1"
+__version__ = "0.14.1"
 
 
 def get_version() -> str:

judgeval-0.13.1/.github/workflows/ci.yaml

@@ -1,176 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-permissions: read-all
-
-jobs:
-  validate-branch:
-    uses: ./.github/workflows/merge-branch-check.yaml
-
-  run-tests:
-    needs: [validate-branch]
-    if: needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version:
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-    name: Unit Tests
-    runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: "."
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      JUDGMENT_DEV: true
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        run: |
-          pip install uv
-          uv sync --dev
-
-      - name: Run tests
-        run: |
-          cd src
-          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
-          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
-          uv run pytest tests
-
-  run-e2e-tests-staging:
-    needs: [validate-branch]
-    if: "github.base_ref == 'staging' && !contains(github.actor, '[bot]') && (needs.validate-branch.result == 'success' || needs.validate-branch.result == 'skipped')"
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-    name: Staging E2E Tests
-    runs-on: ubuntu-latest
-    env:
-      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-1
-          
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-         
-      - name: Install judgeval dependencies
-        run: |
-          pip install uv
-          uv sync --dev
-
-      - name: Check if server is running
-        run: |
-          if ! curl -s https://staging.api.judgmentlabs.ai/health > /dev/null; then
-            echo "Staging Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
-            exit 1
-          else
-            echo "Staging server is running."
-          fi
-        
-      - name: Run E2E tests
-        working-directory: src
-        run: |      
-          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id stg/api-keys/e2e-tests --query SecretString --output text)
-          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
-          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
-          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
-          export JUDGMENT_API_URL=https://staging.api.judgmentlabs.ai
-          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
-
-      - name: Upload coverage HTML report (staging)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-html-staging-${{ matrix.python-version }}
-          path: src/htmlcov
-
-  run-e2e-tests-main:
-    needs: [validate-branch]
-    if: "github.base_ref == 'main' && !contains(github.actor, '[bot]') && needs.validate-branch.result == 'success'"
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-    name: Production E2E Tests
-    runs-on: ubuntu-latest
-    env:
-      TEST_TIMEOUT_SECONDS: ${{ secrets.TEST_TIMEOUT_SECONDS }}
-    steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-1
-          
-      - name: Checkout code
-        uses: actions/checkout@v4
-        
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-         
-      - name: Install judgeval dependencies
-        run: |
-          pip install uv
-          uv sync --dev
-
-      - name: Check if server is running
-        run: |
-          if ! curl -s https://api.judgmentlabs.ai/health > /dev/null; then
-            echo "Production Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
-            exit 1
-          else
-            echo "Production server is running."
-          fi
-        
-      - name: Run E2E tests
-        working-directory: src
-        run: |      
-          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id prod/api-keys/e2e-tests --query SecretString --output text)
-          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
-          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
-          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
-          export JUDGMENT_API_URL=https://api.judgmentlabs.ai
-          timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
-
-      - name: Upload coverage HTML report (production)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: coverage-html-production-${{ matrix.python-version }}
-          path: src/htmlcov