judgeval 0.0.29__tar.gz → 0.0.34__tar.gz

judgeval-0.0.34/.github/pull_request_template.md

@@ -0,0 +1,31 @@
+## 📝 Summary
+
+<!-- Provide a brief description of the changes introduced by this PR -->
+
+## 🎯 Purpose
+
+<!-- Explain the motivation behind these changes. Why are they necessary? -->
+
+## 🎥 Demo of Changes
+
+<!-- Add a short 1-3 minute video describing/demoing the changes -->
+
+## 🧪 Testing
+
+<!-- Describe how the changes were tested (unit/manual) -->
+
+## ✅ Checklist
+
+- [ ] Self-review
+- [ ] Video demo of changes
+- [ ] Unit Tests and CI/CD tests are passing
+- [ ] Reviewers assigned
+
+
+## 📌 Linear Issue
+
+<!-- Reference to associated Linear ticket, e.g., ABC-123 -->
+
+## ✏️ Additional Notes
+
+<!-- Any additional information that doesn't fit into the other sections -->

judgeval-0.0.34/.github/workflows/ci.yaml

@@ -0,0 +1,91 @@
+name: CI
+
+on:
+  pull_request_review:
+    types: [submitted]
+    branches:
+      - main
+
+jobs:
+  run-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version:
+          - "3.11"
+    name: Test
+    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: "."
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install pipenv
+          pipenv install --dev
+          
+
+      - name: Run tests
+        run: |
+          cd src
+          pipenv run pytest
+
+  run-e2e-tests:
+    if: "!contains(github.actor, '[bot]')"  # Exclude if the actor is a bot
+    concurrency:
+      group: e2e-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version:
+          - "3.11"
+    name: E2E Tests
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+          
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+         
+      - name: Install judgeval dependencies
+        run: |
+          pip install pipenv
+          pipenv install --dev
+
+      - name: Check if server is running
+        run: |
+          if ! curl -s http://api.judgmentlabs.ai/health > /dev/null; then
+            echo "Production Judgment server is not running properly. Check logs on AWS CloudWatch for more details."
+            exit 1
+          else
+            echo "Server is running."
+          fi
+        
+      - name: Run E2E tests
+        working-directory: src
+        run: |      
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions/api-keys/judgeval --query SecretString --output text)
+          export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          pipenv run pytest ./e2etests

{judgeval-0.0.29 → judgeval-0.0.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.29
+Version: 0.0.34
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -12,12 +12,13 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Requires-Dist: anthropic
 Requires-Dist: fastapi
+Requires-Dist: google-genai
 Requires-Dist: langchain
 Requires-Dist: langchain-anthropic
 Requires-Dist: langchain-core
 Requires-Dist: langchain-huggingface
 Requires-Dist: langchain-openai
-Requires-Dist: litellm
+Requires-Dist: litellm==1.38.12
 Requires-Dist: nest-asyncio
 Requires-Dist: openai
 Requires-Dist: openpyxl
@@ -94,9 +95,21 @@ Create a file named `traces.py` with the following code:
     from judgeval.common.tracer import Tracer, wrap
     from openai import OpenAI
 
+    # Basic initialization
     client = wrap(OpenAI())
     judgment = Tracer(project_name="my_project")
 
+    # Or with S3 storage enabled
+    # NOTE: Make sure AWS creds correspond to an account with write access to the specified S3 bucket
+    judgment = Tracer(
+        project_name="my_project",
+        use_s3=True,
+        s3_bucket_name="my-traces-bucket", # Bucket created automatically if it doesn't exist
+        s3_aws_access_key_id="your-access-key",  # Optional: defaults to AWS_ACCESS_KEY_ID env var
+        s3_aws_secret_access_key="your-secret-key",  # Optional: defaults to AWS_SECRET_ACCESS_KEY env var
+        s3_region_name="us-west-1"  # Optional: defaults to AWS_REGION env var or "us-west-1"
+    )
+
     @judgment.observe(span_type="tool")
     def my_tool():
         return "Hello world!"

{judgeval-0.0.29 → judgeval-0.0.34}/Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-litellm = "*"
+litellm = "==1.38.12"
 python-dotenv = "==1.0.1"
 fastapi = "*"
 uvicorn = "*"
@@ -24,6 +24,9 @@ langchain-openai = "*"
 langchain-anthropic = "*"
 langchain-core = "*"
 langchain-community = "*"
+langgraph = "*"
+google-genai = "*"
+boto3 = "*"
 
 [dev-packages]
 pytest = "*"