judgeval 0.8.0__tar.gz → 0.9.0__tar.gz

judgeval-0.9.0/.github/workflows/claude-code-review.yml

@@ -0,0 +1,35 @@
+name: Claude Code Review
+
+on:
+  issue_comment:
+    types: [created]
+jobs:
+  claude-review:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/claude review')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        id: claude-review
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          direct_prompt: |
+            Please review this pull request and provide feedback on:
+            - Code quality and best practices
+            - Potential bugs or issues
+            - Performance considerations
+            - Security concerns
+            - Test coverage
+            
+            Be constructive and helpful in your feedback.

judgeval-0.9.0/.github/workflows/claude.yml

@@ -0,0 +1,40 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          additional_permissions: |
+            actions: read

{judgeval-0.8.0 → judgeval-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.8.0
+Version: 0.9.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -10,27 +10,25 @@ License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
-Requires-Dist: boto3
+Requires-Dist: boto3>=1.40.11
 Requires-Dist: click<8.2.0
-Requires-Dist: fireworks-ai>=0.19.18
-Requires-Dist: langchain-anthropic
-Requires-Dist: langchain-core
-Requires-Dist: langchain-huggingface
-Requires-Dist: langchain-openai
-Requires-Dist: litellm>=1.61.15
-Requires-Dist: nest-asyncio>=1.6.0
-Requires-Dist: opentelemetry-api>=1.34.1
-Requires-Dist: opentelemetry-sdk>=1.34.1
+Requires-Dist: dotenv
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: litellm<1.75.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.36.0
+Requires-Dist: opentelemetry-sdk>=1.36.0
+Requires-Dist: opentelemetry-semantic-conventions>=0.57b0
 Requires-Dist: orjson>=3.9.0
-Requires-Dist: python-dotenv
-Requires-Dist: requests
-Requires-Dist: rich
 Requires-Dist: typer>=0.9.0
 Provides-Extra: langchain
 Requires-Dist: langchain-anthropic; extra == 'langchain'
 Requires-Dist: langchain-core; extra == 'langchain'
 Requires-Dist: langchain-huggingface; extra == 'langchain'
 Requires-Dist: langchain-openai; extra == 'langchain'
+Provides-Extra: s3
+Requires-Dist: boto3>=1.40.11; extra == 's3'
+Provides-Extra: trainer
+Requires-Dist: fireworks-ai>=0.19.18; extra == 'trainer'
 Description-Content-Type: text/markdown
 
 <div align="center">

judgeval-0.9.0/pyproject.toml

@@ -0,0 +1,94 @@
+[project]
+name = "judgeval"
+version = "0.9.0"
+authors = [
+    { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
+    { name = "Alex Shan", email = "alex@judgmentlabs.ai" },
+    { name = "Joseph Camyre", email = "joseph@judgmentlabs.ai" },
+]
+description = "Judgeval Package"
+readme = "README.md"
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "Apache-2.0"
+license-files = ["LICENSE.md"]
+
+dependencies = [
+    "dotenv",
+    "httpx>=0.28.1",
+    "litellm<1.75.0", # https://github.com/BerriAI/litellm/issues/13081
+    "opentelemetry-exporter-otlp>=1.36.0",
+    "opentelemetry-sdk>=1.36.0",
+    "opentelemetry-semantic-conventions>=0.57b0",
+    "orjson>=3.9.0",
+    "click<8.2.0",
+    "typer>=0.9.0",
+    "boto3>=1.40.11",
+]
+
+[project.urls]
+Homepage = "https://github.com/JudgmentLabs/judgeval"
+Issues = "https://github.com/JudgmentLabs/judgeval/issues"
+
+[project.scripts]
+judgeval = "judgeval.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/judgeval"]
+include = ["/src/judgeval", "/src/judgeval/**/*.py"]
+
+[project.optional-dependencies]
+langchain = [
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
+]
+s3 = ["boto3>=1.40.11"]
+trainer = ["fireworks-ai>=0.19.18"]
+
+
+[dependency-groups]
+dev = [
+    "anthropic>=0.61.0",
+    "boto3-stubs[s3]>=1.40.11",
+    "datamodel-code-generator>=0.32.0",
+    "google-genai>=1.28.0",
+    "groq>=0.30.0",
+    "langchain-core>=0.3.72",
+    "langgraph>=0.6.4",
+    "mypy>=1.17.1",
+    "openai>=1.78.1",
+    "opentelemetry-instrumentation-openai>=0.44.1",
+    "ruff>=0.9.1,<0.10.0",
+    "together>=1.5.21",
+    "types-pyyaml>=6.0.12.20250516",
+    "pre-commit>=4.2.0",
+    "pytest>=8.4.1",
+    "pytest-cov>=6.2.1",
+    "types-tqdm>=4.67.0.20250809",
+    "pytest-asyncio>=1.1.0",
+]
+
+
+[tool.hatch.build]
+directory = "dist"
+artifacts = ["src/judgeval/**/*.py"]
+exclude = ["src/e2etests/*", "src/tests/*", "src/demo/*"]
+
+[tool.ruff]
+exclude = ["docs"]
+
+[tool.ruff.lint]
+ignore = [
+    "F403",
+    "F405",
+    "E402",
+] # F403: star import, F405: undefined name from star import

judgeval-0.9.0/scripts/api_generator.py

@@ -0,0 +1,360 @@
+from __future__ import annotations
+
+import orjson
+import sys
+from typing import Any, Dict, List, Optional
+import httpx
+import re
+
+spec_file = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000/openapi.json"
+
+if spec_file.startswith("http"):
+    r = httpx.get(spec_file)
+    r.raise_for_status()
+    SPEC = r.json()
+else:
+    with open(spec_file, "rb") as f:
+        SPEC = orjson.loads(f.read())
+
+JUDGEVAL_PATHS: List[str] = [
+    "/traces/spans/batch/",
+    "/traces/evaluation_runs/batch/",
+    "/traces/fetch/",
+    "/traces/upsert/",
+    "/traces/add_to_dataset/",
+    "/projects/add/",
+    "/projects/delete_from_judgeval/",
+    "/evaluate/traces",
+    "/evaluate/examples",
+    "/evaluate_trace/",
+    "/log_eval_results/",
+    "/fetch_experiment_run/",
+    "/add_to_run_eval_queue/examples",
+    "/add_to_run_eval_queue/traces",
+    "/get_evaluation_status/",
+    "/save_scorer/",
+    "/fetch_scorer/",
+    "/scorer_exists/",
+    "/upload_custom_scorer/",
+    "/datasets/push/",
+    "/datasets/insert_examples/",
+    "/datasets/pull_for_judgeval/",
+    "/datasets/fetch_stats_by_project/",
+    "/projects/resolve/",
+    "/e2e_fetch_trace/",
+    "/e2e_fetch_span_score/",
+]
+
+
+def resolve_ref(ref: str) -> str:
+    assert ref.startswith("#/components/schemas/"), (
+        "Reference must start with #/components/schemas/"
+    )
+    return ref.replace("#/components/schemas/", "")
+
+
+def to_snake_case(name: str) -> str:
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower()
+
+
+def get_method_name_from_path(path: str, method: str) -> str:
+    return path.strip("/").replace("/", "_").replace("-", "_")
+
+
+def get_query_parameters(operation: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extract query parameters from the operation."""
+    parameters = operation.get("parameters", [])
+    query_params = []
+
+    for param in parameters:
+        if param.get("in") == "query":
+            param_info = {
+                "name": param["name"],
+                "required": param.get("required", False),
+                "type": param.get("schema", {}).get("type", "str"),
+            }
+            query_params.append(param_info)
+
+    return query_params
+
+
+def get_request_schema(operation: Dict[str, Any]) -> Optional[str]:
+    request_body = operation.get("requestBody", {})
+    if not request_body:
+        return None
+
+    content = request_body.get("content", {})
+    if "application/json" in content:
+        schema = content["application/json"].get("schema", {})
+        if "$ref" in schema:
+            return resolve_ref(schema["$ref"])
+
+    return None
+
+
+def get_response_schema(operation: Dict[str, Any]) -> Optional[str]:
+    responses = operation.get("responses", {})
+    for status_code in ["200", "201"]:
+        if status_code in responses:
+            response = responses[status_code]
+            content = response.get("content", {})
+            if "application/json" in content:
+                schema = content["application/json"].get("schema", {})
+                if "$ref" in schema:
+                    return resolve_ref(schema["$ref"])
+
+    return None
+
+
+def generate_method_signature(
+    method_name: str,
+    request_type: Optional[str],
+    query_params: List[Dict[str, Any]],
+    response_type: str,
+    is_async: bool = False,
+) -> str:
+    async_prefix = "async " if is_async else ""
+
+    params = ["self"]
+
+    # Add required query parameters first
+    for param in query_params:
+        if param["required"]:
+            param_name = param["name"]
+            param_type = "str"  # Default to str for simplicity
+            params.append(f"{param_name}: {param_type}")
+
+    # Add request body parameter if it exists
+    if request_type:
+        params.append(f"payload: {request_type}")
+
+    # Add optional query parameters last
+    for param in query_params:
+        if not param["required"]:
+            param_name = param["name"]
+            param_type = "str"  # Default to str for simplicity
+            params.append(f"{param_name}: Optional[{param_type}] = None")
+
+    params_str = ", ".join(params)
+    return f"{async_prefix}def {method_name}({params_str}) -> {response_type}:"
+
+
+def generate_method_body(
+    method_name: str,
+    path: str,
+    method: str,
+    request_type: Optional[str],
+    query_params: List[Dict[str, Any]],
+    is_async: bool = False,
+) -> str:
+    async_prefix = "await " if is_async else ""
+
+    # Build query parameters dict if they exist
+    if query_params:
+        query_lines = ["query_params = {}"]
+        for param in query_params:
+            param_name = param["name"]
+            if param["required"]:
+                query_lines.append(f"query_params['{param_name}'] = {param_name}")
+            else:
+                query_lines.append(f"if {param_name} is not None:")
+                query_lines.append(f"    query_params['{param_name}'] = {param_name}")
+        query_setup = "\n        ".join(query_lines)
+        query_param = "query_params"
+    else:
+        query_setup = ""
+        query_param = "{}"
+
+    if method == "GET":
+        if query_setup:
+            return f'{query_setup}\n        return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            {query_param},\n        )'
+        else:
+            return f'return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            {{}},\n        )'
+    else:
+        if request_type:
+            if query_setup:
+                return f'{query_setup}\n        return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            payload,\n            params={query_param},\n        )'
+            else:
+                return f'return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            payload,\n        )'
+        else:
+            if query_setup:
+                return f'{query_setup}\n        return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            {{}},\n            params={query_param},\n        )'
+            else:
+                return f'return {async_prefix}self._request(\n            "{method}",\n            url_for("{path}"),\n            {{}},\n        )'
+
+
+def generate_client_class(
+    class_name: str, methods: List[Dict[str, Any]], is_async: bool = False
+) -> str:
+    lines = [f"class {class_name}:"]
+    lines.append('    __slots__ = ("api_key", "organization_id", "client")')
+    lines.append("")
+
+    lines.append("    def __init__(self, api_key: str, organization_id: str):")
+    lines.append("        self.api_key = api_key")
+    lines.append("        self.organization_id = organization_id")
+    client_type = "httpx.AsyncClient" if is_async else "httpx.Client"
+    lines.append(f"        self.client = {client_type}(timeout=30)")
+    lines.append("")
+
+    request_method = "async def _request" if is_async else "def _request"
+    lines.append(f"    {request_method}(")
+    lines.append(
+        '        self, method: Literal["POST", "PATCH", "GET", "DELETE"], url: str, payload: Any, params: Optional[Dict[str, Any]] = None'
+    )
+    lines.append("    ) -> Any:")
+    lines.append('        if method == "GET":')
+    lines.append("            r = self.client.request(")
+    lines.append("                method,")
+    lines.append("                url,")
+    lines.append("                params=payload if params is None else params,")
+    lines.append(
+        "                headers=_headers(self.api_key, self.organization_id),"
+    )
+    lines.append("            )")
+    lines.append("        else:")
+    lines.append("            r = self.client.request(")
+    lines.append("                method,")
+    lines.append("                url,")
+    lines.append("                json=json_encoder(payload),")
+    lines.append("                params=params,")
+    lines.append(
+        "                headers=_headers(self.api_key, self.organization_id),"
+    )
+    lines.append("            )")
+    if is_async:
+        lines.append("        return _handle_response(await r)")
+    else:
+        lines.append("        return _handle_response(r)")
+    lines.append("")
+
+    for method_info in methods:
+        method_name = method_info["name"]
+        path = method_info["path"]
+        http_method = method_info["method"]
+        request_type = method_info["request_type"]
+        query_params = method_info["query_params"]
+        response_type = method_info["response_type"]
+
+        signature = generate_method_signature(
+            method_name, request_type, query_params, response_type, is_async
+        )
+        lines.append(f"    {signature}")
+
+        body = generate_method_body(
+            method_name, path, http_method, request_type, query_params, is_async
+        )
+        lines.append(f"        {body}")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_api_file() -> str:
+    lines = [
+        "from typing import List, Dict, Any, Mapping, Literal, Optional",
+        "import httpx",
+        "from httpx import Response",
+        "from judgeval.exceptions import JudgmentAPIError",
+        "from judgeval.utils.url import url_for",
+        "from judgeval.utils.serialize import json_encoder",
+        "from judgeval.api.api_types import *",
+        "",
+        "",
+        "def _headers(api_key: str, organization_id: str) -> Mapping[str, str]:",
+        "    return {",
+        '        "Content-Type": "application/json",',
+        '        "Authorization": f"Bearer {api_key}",',
+        '        "X-Organization-Id": organization_id,',
+        "    }",
+        "",
+        "",
+        "def _handle_response(r: Response) -> Any:",
+        "    if r.status_code >= 400:",
+        "        try:",
+        '            detail = r.json().get("detail", "")',
+        "        except Exception:",
+        "            detail = r.text",
+        "        raise JudgmentAPIError(r.status_code, detail, r)",
+        "    return r.json()",
+        "",
+        "",
+    ]
+
+    filtered_paths = {
+        path: spec_data
+        for path, spec_data in SPEC["paths"].items()
+        if path in JUDGEVAL_PATHS
+    }
+
+    for path in JUDGEVAL_PATHS:
+        if path not in SPEC["paths"]:
+            print(f"Path {path} not found in OpenAPI spec", file=sys.stderr)
+
+    sync_methods = []
+    async_methods = []
+
+    for path, path_data in filtered_paths.items():
+        for method, operation in path_data.items():
+            if method.upper() in ["GET", "POST", "PUT", "PATCH", "DELETE"]:
+                method_name = get_method_name_from_path(path, method.upper())
+                request_schema = get_request_schema(operation)
+                response_schema = get_response_schema(operation)
+                query_params = get_query_parameters(operation)
+
+                print(
+                    method_name,
+                    request_schema,
+                    response_schema,
+                    query_params,
+                    file=sys.stderr,
+                )
+
+                if not request_schema:
+                    print(f"No request type found for {method_name}", file=sys.stderr)
+
+                if not response_schema:
+                    print(
+                        f"No response schema found for {method_name}", file=sys.stderr
+                    )
+
+                request_type = request_schema if request_schema else None
+                response_type = response_schema if response_schema else "Any"
+
+                method_info = {
+                    "name": method_name,
+                    "path": path,
+                    "method": method.upper(),
+                    "request_type": request_type,
+                    "query_params": query_params,
+                    "response_type": response_type,
+                }
+
+                sync_methods.append(method_info)
+                async_methods.append(method_info)
+
+    sync_client = generate_client_class(
+        "JudgmentSyncClient", sync_methods, is_async=False
+    )
+    async_client = generate_client_class(
+        "JudgmentAsyncClient", async_methods, is_async=True
+    )
+
+    lines.append(sync_client)
+    lines.append("")
+    lines.append("")
+    lines.append(async_client)
+    lines.append("")
+    lines.append("")
+    lines.append("__all__ = [")
+    lines.append('    "JudgmentSyncClient",')
+    lines.append('    "JudgmentAsyncClient",')
+    lines.append("]")
+
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    api_code = generate_api_file()
+    print(api_code)