judgeval 0.9.4__tar.gz → 0.10.0__tar.gz

{judgeval-0.9.4 → judgeval-0.10.0}/.github/workflows/ci.yaml

@@ -97,8 +97,11 @@ jobs:
       - name: Run E2E tests
         working-directory: src
         run: |      
-          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-stg-judgeval/api-keys/judgeval --query SecretString --output text)
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id stg/api-keys/e2e-tests --query SecretString --output text)
           export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          export JUDGMENT_API_URL=https://staging.api.judgmentlabs.ai
           timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
 
       - name: Upload coverage HTML report (staging)
@@ -156,8 +159,11 @@ jobs:
       - name: Run E2E tests
         working-directory: src
         run: |      
-          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id gh-actions-judgeval/api-keys/judgeval --query SecretString --output text)
+          SECRET_VARS=$(aws secretsmanager get-secret-value --secret-id prod/api-keys/e2e-tests --query SecretString --output text)
           export $(echo "$SECRET_VARS" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"')
+          export JUDGMENT_API_KEY="$JUDGEVAL_GH_JUDGMENT_API_KEY"
+          export JUDGMENT_ORG_ID="$JUDGEVAL_GH_JUDGMENT_ORG_ID"
+          export JUDGMENT_API_URL=https://api.judgmentlabs.ai
           timeout ${TEST_TIMEOUT_SECONDS}s uv run pytest -n auto --dist=loadfile --durations=0 --cov=. --cov-config=.coveragerc --cov-report=html ./e2etests
 
       - name: Upload coverage HTML report (production)

{judgeval-0.9.4 → judgeval-0.10.0}/.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.8.0
+    rev: 0.8.15
     hooks:
       - id: uv-lock
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.4
+    rev: v0.12.12
     hooks:
       - id: ruff
         name: ruff (linter)

{judgeval-0.9.4 → judgeval-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.9.4
+Version: 0.10.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.9.4 → judgeval-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.9.4"
+version = "0.10.0"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.9.4 → judgeval-0.10.0}/scripts/api_generator.py

@@ -36,13 +36,13 @@ JUDGEVAL_PATHS: List[str] = [
     "/fetch_scorer/",
     "/scorer_exists/",
     "/upload_custom_scorer/",
-    "/datasets/push/",
-    "/datasets/insert_examples/",
+    "/datasets/create_for_judgeval/",
+    "/datasets/insert_examples_for_judgeval/",
     "/datasets/pull_for_judgeval/",
-    "/datasets/fetch_stats_by_project/",
     "/projects/resolve/",
     "/e2e_fetch_trace/",
     "/e2e_fetch_span_score/",
+    "/e2e_fetch_trace_scorer_span_score/",
 ]
 
 
@@ -253,7 +253,7 @@ def generate_client_class(
 
 def generate_api_file() -> str:
     lines = [
-        "from typing import List, Dict, Any, Mapping, Literal, Optional",
+        "from typing import Dict, Any, Mapping, Literal, Optional",
         "import httpx",
         "from httpx import Response",
         "from judgeval.exceptions import JudgmentAPIError",

{judgeval-0.9.4 → judgeval-0.10.0}/scripts/openapi_transform.py

@@ -35,10 +35,9 @@ JUDGEVAL_PATHS: List[str] = [
     "/fetch_scorer/",
     "/scorer_exists/",
     "/upload_custom_scorer/",
-    "/datasets/push/",
-    "/datasets/insert_examples/",
+    "/datasets/create_for_judgeval/",
+    "/datasets/insert_examples_for_judgeval/",
     "/datasets/pull_for_judgeval/",
-    "/datasets/fetch_stats_by_project/",
     "/projects/resolve/",
     "/e2e_fetch_trace/",
     "/e2e_fetch_span_score/",

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/__init__.py

@@ -6,7 +6,7 @@ from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
 from typing import List, Optional, Union
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import BaseScorer, ExampleAPIScorerConfig
 from judgeval.data.example import Example
 from judgeval.logger import judgeval_logger
 from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_DEFAULT_GPT_MODEL, JUDGMENT_ORG_ID
@@ -38,7 +38,7 @@ class JudgmentClient(metaclass=SingletonMeta):
     def run_evaluation(
         self,
         examples: List[Example],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        scorers: List[Union[ExampleAPIScorerConfig, BaseScorer]],
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/api/__init__.py

@@ -71,13 +71,6 @@ class JudgmentSyncClient:
             payload,
         )
 
-    def evaluate_trace(self, payload: TraceRun) -> Any:
-        return self._request(
-            "POST",
-            url_for("/evaluate_trace/"),
-            payload,
-        )
-
     def evaluate_examples(
         self, payload: ExampleEvaluationRun, stream: Optional[str] = None
     ) -> Any:
@@ -128,59 +121,26 @@ class JudgmentSyncClient:
             query_params,
         )
 
-    def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+    def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
         return self._request(
             "POST",
-            url_for("/datasets/insert_examples/"),
+            url_for("/datasets/insert_examples_for_judgeval/"),
             payload,
         )
 
-    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
         return self._request(
             "POST",
             url_for("/datasets/pull_for_judgeval/"),
             payload,
         )
 
-    def datasets_push(self, payload: DatasetPush) -> Any:
+    def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return self._request(
             "POST",
-            url_for("/datasets/push/"),
-            payload,
-        )
-
-    def traces_upsert(self, payload: TraceSave) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/upsert/"),
-            payload,
-        )
-
-    def traces_fetch(self, payload: TraceFetch) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/fetch/"),
-            payload,
-        )
-
-    def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/add_to_dataset/"),
-            payload,
-        )
-
-    def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/spans/batch/"),
-            payload,
-        )
-
-    def traces_evaluation_runs_batch(self, payload: EvaluationRunsBatchRequest) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/evaluation_runs/batch/"),
+            url_for("/datasets/create_for_judgeval/"),
             payload,
         )
 
@@ -255,6 +215,13 @@ class JudgmentSyncClient:
             payload,
         )
 
+    def e2e_fetch_trace_scorer_span_score(self, payload: SpanScoreRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_trace_scorer_span_score/"),
+            payload,
+        )
+
 
 class JudgmentAsyncClient:
     __slots__ = ("api_key", "organization_id", "client")
@@ -304,13 +271,6 @@ class JudgmentAsyncClient:
             payload,
         )
 
-    async def evaluate_trace(self, payload: TraceRun) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/evaluate_trace/"),
-            payload,
-        )
-
     async def evaluate_examples(
         self, payload: ExampleEvaluationRun, stream: Optional[str] = None
     ) -> Any:
@@ -363,61 +323,26 @@ class JudgmentAsyncClient:
             query_params,
         )
 
-    async def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+    async def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
         return await self._request(
             "POST",
-            url_for("/datasets/insert_examples/"),
+            url_for("/datasets/insert_examples_for_judgeval/"),
             payload,
         )
 
-    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
         return await self._request(
             "POST",
             url_for("/datasets/pull_for_judgeval/"),
             payload,
         )
 
-    async def datasets_push(self, payload: DatasetPush) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/datasets/push/"),
-            payload,
-        )
-
-    async def traces_upsert(self, payload: TraceSave) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/upsert/"),
-            payload,
-        )
-
-    async def traces_fetch(self, payload: TraceFetch) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/fetch/"),
-            payload,
-        )
-
-    async def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/add_to_dataset/"),
-            payload,
-        )
-
-    async def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/spans/batch/"),
-            payload,
-        )
-
-    async def traces_evaluation_runs_batch(
-        self, payload: EvaluationRunsBatchRequest
-    ) -> Any:
+    async def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return await self._request(
             "POST",
-            url_for("/traces/evaluation_runs/batch/"),
+            url_for("/datasets/create_for_judgeval/"),
             payload,
         )
 
@@ -494,6 +419,13 @@ class JudgmentAsyncClient:
             payload,
         )
 
+    async def e2e_fetch_trace_scorer_span_score(self, payload: SpanScoreRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_trace_scorer_span_score/"),
+            payload,
+        )
+
 
 __all__ = [
     "JudgmentSyncClient",

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/api/api_types.py

@@ -1,9 +1,9 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-08-29T04:49:39+00:00
+#   timestamp: 2025-09-10T17:42:12+00:00
 
 from __future__ import annotations
-from typing import Any, Dict, List, Optional, TypedDict, Union
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 from typing_extensions import NotRequired
 
 
@@ -16,40 +16,10 @@ class EvalResultsFetch(TypedDict):
 
 
 class DatasetFetch(TypedDict):
-    dataset_alias: str
+    dataset_name: str
     project_name: str
 
 
-class TraceSave(TypedDict):
-    project_name: str
-    trace_id: str
-    name: str
-    created_at: str
-    duration: float
-    offline_mode: NotRequired[bool]
-    has_notification: NotRequired[bool]
-    customer_id: NotRequired[Optional[str]]
-    tags: NotRequired[List[str]]
-    metadata: NotRequired[Dict[str, Any]]
-    update_id: NotRequired[int]
-
-
-class TraceFetch(TypedDict):
-    trace_id: str
-
-
-class TraceAddToDataset(TypedDict):
-    trace_id: str
-    trace_span_id: str
-    dataset_alias: str
-    project_name: str
-
-
-class EvaluationRunsBatchRequest(TypedDict):
-    organization_id: str
-    evaluation_entries: List[Dict[str, Any]]
-
-
 class ProjectAdd(TypedDict):
     project_name: str
 
@@ -149,8 +119,8 @@ class ScorerConfig(TypedDict):
 
 
 class Example(TypedDict):
-    example_id: str
-    created_at: str
+    example_id: NotRequired[str]
+    created_at: NotRequired[str]
     name: NotRequired[Optional[str]]
 
 
@@ -160,28 +130,7 @@ class ValidationError(TypedDict):
     type: str
 
 
-class SpanBatchItem(TypedDict):
-    span_id: str
-    trace_id: str
-    function: str
-    created_at: NotRequired[Any]
-    parent_span_id: NotRequired[Optional[str]]
-    span_type: NotRequired[Optional[str]]
-    inputs: NotRequired[Optional[Dict[str, Any]]]
-    output: NotRequired[Any]
-    error: NotRequired[Optional[Dict[str, Any]]]
-    usage: NotRequired[Optional[Dict[str, Any]]]
-    duration: NotRequired[Optional[float]]
-    expected_tools: NotRequired[Optional[List[Dict[str, Any]]]]
-    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
-    has_evaluation: NotRequired[Optional[bool]]
-    agent_name: NotRequired[Optional[str]]
-    class_name: NotRequired[Optional[str]]
-    state_before: NotRequired[Optional[Dict[str, Any]]]
-    state_after: NotRequired[Optional[Dict[str, Any]]]
-    span_state: str
-    update_id: NotRequired[int]
-    queued_at: float
+DatasetKind = Literal["trace", "example"]
 
 
 class PromptScorer(TypedDict):
@@ -195,36 +144,45 @@ class PromptScorer(TypedDict):
 
 
 class ScorerData(TypedDict):
+    id: NotRequired[str]
     name: str
     threshold: float
     success: bool
     score: NotRequired[Optional[float]]
     reason: NotRequired[Optional[str]]
     strict_mode: NotRequired[Optional[bool]]
-    evaluation_model: NotRequired[Union[List[str], str]]
+    evaluation_model: NotRequired[str]
     error: NotRequired[Optional[str]]
     additional_metadata: NotRequired[Optional[Dict[str, Any]]]
 
 
-class TraceUsage(TypedDict):
-    prompt_tokens: NotRequired[Optional[int]]
-    completion_tokens: NotRequired[Optional[int]]
-    cache_creation_input_tokens: NotRequired[Optional[int]]
-    cache_read_input_tokens: NotRequired[Optional[int]]
-    total_tokens: NotRequired[Optional[int]]
-    prompt_tokens_cost_usd: NotRequired[Optional[float]]
-    completion_tokens_cost_usd: NotRequired[Optional[float]]
-    total_cost_usd: NotRequired[Optional[float]]
-    model_name: NotRequired[Optional[str]]
-
-
-class Tool(TypedDict):
-    tool_name: str
-    parameters: NotRequired[Optional[Dict[str, Any]]]
-    agent_name: NotRequired[Optional[str]]
-    result_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
-    action_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
-    require_all: NotRequired[Optional[bool]]
+class OtelTraceSpan(TypedDict):
+    organization_id: str
+    project_id: NotRequired[Optional[str]]
+    user_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[str]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[List[Dict[str, Any]]]]
+    legacy_span_id: NotRequired[Optional[str]]
+    inputs: NotRequired[Optional[Dict[str, Any]]]
+    output: Any
+    error: NotRequired[Optional[Dict[str, Any]]]
+    agent_id: NotRequired[Optional[str]]
+    cumulative_llm_cost: NotRequired[Optional[float]]
+    state_after: NotRequired[Optional[Dict[str, Any]]]
+    state_before: NotRequired[Optional[Dict[str, Any]]]
 
 
 class ExampleEvaluationRun(TypedDict):
@@ -257,88 +215,39 @@ class TraceEvaluationRun(TypedDict):
 
 
 class DatasetInsertExamples(TypedDict):
-    dataset_alias: str
+    dataset_name: str
     examples: List[Example]
     project_name: str
 
 
-class SpansBatchRequest(TypedDict):
-    spans: List[SpanBatchItem]
-    organization_id: str
-
-
-class FetchPromptScorerResponse(TypedDict):
-    scorer: PromptScorer
+class DatasetReturn(TypedDict):
+    name: str
+    project_name: str
+    examples: NotRequired[Optional[List[Example]]]
 
 
-class TraceSpan(TypedDict):
-    span_id: str
-    trace_id: str
-    function: str
-    created_at: NotRequired[Any]
-    parent_span_id: NotRequired[Optional[str]]
-    span_type: NotRequired[Optional[str]]
-    inputs: NotRequired[Optional[Dict[str, Any]]]
-    error: NotRequired[Optional[Dict[str, Any]]]
-    output: NotRequired[Any]
-    usage: NotRequired[Optional[TraceUsage]]
-    duration: NotRequired[Optional[float]]
-    expected_tools: NotRequired[Optional[List[Tool]]]
-    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
-    has_evaluation: NotRequired[Optional[bool]]
-    agent_name: NotRequired[Optional[str]]
-    class_name: NotRequired[Optional[str]]
-    state_before: NotRequired[Optional[Dict[str, Any]]]
-    state_after: NotRequired[Optional[Dict[str, Any]]]
-    update_id: NotRequired[int]
+class DatasetCreate(TypedDict):
+    name: str
+    dataset_kind: DatasetKind
+    project_name: str
+    examples: NotRequired[Optional[List[Example]]]
+    overwrite: NotRequired[Optional[bool]]
 
 
-class Trace(TypedDict):
-    trace_id: str
-    name: str
-    created_at: str
-    duration: float
-    trace_spans: List[TraceSpan]
-    offline_mode: NotRequired[bool]
-    rules: NotRequired[Dict[str, Any]]
-    has_notification: NotRequired[bool]
-    customer_id: NotRequired[Optional[str]]
-    tags: NotRequired[List[str]]
-    metadata: NotRequired[Dict[str, Any]]
-    update_id: NotRequired[int]
+class FetchPromptScorerResponse(TypedDict):
+    scorer: PromptScorer
 
 
 class ScoringResult(TypedDict):
     success: bool
     scorers_data: Optional[List[ScorerData]]
     name: NotRequired[Optional[str]]
-    data_object: NotRequired[Optional[Union[TraceSpan, Example]]]
+    data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
     trace_id: NotRequired[Optional[str]]
     run_duration: NotRequired[Optional[float]]
     evaluation_cost: NotRequired[Optional[float]]
 
 
-class TraceRun(TypedDict):
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
-    traces: List[Trace]
-    scorers: List[ScorerConfig]
-    model: str
-    trace_span_id: NotRequired[Optional[str]]
-    tools: NotRequired[Optional[List[Dict[str, Any]]]]
-
-
 class EvalResults(TypedDict):
     results: List[ScoringResult]
     run: Union[ExampleEvaluationRun, TraceEvaluationRun]
-
-
-class DatasetPush(TypedDict):
-    dataset_alias: str
-    comments: NotRequired[Optional[str]]
-    source_file: NotRequired[Optional[str]]
-    examples: NotRequired[Optional[List[Example]]]
-    traces: NotRequired[Optional[List[Trace]]]
-    is_trace: NotRequired[bool]
-    project_name: str
-    overwrite: NotRequired[Optional[bool]]

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/constants.py

@@ -14,16 +14,12 @@ class APIScorerType(str, Enum):
     """
 
     PROMPT_SCORER = "Prompt Scorer"
+    TRACE_PROMPT_SCORER = "Trace Prompt Scorer"
     FAITHFULNESS = "Faithfulness"
     ANSWER_RELEVANCY = "Answer Relevancy"
     ANSWER_CORRECTNESS = "Answer Correctness"
     INSTRUCTION_ADHERENCE = "Instruction Adherence"
     EXECUTION_ORDER = "Execution Order"
-    DERAILMENT = "Derailment"
-    TOOL_ORDER = "Tool Order"
-    MOCK_TRACE_SCORER = "Mock Trace Scorer"
-    CLASSIFIER = "Classifier"
-    TOOL_DEPENDENCY = "Tool Dependency"
     CUSTOM = "Custom"
 
     @classmethod

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/data/__init__.py

@@ -1,7 +1,7 @@
 from judgeval.data.example import Example, ExampleParams
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.trace import Trace, TraceSpan, TraceUsage
+from judgeval.data.trace import TraceUsage
 
 
 __all__ = [
@@ -11,7 +11,5 @@ __all__ = [
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
-    "Trace",
-    "TraceSpan",
     "TraceUsage",
 ]

{judgeval-0.9.4 → judgeval-0.10.0}/src/judgeval/data/example.py

@@ -6,6 +6,8 @@ from enum import Enum
 from datetime import datetime
 from typing import Dict, Any, Optional
 from judgeval.data.judgment_types import Example as JudgmentExample
+from uuid import uuid4
+from pydantic import Field
 
 
 class ExampleParams(str, Enum):
@@ -20,8 +22,8 @@ class ExampleParams(str, Enum):
 
 
 class Example(JudgmentExample):
-    example_id: str = ""
-    created_at: str = datetime.now().isoformat()
+    example_id: str = Field(default_factory=lambda: str(uuid4()))
+    created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
     name: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Any]: