judgeval 0.9.4__py3-none-any.whl → 0.10.0__py3-none-any.whl

judgeval/__init__.py

@@ -6,7 +6,7 @@ from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
 from typing import List, Optional, Union
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import BaseScorer, ExampleAPIScorerConfig
 from judgeval.data.example import Example
 from judgeval.logger import judgeval_logger
 from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_DEFAULT_GPT_MODEL, JUDGMENT_ORG_ID
@@ -38,7 +38,7 @@ class JudgmentClient(metaclass=SingletonMeta):
     def run_evaluation(
         self,
         examples: List[Example],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        scorers: List[Union[ExampleAPIScorerConfig, BaseScorer]],
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,

judgeval/api/__init__.py

@@ -71,13 +71,6 @@ class JudgmentSyncClient:
             payload,
         )
 
-    def evaluate_trace(self, payload: TraceRun) -> Any:
-        return self._request(
-            "POST",
-            url_for("/evaluate_trace/"),
-            payload,
-        )
-
     def evaluate_examples(
         self, payload: ExampleEvaluationRun, stream: Optional[str] = None
     ) -> Any:
@@ -128,59 +121,26 @@ class JudgmentSyncClient:
             query_params,
         )
 
-    def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+    def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
         return self._request(
             "POST",
-            url_for("/datasets/insert_examples/"),
+            url_for("/datasets/insert_examples_for_judgeval/"),
             payload,
         )
 
-    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
         return self._request(
             "POST",
             url_for("/datasets/pull_for_judgeval/"),
             payload,
         )
 
-    def datasets_push(self, payload: DatasetPush) -> Any:
+    def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return self._request(
             "POST",
-            url_for("/datasets/push/"),
-            payload,
-        )
-
-    def traces_upsert(self, payload: TraceSave) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/upsert/"),
-            payload,
-        )
-
-    def traces_fetch(self, payload: TraceFetch) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/fetch/"),
-            payload,
-        )
-
-    def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/add_to_dataset/"),
-            payload,
-        )
-
-    def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/spans/batch/"),
-            payload,
-        )
-
-    def traces_evaluation_runs_batch(self, payload: EvaluationRunsBatchRequest) -> Any:
-        return self._request(
-            "POST",
-            url_for("/traces/evaluation_runs/batch/"),
+            url_for("/datasets/create_for_judgeval/"),
             payload,
         )
 
@@ -255,6 +215,13 @@ class JudgmentSyncClient:
             payload,
         )
 
+    def e2e_fetch_trace_scorer_span_score(self, payload: SpanScoreRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_trace_scorer_span_score/"),
+            payload,
+        )
+
 
 class JudgmentAsyncClient:
     __slots__ = ("api_key", "organization_id", "client")
@@ -304,13 +271,6 @@ class JudgmentAsyncClient:
             payload,
         )
 
-    async def evaluate_trace(self, payload: TraceRun) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/evaluate_trace/"),
-            payload,
-        )
-
     async def evaluate_examples(
         self, payload: ExampleEvaluationRun, stream: Optional[str] = None
     ) -> Any:
@@ -363,61 +323,26 @@ class JudgmentAsyncClient:
             query_params,
         )
 
-    async def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+    async def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
         return await self._request(
             "POST",
-            url_for("/datasets/insert_examples/"),
+            url_for("/datasets/insert_examples_for_judgeval/"),
             payload,
         )
 
-    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
         return await self._request(
             "POST",
             url_for("/datasets/pull_for_judgeval/"),
             payload,
         )
 
-    async def datasets_push(self, payload: DatasetPush) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/datasets/push/"),
-            payload,
-        )
-
-    async def traces_upsert(self, payload: TraceSave) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/upsert/"),
-            payload,
-        )
-
-    async def traces_fetch(self, payload: TraceFetch) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/fetch/"),
-            payload,
-        )
-
-    async def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/add_to_dataset/"),
-            payload,
-        )
-
-    async def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
-        return await self._request(
-            "POST",
-            url_for("/traces/spans/batch/"),
-            payload,
-        )
-
-    async def traces_evaluation_runs_batch(
-        self, payload: EvaluationRunsBatchRequest
-    ) -> Any:
+    async def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
         return await self._request(
             "POST",
-            url_for("/traces/evaluation_runs/batch/"),
+            url_for("/datasets/create_for_judgeval/"),
             payload,
         )
 
@@ -494,6 +419,13 @@ class JudgmentAsyncClient:
             payload,
         )
 
+    async def e2e_fetch_trace_scorer_span_score(self, payload: SpanScoreRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_trace_scorer_span_score/"),
+            payload,
+        )
+
 
 __all__ = [
     "JudgmentSyncClient",

judgeval/api/api_types.py

@@ -1,9 +1,9 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-08-29T04:49:39+00:00
+#   timestamp: 2025-09-10T17:42:12+00:00
 
 from __future__ import annotations
-from typing import Any, Dict, List, Optional, TypedDict, Union
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 from typing_extensions import NotRequired
 
 
@@ -16,40 +16,10 @@ class EvalResultsFetch(TypedDict):
 
 
 class DatasetFetch(TypedDict):
-    dataset_alias: str
+    dataset_name: str
     project_name: str
 
 
-class TraceSave(TypedDict):
-    project_name: str
-    trace_id: str
-    name: str
-    created_at: str
-    duration: float
-    offline_mode: NotRequired[bool]
-    has_notification: NotRequired[bool]
-    customer_id: NotRequired[Optional[str]]
-    tags: NotRequired[List[str]]
-    metadata: NotRequired[Dict[str, Any]]
-    update_id: NotRequired[int]
-
-
-class TraceFetch(TypedDict):
-    trace_id: str
-
-
-class TraceAddToDataset(TypedDict):
-    trace_id: str
-    trace_span_id: str
-    dataset_alias: str
-    project_name: str
-
-
-class EvaluationRunsBatchRequest(TypedDict):
-    organization_id: str
-    evaluation_entries: List[Dict[str, Any]]
-
-
 class ProjectAdd(TypedDict):
     project_name: str
 
@@ -149,8 +119,8 @@ class ScorerConfig(TypedDict):
 
 
 class Example(TypedDict):
-    example_id: str
-    created_at: str
+    example_id: NotRequired[str]
+    created_at: NotRequired[str]
     name: NotRequired[Optional[str]]
 
 
@@ -160,28 +130,7 @@ class ValidationError(TypedDict):
     type: str
 
 
-class SpanBatchItem(TypedDict):
-    span_id: str
-    trace_id: str
-    function: str
-    created_at: NotRequired[Any]
-    parent_span_id: NotRequired[Optional[str]]
-    span_type: NotRequired[Optional[str]]
-    inputs: NotRequired[Optional[Dict[str, Any]]]
-    output: NotRequired[Any]
-    error: NotRequired[Optional[Dict[str, Any]]]
-    usage: NotRequired[Optional[Dict[str, Any]]]
-    duration: NotRequired[Optional[float]]
-    expected_tools: NotRequired[Optional[List[Dict[str, Any]]]]
-    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
-    has_evaluation: NotRequired[Optional[bool]]
-    agent_name: NotRequired[Optional[str]]
-    class_name: NotRequired[Optional[str]]
-    state_before: NotRequired[Optional[Dict[str, Any]]]
-    state_after: NotRequired[Optional[Dict[str, Any]]]
-    span_state: str
-    update_id: NotRequired[int]
-    queued_at: float
+DatasetKind = Literal["trace", "example"]
 
 
 class PromptScorer(TypedDict):
@@ -195,36 +144,45 @@ class PromptScorer(TypedDict):
 
 
 class ScorerData(TypedDict):
+    id: NotRequired[str]
     name: str
     threshold: float
     success: bool
     score: NotRequired[Optional[float]]
     reason: NotRequired[Optional[str]]
     strict_mode: NotRequired[Optional[bool]]
-    evaluation_model: NotRequired[Union[List[str], str]]
+    evaluation_model: NotRequired[str]
     error: NotRequired[Optional[str]]
     additional_metadata: NotRequired[Optional[Dict[str, Any]]]
 
 
-class TraceUsage(TypedDict):
-    prompt_tokens: NotRequired[Optional[int]]
-    completion_tokens: NotRequired[Optional[int]]
-    cache_creation_input_tokens: NotRequired[Optional[int]]
-    cache_read_input_tokens: NotRequired[Optional[int]]
-    total_tokens: NotRequired[Optional[int]]
-    prompt_tokens_cost_usd: NotRequired[Optional[float]]
-    completion_tokens_cost_usd: NotRequired[Optional[float]]
-    total_cost_usd: NotRequired[Optional[float]]
-    model_name: NotRequired[Optional[str]]
-
-
-class Tool(TypedDict):
-    tool_name: str
-    parameters: NotRequired[Optional[Dict[str, Any]]]
-    agent_name: NotRequired[Optional[str]]
-    result_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
-    action_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
-    require_all: NotRequired[Optional[bool]]
+class OtelTraceSpan(TypedDict):
+    organization_id: str
+    project_id: NotRequired[Optional[str]]
+    user_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[str]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[List[Dict[str, Any]]]]
+    legacy_span_id: NotRequired[Optional[str]]
+    inputs: NotRequired[Optional[Dict[str, Any]]]
+    output: Any
+    error: NotRequired[Optional[Dict[str, Any]]]
+    agent_id: NotRequired[Optional[str]]
+    cumulative_llm_cost: NotRequired[Optional[float]]
+    state_after: NotRequired[Optional[Dict[str, Any]]]
+    state_before: NotRequired[Optional[Dict[str, Any]]]
 
 
 class ExampleEvaluationRun(TypedDict):
@@ -257,88 +215,39 @@ class TraceEvaluationRun(TypedDict):
 
 
 class DatasetInsertExamples(TypedDict):
-    dataset_alias: str
+    dataset_name: str
     examples: List[Example]
     project_name: str
 
 
-class SpansBatchRequest(TypedDict):
-    spans: List[SpanBatchItem]
-    organization_id: str
-
-
-class FetchPromptScorerResponse(TypedDict):
-    scorer: PromptScorer
+class DatasetReturn(TypedDict):
+    name: str
+    project_name: str
+    examples: NotRequired[Optional[List[Example]]]
 
 
-class TraceSpan(TypedDict):
-    span_id: str
-    trace_id: str
-    function: str
-    created_at: NotRequired[Any]
-    parent_span_id: NotRequired[Optional[str]]
-    span_type: NotRequired[Optional[str]]
-    inputs: NotRequired[Optional[Dict[str, Any]]]
-    error: NotRequired[Optional[Dict[str, Any]]]
-    output: NotRequired[Any]
-    usage: NotRequired[Optional[TraceUsage]]
-    duration: NotRequired[Optional[float]]
-    expected_tools: NotRequired[Optional[List[Tool]]]
-    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
-    has_evaluation: NotRequired[Optional[bool]]
-    agent_name: NotRequired[Optional[str]]
-    class_name: NotRequired[Optional[str]]
-    state_before: NotRequired[Optional[Dict[str, Any]]]
-    state_after: NotRequired[Optional[Dict[str, Any]]]
-    update_id: NotRequired[int]
+class DatasetCreate(TypedDict):
+    name: str
+    dataset_kind: DatasetKind
+    project_name: str
+    examples: NotRequired[Optional[List[Example]]]
+    overwrite: NotRequired[Optional[bool]]
 
 
-class Trace(TypedDict):
-    trace_id: str
-    name: str
-    created_at: str
-    duration: float
-    trace_spans: List[TraceSpan]
-    offline_mode: NotRequired[bool]
-    rules: NotRequired[Dict[str, Any]]
-    has_notification: NotRequired[bool]
-    customer_id: NotRequired[Optional[str]]
-    tags: NotRequired[List[str]]
-    metadata: NotRequired[Dict[str, Any]]
-    update_id: NotRequired[int]
+class FetchPromptScorerResponse(TypedDict):
+    scorer: PromptScorer
 
 
 class ScoringResult(TypedDict):
     success: bool
     scorers_data: Optional[List[ScorerData]]
     name: NotRequired[Optional[str]]
-    data_object: NotRequired[Optional[Union[TraceSpan, Example]]]
+    data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
     trace_id: NotRequired[Optional[str]]
     run_duration: NotRequired[Optional[float]]
     evaluation_cost: NotRequired[Optional[float]]
 
 
-class TraceRun(TypedDict):
-    project_name: NotRequired[Optional[str]]
-    eval_name: NotRequired[Optional[str]]
-    traces: List[Trace]
-    scorers: List[ScorerConfig]
-    model: str
-    trace_span_id: NotRequired[Optional[str]]
-    tools: NotRequired[Optional[List[Dict[str, Any]]]]
-
-
 class EvalResults(TypedDict):
     results: List[ScoringResult]
     run: Union[ExampleEvaluationRun, TraceEvaluationRun]
-
-
-class DatasetPush(TypedDict):
-    dataset_alias: str
-    comments: NotRequired[Optional[str]]
-    source_file: NotRequired[Optional[str]]
-    examples: NotRequired[Optional[List[Example]]]
-    traces: NotRequired[Optional[List[Trace]]]
-    is_trace: NotRequired[bool]
-    project_name: str
-    overwrite: NotRequired[Optional[bool]]

judgeval/constants.py

@@ -14,16 +14,12 @@ class APIScorerType(str, Enum):
     """
 
     PROMPT_SCORER = "Prompt Scorer"
+    TRACE_PROMPT_SCORER = "Trace Prompt Scorer"
     FAITHFULNESS = "Faithfulness"
     ANSWER_RELEVANCY = "Answer Relevancy"
     ANSWER_CORRECTNESS = "Answer Correctness"
     INSTRUCTION_ADHERENCE = "Instruction Adherence"
     EXECUTION_ORDER = "Execution Order"
-    DERAILMENT = "Derailment"
-    TOOL_ORDER = "Tool Order"
-    MOCK_TRACE_SCORER = "Mock Trace Scorer"
-    CLASSIFIER = "Classifier"
-    TOOL_DEPENDENCY = "Tool Dependency"
     CUSTOM = "Custom"
 
     @classmethod

judgeval/data/__init__.py

@@ -1,7 +1,7 @@
 from judgeval.data.example import Example, ExampleParams
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.trace import Trace, TraceSpan, TraceUsage
+from judgeval.data.trace import TraceUsage
 
 
 __all__ = [
@@ -11,7 +11,5 @@ __all__ = [
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
-    "Trace",
-    "TraceSpan",
     "TraceUsage",
 ]

judgeval/data/example.py

@@ -6,6 +6,8 @@ from enum import Enum
 from datetime import datetime
 from typing import Dict, Any, Optional
 from judgeval.data.judgment_types import Example as JudgmentExample
+from uuid import uuid4
+from pydantic import Field
 
 
 class ExampleParams(str, Enum):
@@ -20,8 +22,8 @@ class ExampleParams(str, Enum):
 
 
 class Example(JudgmentExample):
-    example_id: str = ""
-    created_at: str = datetime.now().isoformat()
+    example_id: str = Field(default_factory=lambda: str(uuid4()))
+    created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
     name: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Any]: