judgeval 0.8.0__py3-none-any.whl → 0.9.1__py3-none-any.whl

judgeval/api/api_types.py

@@ -0,0 +1,344 @@
+# generated by datamodel-codegen:
+#   filename:  .openapi.json
+#   timestamp: 2025-08-29T04:49:39+00:00
+
+from __future__ import annotations
+from typing import Any, Dict, List, Optional, TypedDict, Union
+from typing_extensions import NotRequired
+
+
+TraceAndSpanId = List
+
+
+class EvalResultsFetch(TypedDict):
+    experiment_run_id: str
+    project_name: str
+
+
+class DatasetFetch(TypedDict):
+    dataset_alias: str
+    project_name: str
+
+
+class TraceSave(TypedDict):
+    project_name: str
+    trace_id: str
+    name: str
+    created_at: str
+    duration: float
+    offline_mode: NotRequired[bool]
+    has_notification: NotRequired[bool]
+    customer_id: NotRequired[Optional[str]]
+    tags: NotRequired[List[str]]
+    metadata: NotRequired[Dict[str, Any]]
+    update_id: NotRequired[int]
+
+
+class TraceFetch(TypedDict):
+    trace_id: str
+
+
+class TraceAddToDataset(TypedDict):
+    trace_id: str
+    trace_span_id: str
+    dataset_alias: str
+    project_name: str
+
+
+class EvaluationRunsBatchRequest(TypedDict):
+    organization_id: str
+    evaluation_entries: List[Dict[str, Any]]
+
+
+class ProjectAdd(TypedDict):
+    project_name: str
+
+
+class ProjectAddResponse(TypedDict):
+    project_id: str
+
+
+class ProjectDeleteFromJudgevalResponse(TypedDict):
+    project_name: str
+
+
+class ProjectDeleteResponse(TypedDict):
+    message: str
+
+
+class ScorerExistsRequest(TypedDict):
+    name: str
+
+
+class ScorerExistsResponse(TypedDict):
+    exists: bool
+
+
+class SavePromptScorerRequest(TypedDict):
+    name: str
+    prompt: str
+    threshold: float
+    options: NotRequired[Optional[Dict[str, float]]]
+    is_trace: NotRequired[Optional[bool]]
+
+
+class SavePromptScorerResponse(TypedDict):
+    message: str
+    name: str
+
+
+class FetchPromptScorerRequest(TypedDict):
+    name: str
+
+
+class CustomScorerUploadPayload(TypedDict):
+    scorer_name: str
+    scorer_code: str
+    requirements_text: str
+
+
+class CustomScorerTemplateResponse(TypedDict):
+    scorer_name: str
+    status: str
+    message: str
+
+
+class ResolveProjectNameRequest(TypedDict):
+    project_name: str
+
+
+class ResolveProjectNameResponse(TypedDict):
+    project_id: str
+
+
+class TraceIdRequest(TypedDict):
+    trace_id: str
+
+
+class SpanScoreRequest(TypedDict):
+    span_id: str
+    trace_id: str
+
+
+class BaseScorer(TypedDict):
+    score_type: str
+    threshold: NotRequired[float]
+    name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    score: NotRequired[Optional[float]]
+    score_breakdown: NotRequired[Optional[Dict[str, Any]]]
+    reason: NotRequired[Optional[str]]
+    using_native_model: NotRequired[Optional[bool]]
+    success: NotRequired[Optional[bool]]
+    model: NotRequired[Optional[str]]
+    model_client: NotRequired[Any]
+    strict_mode: NotRequired[bool]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    user: NotRequired[Optional[str]]
+    server_hosted: NotRequired[bool]
+
+
+class ScorerConfig(TypedDict):
+    score_type: str
+    name: NotRequired[Optional[str]]
+    threshold: NotRequired[float]
+    strict_mode: NotRequired[bool]
+    required_params: NotRequired[List[str]]
+    kwargs: NotRequired[Optional[Dict[str, Any]]]
+
+
+class Example(TypedDict):
+    example_id: str
+    created_at: str
+    name: NotRequired[Optional[str]]
+
+
+class ValidationError(TypedDict):
+    loc: List[Union[str, int]]
+    msg: str
+    type: str
+
+
+class SpanBatchItem(TypedDict):
+    span_id: str
+    trace_id: str
+    function: str
+    created_at: NotRequired[Any]
+    parent_span_id: NotRequired[Optional[str]]
+    span_type: NotRequired[Optional[str]]
+    inputs: NotRequired[Optional[Dict[str, Any]]]
+    output: NotRequired[Any]
+    error: NotRequired[Optional[Dict[str, Any]]]
+    usage: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[float]]
+    expected_tools: NotRequired[Optional[List[Dict[str, Any]]]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    has_evaluation: NotRequired[Optional[bool]]
+    agent_name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    state_before: NotRequired[Optional[Dict[str, Any]]]
+    state_after: NotRequired[Optional[Dict[str, Any]]]
+    span_state: str
+    update_id: NotRequired[int]
+    queued_at: float
+
+
+class PromptScorer(TypedDict):
+    name: str
+    prompt: str
+    threshold: float
+    options: NotRequired[Optional[Dict[str, float]]]
+    created_at: NotRequired[Optional[str]]
+    updated_at: NotRequired[Optional[str]]
+    is_trace: NotRequired[Optional[bool]]
+
+
+class ScorerData(TypedDict):
+    name: str
+    threshold: float
+    success: bool
+    score: NotRequired[Optional[float]]
+    reason: NotRequired[Optional[str]]
+    strict_mode: NotRequired[Optional[bool]]
+    evaluation_model: NotRequired[Union[List[str], str]]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+
+
+class TraceUsage(TypedDict):
+    prompt_tokens: NotRequired[Optional[int]]
+    completion_tokens: NotRequired[Optional[int]]
+    cache_creation_input_tokens: NotRequired[Optional[int]]
+    cache_read_input_tokens: NotRequired[Optional[int]]
+    total_tokens: NotRequired[Optional[int]]
+    prompt_tokens_cost_usd: NotRequired[Optional[float]]
+    completion_tokens_cost_usd: NotRequired[Optional[float]]
+    total_cost_usd: NotRequired[Optional[float]]
+    model_name: NotRequired[Optional[str]]
+
+
+class Tool(TypedDict):
+    tool_name: str
+    parameters: NotRequired[Optional[Dict[str, Any]]]
+    agent_name: NotRequired[Optional[str]]
+    result_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
+    action_dependencies: NotRequired[Optional[List[Dict[str, Any]]]]
+    require_all: NotRequired[Optional[bool]]
+
+
+class ExampleEvaluationRun(TypedDict):
+    id: NotRequired[Optional[str]]
+    project_name: NotRequired[Optional[str]]
+    eval_name: NotRequired[Optional[str]]
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    model: str
+    created_at: NotRequired[Optional[str]]
+    examples: List[Example]
+    trace_span_id: NotRequired[Optional[str]]
+    trace_id: NotRequired[Optional[str]]
+
+
+class HTTPValidationError(TypedDict):
+    detail: NotRequired[List[ValidationError]]
+
+
+class TraceEvaluationRun(TypedDict):
+    id: NotRequired[Optional[str]]
+    project_name: NotRequired[Optional[str]]
+    eval_name: NotRequired[Optional[str]]
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    model: str
+    created_at: NotRequired[Optional[str]]
+    trace_and_span_ids: List[TraceAndSpanId]
+    is_offline: NotRequired[bool]
+
+
+class DatasetInsertExamples(TypedDict):
+    dataset_alias: str
+    examples: List[Example]
+    project_name: str
+
+
+class SpansBatchRequest(TypedDict):
+    spans: List[SpanBatchItem]
+    organization_id: str
+
+
+class FetchPromptScorerResponse(TypedDict):
+    scorer: PromptScorer
+
+
+class TraceSpan(TypedDict):
+    span_id: str
+    trace_id: str
+    function: str
+    created_at: NotRequired[Any]
+    parent_span_id: NotRequired[Optional[str]]
+    span_type: NotRequired[Optional[str]]
+    inputs: NotRequired[Optional[Dict[str, Any]]]
+    error: NotRequired[Optional[Dict[str, Any]]]
+    output: NotRequired[Any]
+    usage: NotRequired[Optional[TraceUsage]]
+    duration: NotRequired[Optional[float]]
+    expected_tools: NotRequired[Optional[List[Tool]]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    has_evaluation: NotRequired[Optional[bool]]
+    agent_name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    state_before: NotRequired[Optional[Dict[str, Any]]]
+    state_after: NotRequired[Optional[Dict[str, Any]]]
+    update_id: NotRequired[int]
+
+
+class Trace(TypedDict):
+    trace_id: str
+    name: str
+    created_at: str
+    duration: float
+    trace_spans: List[TraceSpan]
+    offline_mode: NotRequired[bool]
+    rules: NotRequired[Dict[str, Any]]
+    has_notification: NotRequired[bool]
+    customer_id: NotRequired[Optional[str]]
+    tags: NotRequired[List[str]]
+    metadata: NotRequired[Dict[str, Any]]
+    update_id: NotRequired[int]
+
+
+class ScoringResult(TypedDict):
+    success: bool
+    scorers_data: Optional[List[ScorerData]]
+    name: NotRequired[Optional[str]]
+    data_object: NotRequired[Optional[Union[TraceSpan, Example]]]
+    trace_id: NotRequired[Optional[str]]
+    run_duration: NotRequired[Optional[float]]
+    evaluation_cost: NotRequired[Optional[float]]
+
+
+class TraceRun(TypedDict):
+    project_name: NotRequired[Optional[str]]
+    eval_name: NotRequired[Optional[str]]
+    traces: List[Trace]
+    scorers: List[ScorerConfig]
+    model: str
+    trace_span_id: NotRequired[Optional[str]]
+    tools: NotRequired[Optional[List[Dict[str, Any]]]]
+
+
+class EvalResults(TypedDict):
+    results: List[ScoringResult]
+    run: Union[ExampleEvaluationRun, TraceEvaluationRun]
+
+
+class DatasetPush(TypedDict):
+    dataset_alias: str
+    comments: NotRequired[Optional[str]]
+    source_file: NotRequired[Optional[str]]
+    examples: NotRequired[Optional[List[Example]]]
+    traces: NotRequired[Optional[List[Trace]]]
+    is_trace: NotRequired[bool]
+    project_name: str
+    overwrite: NotRequired[Optional[bool]]

judgeval/cli.py

@@ -3,8 +3,8 @@
 import typer
 from pathlib import Path
 from dotenv import load_dotenv
-from judgeval.common.logger import judgeval_logger
-from judgeval.judgment_client import JudgmentClient
+from judgeval.logger import judgeval_logger
+from judgeval import JudgmentClient
 
 load_dotenv()
 
@@ -61,5 +61,3 @@ def version():
 
 if __name__ == "__main__":
     app()
-
-# judgeval upload_scorer /Users/alanzhang/repo/JudgmentLabs/judgeval/src/demo/profile_match_scorer.py /Users/alanzhang/repo/JudgmentLabs/judgeval/src/demo/requirements.txt

judgeval/constants.py

@@ -1,10 +1,8 @@
-"""
-Constant variables used throughout source code
-"""
+from __future__ import annotations
 
 from enum import Enum
+from typing import Set
 import litellm
-import os
 
 
 class APIScorerType(str, Enum):
@@ -23,30 +21,28 @@ class APIScorerType(str, Enum):
     EXECUTION_ORDER = "Execution Order"
     DERAILMENT = "Derailment"
     TOOL_ORDER = "Tool Order"
+    MOCK_TRACE_SCORER = "Mock Trace Scorer"
     CLASSIFIER = "Classifier"
     TOOL_DEPENDENCY = "Tool Dependency"
     CUSTOM = "Custom"
 
     @classmethod
-    def _missing_(cls, value):
+    def __missing__(cls, value: str) -> APIScorerType:
         # Handle case-insensitive lookup
         for member in cls:
             if member.value == value.lower():
                 return member
 
+        raise ValueError(f"Invalid scorer type: {value}")
 
-UNBOUNDED_SCORERS: set[APIScorerType] = (
+
+UNBOUNDED_SCORERS: Set[APIScorerType] = (
     set()
 )  # scorers whose scores are not bounded between 0-1
 
-# RabbitMQ
-RABBITMQ_HOST = os.getenv(
-    "RABBITMQ_HOST", "rabbitmq-networklb-faa155df16ec9085.elb.us-west-1.amazonaws.com"
-)
-RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672)
-RABBITMQ_QUEUE = os.getenv("RABBITMQ_QUEUE", "task_queue")
-# Models
-LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
+
+LITELLM_SUPPORTED_MODELS: Set[str] = set(litellm.model_list)
+
 
 TOGETHER_SUPPORTED_MODELS = [
     "meta-llama/Meta-Llama-3-70B-Instruct-Turbo",
@@ -104,20 +100,8 @@ TOGETHER_SUPPORTED_MODELS = [
     "mistralai/Mistral-7B-Instruct-v0.1",
 ]
 
-DEFAULT_TOGETHER_MODEL = "meta-llama/Meta-Llama-3-8B-Instruct-Lite"
-DEFAULT_GPT_MODEL = "gpt-4.1"
-
 JUDGMENT_SUPPORTED_MODELS = {"osiris-large", "osiris-mini", "osiris"}
 
 ACCEPTABLE_MODELS = (
     set(litellm.model_list) | set(TOGETHER_SUPPORTED_MODELS) | JUDGMENT_SUPPORTED_MODELS
 )
-
-## System settings
-MAX_WORKER_THREADS = 10
-
-# Maximum number of concurrent operations for evaluation runs
-MAX_CONCURRENT_EVALUATIONS = 50  # Adjust based on system capabilities
-
-# Span lifecycle management
-SPAN_LIFECYCLE_END_UPDATE_ID = 20  # Default ending number for completed spans

judgeval/data/evaluation_run.py

@@ -1,4 +1,5 @@
-from typing import List, Optional, Union
+from typing import List, Optional, Union, Tuple
+from litellm.files.main import BaseModel
 from pydantic import field_validator, model_validator, Field
 from datetime import datetime, timezone
 import uuid
@@ -6,29 +7,22 @@ import uuid
 from judgeval.data import Example
 from judgeval.scorers import BaseScorer, APIScorerConfig
 from judgeval.constants import ACCEPTABLE_MODELS
-from judgeval.data.judgment_types import EvaluationRunJudgmentType
+from judgeval.data.judgment_types import (
+    ExampleEvaluationRun as ExampleEvaluationRunJudgmentType,
+    TraceEvaluationRun as TraceEvaluationRunJudgmentType,
+)
 
 
-class EvaluationRun(EvaluationRunJudgmentType):
-    """
-    Stores example and evaluation scorers together for running an eval task
-
-    Args:
-        project_name (str): The name of the project the evaluation results belong to
-        eval_name (str): A name for this evaluation run
-        examples (List[Example]): The examples to evaluate
-        scorers (List[Union[BaseScorer, APIScorerConfig]]): A list of scorers to use for evaluation
-        model (str): The model used as a judge when using LLM as a Judge
-        metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
-    """
-
+class EvaluationRun(BaseModel):
     id: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
     created_at: Optional[str] = Field(
         default_factory=lambda: datetime.now(timezone.utc).isoformat()
     )
+    organization_id: Optional[str] = None
     custom_scorers: Optional[List[BaseScorer]] = None
     judgment_scorers: Optional[List[APIScorerConfig]] = None
-    organization_id: Optional[str] = None
+    scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None
+    model: str
 
     def __init__(
         self,
@@ -57,19 +51,9 @@ class EvaluationRun(EvaluationRunJudgmentType):
         data = super().model_dump(**kwargs)
         data["custom_scorers"] = [s.model_dump() for s in self.custom_scorers]
         data["judgment_scorers"] = [s.model_dump() for s in self.judgment_scorers]
-        data["examples"] = [example.model_dump() for example in self.examples]
 
         return data
 
-    @field_validator("examples")
-    def validate_examples(cls, v):
-        if not v:
-            raise ValueError("Examples cannot be empty.")
-        for item in v:
-            if not isinstance(item, Example):
-                raise ValueError(f"Item of type {type(item)} is not a Example")
-        return v
-
     @model_validator(mode="after")
     @classmethod
     def validate_scorer_lists(cls, values):
@@ -102,3 +86,42 @@ class EvaluationRun(EvaluationRunJudgmentType):
                     f"Model name {v} not recognized. Please select a valid model name.)"
                 )
             return v
+
+
+class ExampleEvaluationRun(EvaluationRun, ExampleEvaluationRunJudgmentType):  # type: ignore
+    """
+    Stores example and evaluation scorers together for running an eval task
+
+    Args:
+        project_name (str): The name of the project the evaluation results belong to
+        eval_name (str): A name for this evaluation run
+        examples (List[Example]): The examples to evaluate
+        scorers (List[Union[BaseScorer, APIScorerConfig]]): A list of scorers to use for evaluation
+        model (str): The model used as a judge when using LLM as a Judge
+    """
+
+    examples: List[Example]  # type: ignore
+
+    @field_validator("examples")
+    def validate_examples(cls, v):
+        if not v:
+            raise ValueError("Examples cannot be empty.")
+        for item in v:
+            if not isinstance(item, Example):
+                raise ValueError(f"Item of type {type(item)} is not a Example")
+        return v
+
+    def model_dump(self, **kwargs):
+        data = super().model_dump(**kwargs)
+        data["examples"] = [example.model_dump() for example in self.examples]
+        return data
+
+
+class TraceEvaluationRun(EvaluationRun, TraceEvaluationRunJudgmentType):  # type: ignore
+    trace_and_span_ids: List[Tuple[str, str]]  # type: ignore
+
+    @field_validator("trace_and_span_ids")
+    def validate_trace_and_span_ids(cls, v):
+        if not v:
+            raise ValueError("Trace and span IDs are required for trace evaluations.")
+        return v

judgeval/data/example.py

@@ -5,7 +5,7 @@ Classes for representing examples in a dataset.
 from enum import Enum
 from datetime import datetime
 from typing import Dict, Any, Optional
-from judgeval.data.judgment_types import ExampleJudgmentType
+from judgeval.data.judgment_types import Example as JudgmentExample
 
 
 class ExampleParams(str, Enum):
@@ -19,7 +19,7 @@ class ExampleParams(str, Enum):
     ADDITIONAL_METADATA = "additional_metadata"
 
 
-class Example(ExampleJudgmentType):
+class Example(JudgmentExample):
     example_id: str = ""
     created_at: str = datetime.now().isoformat()
     name: Optional[str] = None