judgeval 0.4.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

judgeval/common/tracer/otel_span_processor.py

@@ -21,7 +21,7 @@ from judgeval.common.tracer.otel_exporter import JudgmentAPISpanExporter
 from judgeval.common.tracer.span_processor import SpanProcessorBase
 from judgeval.common.tracer.span_transformer import SpanTransformer
 from judgeval.data import TraceSpan
-from judgeval.evaluation_run import EvaluationRun
+from judgeval.data.evaluation_run import EvaluationRun
 
 
 class SimpleReadableSpan(ReadableSpan):

judgeval/common/tracer/providers.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+import logging
+from typing import Any, TypeAlias
+
+
+logger = logging.getLogger(__name__)
+# TODO: Have functions that assert and return the relevant exports when the client is installed.
+# The method should raise if the user tries to access client information that doesnt exist.
+
+HAS_OPENAI = False
+openai_OpenAI = None
+openai_AsyncOpenAI = None
+openai_ChatCompletion = None
+openai_Response = None
+openai_ParsedChatCompletion = None
+
+try:
+    from openai import OpenAI, AsyncOpenAI
+    from openai.types.chat.chat_completion import ChatCompletion
+    from openai.types.responses.response import Response
+    from openai.types.chat import ParsedChatCompletion
+
+    openai_OpenAI = OpenAI
+    openai_AsyncOpenAI = AsyncOpenAI
+    openai_ChatCompletion = ChatCompletion
+    openai_Response = Response
+    openai_ParsedChatCompletion = ParsedChatCompletion
+    HAS_OPENAI = True
+except ImportError:
+    pass
+
+
+HAS_TOGETHER = False
+together_Together = None
+together_AsyncTogether = None
+
+try:
+    from together import Together, AsyncTogether
+
+    together_Together = Together
+    together_AsyncTogether = AsyncTogether
+    HAS_TOGETHER = True
+except ImportError:
+    pass
+
+
+HAS_ANTHROPIC = False
+anthropic_Anthropic = None
+anthropic_AsyncAnthropic = None
+
+try:
+    from anthropic import Anthropic, AsyncAnthropic
+
+    anthropic_Anthropic = Anthropic
+    anthropic_AsyncAnthropic = AsyncAnthropic
+    HAS_ANTHROPIC = True
+except ImportError:
+    pass
+
+
+HAS_GOOGLE_GENAI = False
+google_genai_Client = None
+google_genai_cleint_AsyncClient = None
+
+try:
+    from google.genai import Client
+    from google.genai.client import AsyncClient
+
+    google_genai_Client = Client
+    google_genai_AsyncClient = AsyncClient
+    HAS_GOOGLE_GENAI = True
+except ImportError:
+    pass
+
+
+HAS_GROQ = False
+groq_Groq = None
+groq_AsyncGroq = None
+
+try:
+    from groq import Groq, AsyncGroq
+
+    groq_Groq = Groq
+    groq_AsyncGroq = AsyncGroq
+    HAS_GROQ = True
+except ImportError:
+    pass
+
+
+# TODO: if we support dependency groups we can have this better type, but during runtime, we do
+# not know which clients an end user might have installed.
+ApiClient: TypeAlias = Any
+
+__all__ = [
+    "ApiClient",
+    # OpenAI
+    "HAS_OPENAI",
+    "openai_OpenAI",
+    "openai_AsyncOpenAI",
+    "openai_ChatCompletion",
+    "openai_Response",
+    "openai_ParsedChatCompletion",
+    # Together
+    "HAS_TOGETHER",
+    "together_Together",
+    "together_AsyncTogether",
+    # Anthropic
+    "HAS_ANTHROPIC",
+    "anthropic_Anthropic",
+    "anthropic_AsyncAnthropic",
+    # Google GenAI
+    "HAS_GOOGLE_GENAI",
+    "google_genai_Client",
+    "google_genai_AsyncClient",
+    # Groq
+    "HAS_GROQ",
+    "groq_Groq",
+    "groq_AsyncGroq",
+]

judgeval/common/tracer/span_processor.py

@@ -7,7 +7,7 @@ When monitoring is enabled, we use JudgmentSpanProcessor which overrides the met
 """
 
 from judgeval.data import TraceSpan
-from judgeval.evaluation_run import EvaluationRun
+from judgeval.data.evaluation_run import EvaluationRun
 
 
 class SpanProcessorBase:

judgeval/common/tracer/span_transformer.py

@@ -9,8 +9,9 @@ from typing import Any, Dict, Mapping, Optional, Union
 from opentelemetry.sdk.trace import ReadableSpan
 from pydantic import BaseModel
 
+from judgeval.common.api.json_encoder import json_encoder
 from judgeval.data import TraceSpan
-from judgeval.evaluation_run import EvaluationRun
+from judgeval.data.evaluation_run import EvaluationRun
 
 
 class SpanTransformer:
@@ -38,21 +39,13 @@ class SpanTransformer:
             return True
 
     @staticmethod
-    def _safe_json_handle(obj: Any, serialize: bool = True) -> Any:
-        if serialize:
-            if obj is None:
-                return None
-            try:
-                return orjson.dumps(obj, default=str).decode("utf-8")
-            except Exception:
-                return orjson.dumps(str(obj)).decode("utf-8")
-        else:
-            if not isinstance(obj, str):
-                return obj
-            try:
-                return orjson.loads(obj)
-            except (orjson.JSONDecodeError, TypeError, ValueError):
-                return obj
+    def _safe_deserialize(obj: Any) -> Any:
+        if not isinstance(obj, str):
+            return obj
+        try:
+            return orjson.loads(obj)
+        except (orjson.JSONDecodeError, TypeError):
+            return obj
 
     @staticmethod
     def _format_timestamp(timestamp: Optional[Union[float, int, str]]) -> str:
@@ -84,15 +77,13 @@ class SpanTransformer:
             if field_name == "created_at":
                 attributes[attr_name] = SpanTransformer._format_timestamp(value)
             elif field_name == "expected_tools" and value:
-                attributes[attr_name] = SpanTransformer._safe_json_handle(
+                attributes[attr_name] = json_encoder(
                     [tool.model_dump() for tool in trace_span.expected_tools]
                 )
             elif field_name == "usage" and value:
-                attributes[attr_name] = SpanTransformer._safe_json_handle(
-                    trace_span.usage.model_dump()
-                )
+                attributes[attr_name] = json_encoder(trace_span.usage)
             elif SpanTransformer._needs_json_serialization(value):
-                attributes[attr_name] = SpanTransformer._safe_json_handle(value)
+                attributes[attr_name] = json_encoder(value)
             else:
                 attributes[attr_name] = value
 
@@ -115,7 +106,7 @@ class SpanTransformer:
             field_name = key[9:]
 
             if isinstance(value, str):
-                deserialized = SpanTransformer._safe_json_handle(value, serialize=False)
+                deserialized = SpanTransformer._safe_deserialize(value)
                 judgment_data[field_name] = deserialized
             else:
                 judgment_data[field_name] = value
@@ -159,6 +150,7 @@ class SpanTransformer:
                 "additional_metadata": judgment_data.get("additional_metadata"),
                 "has_evaluation": judgment_data.get("has_evaluation", False),
                 "agent_name": judgment_data.get("agent_name"),
+                "class_name": judgment_data.get("class_name"),
                 "state_before": judgment_data.get("state_before"),
                 "state_after": judgment_data.get("state_after"),
                 "update_id": judgment_data.get("update_id", 1),
@@ -174,9 +166,7 @@ class SpanTransformer:
         attributes = {
             "judgment.evaluation_run": True,
             "judgment.associated_span_id": span_id,
-            "judgment.span_data": SpanTransformer._safe_json_handle(
-                span_data.model_dump()
-            ),
+            "judgment.span_data": json_encoder(span_data),
         }
 
         eval_data = evaluation_run.model_dump()
@@ -186,7 +176,7 @@ class SpanTransformer:
 
             attr_name = f"judgment.{key}"
             if SpanTransformer._needs_json_serialization(value):
-                attributes[attr_name] = SpanTransformer._safe_json_handle(value)
+                attributes[attr_name] = json_encoder(value)
             else:
                 attributes[attr_name] = value
 

judgeval/constants.py

@@ -105,6 +105,7 @@ TOGETHER_SUPPORTED_MODELS = [
 ]
 
 DEFAULT_TOGETHER_MODEL = "meta-llama/Meta-Llama-3-8B-Instruct-Lite"
+DEFAULT_GPT_MODEL = "gpt-4.1"
 
 JUDGMENT_SUPPORTED_MODELS = {"osiris-large", "osiris-mini", "osiris"}
 

judgeval/data/evaluation_run.py

@@ -0,0 +1,104 @@
+from typing import List, Optional, Union
+from pydantic import field_validator, model_validator, Field
+from datetime import datetime, timezone
+import uuid
+
+from judgeval.data import Example
+from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.constants import ACCEPTABLE_MODELS
+from judgeval.data.judgment_types import EvaluationRunJudgmentType
+
+
+class EvaluationRun(EvaluationRunJudgmentType):
+    """
+    Stores example and evaluation scorers together for running an eval task
+
+    Args:
+        project_name (str): The name of the project the evaluation results belong to
+        eval_name (str): A name for this evaluation run
+        examples (List[Example]): The examples to evaluate
+        scorers (List[Union[BaseScorer, APIScorerConfig]]): A list of scorers to use for evaluation
+        model (str): The model used as a judge when using LLM as a Judge
+        metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
+    """
+
+    id: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
+    created_at: Optional[str] = Field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    custom_scorers: Optional[List[BaseScorer]] = None
+    judgment_scorers: Optional[List[APIScorerConfig]] = None
+    organization_id: Optional[str] = None
+
+    def __init__(
+        self,
+        scorers: Optional[List[Union[BaseScorer, APIScorerConfig]]] = None,
+        **kwargs,
+    ):
+        """
+        Initialize EvaluationRun with automatic scorer classification.
+
+        Args:
+            scorers: List of scorers that will be automatically sorted into custom_scorers or judgment_scorers
+            **kwargs: Other initialization arguments
+        """
+        if scorers is not None:
+            # Automatically sort scorers into appropriate fields
+            custom_scorers = [s for s in scorers if isinstance(s, BaseScorer)]
+            judgment_scorers = [s for s in scorers if isinstance(s, APIScorerConfig)]
+
+            # Always set both fields as lists (even if empty) to satisfy validation
+            kwargs["custom_scorers"] = custom_scorers
+            kwargs["judgment_scorers"] = judgment_scorers
+
+        super().__init__(**kwargs)
+
+    def model_dump(self, **kwargs):
+        data = super().model_dump(**kwargs)
+        data["custom_scorers"] = [s.model_dump() for s in self.custom_scorers]
+        data["judgment_scorers"] = [s.model_dump() for s in self.judgment_scorers]
+        data["examples"] = [example.model_dump() for example in self.examples]
+
+        return data
+
+    @field_validator("examples")
+    def validate_examples(cls, v):
+        if not v:
+            raise ValueError("Examples cannot be empty.")
+        for item in v:
+            if not isinstance(item, Example):
+                raise ValueError(f"Item of type {type(item)} is not a Example")
+        return v
+
+    @model_validator(mode="after")
+    @classmethod
+    def validate_scorer_lists(cls, values):
+        custom_scorers = values.custom_scorers
+        judgment_scorers = values.judgment_scorers
+
+        # Check that both lists are not empty
+        if not custom_scorers and not judgment_scorers:
+            raise ValueError(
+                "At least one of custom_scorers or judgment_scorers must be provided."
+            )
+
+        # Check that only one list is filled
+        if custom_scorers and judgment_scorers:
+            raise ValueError(
+                "Only one of custom_scorers or judgment_scorers can be provided, not both."
+            )
+
+        return values
+
+    @field_validator("model")
+    def validate_model(cls, v, values):
+        if not v:
+            raise ValueError("Model cannot be empty.")
+
+        # Check if model is string or list of strings
+        if isinstance(v, str):
+            if v not in ACCEPTABLE_MODELS:
+                raise ValueError(
+                    f"Model name {v} not recognized. Please select a valid model name.)"
+                )
+            return v

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  openapi_new.json
-#   timestamp: 2025-07-29T18:13:07+00:00
+#   timestamp: 2025-08-08T18:50:51+00:00
 
 from __future__ import annotations
 
@@ -51,6 +51,31 @@ class ScorerConfigJudgmentType(BaseModel):
     kwargs: Annotated[Optional[Dict[str, Any]], Field(title="Kwargs")] = None
 
 
+class BaseScorerJudgmentType(BaseModel):
+    score_type: Annotated[str, Field(title="Score Type")]
+    threshold: Annotated[Optional[float], Field(title="Threshold")] = 0.5
+    name: Annotated[Optional[str], Field(title="Name")] = None
+    class_name: Annotated[Optional[str], Field(title="Class Name")] = None
+    score: Annotated[Optional[float], Field(title="Score")] = None
+    score_breakdown: Annotated[
+        Optional[Dict[str, Any]], Field(title="Score Breakdown")
+    ] = None
+    reason: Annotated[Optional[str], Field(title="Reason")] = ""
+    using_native_model: Annotated[Optional[bool], Field(title="Using Native Model")] = (
+        None
+    )
+    success: Annotated[Optional[bool], Field(title="Success")] = None
+    model: Annotated[Optional[str], Field(title="Model")] = None
+    model_client: Annotated[Any, Field(title="Model Client")] = None
+    strict_mode: Annotated[Optional[bool], Field(title="Strict Mode")] = False
+    error: Annotated[Optional[str], Field(title="Error")] = None
+    additional_metadata: Annotated[
+        Optional[Dict[str, Any]], Field(title="Additional Metadata")
+    ] = None
+    user: Annotated[Optional[str], Field(title="User")] = None
+    server_hosted: Annotated[Optional[bool], Field(title="Server Hosted")] = False
+
+
 class TraceUsageJudgmentType(BaseModel):
     prompt_tokens: Annotated[Optional[int], Field(title="Prompt Tokens")] = None
     completion_tokens: Annotated[Optional[int], Field(title="Completion Tokens")] = None
@@ -90,15 +115,21 @@ class HTTPValidationErrorJudgmentType(BaseModel):
     ] = None
 
 
-class JudgmentEvalJudgmentType(BaseModel):
+class EvaluationRunJudgmentType(BaseModel):
+    id: Annotated[Optional[str], Field(title="Id")] = None
     project_name: Annotated[Optional[str], Field(title="Project Name")] = None
     eval_name: Annotated[Optional[str], Field(title="Eval Name")] = None
     examples: Annotated[List[ExampleJudgmentType], Field(title="Examples")]
-    scorers: Annotated[List[ScorerConfigJudgmentType], Field(title="Scorers")]
+    custom_scorers: Annotated[
+        Optional[List[BaseScorerJudgmentType]], Field(title="Custom Scorers")
+    ] = Field(default_factory=list)
+    judgment_scorers: Annotated[
+        Optional[List[ScorerConfigJudgmentType]], Field(title="Judgment Scorers")
+    ] = Field(default_factory=list)
     model: Annotated[str, Field(title="Model")]
-    append: Annotated[Optional[bool], Field(title="Append")] = False
-    override: Annotated[Optional[bool], Field(title="Override")] = False
     trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
+    trace_id: Annotated[Optional[str], Field(title="Trace Id")] = None
+    created_at: Annotated[Optional[str], Field(title="Created At")] = None
 
 
 class TraceSpanJudgmentType(BaseModel):
@@ -122,6 +153,7 @@ class TraceSpanJudgmentType(BaseModel):
     ] = None
     has_evaluation: Annotated[Optional[bool], Field(title="Has Evaluation")] = False
     agent_name: Annotated[Optional[str], Field(title="Agent Name")] = None
+    class_name: Annotated[Optional[str], Field(title="Class Name")] = None
     state_before: Annotated[Optional[Dict[str, Any]], Field(title="State Before")] = (
         None
     )
@@ -171,8 +203,6 @@ class TraceRunJudgmentType(BaseModel):
     traces: Annotated[List[TraceJudgmentType], Field(title="Traces")]
     scorers: Annotated[List[ScorerConfigJudgmentType], Field(title="Scorers")]
     model: Annotated[str, Field(title="Model")]
-    append: Annotated[Optional[bool], Field(title="Append")] = False
-    override: Annotated[Optional[bool], Field(title="Override")] = False
     trace_span_id: Annotated[Optional[str], Field(title="Trace Span Id")] = None
     tools: Annotated[Optional[List[Dict[str, Any]]], Field(title="Tools")] = None
 
@@ -180,5 +210,5 @@ class TraceRunJudgmentType(BaseModel):
 class EvalResultsJudgmentType(BaseModel):
     results: Annotated[List[ScoringResultJudgmentType], Field(title="Results")]
     run: Annotated[
-        Union[TraceRunJudgmentType, JudgmentEvalJudgmentType], Field(title="Run")
+        Union[TraceRunJudgmentType, EvaluationRunJudgmentType], Field(title="Run")
     ]

judgeval/data/trace.py

@@ -1,7 +1,4 @@
-from typing import Any
-import sys
 import threading
-import orjson
 from datetime import datetime, timezone
 from judgeval.data.judgment_types import (
     TraceUsageJudgmentType,
@@ -9,7 +6,7 @@ from judgeval.data.judgment_types import (
     TraceJudgmentType,
 )
 from judgeval.constants import SPAN_LIFECYCLE_END_UPDATE_ID
-from pydantic import BaseModel
+from judgeval.common.api.json_encoder import json_encoder
 
 
 class TraceUsage(TraceUsageJudgmentType):
@@ -25,9 +22,9 @@ class TraceSpan(TraceSpanJudgmentType):
             "created_at": datetime.fromtimestamp(
                 self.created_at, tz=timezone.utc
             ).isoformat(),
-            "inputs": self._serialize_value(self.inputs),
-            "output": self._serialize_value(self.output),
-            "error": self._serialize_value(self.error),
+            "inputs": json_encoder(self.inputs),
+            "output": json_encoder(self.output),
+            "error": json_encoder(self.error),
             "parent_span_id": self.parent_span_id,
             "function": self.function,
             "duration": self.duration,
@@ -35,9 +32,10 @@ class TraceSpan(TraceSpanJudgmentType):
             "usage": self.usage.model_dump() if self.usage else None,
             "has_evaluation": self.has_evaluation,
             "agent_name": self.agent_name,
+            "class_name": self.class_name,
             "state_before": self.state_before,
             "state_after": self.state_after,
-            "additional_metadata": self._serialize_value(self.additional_metadata),
+            "additional_metadata": json_encoder(self.additional_metadata),
             "update_id": self.update_id,
         }
 
@@ -80,120 +78,6 @@ class TraceSpan(TraceSpanJudgmentType):
         )
         print(f"{indent}→ {self.function} (id: {self.span_id}){parent_info}")
 
-    def _is_json_serializable(self, obj: Any) -> bool:
-        """Helper method to check if an object is JSON serializable."""
-        try:
-            orjson.dumps(obj)
-            return True
-        except (TypeError, OverflowError, ValueError):
-            return False
-
-    def safe_stringify(self, output, function_name):
-        """
-        Safely converts an object to a JSON-serializable structure, handling common object types intelligently.
-        """
-        # Handle Pydantic models
-        if hasattr(output, "model_dump"):
-            try:
-                return output.model_dump()
-            except Exception:
-                pass
-
-        # Handle LangChain messages and similar objects with content/type
-        if hasattr(output, "content") and hasattr(output, "type"):
-            try:
-                result = {"type": output.type, "content": output.content}
-                # Add additional fields if they exist
-                if hasattr(output, "additional_kwargs"):
-                    result["additional_kwargs"] = output.additional_kwargs
-                if hasattr(output, "response_metadata"):
-                    result["response_metadata"] = output.response_metadata
-                if hasattr(output, "name"):
-                    result["name"] = output.name
-                return result
-            except Exception:
-                pass
-
-        if hasattr(output, "dict"):
-            try:
-                return output.dict()
-            except Exception:
-                pass
-
-        if hasattr(output, "to_dict"):
-            try:
-                return output.to_dict()
-            except Exception:
-                pass
-
-        if hasattr(output, "__dataclass_fields__"):
-            try:
-                import dataclasses
-
-                return dataclasses.asdict(output)
-            except Exception:
-                pass
-
-        if hasattr(output, "__dict__"):
-            try:
-                return output.__dict__
-            except Exception:
-                pass
-
-        try:
-            return str(output)
-        except (TypeError, OverflowError, ValueError):
-            pass
-
-        try:
-            return repr(output)
-        except (TypeError, OverflowError, ValueError):
-            pass
-
-        return None
-
-    def _serialize_value(self, value: Any) -> Any:
-        """Helper method to deep serialize a value safely supporting Pydantic Models / regular PyObjects."""
-        if value is None:
-            return None
-
-        recursion_limit = sys.getrecursionlimit()
-        recursion_limit = int(recursion_limit * 0.75)
-
-        def serialize_value(value, current_depth=0):
-            try:
-                if current_depth > recursion_limit:
-                    return {"error": "max_depth_reached: " + type(value).__name__}
-
-                if isinstance(value, BaseModel):
-                    return value.model_dump()
-                elif isinstance(value, dict):
-                    # Recursively serialize dictionary values
-                    return {
-                        k: serialize_value(v, current_depth + 1)
-                        for k, v in value.items()
-                    }
-                elif isinstance(value, (list, tuple)):
-                    # Recursively serialize list/tuple items
-                    return [serialize_value(item, current_depth + 1) for item in value]
-                else:
-                    try:
-                        orjson.dumps(value)
-                        return value
-                    except (TypeError, OverflowError, ValueError):
-                        # Fallback to safe stringification
-                        return self.safe_stringify(value, self.function)
-                    except Exception:
-                        return {"error": "Unable to serialize"}
-            except Exception:
-                return {"error": "Unable to serialize"}
-
-        # Start serialization with the top-level value
-        try:
-            return serialize_value(value, current_depth=0)
-        except Exception:
-            return {"error": "Unable to serialize"}
-
 
 class Trace(TraceJudgmentType):
     pass

judgeval/data/trace_run.py

@@ -3,6 +3,7 @@ from typing import List, Optional, Dict, Any, Union
 from judgeval.data import Trace
 from judgeval.scorers import APIScorerConfig, BaseScorer
 from judgeval.rules import Rule
+from judgeval.constants import DEFAULT_GPT_MODEL
 
 
 class TraceRun(BaseModel):
@@ -26,10 +27,8 @@ class TraceRun(BaseModel):
     eval_name: Optional[str] = None
     traces: Optional[List[Trace]] = None
     scorers: List[Union[APIScorerConfig, BaseScorer]]
-    model: Optional[str] = "gpt-4.1"
+    model: Optional[str] = DEFAULT_GPT_MODEL
     trace_span_id: Optional[str] = None
-    append: Optional[bool] = False
-    override: Optional[bool] = False
     rules: Optional[List[Rule]] = None
     tools: Optional[List[Dict[str, Any]]] = None
 

judgeval/dataset.py

@@ -35,6 +35,7 @@ class Dataset:
         for e in examples:
             if isinstance(e, dict) and isinstance(e.get("data"), dict):
                 e.update(e.pop("data"))
+        judgeval_logger.info(f"Succesfully retrieved dataset {name}!")
         return cls(
             name=name,
             project_name=project_name,
@@ -68,6 +69,7 @@ class Dataset:
             traces=[t.model_dump() for t in traces],
             overwrite=overwrite,
         )
+        judgeval_logger.info(f"Succesfull created dataset {name}!")
         return cls(
             name=name,
             project_name=project_name,

judgeval/integrations/langgraph.py

@@ -133,7 +133,8 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         inputs: Optional[Dict[str, Any]] = None,
     ) -> None:
         """Start tracking a span, ensuring trace client exists"""
-
+        if name.startswith("__") and name.endswith("__"):
+            return
         start_time = time.time()
         span_id = str(uuid.uuid4())
         parent_span_id: Optional[str] = None

judgeval/judges/litellm_judge.py

@@ -7,6 +7,7 @@ from judgeval.common.utils import (
     fetch_litellm_api_response,
 )
 from judgeval.common.logger import judgeval_logger
+from judgeval.constants import DEFAULT_GPT_MODEL
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
@@ -14,7 +15,7 @@ BASE_CONVERSATION = [
 
 
 class LiteLLMJudge(JudgevalJudge):
-    def __init__(self, model: str = "gpt-4.1-mini", **kwargs):
+    def __init__(self, model: str = DEFAULT_GPT_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)

judgeval/judges/mixture_of_judges.py

@@ -14,6 +14,7 @@ from judgeval.common.utils import (
     aget_chat_completion,
 )
 from judgeval.common.logger import judgeval_logger
+from judgeval.constants import DEFAULT_GPT_MODEL
 
 
 def build_dynamic_mixture_prompt(
@@ -161,7 +162,7 @@ class MixtureOfJudges(JudgevalJudge):
             "LLAMA3_70B_INSTRUCT_TURBO",
             "MISTRAL_8x22B_INSTRUCT",
         ],
-        aggregator: str = "gpt-4.1",
+        aggregator: str = DEFAULT_GPT_MODEL,
         **kwargs,
     ):
         """