judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/judges/together_judge.py

@@ -3,14 +3,77 @@ Implementation of using TogetherAI inference for judges.
 """
 
 from pydantic import BaseModel
-from typing import List, Union
-
+from typing import Dict, List, Union, Any, cast
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import (
-    fetch_together_api_response,
-    afetch_together_api_response,
+from judgeval.logger import judgeval_logger
+from judgeval.env import (
+    JUDGMENT_DEFAULT_TOGETHER_MODEL,
+    TOGETHERAI_API_KEY,
+    TOGETHER_API_KEY,
 )
-from judgeval.common.logger import judgeval_logger
+
+together_api_key = TOGETHERAI_API_KEY or TOGETHER_API_KEY
+if together_api_key:
+    try:
+        from together import Together, AsyncTogether  # type: ignore[import-untyped]
+
+        together_client = Together(api_key=together_api_key)
+        async_together_client = AsyncTogether(api_key=together_api_key)
+    except Exception:
+        pass
+
+
+def fetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
+
+async def afetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
@@ -18,44 +81,52 @@ BASE_CONVERSATION = [
 
 
 class TogetherJudge(JudgevalJudge):
-    def __init__(self, model: str = "Qwen/Qwen2.5-72B-Instruct-Turbo", **kwargs):
+    def __init__(self, model: str = JUDGMENT_DEFAULT_TOGETHER_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)
 
-    # TODO: Fix cost for generate and a_generate
-    def generate(self, input: Union[str, List[dict]], schema: BaseModel = None) -> str:
+    def generate(
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
+    ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             return fetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, convo, response_format=response_format
             )
         elif isinstance(input, list):
-            convo = input
+            messages = [dict(msg) for msg in input]
             return fetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, messages, response_format=response_format
             )
         else:
             judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
     async def a_generate(
-        self, input: Union[str, List[dict]], schema: BaseModel = None
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
     ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             res = await afetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, convo, response_format=response_format
             )
             return res
         elif isinstance(input, list):
-            convo = input
+            messages = [dict(msg) for msg in input]
             res = await afetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, messages, response_format=response_format
             )
             return res
         else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
     def load_model(self) -> str:

judgeval/judges/utils.py

@@ -3,21 +3,21 @@ This module contains utility functions for judge models.
 """
 
 import litellm
-from typing import Optional, Union, Tuple, List
+from typing import Optional, Union, Tuple
 
-from judgeval.common.exceptions import InvalidJudgeModelError
-from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges
+from judgeval.exceptions import InvalidJudgeModelError
+from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 from judgeval.constants import (
     TOGETHER_SUPPORTED_MODELS,
     JUDGMENT_SUPPORTED_MODELS,
-    ACCEPTABLE_MODELS,
 )
 
 LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
 
 
 def create_judge(
-    model: Optional[Union[str, List[str], JudgevalJudge]] = None,
+    model: Optional[Union[str, JudgevalJudge]] = None,
 ) -> Tuple[JudgevalJudge, bool]:
     """
     Creates a judge model from string(s) or a judgeval judge object.
@@ -30,28 +30,15 @@ def create_judge(
     If no model is provided, uses GPT4o as the default judge.
     """
     if model is None:  # default option
-        return LiteLLMJudge(model="gpt-4.1"), True
+        return LiteLLMJudge(model=JUDGMENT_DEFAULT_GPT_MODEL), True
     if not isinstance(model, (str, list, JudgevalJudge)):
         raise InvalidJudgeModelError(
             f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead."
         )
     # If model is already a valid judge type, return it and mark native
-    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges)):
+    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge)):
         return model, True
 
-    # Either string or List[str]
-    if isinstance(model, list):
-        for m in model:
-            if m in JUDGMENT_SUPPORTED_MODELS:
-                raise NotImplementedError(
-                    """Judgment models are not yet supported for local scoring.
-                    Please either set the `use_judgment` flag to True or use 
-                    non-Judgment models."""
-                )
-            if m not in ACCEPTABLE_MODELS:
-                raise InvalidJudgeModelError(f"Invalid judge model chosen: {m}")
-        return MixtureOfJudges(models=model), True
-    # If model is a string, check that it corresponds to a valid model
     if model in LITELLM_SUPPORTED_MODELS:
         return LiteLLMJudge(model=model), True
     if model in TOGETHER_SUPPORTED_MODELS:

judgeval/judgment_attribute_keys.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class AttributeKeys(str, Enum):
+    JUDGMENT_SPAN_KIND = "judgment.span_kind"
+    JUDGMENT_INPUT = "judgment.input"
+    JUDGMENT_OUTPUT = "judgment.output"
+    JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"
+    JUDGMENT_UPDATE_ID = "judgment.update_id"
+    JUDGMENT_CUSTOMER_ID = "judgment.customer_id"
+    JUDGMENT_AGENT_ID = "judgment.agent_id"
+    JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"
+    JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"
+    JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"
+    JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"
+    JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"
+    JUDGMENT_STATE_BEFORE = "judgment.state_before"
+    JUDGMENT_STATE_AFTER = "judgment.state_after"
+    JUDGMENT_PENDING_TRACE_EVAL = "judgment.pending_trace_eval"
+    JUDGMENT_USAGE_METADATA = "judgment.usage.metadata"
+
+    JUDGMENT_LLM_PROVIDER = "judgment.llm.provider"
+    JUDGMENT_LLM_MODEL_NAME = "judgment.llm.model"
+    JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS = "judgment.usage.non_cached_input_tokens"
+    JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS = (
+        "judgment.usage.cache_creation_input_tokens"
+    )
+    JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS = "judgment.usage.cache_read_input_tokens"
+    JUDGMENT_USAGE_OUTPUT_TOKENS = "judgment.usage.output_tokens"
+    JUDGMENT_USAGE_TOTAL_COST_USD = "judgment.usage.total_cost_usd"
+
+    GEN_AI_PROMPT = "gen_ai.prompt"
+    GEN_AI_COMPLETION = "gen_ai.completion"
+    GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+    GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
+    GEN_AI_SYSTEM = "gen_ai.system"
+    GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
+    GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
+    GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = (
+        "gen_ai.usage.cache_creation_input_tokens"
+    )
+    GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"
+    GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
+    GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
+    GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"
+
+
+class ResourceKeys(str, Enum):
+    SERVICE_NAME = "service.name"
+    TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"
+    TELEMETRY_SDK_NAME = "telemetry.sdk.name"
+    TELEMETRY_SDK_VERSION = "telemetry.sdk.version"
+    JUDGMENT_PROJECT_ID = "judgment.project_id"

judgeval/{common/logger.py → logger.py}

@@ -1,10 +1,9 @@
-# logger.py
-
 import logging
 import sys
-import os
 
-# ANSI escape sequences
+from judgeval.env import JUDGMENT_NO_COLOR, JUDGMENT_LOG_LEVEL
+from judgeval.utils.decorators.use_once import use_once
+
 RESET = "\033[0m"
 RED = "\033[31m"
 YELLOW = "\033[33m"
@@ -38,10 +37,25 @@ class ColorFormatter(logging.Formatter):
         return message
 
 
+def _parse_log_level(level_str: str) -> int:
+    level_map = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "warning": logging.WARNING,
+        "warn": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL,
+    }
+    return level_map.get(level_str.lower(), logging.WARNING)
+
+
+@use_once
 def _setup_judgeval_logger():
-    use_color = sys.stdout.isatty() and os.getenv("NO_COLOR") is None
+    use_color = sys.stdout.isatty() and JUDGMENT_NO_COLOR is None
+    log_level = _parse_log_level(JUDGMENT_LOG_LEVEL)
+
     handler = logging.StreamHandler(sys.stdout)
-    handler.setLevel(logging.DEBUG)
+    handler.setLevel(log_level)
     handler.setFormatter(
         ColorFormatter(
             fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
@@ -51,10 +65,12 @@ def _setup_judgeval_logger():
     )
 
     logger = logging.getLogger("judgeval")
-    logger.setLevel(logging.DEBUG)
+    logger.setLevel(log_level)
     logger.addHandler(handler)
     return logger
 
 
-# Global logger you can import elsewhere
 judgeval_logger = _setup_judgeval_logger()
+
+
+__all__ = ("judgeval_logger",)

judgeval/prompt/__init__.py

@@ -0,0 +1,330 @@
+from typing import List, Optional, Dict
+from judgeval.api import JudgmentSyncClient
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.api.api_types import (
+    PromptCommitInfo,
+    PromptTagResponse,
+    PromptUntagResponse,
+    PromptVersionsResponse,
+)
+from dataclasses import dataclass, field
+import re
+from string import Template
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
+from judgeval.utils.project import _resolve_project_id
+
+
+def push_prompt(
+    project_name: str,
+    name: str,
+    prompt: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> tuple[str, Optional[str], str]:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        r = client.prompts_insert(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "prompt": prompt,
+                "tags": tags,
+            }
+        )
+        return r["commit_id"], r.get("parent_commit_id"), r["created_at"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to save prompt: {e.detail}",
+            response=e.response,
+        )
+
+
+def fetch_prompt(
+    project_name: str,
+    name: str,
+    commit_id: Optional[str] = None,
+    tag: Optional[str] = None,
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> Optional[PromptCommitInfo]:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_fetch(
+            name=name,
+            project_id=project_id,
+            commit_id=commit_id,
+            tag=tag,
+        )
+        return prompt_config["commit"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to fetch prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def tag_prompt(
+    project_name: str,
+    name: str,
+    commit_id: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptTagResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_tag(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "commit_id": commit_id,
+                "tags": tags,
+            }
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to tag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def untag_prompt(
+    project_name: str,
+    name: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptUntagResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_untag(
+            payload={"project_id": project_id, "name": name, "tags": tags}
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to untag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def list_prompt(
+    project_name: str,
+    name: str,
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptVersionsResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_get_prompt_versions(
+            project_id=project_id, name=name
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to list prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+@dataclass
+class Prompt:
+    name: str
+    prompt: str
+    created_at: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: Optional[str] = None
+    metadata: Dict[str, str] = field(default_factory=dict)
+    _template: Template = field(init=False, repr=False)
+
+    def __post_init__(self):
+        template_str = re.sub(r"\{\{([^}]+)\}\}", r"$\1", self.prompt)
+        self._template = Template(template_str)
+
+    @classmethod
+    def create(
+        cls,
+        project_name: str,
+        name: str,
+        prompt: str,
+        tags: Optional[List[str]] = None,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        if tags is None:
+            tags = []
+        commit_id, parent_commit_id, created_at = push_prompt(
+            project_name, name, prompt, tags, judgment_api_key, organization_id
+        )
+        return cls(
+            name=name,
+            prompt=prompt,
+            created_at=created_at,
+            tags=tags,
+            commit_id=commit_id,
+            parent_commit_id=parent_commit_id,
+        )
+
+    @classmethod
+    def get(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        if commit_id is not None and tag is not None:
+            raise ValueError(
+                "You cannot fetch a prompt by both commit_id and tag at the same time"
+            )
+        prompt_config = fetch_prompt(
+            project_name, name, commit_id, tag, judgment_api_key, organization_id
+        )
+        if prompt_config is None:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Prompt '{name}' not found in project '{project_name}'",
+                response=None,  # type: ignore
+            )
+        return cls(
+            name=prompt_config["name"],
+            prompt=prompt_config["prompt"],
+            created_at=prompt_config["created_at"],
+            tags=prompt_config["tags"],
+            commit_id=prompt_config["commit_id"],
+            parent_commit_id=prompt_config.get("parent_commit_id"),
+            metadata={
+                "creator_first_name": prompt_config["first_name"],
+                "creator_last_name": prompt_config["last_name"],
+                "creator_email": prompt_config["user_email"],
+            },
+        )
+
+    @classmethod
+    def tag(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: str,
+        tags: List[str],
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = tag_prompt(
+            project_name, name, commit_id, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_id"]
+
+    @classmethod
+    def untag(
+        cls,
+        project_name: str,
+        name: str,
+        tags: List[str],
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = untag_prompt(
+            project_name, name, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_ids"]
+
+    @classmethod
+    def list(
+        cls,
+        project_name: str,
+        name: str,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_configs = list_prompt(
+            project_name, name, judgment_api_key, organization_id
+        )["versions"]
+        return [
+            cls(
+                name=prompt_config["name"],
+                prompt=prompt_config["prompt"],
+                tags=prompt_config["tags"],
+                created_at=prompt_config["created_at"],
+                commit_id=prompt_config["commit_id"],
+                parent_commit_id=prompt_config.get("parent_commit_id"),
+                metadata={
+                    "creator_first_name": prompt_config["first_name"],
+                    "creator_last_name": prompt_config["last_name"],
+                    "creator_email": prompt_config["user_email"],
+                },
+            )
+            for prompt_config in prompt_configs
+        ]
+
+    def compile(self, **kwargs) -> str:
+        try:
+            return self._template.substitute(**kwargs)
+        except KeyError as e:
+            missing_var = str(e).strip("'")
+            raise ValueError(f"Missing required variable: {missing_var}")

judgeval/scorers/__init__.py

@@ -1,29 +1,29 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import (
+    APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
+)
 from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.scorers.judgeval_scorers.api_scorers import (
-    ExecutionOrderScorer,
-    HallucinationScorer,
     FaithfulnessScorer,
     AnswerRelevancyScorer,
     AnswerCorrectnessScorer,
     InstructionAdherenceScorer,
-    DerailmentScorer,
-    ToolOrderScorer,
+    TracePromptScorer,
     PromptScorer,
-    ToolDependencyScorer,
 )
 
 __all__ = [
     "APIScorerConfig",
+    "ExampleAPIScorerConfig",
+    "TraceAPIScorerConfig",
     "BaseScorer",
+    "ExampleScorer",
+    "TracePromptScorer",
     "PromptScorer",
-    "ExecutionOrderScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
     "InstructionAdherenceScorer",
-    "DerailmentScorer",
-    "ToolOrderScorer",
-    "ToolDependencyScorer",
 ]