judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/prompt/__init__.py

@@ -0,0 +1,330 @@
+from typing import List, Optional, Dict
+from judgeval.api import JudgmentSyncClient
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.api.api_types import (
+    PromptCommitInfo,
+    PromptTagResponse,
+    PromptUntagResponse,
+    PromptVersionsResponse,
+)
+from dataclasses import dataclass, field
+import re
+from string import Template
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
+from judgeval.utils.project import _resolve_project_id
+
+
+def push_prompt(
+    project_name: str,
+    name: str,
+    prompt: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> tuple[str, Optional[str], str]:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        r = client.prompts_insert(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "prompt": prompt,
+                "tags": tags,
+            }
+        )
+        return r["commit_id"], r.get("parent_commit_id"), r["created_at"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to save prompt: {e.detail}",
+            response=e.response,
+        )
+
+
+def fetch_prompt(
+    project_name: str,
+    name: str,
+    commit_id: Optional[str] = None,
+    tag: Optional[str] = None,
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> Optional[PromptCommitInfo]:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_fetch(
+            name=name,
+            project_id=project_id,
+            commit_id=commit_id,
+            tag=tag,
+        )
+        return prompt_config["commit"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to fetch prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def tag_prompt(
+    project_name: str,
+    name: str,
+    commit_id: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptTagResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_tag(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "commit_id": commit_id,
+                "tags": tags,
+            }
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to tag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def untag_prompt(
+    project_name: str,
+    name: str,
+    tags: List[str],
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptUntagResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_untag(
+            payload={"project_id": project_id, "name": name, "tags": tags}
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to untag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def list_prompt(
+    project_name: str,
+    name: str,
+    judgment_api_key: str | None = JUDGMENT_API_KEY,
+    organization_id: str | None = JUDGMENT_ORG_ID,
+) -> PromptVersionsResponse:
+    if not judgment_api_key or not organization_id:
+        raise ValueError("Judgment API key and organization ID are required")
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_get_prompt_versions(
+            project_id=project_id, name=name
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to list prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+@dataclass
+class Prompt:
+    name: str
+    prompt: str
+    created_at: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: Optional[str] = None
+    metadata: Dict[str, str] = field(default_factory=dict)
+    _template: Template = field(init=False, repr=False)
+
+    def __post_init__(self):
+        template_str = re.sub(r"\{\{([^}]+)\}\}", r"$\1", self.prompt)
+        self._template = Template(template_str)
+
+    @classmethod
+    def create(
+        cls,
+        project_name: str,
+        name: str,
+        prompt: str,
+        tags: Optional[List[str]] = None,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        if tags is None:
+            tags = []
+        commit_id, parent_commit_id, created_at = push_prompt(
+            project_name, name, prompt, tags, judgment_api_key, organization_id
+        )
+        return cls(
+            name=name,
+            prompt=prompt,
+            created_at=created_at,
+            tags=tags,
+            commit_id=commit_id,
+            parent_commit_id=parent_commit_id,
+        )
+
+    @classmethod
+    def get(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        if commit_id is not None and tag is not None:
+            raise ValueError(
+                "You cannot fetch a prompt by both commit_id and tag at the same time"
+            )
+        prompt_config = fetch_prompt(
+            project_name, name, commit_id, tag, judgment_api_key, organization_id
+        )
+        if prompt_config is None:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Prompt '{name}' not found in project '{project_name}'",
+                response=None,  # type: ignore
+            )
+        return cls(
+            name=prompt_config["name"],
+            prompt=prompt_config["prompt"],
+            created_at=prompt_config["created_at"],
+            tags=prompt_config["tags"],
+            commit_id=prompt_config["commit_id"],
+            parent_commit_id=prompt_config.get("parent_commit_id"),
+            metadata={
+                "creator_first_name": prompt_config["first_name"],
+                "creator_last_name": prompt_config["last_name"],
+                "creator_email": prompt_config["user_email"],
+            },
+        )
+
+    @classmethod
+    def tag(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: str,
+        tags: List[str],
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = tag_prompt(
+            project_name, name, commit_id, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_id"]
+
+    @classmethod
+    def untag(
+        cls,
+        project_name: str,
+        name: str,
+        tags: List[str],
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = untag_prompt(
+            project_name, name, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_ids"]
+
+    @classmethod
+    def list(
+        cls,
+        project_name: str,
+        name: str,
+        judgment_api_key: str | None = JUDGMENT_API_KEY,
+        organization_id: str | None = JUDGMENT_ORG_ID,
+    ):
+        prompt_configs = list_prompt(
+            project_name, name, judgment_api_key, organization_id
+        )["versions"]
+        return [
+            cls(
+                name=prompt_config["name"],
+                prompt=prompt_config["prompt"],
+                tags=prompt_config["tags"],
+                created_at=prompt_config["created_at"],
+                commit_id=prompt_config["commit_id"],
+                parent_commit_id=prompt_config.get("parent_commit_id"),
+                metadata={
+                    "creator_first_name": prompt_config["first_name"],
+                    "creator_last_name": prompt_config["last_name"],
+                    "creator_email": prompt_config["user_email"],
+                },
+            )
+            for prompt_config in prompt_configs
+        ]
+
+    def compile(self, **kwargs) -> str:
+        try:
+            return self._template.substitute(**kwargs)
+        except KeyError as e:
+            missing_var = str(e).strip("'")
+            raise ValueError(f"Missing required variable: {missing_var}")

judgeval/scorers/__init__.py

@@ -1,36 +1,29 @@
-from judgeval.scorers.api_scorer import APIJudgmentScorer
-from judgeval.scorers.judgeval_scorer import JudgevalScorer
-from judgeval.scorers.prompt_scorer import PromptScorer, ClassifierScorer
-from judgeval.scorers.judgeval_scorers import (
-    ToolCorrectnessScorer,
-    JSONCorrectnessScorer,
-    SummarizationScorer,
-    HallucinationScorer,
+from judgeval.scorers.api_scorer import (
+    APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
+)
+from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.scorers.example_scorer import ExampleScorer
+from judgeval.scorers.judgeval_scorers.api_scorers import (
     FaithfulnessScorer,
-    ContextualRelevancyScorer,
-    ContextualPrecisionScorer,
-    ContextualRecallScorer,
     AnswerRelevancyScorer,
-    ScorerWrapper,
     AnswerCorrectnessScorer,
-    Text2SQLScorer,
+    InstructionAdherenceScorer,
+    TracePromptScorer,
+    PromptScorer,
 )
 
 __all__ = [
-    "APIJudgmentScorer",
-    "JudgevalScorer",
+    "APIScorerConfig",
+    "ExampleAPIScorerConfig",
+    "TraceAPIScorerConfig",
+    "BaseScorer",
+    "ExampleScorer",
+    "TracePromptScorer",
     "PromptScorer",
-    "ClassifierScorer",
-    "ToolCorrectnessScorer",
-    "JSONCorrectnessScorer",
-    "SummarizationScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
-    "ContextualRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
     "AnswerRelevancyScorer",
-    "ScorerWrapper",
     "AnswerCorrectnessScorer",
-    "Text2SQLScorer",
+    "InstructionAdherenceScorer",
 ]

judgeval/scorers/agent_scorer.py

@@ -0,0 +1,17 @@
+# from judgeval.scorers.base_scorer import BaseScorer
+# from judgeval.data.judgment_types import Trace as JudgmentTrace
+# from typing import List, Optional
+# from abc import abstractmethod
+
+
+# class TraceScorer(BaseScorer):
+#     @abstractmethod
+#     async def a_score_trace(
+#         self, trace: JudgmentTrace, tools: Optional[List] = None, *args, **kwargs
+#     ) -> float:
+#         """
+#         Asynchronously measures the score on a trace
+#         """
+#         raise NotImplementedError(
+#             "You must implement the `a_score_trace` method in your custom scorer"
+#         )

judgeval/scorers/api_scorer.py

@@ -4,61 +4,65 @@ Judgment Scorer class.
 Scores `Example`s using ready-made Judgment evaluators.
 """
 
-from pydantic import BaseModel, field_validator
-from judgeval.common.logger import debug, info, warning, error
+from __future__ import annotations
 
-from judgeval.constants import APIScorer
+from pydantic import BaseModel, field_validator
+from typing import List
+from judgeval.constants import APIScorerType
+from judgeval.data.example import ExampleParams
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 
 
-class APIJudgmentScorer(BaseModel):
+class APIScorerConfig(BaseModel):
     """
-    Class for ready-made, "out-of-the-box" scorer that uses Judgment evaluators to score `Example`s.
+    Scorer config that is used to send to our Judgment server.
 
     Args:
         score_type (APIScorer): The Judgment metric to use for scoring `Example`s
+        name (str): The name of the scorer, usually this is the same as the score_type
         threshold (float): A value between 0 and 1 that determines the scoring threshold
+        strict_mode (bool): Whether to use strict mode for the scorer
+        required_params (List[ExampleParams]): List of the required parameters on examples for the scorer
+        kwargs (dict): Additional keyword arguments to pass to the scorer
     """
-    threshold: float
-    score_type: APIScorer
 
-    @field_validator('threshold')
-    def validate_threshold(cls, v):
+    score_type: APIScorerType
+    name: str = ""
+    threshold: float = 0.5
+    strict_mode: bool = False
+    model: str = JUDGMENT_DEFAULT_GPT_MODEL
+
+    required_params: List[ExampleParams] = []
+
+    kwargs: dict = {}
+
+    @field_validator("threshold")
+    @classmethod
+    def validate_threshold(cls, v, info):
         """
         Validates that the threshold is between 0 and 1 inclusive.
         """
+        score_type = info.data.get("score_type")
         if not 0 <= v <= 1:
-            error(f"Threshold must be between 0 and 1, got: {v}")
-            raise ValueError(f"Threshold must be between 0 and 1, got: {v}")
+            raise ValueError(
+                f"Threshold for {score_type} must be between 0 and 1, got: {v}"
+            )
+        return v
+
+    @field_validator("name", mode="after")
+    @classmethod
+    def set_name_to_score_type_if_none(cls, v, info):
+        if v is None:
+            return info.data.get("score_type")
         return v
 
-    @field_validator('score_type')
-    def convert_to_enum_value(cls, v):
-        """
-        Validates that the `score_type` is a valid `JudgmentMetric` enum value.
-        Converts string values to `JudgmentMetric` enum values.
-        """
-        debug(f"Attempting to convert score_type value: {v}")
-        if isinstance(v, APIScorer):
-            info(f"Using existing JudgmentMetric: {v.value}")
-            return v.value
-        elif isinstance(v, str):
-            debug(f"Converting string value to JudgmentMetric enum: {v}")
-            return APIScorer[v.upper()].value
-        error(f"Invalid score_type value: {v}")
-        raise ValueError(f"Invalid value for score_type: {v}")
-    
     def __str__(self):
-        return f"JudgmentScorer(score_type={self.score_type}, threshold={self.threshold})"
-    
-    def to_dict(self) -> dict:
-        """
-        Converts the scorer configuration to a dictionary format.
-        
-        Returns:
-            dict: A dictionary containing the scorer's configuration
-        """
-        return {
-            "score_type": self.score_type,
-            "threshold": self.threshold
-        }
-    
+        return f"JudgmentScorer(score_type={self.score_type.value}, threshold={self.threshold})"
+
+
+class ExampleAPIScorerConfig(APIScorerConfig):
+    pass
+
+
+class TraceAPIScorerConfig(APIScorerConfig):
+    pass

judgeval/scorers/base_scorer.py

@@ -1,52 +1,97 @@
 """
-Judgment Scorer class.
-
-Scores `Example`s using ready-made Judgment evaluators.
+Base class for all scorers.
 """
 
-from pydantic import BaseModel, field_validator
-from judgeval.common.logger import debug, info, warning, error
+from __future__ import annotations
+from typing import Dict, Optional
 
-from judgeval.constants import APIScorer
+from pydantic import BaseModel
 
 
-class APIJudgmentScorer(BaseModel):
-    """
-    Class for ready-made, "out-of-the-box" scorer that uses Judgment evaluators to score `Example`s.
+from judgeval.judges.utils import create_judge
+from typing import Any
+from pydantic import model_validator, Field
+
 
-    Args:
-        score_type (APIScorer): The Judgment metric to use for scoring `Example`s
-        threshold (float): A value between 0 and 1 that determines the scoring threshold
+class BaseScorer(BaseModel):
     """
-    threshold: float
-    score_type: APIScorer
+    If you want to create a scorer that does not fall under any of the ready-made Judgment scorers,
+    you can create a custom scorer by extending this class. This is best used for special use cases
+    where none of Judgment's scorers are suitable.
+    """
+
+    # type of your scorer (Faithfulness, PromptScorer)
+    score_type: str
+
+    # The threshold to pass a test while using this scorer as a scorer
+    threshold: float = 0.5
+
+    # name of your scorer (Faithfulness, PromptScorer-randomslug)
+    name: str = ""
+
+    # The name of the class of the scorer
+    class_name: Optional[str] = None
+
+    # The float score of the scorer run on the test case
+    score: Optional[float] = None
+
+    score_breakdown: Optional[Dict] = None
+    reason: Optional[str] = ""
+
+    # Whether the model is a native model
+    using_native_model: Optional[bool] = None
 
-    @field_validator('threshold')
-    def validate_threshold(cls, v):
+    # Whether the test case passed or failed
+    success: bool = False
+
+    # The name of the model used to evaluate the test case
+    model: Optional[str] = None
+
+    # The model used to evaluate the test case
+    model_client: Optional[Any] = Field(default=None, exclude=True)
+
+    # Whether to run the scorer in strict mode
+    strict_mode: bool = False
+
+    # The error message if the scorer failed
+    error: Optional[str] = None
+
+    # Additional metadata for the scorer
+    additional_metadata: Optional[Dict] = None
+
+    # The user ID of the scorer
+    user: Optional[str] = None
+
+    # Whether the scorer is hosted on the server
+    server_hosted: bool = False
+
+    @model_validator(mode="after")
+    def enforce_strict_threshold(self):
+        if self.strict_mode:
+            self.threshold = 1.0
+        return self
+
+    @model_validator(mode="after")
+    def default_name(self):
+        self.class_name = self.__class__.__name__
+        if not self.name:
+            self.name = self.class_name
+        return self
+
+    def _add_model(self, model: str):
         """
-        Validates that the threshold is between 0 and 1 inclusive.
+        Adds the evaluation model to the BaseScorer instance
+
+        This method is used at eval time
         """
-        if not 0 <= v <= 1:
-            error(f"Threshold must be between 0 and 1, got: {v}")
-            raise ValueError(f"Threshold must be between 0 and 1, got: {v}")
-        return v
+        self.model_client, self.using_native_model = create_judge(model)
 
-    @field_validator('score_type')
-    def convert_to_enum_value(cls, v):
+    def success_check(self) -> bool:
         """
-        Validates that the `score_type` is a valid `JudgmentMetric` enum value.
-        Converts string values to `JudgmentMetric` enum values.
+        For unit testing, determines whether the test case passes or fails
         """
-        debug(f"Attempting to convert score_type value: {v}")
-        if isinstance(v, APIScorer):
-            info(f"Using existing JudgmentMetric: {v.value}")
-            return v.value
-        elif isinstance(v, str):
-            debug(f"Converting string value to JudgmentMetric enum: {v}")
-            return APIScorer[v.upper()].value
-        error(f"Invalid score_type value: {v}")
-        raise ValueError(f"Invalid value for score_type: {v}")
-    
-    def __str__(self):
-        return f"JudgmentScorer(score_type={self.score_type}, threshold={self.threshold})"
-    
+        if self.error:
+            return False
+        if self.score is None:
+            return False
+        return self.score >= self.threshold

judgeval/scorers/example_scorer.py

@@ -0,0 +1,17 @@
+from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.data import Example
+from typing import List
+from pydantic import Field
+
+
+class ExampleScorer(BaseScorer):
+    score_type: str = "Custom"
+    required_params: List[str] = Field(default_factory=list)
+
+    async def a_score_example(self, example: Example, *args, **kwargs) -> float:
+        """
+        Asynchronously measures the score on a single example
+        """
+        raise NotImplementedError(
+            "You must implement the `a_score_example` method in your custom scorer"
+        )

judgeval/scorers/exceptions.py

@@ -8,4 +8,5 @@ class MissingExampleParamsError(Exception):
     """
     Error raised when a scorer is missing required example parameters.
     """
+
     pass