judgeval 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

judgeval/__init__.py

@@ -6,6 +6,7 @@ from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
 from typing import List, Optional, Union, Sequence
+import ast
 from judgeval.scorers import ExampleAPIScorerConfig
 from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.data.example import Example
@@ -81,6 +82,7 @@ class JudgmentClient(metaclass=SingletonMeta):
         scorer_file_path: str,
         requirements_file_path: Optional[str] = None,
         unique_name: Optional[str] = None,
+        overwrite: bool = False,
     ) -> bool:
         """
         Upload custom ExampleScorer from files to backend.
@@ -89,6 +91,7 @@ class JudgmentClient(metaclass=SingletonMeta):
             scorer_file_path: Path to Python file containing CustomScorer class
             requirements_file_path: Optional path to requirements.txt
             unique_name: Optional unique identifier (auto-detected from scorer.name if not provided)
+            overwrite: Whether to overwrite existing scorer if it already exists
 
         Returns:
             bool: True if upload successful
@@ -111,6 +114,31 @@ class JudgmentClient(metaclass=SingletonMeta):
         with open(scorer_file_path, "r") as f:
             scorer_code = f.read()
 
+        try:
+            tree = ast.parse(scorer_code, filename=scorer_file_path)
+        except SyntaxError as e:
+            error_msg = f"Invalid Python syntax in {scorer_file_path}: {e}"
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        scorer_classes = []
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                for base in node.bases:
+                    if (isinstance(base, ast.Name) and base.id == "ExampleScorer") or (
+                        isinstance(base, ast.Attribute) and base.attr == "ExampleScorer"
+                    ):
+                        scorer_classes.append(node.name)
+
+        if len(scorer_classes) > 1:
+            error_msg = f"Multiple ExampleScorer classes found in {scorer_file_path}: {scorer_classes}. Please only upload one scorer class per file."
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+        elif len(scorer_classes) == 0:
+            error_msg = f"No ExampleScorer class was found in {scorer_file_path}. Please ensure the file contains a valid scorer class that inherits from ExampleScorer."
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+
         # Read requirements (optional)
         requirements_text = ""
         if requirements_file_path and os.path.exists(requirements_file_path):
@@ -127,6 +155,7 @@ class JudgmentClient(metaclass=SingletonMeta):
                     "scorer_name": unique_name,
                     "scorer_code": scorer_code,
                     "requirements_text": requirements_text,
+                    "overwrite": overwrite,
                 }
             )
 

judgeval/api/__init__.py

@@ -189,6 +189,59 @@ class JudgmentSyncClient:
             payload,
         )
 
+    def prompts_insert(self, payload: PromptInsertRequest) -> PromptInsertResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/insert/"),
+            payload,
+        )
+
+    def prompts_tag(self, payload: PromptTagRequest) -> PromptTagResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/tag/"),
+            payload,
+        )
+
+    def prompts_untag(self, payload: PromptUntagRequest) -> PromptUntagResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/untag/"),
+            payload,
+        )
+
+    def prompts_fetch(
+        self,
+        project_id: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+    ) -> PromptFetchResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        if commit_id is not None:
+            query_params["commit_id"] = commit_id
+        if tag is not None:
+            query_params["tag"] = tag
+        return self._request(
+            "GET",
+            url_for("/prompts/fetch/"),
+            query_params,
+        )
+
+    def prompts_get_prompt_versions(
+        self, project_id: str, name: str
+    ) -> PromptVersionsResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        return self._request(
+            "GET",
+            url_for("/prompts/get_prompt_versions/"),
+            query_params,
+        )
+
     def projects_resolve(
         self, payload: ResolveProjectNameRequest
     ) -> ResolveProjectNameResponse:
@@ -381,6 +434,61 @@ class JudgmentAsyncClient:
             payload,
         )
 
+    async def prompts_insert(
+        self, payload: PromptInsertRequest
+    ) -> PromptInsertResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/insert/"),
+            payload,
+        )
+
+    async def prompts_tag(self, payload: PromptTagRequest) -> PromptTagResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/tag/"),
+            payload,
+        )
+
+    async def prompts_untag(self, payload: PromptUntagRequest) -> PromptUntagResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/untag/"),
+            payload,
+        )
+
+    async def prompts_fetch(
+        self,
+        project_id: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+    ) -> PromptFetchResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        if commit_id is not None:
+            query_params["commit_id"] = commit_id
+        if tag is not None:
+            query_params["tag"] = tag
+        return await self._request(
+            "GET",
+            url_for("/prompts/fetch/"),
+            query_params,
+        )
+
+    async def prompts_get_prompt_versions(
+        self, project_id: str, name: str
+    ) -> PromptVersionsResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        return await self._request(
+            "GET",
+            url_for("/prompts/get_prompt_versions/"),
+            query_params,
+        )
+
     async def projects_resolve(
         self, payload: ResolveProjectNameRequest
     ) -> ResolveProjectNameResponse:

judgeval/api/api_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-15T19:25:00+00:00
+#   timestamp: 2025-10-21T01:37:42+00:00
 
 from __future__ import annotations
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -80,6 +80,7 @@ class CustomScorerUploadPayload(TypedDict):
     scorer_name: str
     scorer_code: str
     requirements_text: str
+    overwrite: NotRequired[bool]
 
 
 class CustomScorerTemplateResponse(TypedDict):
@@ -88,6 +89,40 @@ class CustomScorerTemplateResponse(TypedDict):
     message: str
 
 
+class PromptInsertRequest(TypedDict):
+    project_id: str
+    name: str
+    prompt: str
+    tags: List[str]
+
+
+class PromptInsertResponse(TypedDict):
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+
+
+class PromptTagRequest(TypedDict):
+    project_id: str
+    name: str
+    commit_id: str
+    tags: List[str]
+
+
+class PromptTagResponse(TypedDict):
+    commit_id: str
+
+
+class PromptUntagRequest(TypedDict):
+    project_id: str
+    name: str
+    tags: List[str]
+
+
+class PromptUntagResponse(TypedDict):
+    commit_ids: List[str]
+
+
 class ResolveProjectNameRequest(TypedDict):
     project_name: str
 
@@ -169,6 +204,18 @@ class PromptScorer(TypedDict):
     is_trace: NotRequired[Optional[bool]]
 
 
+class PromptCommitInfo(TypedDict):
+    name: str
+    prompt: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+    first_name: str
+    last_name: str
+    user_email: str
+
+
 class ScorerData(TypedDict):
     id: NotRequired[str]
     name: str
@@ -265,6 +312,14 @@ class FetchPromptScorersResponse(TypedDict):
     scorers: List[PromptScorer]
 
 
+class PromptFetchResponse(TypedDict):
+    commit: NotRequired[Optional[PromptCommitInfo]]
+
+
+class PromptVersionsResponse(TypedDict):
+    versions: List[PromptCommitInfo]
+
+
 class ScoringResult(TypedDict):
     success: bool
     scorers_data: List[ScorerData]

judgeval/cli.py

@@ -26,6 +26,12 @@ def upload_scorer(
     unique_name: str = typer.Option(
         None, help="Custom name for the scorer (auto-detected if not provided)"
     ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        "-o",
+        help="Overwrite existing scorer if it already exists",
+    ),
 ):
     # Validate file paths
     if not Path(scorer_file_path).exists():
@@ -43,6 +49,7 @@ def upload_scorer(
             scorer_file_path=scorer_file_path,
             requirements_file_path=requirements_file_path,
             unique_name=unique_name,
+            overwrite=overwrite,
         )
 
         if not result:

judgeval/data/judgment_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  .openapi.json
-#   timestamp: 2025-10-15T19:24:59+00:00
+#   timestamp: 2025-10-21T01:37:41+00:00
 
 from __future__ import annotations
 from typing import Annotated, Any, Dict, List, Optional, Union
@@ -87,6 +87,7 @@ class CustomScorerUploadPayload(BaseModel):
     scorer_name: Annotated[str, Field(title="Scorer Name")]
     scorer_code: Annotated[str, Field(title="Scorer Code")]
     requirements_text: Annotated[str, Field(title="Requirements Text")]
+    overwrite: Annotated[Optional[bool], Field(title="Overwrite")] = False
 
 
 class CustomScorerTemplateResponse(BaseModel):
@@ -95,6 +96,40 @@ class CustomScorerTemplateResponse(BaseModel):
     message: Annotated[str, Field(title="Message")]
 
 
+class PromptInsertRequest(BaseModel):
+    project_id: Annotated[str, Field(title="Project Id")]
+    name: Annotated[str, Field(title="Name")]
+    prompt: Annotated[str, Field(title="Prompt")]
+    tags: Annotated[List[str], Field(title="Tags")]
+
+
+class PromptInsertResponse(BaseModel):
+    commit_id: Annotated[str, Field(title="Commit Id")]
+    parent_commit_id: Annotated[Optional[str], Field(title="Parent Commit Id")] = None
+    created_at: Annotated[str, Field(title="Created At")]
+
+
+class PromptTagRequest(BaseModel):
+    project_id: Annotated[str, Field(title="Project Id")]
+    name: Annotated[str, Field(title="Name")]
+    commit_id: Annotated[str, Field(title="Commit Id")]
+    tags: Annotated[List[str], Field(title="Tags")]
+
+
+class PromptTagResponse(BaseModel):
+    commit_id: Annotated[str, Field(title="Commit Id")]
+
+
+class PromptUntagRequest(BaseModel):
+    project_id: Annotated[str, Field(title="Project Id")]
+    name: Annotated[str, Field(title="Name")]
+    tags: Annotated[List[str], Field(title="Tags")]
+
+
+class PromptUntagResponse(BaseModel):
+    commit_ids: Annotated[List[str], Field(title="Commit Ids")]
+
+
 class ResolveProjectNameRequest(BaseModel):
     project_name: Annotated[str, Field(title="Project Name")]
 
@@ -187,6 +222,18 @@ class PromptScorer(BaseModel):
     is_trace: Annotated[Optional[bool], Field(title="Is Trace")] = False
 
 
+class PromptCommitInfo(BaseModel):
+    name: Annotated[str, Field(title="Name")]
+    prompt: Annotated[str, Field(title="Prompt")]
+    tags: Annotated[List[str], Field(title="Tags")]
+    commit_id: Annotated[str, Field(title="Commit Id")]
+    parent_commit_id: Annotated[Optional[str], Field(title="Parent Commit Id")] = None
+    created_at: Annotated[str, Field(title="Created At")]
+    first_name: Annotated[str, Field(title="First Name")]
+    last_name: Annotated[str, Field(title="Last Name")]
+    user_email: Annotated[str, Field(title="User Email")]
+
+
 class ScorerData(BaseModel):
     id: Annotated[Optional[str], Field(title="Id")] = None
     name: Annotated[str, Field(title="Name")]
@@ -299,6 +346,14 @@ class FetchPromptScorersResponse(BaseModel):
     scorers: Annotated[List[PromptScorer], Field(title="Scorers")]
 
 
+class PromptFetchResponse(BaseModel):
+    commit: Optional[PromptCommitInfo] = None
+
+
+class PromptVersionsResponse(BaseModel):
+    versions: Annotated[List[PromptCommitInfo], Field(title="Versions")]
+
+
 class ScoringResult(BaseModel):
     success: Annotated[bool, Field(title="Success")]
     scorers_data: Annotated[List[ScorerData], Field(title="Scorers Data")]

judgeval/prompts/prompt.py

@@ -0,0 +1,320 @@
+from typing import List, Optional, Dict
+from judgeval.api import JudgmentSyncClient
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.api.api_types import (
+    PromptCommitInfo,
+    PromptTagResponse,
+    PromptUntagResponse,
+    PromptVersionsResponse,
+)
+from dataclasses import dataclass, field
+import re
+from string import Template
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
+from judgeval.utils.project import _resolve_project_id
+
+
+def push_prompt(
+    project_name: str,
+    name: str,
+    prompt: str,
+    tags: List[str],
+    judgment_api_key: str = JUDGMENT_API_KEY,
+    organization_id: str = JUDGMENT_ORG_ID,
+) -> tuple[str, Optional[str], str]:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        r = client.prompts_insert(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "prompt": prompt,
+                "tags": tags,
+            }
+        )
+        return r["commit_id"], r.get("parent_commit_id"), r["created_at"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to save prompt: {e.detail}",
+            response=e.response,
+        )
+
+
+def fetch_prompt(
+    project_name: str,
+    name: str,
+    commit_id: Optional[str] = None,
+    tag: Optional[str] = None,
+    judgment_api_key: str = JUDGMENT_API_KEY,
+    organization_id: str = JUDGMENT_ORG_ID,
+) -> Optional[PromptCommitInfo]:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_fetch(
+            name=name,
+            project_id=project_id,
+            commit_id=commit_id,
+            tag=tag,
+        )
+        return prompt_config["commit"]
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to fetch prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def tag_prompt(
+    project_name: str,
+    name: str,
+    commit_id: str,
+    tags: List[str],
+    judgment_api_key: str = JUDGMENT_API_KEY,
+    organization_id: str = JUDGMENT_ORG_ID,
+) -> PromptTagResponse:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_tag(
+            payload={
+                "project_id": project_id,
+                "name": name,
+                "commit_id": commit_id,
+                "tags": tags,
+            }
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to tag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def untag_prompt(
+    project_name: str,
+    name: str,
+    tags: List[str],
+    judgment_api_key: str = JUDGMENT_API_KEY,
+    organization_id: str = JUDGMENT_ORG_ID,
+) -> PromptUntagResponse:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_untag(
+            payload={"project_id": project_id, "name": name, "tags": tags}
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to untag prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+def list_prompt(
+    project_name: str,
+    name: str,
+    judgment_api_key: str = JUDGMENT_API_KEY,
+    organization_id: str = JUDGMENT_ORG_ID,
+) -> PromptVersionsResponse:
+    client = JudgmentSyncClient(judgment_api_key, organization_id)
+    try:
+        project_id = _resolve_project_id(
+            project_name, judgment_api_key, organization_id
+        )
+        if not project_id:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Project '{project_name}' not found",
+                response=None,  # type: ignore
+            )
+        prompt_config = client.prompts_get_prompt_versions(
+            project_id=project_id, name=name
+        )
+        return prompt_config
+    except JudgmentAPIError as e:
+        raise JudgmentAPIError(
+            status_code=e.status_code,
+            detail=f"Failed to list prompt '{name}': {e.detail}",
+            response=e.response,
+        )
+
+
+@dataclass
+class Prompt:
+    name: str
+    prompt: str
+    created_at: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: Optional[str] = None
+    metadata: Dict[str, str] = field(default_factory=dict)
+    _template: Template = field(init=False, repr=False)
+
+    def __post_init__(self):
+        template_str = re.sub(r"\{\{([^}]+)\}\}", r"$\1", self.prompt)
+        self._template = Template(template_str)
+
+    @classmethod
+    def create(
+        cls,
+        project_name: str,
+        name: str,
+        prompt: str,
+        tags: Optional[List[str]] = None,
+        judgment_api_key: str = JUDGMENT_API_KEY,
+        organization_id: str = JUDGMENT_ORG_ID,
+    ):
+        if tags is None:
+            tags = []
+        commit_id, parent_commit_id, created_at = push_prompt(
+            project_name, name, prompt, tags, judgment_api_key, organization_id
+        )
+        return cls(
+            name=name,
+            prompt=prompt,
+            created_at=created_at,
+            tags=tags,
+            commit_id=commit_id,
+            parent_commit_id=parent_commit_id,
+        )
+
+    @classmethod
+    def get(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+        judgment_api_key: str = JUDGMENT_API_KEY,
+        organization_id: str = JUDGMENT_ORG_ID,
+    ):
+        if commit_id is not None and tag is not None:
+            raise ValueError(
+                "You cannot fetch a prompt by both commit_id and tag at the same time"
+            )
+        prompt_config = fetch_prompt(
+            project_name, name, commit_id, tag, judgment_api_key, organization_id
+        )
+        if prompt_config is None:
+            raise JudgmentAPIError(
+                status_code=404,
+                detail=f"Prompt '{name}' not found in project '{project_name}'",
+                response=None,  # type: ignore
+            )
+        return cls(
+            name=prompt_config["name"],
+            prompt=prompt_config["prompt"],
+            created_at=prompt_config["created_at"],
+            tags=prompt_config["tags"],
+            commit_id=prompt_config["commit_id"],
+            parent_commit_id=prompt_config.get("parent_commit_id"),
+            metadata={
+                "creator_first_name": prompt_config["first_name"],
+                "creator_last_name": prompt_config["last_name"],
+                "creator_email": prompt_config["user_email"],
+            },
+        )
+
+    @classmethod
+    def tag(
+        cls,
+        project_name: str,
+        name: str,
+        commit_id: str,
+        tags: List[str],
+        judgment_api_key: str = JUDGMENT_API_KEY,
+        organization_id: str = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = tag_prompt(
+            project_name, name, commit_id, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_id"]
+
+    @classmethod
+    def untag(
+        cls,
+        project_name: str,
+        name: str,
+        tags: List[str],
+        judgment_api_key: str = JUDGMENT_API_KEY,
+        organization_id: str = JUDGMENT_ORG_ID,
+    ):
+        prompt_config = untag_prompt(
+            project_name, name, tags, judgment_api_key, organization_id
+        )
+        return prompt_config["commit_ids"]
+
+    @classmethod
+    def list(
+        cls,
+        project_name: str,
+        name: str,
+        judgment_api_key: str = JUDGMENT_API_KEY,
+        organization_id: str = JUDGMENT_ORG_ID,
+    ):
+        prompt_configs = list_prompt(
+            project_name, name, judgment_api_key, organization_id
+        )["versions"]
+        return [
+            cls(
+                name=prompt_config["name"],
+                prompt=prompt_config["prompt"],
+                tags=prompt_config["tags"],
+                created_at=prompt_config["created_at"],
+                commit_id=prompt_config["commit_id"],
+                parent_commit_id=prompt_config.get("parent_commit_id"),
+                metadata={
+                    "creator_first_name": prompt_config["first_name"],
+                    "creator_last_name": prompt_config["last_name"],
+                    "creator_email": prompt_config["user_email"],
+                },
+            )
+            for prompt_config in prompt_configs
+        ]
+
+    def compile(self, **kwargs) -> str:
+        try:
+            return self._template.substitute(**kwargs)
+        except KeyError as e:
+            missing_var = str(e).strip("'")
+            raise ValueError(f"Missing required variable: {missing_var}")