judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/v1/internal/api/api_types.py

@@ -0,0 +1,413 @@
+# generated by datamodel-codegen:
+#   filename:  tmp5p8bhyaw.json
+#   timestamp: 2025-11-18T18:52:12+00:00
+
+from __future__ import annotations
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+from typing_extensions import NotRequired
+
+
+TraceAndSpanId = List
+
+
+class LogEvalResultsResponse(TypedDict):
+    ui_results_url: str
+
+
+class EvalResultsFetch(TypedDict):
+    experiment_run_id: str
+    project_name: str
+
+
+class FetchExperimentRunResponse(TypedDict):
+    results: NotRequired[Optional[List]]
+    ui_results_url: NotRequired[Optional[str]]
+
+
+class DatasetFetch(TypedDict):
+    dataset_name: str
+    project_name: str
+
+
+class DatasetsFetch(TypedDict):
+    project_name: str
+
+
+class ProjectAdd(TypedDict):
+    project_name: str
+
+
+class ProjectAddResponse(TypedDict):
+    project_id: str
+
+
+class ProjectDeleteFromJudgevalResponse(TypedDict):
+    project_name: str
+
+
+class ProjectDeleteResponse(TypedDict):
+    message: str
+
+
+class ScorerExistsRequest(TypedDict):
+    name: str
+
+
+class ScorerExistsResponse(TypedDict):
+    exists: bool
+
+
+class SavePromptScorerRequest(TypedDict):
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+
+
+class FetchPromptScorersRequest(TypedDict):
+    names: NotRequired[Optional[List[str]]]
+    is_trace: NotRequired[Optional[bool]]
+
+
+class CustomScorerUploadPayload(TypedDict):
+    scorer_name: str
+    scorer_code: str
+    requirements_text: str
+    overwrite: NotRequired[bool]
+
+
+class CustomScorerTemplateResponse(TypedDict):
+    scorer_name: str
+    status: str
+    message: str
+
+
+class PromptInsertRequest(TypedDict):
+    project_id: str
+    name: str
+    prompt: str
+    tags: List[str]
+
+
+class PromptInsertResponse(TypedDict):
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+
+
+class PromptTagRequest(TypedDict):
+    project_id: str
+    name: str
+    commit_id: str
+    tags: List[str]
+
+
+class PromptTagResponse(TypedDict):
+    commit_id: str
+
+
+class PromptUntagRequest(TypedDict):
+    project_id: str
+    name: str
+    tags: List[str]
+
+
+class PromptUntagResponse(TypedDict):
+    commit_ids: List[str]
+
+
+class ResolveProjectNameRequest(TypedDict):
+    project_name: str
+
+
+class ResolveProjectNameResponse(TypedDict):
+    project_id: str
+
+
+class TraceIdRequest(TypedDict):
+    trace_id: str
+
+
+class SpanScoreRequest(TypedDict):
+    span_id: str
+    trace_id: str
+
+
+class BaseScorer(TypedDict):
+    score_type: str
+    threshold: NotRequired[float]
+    name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    score: NotRequired[Optional[float]]
+    score_breakdown: NotRequired[Optional[Dict[str, Any]]]
+    reason: NotRequired[Optional[str]]
+    using_native_model: NotRequired[Optional[bool]]
+    success: NotRequired[Optional[bool]]
+    model: NotRequired[Optional[str]]
+    model_client: NotRequired[Any]
+    strict_mode: NotRequired[bool]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    user: NotRequired[Optional[str]]
+    server_hosted: NotRequired[bool]
+
+
+class ScorerConfig(TypedDict):
+    score_type: str
+    name: NotRequired[Optional[str]]
+    threshold: NotRequired[float]
+    strict_mode: NotRequired[bool]
+    required_params: NotRequired[List[str]]
+    kwargs: NotRequired[Optional[Dict[str, Any]]]
+
+
+class Example(TypedDict):
+    example_id: NotRequired[str]
+    created_at: NotRequired[str]
+    name: NotRequired[Optional[str]]
+
+
+class ValidationError(TypedDict):
+    loc: List[Union[str, int]]
+    msg: str
+    type: str
+
+
+class UsageInfo(TypedDict):
+    total_judgees: int
+    regular_use: int
+    pay_as_you_go_use: int
+    remaining_regular: int
+    remaining_after: int
+
+
+DatasetKind = Literal["trace", "example"]
+
+
+class PromptScorer(TypedDict):
+    id: str
+    user_id: str
+    organization_id: str
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+    created_at: NotRequired[Optional[str]]
+    updated_at: NotRequired[Optional[str]]
+    is_trace: NotRequired[Optional[bool]]
+    is_bucket_rubric: NotRequired[Optional[bool]]
+
+
+class PromptCommitInfo(TypedDict):
+    name: str
+    prompt: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+    first_name: str
+    last_name: str
+    user_email: str
+
+
+class ScorerData(TypedDict):
+    id: NotRequired[str]
+    name: str
+    threshold: float
+    success: bool
+    score: NotRequired[Optional[float]]
+    reason: NotRequired[Optional[str]]
+    strict_mode: NotRequired[Optional[bool]]
+    evaluation_model: NotRequired[Optional[str]]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+
+
+class OtelTraceSpan(TypedDict):
+    organization_id: str
+    project_id: NotRequired[Optional[str]]
+    user_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[List[Dict[str, Any]]]]
+
+
+class OtelSpanListItemScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+
+
+class OtelSpanDetailScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+    example_id: NotRequired[Optional[str]]
+
+
+class ExampleEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    created_at: NotRequired[str]
+    examples: List[Example]
+    trace_span_id: NotRequired[Optional[str]]
+    trace_id: NotRequired[Optional[str]]
+
+
+class HTTPValidationError(TypedDict):
+    detail: NotRequired[List[ValidationError]]
+
+
+class TraceEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    created_at: NotRequired[str]
+    trace_and_span_ids: List[TraceAndSpanId]
+    is_offline: NotRequired[bool]
+    is_bucket_run: NotRequired[bool]
+
+
+class DatasetInsertExamples(TypedDict):
+    dataset_name: str
+    examples: List[Example]
+    project_name: str
+
+
+class DatasetInfo(TypedDict):
+    dataset_id: str
+    name: str
+    created_at: str
+    kind: DatasetKind
+    entries: int
+    creator: str
+
+
+class DatasetCreate(TypedDict):
+    name: str
+    dataset_kind: DatasetKind
+    project_name: str
+    examples: List[Example]
+    overwrite: bool
+
+
+class SavePromptScorerResponse(TypedDict):
+    scorer_response: PromptScorer
+
+
+class FetchPromptScorersResponse(TypedDict):
+    scorers: List[PromptScorer]
+
+
+class PromptFetchResponse(TypedDict):
+    commit: NotRequired[Optional[PromptCommitInfo]]
+
+
+class PromptVersionsResponse(TypedDict):
+    versions: List[PromptCommitInfo]
+
+
+class ScoringResult(TypedDict):
+    success: bool
+    scorers_data: List[ScorerData]
+    name: NotRequired[Optional[str]]
+    data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
+    trace_id: NotRequired[Optional[str]]
+    run_duration: NotRequired[Optional[float]]
+    evaluation_cost: NotRequired[Optional[float]]
+
+
+class OtelTraceListItem(TypedDict):
+    organization_id: str
+    project_id: str
+    trace_id: str
+    created_at: str
+    duration: NotRequired[Optional[int]]
+    tags: NotRequired[Optional[List[str]]]
+    experiment_run_id: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    llm_cost: NotRequired[Optional[float]]
+    error: NotRequired[str]
+    scores: NotRequired[List[OtelSpanListItemScores]]
+    rules_invoked: NotRequired[List[str]]
+    customer_id: NotRequired[Optional[str]]
+    input: NotRequired[Optional[str]]
+    output: NotRequired[Optional[str]]
+    input_preview: NotRequired[Optional[str]]
+    output_preview: NotRequired[Optional[str]]
+    annotation_count: NotRequired[int]
+    span_id: str
+    rule_id: NotRequired[Optional[str]]
+
+
+class OtelSpanDetail(TypedDict):
+    organization_id: str
+    project_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]]
+    llm_cost: NotRequired[Optional[float]]
+    prompt_tokens: NotRequired[Optional[int]]
+    completion_tokens: NotRequired[Optional[int]]
+    scores: NotRequired[Optional[List[OtelSpanDetailScores]]]
+
+
+class EvaluateResponse(TypedDict):
+    status: str
+    results: List[ScoringResult]
+    resource_usage: NotRequired[Optional[UsageInfo]]
+
+
+class EvalResults(TypedDict):
+    results: List[ScoringResult]
+    run: Union[ExampleEvaluationRun, TraceEvaluationRun]
+
+
+class DatasetTraceWithSpans(TypedDict):
+    dataset_id: str
+    trace_detail: OtelTraceListItem
+    spans: List[OtelSpanDetail]
+
+
+class DatasetReturn(TypedDict):
+    name: str
+    project_name: str
+    dataset_kind: DatasetKind
+    examples: NotRequired[List[Example]]
+    traces: NotRequired[Optional[List[DatasetTraceWithSpans]]]

judgeval/v1/prompts/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.prompts.prompt import Prompt
+from judgeval.v1.prompts.prompt_factory import PromptFactory
+
+__all__ = ["Prompt", "PromptFactory"]

judgeval/v1/prompts/prompt.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from string import Template
+from typing import Dict, List, Optional
+
+
+@dataclass
+class Prompt:
+    name: str
+    prompt: str
+    created_at: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: Optional[str] = None
+    metadata: Dict[str, str] = field(default_factory=dict)
+    _template: Template = field(init=False, repr=False)
+
+    def __post_init__(self):
+        template_str = re.sub(r"\{\{([^}]+)\}\}", r"$\1", self.prompt)
+        self._template = Template(template_str)
+
+    def compile(self, **kwargs) -> str:
+        try:
+            return self._template.substitute(**kwargs)
+        except KeyError as e:
+            missing_var = str(e).strip("'")
+            raise ValueError(f"Missing required variable: {missing_var}")

judgeval/v1/prompts/prompt_factory.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from typing import List, Optional, overload
+
+from judgeval.logger import judgeval_logger
+from judgeval.utils.decorators.dont_throw import dont_throw
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.v1.prompts.prompt import Prompt
+from judgeval.v1.utils import resolve_project_id
+
+
+class PromptFactory:
+    __slots__ = "_client"
+
+    def __init__(self, client: JudgmentSyncClient):
+        self._client = client
+
+    def create(
+        self,
+        project_name: str,
+        name: str,
+        prompt: str,
+        tags: Optional[List[str]] = None,
+    ) -> Prompt:
+        try:
+            if tags is None:
+                tags = []
+
+            project_id = resolve_project_id(self._client, project_name)
+            assert project_id is not None
+            response = self._client.prompts_insert(
+                {
+                    "project_id": project_id,
+                    "name": name,
+                    "prompt": prompt,
+                    "tags": tags,
+                }
+            )
+            return Prompt(
+                name=name,
+                prompt=prompt,
+                created_at=response["created_at"],
+                tags=tags,
+                commit_id=response["commit_id"],
+                parent_commit_id=response.get("parent_commit_id"),
+            )
+        except Exception as e:
+            judgeval_logger.error(f"Failed to create prompt: {str(e)}")
+            raise
+
+    @overload
+    def get(
+        self,
+        /,
+        *,
+        project_name: str,
+        name: str,
+        commit_id: str,
+    ) -> Optional[Prompt]: ...
+
+    @overload
+    def get(
+        self,
+        /,
+        *,
+        project_name: str,
+        name: str,
+        tag: str,
+    ) -> Optional[Prompt]: ...
+
+    @dont_throw
+    def get(
+        self,
+        /,
+        *,
+        project_name: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+    ) -> Optional[Prompt]:
+        if commit_id is not None and tag is not None:
+            judgeval_logger.error("Cannot fetch prompt by both commit_id and tag")
+            return None
+
+        project_id = resolve_project_id(self._client, project_name)
+        if project_id is None:
+            return None
+
+        response = self._client.prompts_fetch(
+            project_id=project_id,
+            name=name,
+            commit_id=commit_id,
+            tag=tag,
+        )
+
+        prompt_config = response.get("commit")
+        if prompt_config is None:
+            return None
+
+        return Prompt(
+            name=prompt_config["name"],
+            prompt=prompt_config["prompt"],
+            created_at=prompt_config["created_at"],
+            tags=prompt_config["tags"],
+            commit_id=prompt_config["commit_id"],
+            parent_commit_id=prompt_config.get("parent_commit_id"),
+            metadata={
+                "creator_first_name": prompt_config["first_name"],
+                "creator_last_name": prompt_config["last_name"],
+                "creator_email": prompt_config["user_email"],
+            },
+        )
+
+    def tag(
+        self,
+        project_name: str,
+        name: str,
+        commit_id: str,
+        tags: List[str],
+    ) -> str:
+        try:
+            project_id = resolve_project_id(self._client, project_name)
+            assert project_id is not None
+            response = self._client.prompts_tag(
+                {
+                    "project_id": project_id,
+                    "name": name,
+                    "commit_id": commit_id,
+                    "tags": tags,
+                }
+            )
+            return response["commit_id"]
+        except Exception as e:
+            judgeval_logger.error(f"Failed to tag prompt: {str(e)}")
+            raise
+
+    def untag(
+        self,
+        project_name: str,
+        name: str,
+        tags: List[str],
+    ) -> List[str]:
+        try:
+            project_id = resolve_project_id(self._client, project_name)
+            assert project_id is not None
+            response = self._client.prompts_untag(
+                {
+                    "project_id": project_id,
+                    "name": name,
+                    "tags": tags,
+                }
+            )
+            return response["commit_ids"]
+        except Exception as e:
+            judgeval_logger.error(f"Failed to untag prompt: {str(e)}")
+            raise
+
+    def list(
+        self,
+        project_name: str,
+        name: str,
+    ) -> List[Prompt]:
+        try:
+            project_id = resolve_project_id(self._client, project_name)
+            assert project_id is not None
+            response = self._client.prompts_get_prompt_versions(
+                project_id=project_id,
+                name=name,
+            )
+
+            return [
+                Prompt(
+                    name=prompt_config["name"],
+                    prompt=prompt_config["prompt"],
+                    tags=prompt_config["tags"],
+                    created_at=prompt_config["created_at"],
+                    commit_id=prompt_config["commit_id"],
+                    parent_commit_id=prompt_config.get("parent_commit_id"),
+                    metadata={
+                        "creator_first_name": prompt_config["first_name"],
+                        "creator_last_name": prompt_config["last_name"],
+                        "creator_email": prompt_config["user_email"],
+                    },
+                )
+                for prompt_config in response["versions"]
+            ]
+        except Exception as e:
+            judgeval_logger.error(f"Failed to list prompt versions: {str(e)}")
+            raise

judgeval/v1/scorers/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.scorers.base_scorer import BaseScorer
+from judgeval.v1.scorers.scorers_factory import ScorersFactory
+
+__all__ = ["BaseScorer", "ScorersFactory"]