judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/__init__.py

@@ -1,13 +1,176 @@
-# Import key components that should be publicly accessible
-from judgeval.clients import client, together_client
-from judgeval.judgment_client import JudgmentClient
-from judgeval.version_check import check_latest_version
+from __future__ import annotations
+
+from judgeval.data.result import ScoringResult
+from judgeval.evaluation import run_eval
+from judgeval.data.evaluation_run import ExampleEvaluationRun
+
+
+from typing import List, Optional, Union, Sequence
+import ast
+from judgeval.scorers import ExampleAPIScorerConfig
+from judgeval.scorers.example_scorer import ExampleScorer
+from judgeval.data.example import Example
+from judgeval.logger import judgeval_logger
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_ORG_ID
+from judgeval.utils.meta import SingletonMeta
+from judgeval.exceptions import JudgmentRuntimeError, JudgmentTestError
+from judgeval.api import JudgmentSyncClient
+from judgeval.utils.file_utils import extract_scorer_name
+from judgeval.utils.guards import expect_api_key, expect_organization_id
+from judgeval.utils.version_check import check_latest_version
+from judgeval.utils.testing import assert_test_results
+from judgeval.v1 import Judgeval
 
 check_latest_version()
 
-__all__ = [
-    # Clients
-    "client",
-    "together_client",
-    "JudgmentClient",
-]
+
+class JudgmentClient(metaclass=SingletonMeta):
+    __slots__ = ("api_key", "organization_id")
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        organization_id: Optional[str] = None,
+    ):
+        _api_key = api_key or JUDGMENT_API_KEY
+        _organization_id = organization_id or JUDGMENT_ORG_ID
+
+        self.api_key = expect_api_key(_api_key)
+        self.organization_id = expect_organization_id(_organization_id)
+
+    def run_evaluation(
+        self,
+        examples: List[Example],
+        scorers: Sequence[Union[ExampleAPIScorerConfig, ExampleScorer, None]],
+        project_name: str = "default_project",
+        eval_run_name: str = "default_eval_run",
+        assert_test: bool = False,
+    ) -> List[ScoringResult]:
+        try:
+            for scorer in scorers:
+                if scorer is None:
+                    raise ValueError(
+                        "Failed to run evaluation: At least one Prompt Scorer was not successfuly retrieved."
+                    )
+            eval = ExampleEvaluationRun(
+                project_name=project_name,
+                eval_name=eval_run_name,
+                examples=examples,
+                scorers=scorers,  # type: ignore
+            )
+
+            results = run_eval(eval)
+            if assert_test:
+                assert_test_results(results)
+
+            return results
+
+        except JudgmentTestError as e:
+            raise JudgmentTestError(e)
+        except ValueError as e:
+            raise ValueError(
+                f"Please check your EvaluationRun object, one or more fields are invalid: \n{e}"
+            )
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"An unexpected error occured during evaluation: {e}"
+            ) from e
+
+    def upload_custom_scorer(
+        self,
+        scorer_file_path: str,
+        requirements_file_path: Optional[str] = None,
+        unique_name: Optional[str] = None,
+        overwrite: bool = False,
+    ) -> bool:
+        """
+        Upload custom ExampleScorer from files to backend.
+
+        Args:
+            scorer_file_path: Path to Python file containing CustomScorer class
+            requirements_file_path: Optional path to requirements.txt
+            unique_name: Optional unique identifier (auto-detected from scorer.name if not provided)
+            overwrite: Whether to overwrite existing scorer if it already exists
+
+        Returns:
+            bool: True if upload successful
+
+        Raises:
+            ValueError: If scorer file is invalid
+            FileNotFoundError: If scorer file doesn't exist
+        """
+        import os
+
+        if not os.path.exists(scorer_file_path):
+            raise FileNotFoundError(f"Scorer file not found: {scorer_file_path}")
+
+        # Auto-detect scorer name if not provided
+        if unique_name is None:
+            unique_name = extract_scorer_name(scorer_file_path)
+            judgeval_logger.info(f"Auto-detected scorer name: '{unique_name}'")
+
+        # Read scorer code
+        with open(scorer_file_path, "r") as f:
+            scorer_code = f.read()
+
+        try:
+            tree = ast.parse(scorer_code, filename=scorer_file_path)
+        except SyntaxError as e:
+            error_msg = f"Invalid Python syntax in {scorer_file_path}: {e}"
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        scorer_classes = []
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                for base in node.bases:
+                    if (isinstance(base, ast.Name) and base.id == "ExampleScorer") or (
+                        isinstance(base, ast.Attribute) and base.attr == "ExampleScorer"
+                    ):
+                        scorer_classes.append(node.name)
+
+        if len(scorer_classes) > 1:
+            error_msg = f"Multiple ExampleScorer classes found in {scorer_file_path}: {scorer_classes}. Please only upload one scorer class per file."
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+        elif len(scorer_classes) == 0:
+            error_msg = f"No ExampleScorer class was found in {scorer_file_path}. Please ensure the file contains a valid scorer class that inherits from ExampleScorer."
+            judgeval_logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # Read requirements (optional)
+        requirements_text = ""
+        if requirements_file_path and os.path.exists(requirements_file_path):
+            with open(requirements_file_path, "r") as f:
+                requirements_text = f.read()
+
+        try:
+            if not self.api_key or not self.organization_id:
+                raise ValueError("Judgment API key and organization ID are required")
+            client = JudgmentSyncClient(
+                api_key=self.api_key,
+                organization_id=self.organization_id,
+            )
+            response = client.upload_custom_scorer(
+                payload={
+                    "scorer_name": unique_name,
+                    "scorer_code": scorer_code,
+                    "requirements_text": requirements_text,
+                    "overwrite": overwrite,
+                }
+            )
+
+            if response.get("status") == "success":
+                judgeval_logger.info(
+                    f"Successfully uploaded custom scorer: {unique_name}"
+                )
+                return True
+            else:
+                judgeval_logger.error(f"Failed to upload custom scorer: {unique_name}")
+                return False
+
+        except Exception:
+            raise
+
+
+__all__ = ("JudgmentClient", "Judgeval")

judgeval/api/__init__.py

@@ -0,0 +1,523 @@
+from typing import Dict, Any, Mapping, Literal, Optional
+import httpx
+from httpx import Response
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.utils.url import url_for
+from judgeval.utils.serialize import json_encoder
+from judgeval.api.api_types import *
+
+
+def _headers(api_key: str, organization_id: str) -> Mapping[str, str]:
+    return {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+        "X-Organization-Id": organization_id,
+    }
+
+
+def _handle_response(r: Response) -> Any:
+    if r.status_code >= 400:
+        try:
+            detail = r.json().get("detail", "")
+        except Exception:
+            detail = r.text
+        raise JudgmentAPIError(r.status_code, detail, r)
+    return r.json()
+
+
+class JudgmentSyncClient:
+    __slots__ = ("api_key", "organization_id", "client")
+
+    def __init__(self, api_key: str, organization_id: str):
+        self.api_key = api_key
+        self.organization_id = organization_id
+        self.client = httpx.Client(timeout=30)
+
+    def _request(
+        self,
+        method: Literal["POST", "PATCH", "GET", "DELETE"],
+        url: str,
+        payload: Any,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        if method == "GET":
+            r = self.client.request(
+                method,
+                url,
+                params=payload if params is None else params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        else:
+            r = self.client.request(
+                method,
+                url,
+                json=json_encoder(payload),
+                params=params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        return _handle_response(r)
+
+    def add_to_run_eval_queue_examples(self, payload: ExampleEvaluationRun) -> Any:
+        return self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/examples"),
+            payload,
+        )
+
+    def add_to_run_eval_queue_traces(self, payload: TraceEvaluationRun) -> Any:
+        return self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/traces"),
+            payload,
+        )
+
+    def evaluate_examples(
+        self, payload: ExampleEvaluationRun, stream: Optional[str] = None
+    ) -> EvaluateResponse:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return self._request(
+            "POST",
+            url_for("/evaluate/examples"),
+            payload,
+            params=query_params,
+        )
+
+    def evaluate_traces(
+        self, payload: TraceEvaluationRun, stream: Optional[str] = None
+    ) -> EvaluateResponse:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return self._request(
+            "POST",
+            url_for("/evaluate/traces"),
+            payload,
+            params=query_params,
+        )
+
+    def log_eval_results(self, payload: EvalResults) -> LogEvalResultsResponse:
+        return self._request(
+            "POST",
+            url_for("/log_eval_results/"),
+            payload,
+        )
+
+    def fetch_experiment_run(
+        self, payload: EvalResultsFetch
+    ) -> FetchExperimentRunResponse:
+        return self._request(
+            "POST",
+            url_for("/fetch_experiment_run/"),
+            payload,
+        )
+
+    def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/insert_examples_for_judgeval/"),
+            payload,
+        )
+
+    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
+        return self._request(
+            "POST",
+            url_for("/datasets/pull_for_judgeval/"),
+            payload,
+        )
+
+    def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/pull_all_for_judgeval/"),
+            payload,
+        )
+
+    def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/create_for_judgeval/"),
+            payload,
+        )
+
+    def projects_add(self, payload: ProjectAdd) -> ProjectAddResponse:
+        return self._request(
+            "POST",
+            url_for("/projects/add/"),
+            payload,
+        )
+
+    def projects_delete_from_judgeval(
+        self, payload: ProjectDeleteFromJudgevalResponse
+    ) -> ProjectDeleteResponse:
+        return self._request(
+            "DELETE",
+            url_for("/projects/delete_from_judgeval/"),
+            payload,
+        )
+
+    def scorer_exists(self, payload: ScorerExistsRequest) -> ScorerExistsResponse:
+        return self._request(
+            "POST",
+            url_for("/scorer_exists/"),
+            payload,
+        )
+
+    def save_scorer(self, payload: SavePromptScorerRequest) -> SavePromptScorerResponse:
+        return self._request(
+            "POST",
+            url_for("/save_scorer/"),
+            payload,
+        )
+
+    def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
+        return self._request(
+            "POST",
+            url_for("/fetch_scorers/"),
+            payload,
+        )
+
+    def upload_custom_scorer(
+        self, payload: CustomScorerUploadPayload
+    ) -> CustomScorerTemplateResponse:
+        return self._request(
+            "POST",
+            url_for("/upload_custom_scorer/"),
+            payload,
+        )
+
+    def prompts_insert(self, payload: PromptInsertRequest) -> PromptInsertResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/insert/"),
+            payload,
+        )
+
+    def prompts_tag(self, payload: PromptTagRequest) -> PromptTagResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/tag/"),
+            payload,
+        )
+
+    def prompts_untag(self, payload: PromptUntagRequest) -> PromptUntagResponse:
+        return self._request(
+            "POST",
+            url_for("/prompts/untag/"),
+            payload,
+        )
+
+    def prompts_fetch(
+        self,
+        project_id: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+    ) -> PromptFetchResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        if commit_id is not None:
+            query_params["commit_id"] = commit_id
+        if tag is not None:
+            query_params["tag"] = tag
+        return self._request(
+            "GET",
+            url_for("/prompts/fetch/"),
+            query_params,
+        )
+
+    def prompts_get_prompt_versions(
+        self, project_id: str, name: str
+    ) -> PromptVersionsResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        return self._request(
+            "GET",
+            url_for("/prompts/get_prompt_versions/"),
+            query_params,
+        )
+
+    def projects_resolve(
+        self, payload: ResolveProjectNameRequest
+    ) -> ResolveProjectNameResponse:
+        return self._request(
+            "POST",
+            url_for("/projects/resolve/"),
+            payload,
+        )
+
+    def e2e_fetch_trace(self, payload: TraceIdRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_trace/"),
+            payload,
+        )
+
+    def e2e_fetch_span_score(self, payload: SpanScoreRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_span_score/"),
+            payload,
+        )
+
+
+class JudgmentAsyncClient:
+    __slots__ = ("api_key", "organization_id", "client")
+
+    def __init__(self, api_key: str, organization_id: str):
+        self.api_key = api_key
+        self.organization_id = organization_id
+        self.client = httpx.AsyncClient(timeout=30)
+
+    async def _request(
+        self,
+        method: Literal["POST", "PATCH", "GET", "DELETE"],
+        url: str,
+        payload: Any,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        if method == "GET":
+            r = self.client.request(
+                method,
+                url,
+                params=payload if params is None else params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        else:
+            r = self.client.request(
+                method,
+                url,
+                json=json_encoder(payload),
+                params=params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        return _handle_response(await r)
+
+    async def add_to_run_eval_queue_examples(
+        self, payload: ExampleEvaluationRun
+    ) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/examples"),
+            payload,
+        )
+
+    async def add_to_run_eval_queue_traces(self, payload: TraceEvaluationRun) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/traces"),
+            payload,
+        )
+
+    async def evaluate_examples(
+        self, payload: ExampleEvaluationRun, stream: Optional[str] = None
+    ) -> EvaluateResponse:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return await self._request(
+            "POST",
+            url_for("/evaluate/examples"),
+            payload,
+            params=query_params,
+        )
+
+    async def evaluate_traces(
+        self, payload: TraceEvaluationRun, stream: Optional[str] = None
+    ) -> EvaluateResponse:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return await self._request(
+            "POST",
+            url_for("/evaluate/traces"),
+            payload,
+            params=query_params,
+        )
+
+    async def log_eval_results(self, payload: EvalResults) -> LogEvalResultsResponse:
+        return await self._request(
+            "POST",
+            url_for("/log_eval_results/"),
+            payload,
+        )
+
+    async def fetch_experiment_run(
+        self, payload: EvalResultsFetch
+    ) -> FetchExperimentRunResponse:
+        return await self._request(
+            "POST",
+            url_for("/fetch_experiment_run/"),
+            payload,
+        )
+
+    async def datasets_insert_examples_for_judgeval(
+        self, payload: DatasetInsertExamples
+    ) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/insert_examples_for_judgeval/"),
+            payload,
+        )
+
+    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> DatasetReturn:
+        return await self._request(
+            "POST",
+            url_for("/datasets/pull_for_judgeval/"),
+            payload,
+        )
+
+    async def datasets_pull_all_for_judgeval(self, payload: DatasetsFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/pull_all_for_judgeval/"),
+            payload,
+        )
+
+    async def datasets_create_for_judgeval(self, payload: DatasetCreate) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/create_for_judgeval/"),
+            payload,
+        )
+
+    async def projects_add(self, payload: ProjectAdd) -> ProjectAddResponse:
+        return await self._request(
+            "POST",
+            url_for("/projects/add/"),
+            payload,
+        )
+
+    async def projects_delete_from_judgeval(
+        self, payload: ProjectDeleteFromJudgevalResponse
+    ) -> ProjectDeleteResponse:
+        return await self._request(
+            "DELETE",
+            url_for("/projects/delete_from_judgeval/"),
+            payload,
+        )
+
+    async def scorer_exists(self, payload: ScorerExistsRequest) -> ScorerExistsResponse:
+        return await self._request(
+            "POST",
+            url_for("/scorer_exists/"),
+            payload,
+        )
+
+    async def save_scorer(
+        self, payload: SavePromptScorerRequest
+    ) -> SavePromptScorerResponse:
+        return await self._request(
+            "POST",
+            url_for("/save_scorer/"),
+            payload,
+        )
+
+    async def fetch_scorers(
+        self, payload: FetchPromptScorersRequest
+    ) -> FetchPromptScorersResponse:
+        return await self._request(
+            "POST",
+            url_for("/fetch_scorers/"),
+            payload,
+        )
+
+    async def upload_custom_scorer(
+        self, payload: CustomScorerUploadPayload
+    ) -> CustomScorerTemplateResponse:
+        return await self._request(
+            "POST",
+            url_for("/upload_custom_scorer/"),
+            payload,
+        )
+
+    async def prompts_insert(
+        self, payload: PromptInsertRequest
+    ) -> PromptInsertResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/insert/"),
+            payload,
+        )
+
+    async def prompts_tag(self, payload: PromptTagRequest) -> PromptTagResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/tag/"),
+            payload,
+        )
+
+    async def prompts_untag(self, payload: PromptUntagRequest) -> PromptUntagResponse:
+        return await self._request(
+            "POST",
+            url_for("/prompts/untag/"),
+            payload,
+        )
+
+    async def prompts_fetch(
+        self,
+        project_id: str,
+        name: str,
+        commit_id: Optional[str] = None,
+        tag: Optional[str] = None,
+    ) -> PromptFetchResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        if commit_id is not None:
+            query_params["commit_id"] = commit_id
+        if tag is not None:
+            query_params["tag"] = tag
+        return await self._request(
+            "GET",
+            url_for("/prompts/fetch/"),
+            query_params,
+        )
+
+    async def prompts_get_prompt_versions(
+        self, project_id: str, name: str
+    ) -> PromptVersionsResponse:
+        query_params = {}
+        query_params["project_id"] = project_id
+        query_params["name"] = name
+        return await self._request(
+            "GET",
+            url_for("/prompts/get_prompt_versions/"),
+            query_params,
+        )
+
+    async def projects_resolve(
+        self, payload: ResolveProjectNameRequest
+    ) -> ResolveProjectNameResponse:
+        return await self._request(
+            "POST",
+            url_for("/projects/resolve/"),
+            payload,
+        )
+
+    async def e2e_fetch_trace(self, payload: TraceIdRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_trace/"),
+            payload,
+        )
+
+    async def e2e_fetch_span_score(self, payload: SpanScoreRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_span_score/"),
+            payload,
+        )
+
+
+__all__ = [
+    "JudgmentSyncClient",
+    "JudgmentAsyncClient",
+]