judgeval 0.7.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

judgeval/__init__.py

@@ -1,15 +1,142 @@
-# Import key components that should be publicly accessible
-from judgeval.clients import client, together_client
-from judgeval.judgment_client import JudgmentClient
-from judgeval.version_check import check_latest_version
-from judgeval.local_eval_queue import LocalEvaluationQueue
+from __future__ import annotations
+
+from judgeval.data.result import ScoringResult
+from judgeval.evaluation import run_eval
+from judgeval.data.evaluation_run import ExampleEvaluationRun
+
+
+from typing import List, Optional, Union
+from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.data.example import Example
+from judgeval.logger import judgeval_logger
+from judgeval.env import JUDGMENT_API_KEY, JUDGMENT_DEFAULT_GPT_MODEL, JUDGMENT_ORG_ID
+from judgeval.utils.meta import SingletonMeta
+from judgeval.exceptions import JudgmentRuntimeError, JudgmentTestError
+from judgeval.api import JudgmentSyncClient
+from judgeval.utils.file_utils import extract_scorer_name
+from judgeval.utils.guards import expect_api_key, expect_organization_id
+from judgeval.utils.version_check import check_latest_version
+from judgeval.utils.testing import assert_test_results
 
 check_latest_version()
 
-__all__ = [
-    # Clients
-    "client",
-    "together_client",
-    "JudgmentClient",
-    "LocalEvaluationQueue",
-]
+
+class JudgmentClient(metaclass=SingletonMeta):
+    __slots__ = ("api_key", "organization_id")
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        organization_id: Optional[str] = None,
+    ):
+        _api_key = api_key or JUDGMENT_API_KEY
+        _organization_id = organization_id or JUDGMENT_ORG_ID
+
+        self.api_key = expect_api_key(_api_key)
+        self.organization_id = expect_organization_id(_organization_id)
+
+    def run_evaluation(
+        self,
+        examples: List[Example],
+        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        project_name: str,
+        eval_run_name: str,
+        model: str = JUDGMENT_DEFAULT_GPT_MODEL,
+        assert_test: bool = False,
+    ) -> List[ScoringResult]:
+        try:
+            eval = ExampleEvaluationRun(
+                project_name=project_name,
+                eval_name=eval_run_name,
+                examples=examples,
+                scorers=scorers,
+                model=model,
+                organization_id=self.organization_id,
+            )
+
+            results = run_eval(eval, self.api_key)
+            if assert_test:
+                assert_test_results(results)
+
+            return results
+
+        except JudgmentTestError as e:
+            raise JudgmentTestError(e)
+        except ValueError as e:
+            raise ValueError(
+                f"Please check your EvaluationRun object, one or more fields are invalid: \n{e}"
+            )
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"An unexpected error occured during evaluation: {e}"
+            ) from e
+
+    def upload_custom_scorer(
+        self,
+        scorer_file_path: str,
+        requirements_file_path: Optional[str] = None,
+        unique_name: Optional[str] = None,
+    ) -> bool:
+        """
+        Upload custom ExampleScorer from files to backend.
+
+        Args:
+            scorer_file_path: Path to Python file containing CustomScorer class
+            requirements_file_path: Optional path to requirements.txt
+            unique_name: Optional unique identifier (auto-detected from scorer.name if not provided)
+
+        Returns:
+            bool: True if upload successful
+
+        Raises:
+            ValueError: If scorer file is invalid
+            FileNotFoundError: If scorer file doesn't exist
+        """
+        import os
+
+        if not os.path.exists(scorer_file_path):
+            raise FileNotFoundError(f"Scorer file not found: {scorer_file_path}")
+
+        # Auto-detect scorer name if not provided
+        if unique_name is None:
+            unique_name = extract_scorer_name(scorer_file_path)
+            judgeval_logger.info(f"Auto-detected scorer name: '{unique_name}'")
+
+        # Read scorer code
+        with open(scorer_file_path, "r") as f:
+            scorer_code = f.read()
+
+        # Read requirements (optional)
+        requirements_text = ""
+        if requirements_file_path and os.path.exists(requirements_file_path):
+            with open(requirements_file_path, "r") as f:
+                requirements_text = f.read()
+
+        try:
+            client = JudgmentSyncClient(
+                api_key=self.api_key,
+                organization_id=self.organization_id,
+            )
+            response = client.upload_custom_scorer(
+                payload={
+                    "scorer_name": unique_name,
+                    "scorer_code": scorer_code,
+                    "requirements_text": requirements_text,
+                }
+            )
+
+            if response.get("status") == "success":
+                judgeval_logger.info(
+                    f"Successfully uploaded custom scorer: {unique_name}"
+                )
+                return True
+            else:
+                judgeval_logger.error(f"Failed to upload custom scorer: {unique_name}")
+                return False
+
+        except Exception as e:
+            judgeval_logger.error(f"Error uploading custom scorer: {e}")
+            raise
+
+
+__all__ = ("JudgmentClient",)

judgeval/api/__init__.py

@@ -0,0 +1,501 @@
+from typing import Dict, Any, Mapping, Literal, Optional
+import httpx
+from httpx import Response
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.utils.url import url_for
+from judgeval.utils.serialize import json_encoder
+from judgeval.api.api_types import *
+
+
+def _headers(api_key: str, organization_id: str) -> Mapping[str, str]:
+    return {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+        "X-Organization-Id": organization_id,
+    }
+
+
+def _handle_response(r: Response) -> Any:
+    if r.status_code >= 400:
+        try:
+            detail = r.json().get("detail", "")
+        except Exception:
+            detail = r.text
+        raise JudgmentAPIError(r.status_code, detail, r)
+    return r.json()
+
+
+class JudgmentSyncClient:
+    __slots__ = ("api_key", "organization_id", "client")
+
+    def __init__(self, api_key: str, organization_id: str):
+        self.api_key = api_key
+        self.organization_id = organization_id
+        self.client = httpx.Client(timeout=30)
+
+    def _request(
+        self,
+        method: Literal["POST", "PATCH", "GET", "DELETE"],
+        url: str,
+        payload: Any,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        if method == "GET":
+            r = self.client.request(
+                method,
+                url,
+                params=payload if params is None else params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        else:
+            r = self.client.request(
+                method,
+                url,
+                json=json_encoder(payload),
+                params=params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        return _handle_response(r)
+
+    def add_to_run_eval_queue_examples(self, payload: ExampleEvaluationRun) -> Any:
+        return self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/examples"),
+            payload,
+        )
+
+    def add_to_run_eval_queue_traces(self, payload: TraceEvaluationRun) -> Any:
+        return self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/traces"),
+            payload,
+        )
+
+    def evaluate_trace(self, payload: TraceRun) -> Any:
+        return self._request(
+            "POST",
+            url_for("/evaluate_trace/"),
+            payload,
+        )
+
+    def evaluate_examples(
+        self, payload: ExampleEvaluationRun, stream: Optional[str] = None
+    ) -> Any:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return self._request(
+            "POST",
+            url_for("/evaluate/examples"),
+            payload,
+            params=query_params,
+        )
+
+    def evaluate_traces(
+        self, payload: TraceEvaluationRun, stream: Optional[str] = None
+    ) -> Any:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return self._request(
+            "POST",
+            url_for("/evaluate/traces"),
+            payload,
+            params=query_params,
+        )
+
+    def log_eval_results(self, payload: EvalResults) -> Any:
+        return self._request(
+            "POST",
+            url_for("/log_eval_results/"),
+            payload,
+        )
+
+    def fetch_experiment_run(self, payload: EvalResultsFetch) -> Any:
+        return self._request(
+            "POST",
+            url_for("/fetch_experiment_run/"),
+            payload,
+        )
+
+    def get_evaluation_status(self, experiment_run_id: str, project_name: str) -> Any:
+        query_params = {}
+        query_params["experiment_run_id"] = experiment_run_id
+        query_params["project_name"] = project_name
+        return self._request(
+            "GET",
+            url_for("/get_evaluation_status/"),
+            query_params,
+        )
+
+    def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/insert_examples/"),
+            payload,
+        )
+
+    def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/pull_for_judgeval/"),
+            payload,
+        )
+
+    def datasets_push(self, payload: DatasetPush) -> Any:
+        return self._request(
+            "POST",
+            url_for("/datasets/push/"),
+            payload,
+        )
+
+    def traces_upsert(self, payload: TraceSave) -> Any:
+        return self._request(
+            "POST",
+            url_for("/traces/upsert/"),
+            payload,
+        )
+
+    def traces_fetch(self, payload: TraceFetch) -> Any:
+        return self._request(
+            "POST",
+            url_for("/traces/fetch/"),
+            payload,
+        )
+
+    def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
+        return self._request(
+            "POST",
+            url_for("/traces/add_to_dataset/"),
+            payload,
+        )
+
+    def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/traces/spans/batch/"),
+            payload,
+        )
+
+    def traces_evaluation_runs_batch(self, payload: EvaluationRunsBatchRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/traces/evaluation_runs/batch/"),
+            payload,
+        )
+
+    def projects_add(self, payload: ProjectAdd) -> ProjectAddResponse:
+        return self._request(
+            "POST",
+            url_for("/projects/add/"),
+            payload,
+        )
+
+    def projects_delete_from_judgeval(
+        self, payload: ProjectDeleteFromJudgevalResponse
+    ) -> ProjectDeleteResponse:
+        return self._request(
+            "DELETE",
+            url_for("/projects/delete_from_judgeval/"),
+            payload,
+        )
+
+    def scorer_exists(self, payload: ScorerExistsRequest) -> ScorerExistsResponse:
+        return self._request(
+            "POST",
+            url_for("/scorer_exists/"),
+            payload,
+        )
+
+    def save_scorer(self, payload: SavePromptScorerRequest) -> SavePromptScorerResponse:
+        return self._request(
+            "POST",
+            url_for("/save_scorer/"),
+            payload,
+        )
+
+    def fetch_scorer(
+        self, payload: FetchPromptScorerRequest
+    ) -> FetchPromptScorerResponse:
+        return self._request(
+            "POST",
+            url_for("/fetch_scorer/"),
+            payload,
+        )
+
+    def upload_custom_scorer(
+        self, payload: CustomScorerUploadPayload
+    ) -> CustomScorerTemplateResponse:
+        return self._request(
+            "POST",
+            url_for("/upload_custom_scorer/"),
+            payload,
+        )
+
+    def projects_resolve(
+        self, payload: ResolveProjectNameRequest
+    ) -> ResolveProjectNameResponse:
+        return self._request(
+            "POST",
+            url_for("/projects/resolve/"),
+            payload,
+        )
+
+    def e2e_fetch_trace(self, payload: TraceIdRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_trace/"),
+            payload,
+        )
+
+    def e2e_fetch_span_score(self, payload: SpanScoreRequest) -> Any:
+        return self._request(
+            "POST",
+            url_for("/e2e_fetch_span_score/"),
+            payload,
+        )
+
+
+class JudgmentAsyncClient:
+    __slots__ = ("api_key", "organization_id", "client")
+
+    def __init__(self, api_key: str, organization_id: str):
+        self.api_key = api_key
+        self.organization_id = organization_id
+        self.client = httpx.AsyncClient(timeout=30)
+
+    async def _request(
+        self,
+        method: Literal["POST", "PATCH", "GET", "DELETE"],
+        url: str,
+        payload: Any,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Any:
+        if method == "GET":
+            r = self.client.request(
+                method,
+                url,
+                params=payload if params is None else params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        else:
+            r = self.client.request(
+                method,
+                url,
+                json=json_encoder(payload),
+                params=params,
+                headers=_headers(self.api_key, self.organization_id),
+            )
+        return _handle_response(await r)
+
+    async def add_to_run_eval_queue_examples(
+        self, payload: ExampleEvaluationRun
+    ) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/examples"),
+            payload,
+        )
+
+    async def add_to_run_eval_queue_traces(self, payload: TraceEvaluationRun) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/add_to_run_eval_queue/traces"),
+            payload,
+        )
+
+    async def evaluate_trace(self, payload: TraceRun) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/evaluate_trace/"),
+            payload,
+        )
+
+    async def evaluate_examples(
+        self, payload: ExampleEvaluationRun, stream: Optional[str] = None
+    ) -> Any:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return await self._request(
+            "POST",
+            url_for("/evaluate/examples"),
+            payload,
+            params=query_params,
+        )
+
+    async def evaluate_traces(
+        self, payload: TraceEvaluationRun, stream: Optional[str] = None
+    ) -> Any:
+        query_params = {}
+        if stream is not None:
+            query_params["stream"] = stream
+        return await self._request(
+            "POST",
+            url_for("/evaluate/traces"),
+            payload,
+            params=query_params,
+        )
+
+    async def log_eval_results(self, payload: EvalResults) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/log_eval_results/"),
+            payload,
+        )
+
+    async def fetch_experiment_run(self, payload: EvalResultsFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/fetch_experiment_run/"),
+            payload,
+        )
+
+    async def get_evaluation_status(
+        self, experiment_run_id: str, project_name: str
+    ) -> Any:
+        query_params = {}
+        query_params["experiment_run_id"] = experiment_run_id
+        query_params["project_name"] = project_name
+        return await self._request(
+            "GET",
+            url_for("/get_evaluation_status/"),
+            query_params,
+        )
+
+    async def datasets_insert_examples(self, payload: DatasetInsertExamples) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/insert_examples/"),
+            payload,
+        )
+
+    async def datasets_pull_for_judgeval(self, payload: DatasetFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/pull_for_judgeval/"),
+            payload,
+        )
+
+    async def datasets_push(self, payload: DatasetPush) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/datasets/push/"),
+            payload,
+        )
+
+    async def traces_upsert(self, payload: TraceSave) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/traces/upsert/"),
+            payload,
+        )
+
+    async def traces_fetch(self, payload: TraceFetch) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/traces/fetch/"),
+            payload,
+        )
+
+    async def traces_add_to_dataset(self, payload: TraceAddToDataset) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/traces/add_to_dataset/"),
+            payload,
+        )
+
+    async def traces_spans_batch(self, payload: SpansBatchRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/traces/spans/batch/"),
+            payload,
+        )
+
+    async def traces_evaluation_runs_batch(
+        self, payload: EvaluationRunsBatchRequest
+    ) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/traces/evaluation_runs/batch/"),
+            payload,
+        )
+
+    async def projects_add(self, payload: ProjectAdd) -> ProjectAddResponse:
+        return await self._request(
+            "POST",
+            url_for("/projects/add/"),
+            payload,
+        )
+
+    async def projects_delete_from_judgeval(
+        self, payload: ProjectDeleteFromJudgevalResponse
+    ) -> ProjectDeleteResponse:
+        return await self._request(
+            "DELETE",
+            url_for("/projects/delete_from_judgeval/"),
+            payload,
+        )
+
+    async def scorer_exists(self, payload: ScorerExistsRequest) -> ScorerExistsResponse:
+        return await self._request(
+            "POST",
+            url_for("/scorer_exists/"),
+            payload,
+        )
+
+    async def save_scorer(
+        self, payload: SavePromptScorerRequest
+    ) -> SavePromptScorerResponse:
+        return await self._request(
+            "POST",
+            url_for("/save_scorer/"),
+            payload,
+        )
+
+    async def fetch_scorer(
+        self, payload: FetchPromptScorerRequest
+    ) -> FetchPromptScorerResponse:
+        return await self._request(
+            "POST",
+            url_for("/fetch_scorer/"),
+            payload,
+        )
+
+    async def upload_custom_scorer(
+        self, payload: CustomScorerUploadPayload
+    ) -> CustomScorerTemplateResponse:
+        return await self._request(
+            "POST",
+            url_for("/upload_custom_scorer/"),
+            payload,
+        )
+
+    async def projects_resolve(
+        self, payload: ResolveProjectNameRequest
+    ) -> ResolveProjectNameResponse:
+        return await self._request(
+            "POST",
+            url_for("/projects/resolve/"),
+            payload,
+        )
+
+    async def e2e_fetch_trace(self, payload: TraceIdRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_trace/"),
+            payload,
+        )
+
+    async def e2e_fetch_span_score(self, payload: SpanScoreRequest) -> Any:
+        return await self._request(
+            "POST",
+            url_for("/e2e_fetch_span_score/"),
+            payload,
+        )
+
+
+__all__ = [
+    "JudgmentSyncClient",
+    "JudgmentAsyncClient",
+]