eval-studio-client 1.0.0a1__py3-none-any.whl → 1.0.1__py3-none-any.whl

eval_studio_client/leaderboards.py

@@ -1,6 +1,7 @@
 import dataclasses
 import datetime
 import json
+import os
 import time
 from typing import Dict
 from typing import List
@@ -33,12 +34,14 @@ class Leaderboard:
     update_time: Optional[datetime.datetime] = None
     problems: List[p6s.Problem] = dataclasses.field(default_factory=list)
     insights: List[i6s.Insight] = dataclasses.field(default_factory=list)
+    summary: Optional[str] = None
     existing_collection: Optional[str] = None
     _report: Optional[str] = None
     _leaderboard: Optional[str] = None
     _model_name: Optional[str] = None
     _status: Optional[models.V1LeaderboardStatus] = None
     _client: Optional[api.ApiClient] = None
+    _operation: Optional[str] = None
 
     def __post_init__(self):
         self._evaluator_api = api.EvaluatorServiceApi(self._client)
@@ -85,6 +88,42 @@ class Leaderboard:
         if self._client:
             self._leaderboard_api.leaderboard_service_delete_leaderboard(self.key)
 
+    def download_result(self, dest: str):
+        """Downloads the leaderboard result to a JSON file.
+
+        Args:
+            dest (str): The destination path for the report.
+        """
+        if not os.path.exists(dest):
+            raise ValueError("Destination path does not exist.")
+
+        if os.path.isdir(dest):
+            dest = os.path.join(dest, "results.json")
+
+        if self._client and self.finished:
+            headers: Dict[str, str] = {}
+            url = urljoin(
+                self._client.configuration.host, f"/content/{self.key}/results"
+            )
+            self._client.update_params_for_auth(
+                headers=headers,
+                queries=[],
+                auth_settings=[],
+                resource_path=url,
+                method="GET",
+                body=None,
+            )
+            response = urllib3.request("GET", url, headers=headers)
+
+            if response.status == 200:
+                with open(dest, "wb") as f:
+                    f.write(response.data)
+                    return
+            else:
+                raise RuntimeError("Failed to retrieve leaderboard result.")
+
+        raise ValueError("Cannot download result for unfinished leaderboard.")
+
     def download_report(self, dest: str):
         """Downloads the leaderboard report to a zip file.
 
@@ -113,6 +152,30 @@ class Leaderboard:
 
         raise ValueError("Cannot download report for unfinished leaderboard.")
 
+    def get_result_json(self) -> str:
+        """Retrieves the leaderboard result as a JSON string."""
+        if self._client and self.finished:
+            headers: Dict[str, str] = {}
+            url = urljoin(
+                self._client.configuration.host, f"/content/{self.key}/results"
+            )
+            self._client.update_params_for_auth(
+                headers=headers,
+                queries=[],
+                auth_settings=[],
+                resource_path=url,
+                method="GET",
+                body=None,
+            )
+            response = urllib3.request("GET", url, headers=headers)
+
+            if response.status == 200:
+                return str(response.data)
+            else:
+                raise RuntimeError("Failed to retrieve leaderboard result.")
+
+        raise ValueError("Cannot download result for unfinished leaderboard.")
+
     def get_table(self) -> LeaderboardTable:
         """Retrieves the leaderboard table."""
         if self._client and self.finished:
@@ -169,6 +232,7 @@ class Leaderboard:
         """Refresh the leaderboard with the latest API data."""
         self.key = api_leaderboard.name or ""
         self.update_time = api_leaderboard.update_time
+        self.summary = api_leaderboard.leaderboard_summary
         self._leaderboard = api_leaderboard.leaderboard_table
         self._report = api_leaderboard.leaderboard_report or ""
         self._status = api_leaderboard.status
@@ -191,6 +255,7 @@ class Leaderboard:
             update_time=api_leaderboard.update_time,
             problems=problems,
             insights=insights,
+            summary=api_leaderboard.leaderboard_summary,
             existing_collection=api_leaderboard.h2ogpte_collection or None,
             _evaluator_name=api_leaderboard.evaluator or "",
             _test_names=api_leaderboard.tests or [],
@@ -198,6 +263,7 @@ class Leaderboard:
             _leaderboard=api_leaderboard.leaderboard_table,
             _status=api_leaderboard.status,
             _client=client,
+            _operation=api_leaderboard.create_operation or None,
         )
 
     @staticmethod
@@ -206,3 +272,62 @@ class Leaderboard:
             models.V1LeaderboardStatus.LEADERBOARD_STATUS_COMPLETED,
             models.V1LeaderboardStatus.LEADERBOARD_STATUS_FAILED,
         ]
+
+    @staticmethod
+    def from_operation(
+        operation: models.V1Operation, client: Optional[api.ApiClient]
+    ) -> Optional["Leaderboard"]:
+        """Retrieves the leaderboard from the operation, which created it.
+
+        Args:
+            operation: The operation that created the dashboard.
+            client: The API client to use for the leaderboard retrieval.
+
+        Returns:
+            Leaderboard: The leaderboard instance created by the operation.
+        """
+        if not client:
+            raise RuntimeError("API Client is not provided")
+
+        if not operation.metadata:
+            raise RuntimeError(
+                "Operation metadata missing, it's not possible to retrieve leaderboard from operation"
+            )
+
+        leaderboard_api = api.LeaderboardServiceApi(client)
+        leadeboard_id = operation.metadata.to_dict().get("leaderboard", "")
+        res = leaderboard_api.leaderboard_service_get_leaderboard(str(leadeboard_id))
+        if res and res.leaderboard:
+            return Leaderboard._from_api_leaderboard(res.leaderboard, client)
+
+        return None
+
+
+class _Leaderboards:
+    def __init__(self, client: api.ApiClient):
+        self._client = client
+        self._api = api.LeaderboardServiceApi(client)
+
+    def get(self, key: str) -> Leaderboard:
+        """Gets an individual leaderboard with a given key from Eval Studio.
+
+        Args:
+            key: The leaderboard resource name to retrieve.
+        """
+        res = self._api.leaderboard_service_get_leaderboard(key)
+        if res and res.leaderboard:
+            return Leaderboard._from_api_leaderboard(res.leaderboard, self._client)
+
+        raise KeyError("Leaderboard not found.")
+
+    def list(self) -> List[Leaderboard]:
+        """Lists all user leaderboards in Eval Studio."""
+        res = self._api.leaderboard_service_list_leaderboards()
+        if res:
+            res_leaderboards = res.leaderboards or []
+            return [
+                Leaderboard._from_api_leaderboard(lb, self._client)
+                for lb in res_leaderboards
+            ]
+
+        return []

eval_studio_client/models.py

@@ -168,7 +168,7 @@ class Model:
             )
 
         if res and res.operation:
-            return self._get_leaderboard_from_operation(res.operation)
+            return l10s.Leaderboard.from_operation(res.operation, self._client)
 
         return None
 
@@ -226,7 +226,7 @@ class Model:
         )
 
         if res and res.operation:
-            return self._get_dashboard_from_operation(res.operation)
+            return d8s.Dashboard.from_operation(res.operation, self._client)
 
         return None
 
@@ -257,7 +257,7 @@ class Model:
         )
         res = self._leaderboard_api.leaderboard_service_import_leaderboard(req)
         if res and res.operation:
-            return self._get_leaderboard_from_operation(res.operation)
+            return l10s.Leaderboard.from_operation(res.operation, self._client)
 
         return None
 
@@ -273,45 +273,6 @@ class Model:
 
         raise RuntimeError("Failed to list base models")
 
-    def _get_leaderboard_from_operation(
-        self, operation: models.V1Operation
-    ) -> Optional[l10s.Leaderboard]:
-        """Retrieves the leaderboard from the operation, which created it.
-
-        Args:
-            operation: The operation that created the leaderboard.
-        """
-        if not operation.metadata:
-            raise RuntimeError("Not possible to retrieve leaderboard from operation")
-
-        leadeboard_id = operation.metadata.to_dict().get("leaderboard")
-        res = self._leaderboard_api.leaderboard_service_get_leaderboard(leadeboard_id)
-        if res and res.leaderboard:
-            return l10s.Leaderboard._from_api_leaderboard(res.leaderboard, self._client)
-
-        return None
-
-    def _get_dashboard_from_operation(
-        self, operation: models.V1Operation
-    ) -> Optional[d8s.Dashboard]:
-        """Retrieves the dashboard from the operation, which created it.
-
-        Args:
-            operation: The operation that created the dashboard.
-        """
-        if not self._client:
-            raise RuntimeError("Client is not set.")
-
-        if not operation.metadata:
-            raise RuntimeError("Not possible to retrieve dashboard from operation")
-
-        dashboard_id = operation.metadata.to_dict().get("dashboard")
-        res = self._dashboard_api.dashboard_service_get_dashboard(dashboard_id)
-        if res and res.dashboard:
-            return d8s.Dashboard._from_api_dashboard(res.dashboard, self._client)
-
-        return None
-
     @staticmethod
     def _from_api_model(api_model: models.V1Model, client: api.ApiClient) -> "Model":
         """Converts the API model to the client model."""

eval_studio_client/test_labs.py

@@ -7,7 +7,8 @@ from typing import Union
 import uuid
 
 from eval_studio_client import api
-from eval_studio_client import evaluators
+from eval_studio_client import dashboards
+from eval_studio_client import evaluators as e8s
 from eval_studio_client import leaderboards as l10s
 from eval_studio_client.api import models as apiModels
 
@@ -92,11 +93,56 @@ class TestLab:
         self._models.append(_m)
         return _m
 
-    def evaluate(self, evaluator: evaluators.Evaluator) -> Optional[l10s.Leaderboard]:
+    def evaluate(
+        self,
+        evaluators: Union[e8s.Evaluator, List[e8s.Evaluator]],
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> Optional[dashboards.Dashboard]:
         """Runs an evaluation for the test lab.
 
+        Args:
+            evaluators (Union[e8s.Evaluator, List[e8s.Evaluator]]): One or many evaluators
+                used to evaluate the test lab.
+            name (str, optional): Optional name for the evaluation.
+            description (str, optional): Optional description for the evaluation.
+
+        Returns:
+           Dashboard: Evaluation dashboard instance. In case launching of evaluation
+              fails, `None` is returned.
+        """
+        _evaluators = (
+            [evaluators] if isinstance(evaluators, e8s.Evaluator) else evaluators
+        )
+        name = name or self.name or "Imported Dashboard"
+        description = description or self.description or ""
+        req = apiModels.V1BatchImportLeaderboardRequest(
+            testLabJson=self.json(),
+            evaluators=[e.key for e in _evaluators],
+            model=None,
+            dashboardDisplayName=name,
+            dashboardDescription=description,
+            testDisplayName=f"{name} - Test",
+            testDescription=f"Test suite for {description}",
+        )
+        res = self._leaderboard_api.leaderboard_service_batch_import_leaderboard(req)
+
+        if res and res.operation:
+            return dashboards.Dashboard.from_operation(res.operation, self._client)
+
+        return None
+
+    def create_leaderboard(
+        self, evaluator: e8s.Evaluator
+    ) -> Optional[l10s.Leaderboard]:
+        """Creates a single leaderboard for the test lab.
+
         Args:
             evaluator: The evaluator to use for the evaluation.
+
+        Returns:
+           Leaderboard: Single evaluation leaderboard instance.
+            In case launching of evaluation fails, `None` is returned.
         """
         req = apiModels.V1ImportLeaderboardRequest(
             testLabJson=self.json(),
@@ -109,7 +155,7 @@ class TestLab:
         )
         res = self._leaderboard_api.leaderboard_service_import_leaderboard(req)
         if res and res.operation:
-            return self._get_leaderboard_from_operation(res.operation)
+            return l10s.Leaderboard.from_operation(res.operation, self._client)
 
         return None
 
@@ -131,24 +177,6 @@ class TestLab:
 
         return json.dumps(lab, indent=4, sort_keys=True)
 
-    def _get_leaderboard_from_operation(
-        self, operation: apiModels.V1Operation
-    ) -> Optional[l10s.Leaderboard]:
-        """Retrieves the leaderboard from the operation, which created it.
-
-        Args:
-            operation: The operation that created the leaderboard.
-        """
-        if not operation.metadata:
-            raise RuntimeError("Not possible to retrieve leaderboard from operation")
-
-        leadeboard_id = operation.metadata.to_dict().get("leaderboard")
-        res = self._leaderboard_api.leaderboard_service_get_leaderboard(leadeboard_id)
-        if res and res.leaderboard:
-            return l10s.Leaderboard._from_api_leaderboard(res.leaderboard, self._client)
-
-        return None
-
     def _llm_model_names(self) -> List[str]:
         return [m.llm_model_name for m in self.models]