eval-studio-client 0.8.0a2__py3-none-any.whl → 1.0.0__py3-none-any.whl

eval_studio_client/api/test/{test_v1alpha_update_test_case_response.py → test_v1_update_test_case_response.py}

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 """
-    ai/h2o/eval_studio/v1alpha/collection.proto
+    ai/h2o/eval_studio/v1/collection.proto
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
 
@@ -14,10 +14,10 @@
 
 import unittest
 
-from eval_studio_client.api.models.v1alpha_update_test_case_response import V1alphaUpdateTestCaseResponse
+from eval_studio_client.api.models.v1_update_test_case_response import V1UpdateTestCaseResponse
 
-class TestV1alphaUpdateTestCaseResponse(unittest.TestCase):
-    """V1alphaUpdateTestCaseResponse unit test stubs"""
+class TestV1UpdateTestCaseResponse(unittest.TestCase):
+    """V1UpdateTestCaseResponse unit test stubs"""
 
     def setUp(self):
         pass
@@ -25,17 +25,17 @@ class TestV1alphaUpdateTestCaseResponse(unittest.TestCase):
     def tearDown(self):
         pass
 
-    def make_instance(self, include_optional) -> V1alphaUpdateTestCaseResponse:
-        """Test V1alphaUpdateTestCaseResponse
+    def make_instance(self, include_optional) -> V1UpdateTestCaseResponse:
+        """Test V1UpdateTestCaseResponse
             include_option is a boolean, when False only required
             params are included, when True both required and
             optional params are included """
-        # uncomment below to create an instance of `V1alphaUpdateTestCaseResponse`
+        # uncomment below to create an instance of `V1UpdateTestCaseResponse`
         """
-        model = V1alphaUpdateTestCaseResponse()
+        model = V1UpdateTestCaseResponse()
         if include_optional:
-            return V1alphaUpdateTestCaseResponse(
-                test_case = eval_studio_client.api.models.v1alpha_test_case.v1alphaTestCase(
+            return V1UpdateTestCaseResponse(
+                test_case = eval_studio_client.api.models.v1_test_case.v1TestCase(
                     name = '', 
                     create_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
                     creator = '', 
@@ -52,12 +52,12 @@ class TestV1alphaUpdateTestCaseResponse(unittest.TestCase):
                     condition = '', )
             )
         else:
-            return V1alphaUpdateTestCaseResponse(
+            return V1UpdateTestCaseResponse(
         )
         """
 
-    def testV1alphaUpdateTestCaseResponse(self):
-        """Test V1alphaUpdateTestCaseResponse"""
+    def testV1UpdateTestCaseResponse(self):
+        """Test V1UpdateTestCaseResponse"""
         # inst_req_only = self.make_instance(include_optional=False)
         # inst_req_and_optional = self.make_instance(include_optional=True)
 

eval_studio_client/api/test/test_v1_update_test_response.py

@@ -0,0 +1,67 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_update_test_response import V1UpdateTestResponse
+
+class TestV1UpdateTestResponse(unittest.TestCase):
+    """V1UpdateTestResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1UpdateTestResponse:
+        """Test V1UpdateTestResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1UpdateTestResponse`
+        """
+        model = V1UpdateTestResponse()
+        if include_optional:
+            return V1UpdateTestResponse(
+                test = eval_studio_client.api.models.v1_test.v1Test(
+                    name = '', 
+                    create_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    creator = '', 
+                    update_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    updater = '', 
+                    delete_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    deleter = '', 
+                    display_name = '', 
+                    description = '', 
+                    documents = [
+                        ''
+                        ], 
+                    tags = [
+                        ''
+                        ], 
+                    demo = True, )
+            )
+        else:
+            return V1UpdateTestResponse(
+        )
+        """
+
+    def testV1UpdateTestResponse(self):
+        """Test V1UpdateTestResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/{test_v1alpha_who_am_i_response.py → test_v1_who_am_i_response.py}

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 """
-    ai/h2o/eval_studio/v1alpha/collection.proto
+    ai/h2o/eval_studio/v1/collection.proto
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
 
@@ -14,10 +14,10 @@
 
 import unittest
 
-from eval_studio_client.api.models.v1alpha_who_am_i_response import V1alphaWhoAmIResponse
+from eval_studio_client.api.models.v1_who_am_i_response import V1WhoAmIResponse
 
-class TestV1alphaWhoAmIResponse(unittest.TestCase):
-    """V1alphaWhoAmIResponse unit test stubs"""
+class TestV1WhoAmIResponse(unittest.TestCase):
+    """V1WhoAmIResponse unit test stubs"""
 
     def setUp(self):
         pass
@@ -25,27 +25,27 @@ class TestV1alphaWhoAmIResponse(unittest.TestCase):
     def tearDown(self):
         pass
 
-    def make_instance(self, include_optional) -> V1alphaWhoAmIResponse:
-        """Test V1alphaWhoAmIResponse
+    def make_instance(self, include_optional) -> V1WhoAmIResponse:
+        """Test V1WhoAmIResponse
             include_option is a boolean, when False only required
             params are included, when True both required and
             optional params are included """
-        # uncomment below to create an instance of `V1alphaWhoAmIResponse`
+        # uncomment below to create an instance of `V1WhoAmIResponse`
         """
-        model = V1alphaWhoAmIResponse()
+        model = V1WhoAmIResponse()
         if include_optional:
-            return V1alphaWhoAmIResponse(
+            return V1WhoAmIResponse(
                 sub = '',
                 preferred_username = '',
                 email = ''
             )
         else:
-            return V1alphaWhoAmIResponse(
+            return V1WhoAmIResponse(
         )
         """
 
-    def testV1alphaWhoAmIResponse(self):
-        """Test V1alphaWhoAmIResponse"""
+    def testV1WhoAmIResponse(self):
+        """Test V1WhoAmIResponse"""
         # inst_req_only = self.make_instance(include_optional=False)
         # inst_req_and_optional = self.make_instance(include_optional=True)
 

eval_studio_client/api/test/test_who_am_i_service_api.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 """
-    ai/h2o/eval_studio/v1alpha/collection.proto
+    ai/h2o/eval_studio/v1/collection.proto
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
 

eval_studio_client/dashboards.py

@@ -3,11 +3,13 @@ import datetime
 import time
 from typing import List
 from typing import Optional
+import urllib.parse
 
 from eval_studio_client import api
 from eval_studio_client import insights as i6s
 from eval_studio_client import leaderboards as l10s
 from eval_studio_client import problems as p6s
+from eval_studio_client import utils
 from eval_studio_client.api import models
 
 
@@ -39,6 +41,8 @@ class Dashboard:
         if self._client:
             self._dashboard_api = api.DashboardServiceApi(self._client)
             self._leaderboard_api = api.LeaderboardServiceApi(self._client)
+            self._info_api = api.InfoServiceApi(self._client)
+            self._operation_api = api.OperationServiceApi(self._client)
 
     @property
     def leaderboards(self) -> Optional[List[l10s.Leaderboard]]:
@@ -116,34 +120,71 @@ class Dashboard:
         else:
             raise ValueError("Cannot establish connection to Eval Studio host.")
 
-    def wait_to_finish(self, timeout: Optional[float] = None):
+    def wait_to_finish(self, timeout: Optional[float] = None, verbose: bool = False):
         """Waits for the dashboard to finish.
 
         Args:
             timeout: The maximum time to wait in seconds.
+            verbose (bool): If True, prints the status of the evaluation while waiting.
         """
         timeout = timeout or float("inf")
+        progress_bar = utils.ProgressBar()
         if self.finished:
             return
 
+        if not self._create_operation:
+            # This means that the evaluation has no assigned operation, thus cannot poll.
+            raise RuntimeError("Failed to retrieve running evaluation info.")
+
         if self._client:
             ctr = 0
             while ctr < timeout:
-                lbs = self.leaderboards
-                if lbs:
-                    if all(lb.finished for lb in lbs):
-                        return
-
-                    ctr += 1
-                    time.sleep(1)
+                op = self._operation_api.operation_service_get_operation(
+                    self._create_operation
+                )
+                if not op or not op.operation:
+                    raise RuntimeError(
+                        "Failed to retrieve running evaluation progress."
+                    )
+
+                if verbose:
+                    if not op.operation.metadata:
+                        raise RuntimeError(
+                            "Failed to retrieve running evaluation progress details."
+                        )
+
+                    op_meta = op.operation.metadata.to_dict()
+                    progress = op_meta.get("progress", 0)
+                    progress_msg = op_meta.get("progressMessage", "Running")
+                    progress_bar.update(progress, progress_msg)
+
+                if op.operation.done:
+                    return
+
+                ctr += 1
+                time.sleep(1)
         else:
             raise ValueError("Cannot establish connection to Eval Studio host.")
 
         raise TimeoutError("Waiting timeout has been reached.")
 
+    def show(self) -> str:
+        """Prints the endpoint URL of the evaluation dashboard."""
+        if self._client:
+            info_res = self._info_api.info_service_get_info()
+            if not info_res or not info_res.info:
+                raise RuntimeError("Cannot retrieve server information.")
+
+            host = info_res.info.base_url
+            url = urllib.parse.urljoin(host, self.key)
+            print(f"Open following url to access evaluation dashboard: \n\n{url}")
+            return url
+        else:
+            raise ValueError("Cannot establish connection to Eval Studio host.")
+
     @staticmethod
     def _from_api_dashboard(
-        api_dashboard: models.V1alphaDashboard, client: api.ApiClient
+        api_dashboard: models.V1Dashboard, client: api.ApiClient
     ) -> "Dashboard":
         """Converts the API dashboard to the client dashboard."""
         return Dashboard(

eval_studio_client/documents.py

@@ -36,15 +36,15 @@ class Document:
         """Deletes the document."""
         self._document_api.document_service_delete_document(self.key)
 
-    def to_api_proto(self) -> models.V1alphaDocument:
+    def to_api_proto(self) -> models.V1Document:
         """Converts the client Document to an API Document."""
-        return models.V1alphaDocument(
+        return models.V1Document(
             display_name=self.name, description=self.description, url=self.url
         )
 
     @staticmethod
     def _from_api_document(
-        api_document: models.V1alphaDocument, client: Optional[api.ApiClient]
+        api_document: models.V1Document, client: Optional[api.ApiClient]
     ) -> "Document":
         """Converts an API Document to a client Document."""
         return Document(

eval_studio_client/evaluators.py

@@ -26,7 +26,7 @@ class Evaluator:
     enabled: bool
 
     @staticmethod
-    def _from_api_evaluator(api_evaluator: models.V1alphaEvaluator) -> "Evaluator":
+    def _from_api_evaluator(api_evaluator: models.V1Evaluator) -> "Evaluator":
         """Converts an API Evaluator to a client Evaluator."""
         return Evaluator(
             key=api_evaluator.name or "",