eval-studio-client 1.0.0a1__py3-none-any.whl → 1.0.1__py3-none-any.whl

eval_studio_client/api/test/test_v1_workflow_node_type.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_workflow_node_type import V1WorkflowNodeType
+
+class TestV1WorkflowNodeType(unittest.TestCase):
+    """V1WorkflowNodeType unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testV1WorkflowNodeType(self):
+        """Test V1WorkflowNodeType"""
+        # inst = V1WorkflowNodeType()
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_workflow_node_view.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_workflow_node_view import V1WorkflowNodeView
+
+class TestV1WorkflowNodeView(unittest.TestCase):
+    """V1WorkflowNodeView unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testV1WorkflowNodeView(self):
+        """Test V1WorkflowNodeView"""
+        # inst = V1WorkflowNodeView()
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_workflow_type.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_workflow_type import V1WorkflowType
+
+class TestV1WorkflowType(unittest.TestCase):
+    """V1WorkflowType unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testV1WorkflowType(self):
+        """Test V1WorkflowType"""
+        # inst = V1WorkflowType()
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_workflow_edge_service_api.py

@@ -0,0 +1,38 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.api.workflow_edge_service_api import WorkflowEdgeServiceApi
+
+
+class TestWorkflowEdgeServiceApi(unittest.TestCase):
+    """WorkflowEdgeServiceApi unit test stubs"""
+
+    def setUp(self) -> None:
+        self.api = WorkflowEdgeServiceApi()
+
+    def tearDown(self) -> None:
+        pass
+
+    def test_workflow_edge_service_batch_get_workflow_edges(self) -> None:
+        """Test case for workflow_edge_service_batch_get_workflow_edges
+
+        Retrieves all WorkflowEdges with the specified resource names. If any of the WorkflowEdges do not exist an error is returned. The order of resource names in the request and the returned WorkflowEdges might differ.
+        """
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_workflow_node_service_api.py

@@ -0,0 +1,73 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.api.workflow_node_service_api import WorkflowNodeServiceApi
+
+
+class TestWorkflowNodeServiceApi(unittest.TestCase):
+    """WorkflowNodeServiceApi unit test stubs"""
+
+    def setUp(self) -> None:
+        self.api = WorkflowNodeServiceApi()
+
+    def tearDown(self) -> None:
+        pass
+
+    def test_workflow_node_service_batch_get_workflow_nodes(self) -> None:
+        """Test case for workflow_node_service_batch_get_workflow_nodes
+
+        BatchGetWorkflowNodes retrieves all WorkflowNodes with the specified resource names. If any of the WorkflowNodes do not exist an error is returned. Deleted WorkflowNodes are returned without error. The order of resource names in the request and the returned WorkflowNodes might differ.
+        """
+        pass
+
+    def test_workflow_node_service_delete_workflow_node(self) -> None:
+        """Test case for workflow_node_service_delete_workflow_node
+
+        DeleteWorkflowNode deletes a WorkflowNode by name. If the WorkflowNode does not exist an error is returned.
+        """
+        pass
+
+    def test_workflow_node_service_get_workflow_node(self) -> None:
+        """Test case for workflow_node_service_get_workflow_node
+
+        GetWorkflowNode retrieves a WorkflowNode by name. Deleted WorkflowNode is returned without error, it has a delete_time and deleter fields set.
+        """
+        pass
+
+    def test_workflow_node_service_get_workflow_node_prerequisites(self) -> None:
+        """Test case for workflow_node_service_get_workflow_node_prerequisites
+
+        GetWorkflowNodePrerequisites retrieves the WorkflowNodes and WorkflowEdges that are the prerequisites of the specified WorkflowNode. The list might be empty. Large data might be stored in storage and not returned in the response. It is client's responsibility to retrieve the data from storage using the content handlers. It is intended to be used by the Eval Studio Workers.
+        """
+        pass
+
+    def test_workflow_node_service_process_workflow_node(self) -> None:
+        """Test case for workflow_node_service_process_workflow_node
+
+        ProcessWorkflowNode processes a WorkflowNode.
+        """
+        pass
+
+    def test_workflow_node_service_update_workflow_node(self) -> None:
+        """Test case for workflow_node_service_update_workflow_node
+
+        UpdateWorkflowNode updates a WorkflowNode. The update_mask is used to specify the fields to be updated.
+        """
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_workflow_service_api.py

@@ -0,0 +1,73 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.api.workflow_service_api import WorkflowServiceApi
+
+
+class TestWorkflowServiceApi(unittest.TestCase):
+    """WorkflowServiceApi unit test stubs"""
+
+    def setUp(self) -> None:
+        self.api = WorkflowServiceApi()
+
+    def tearDown(self) -> None:
+        pass
+
+    def test_workflow_service_batch_delete_workflows(self) -> None:
+        """Test case for workflow_service_batch_delete_workflows
+
+        BatchDeleteWorkflows deletes Workflows by names. If any of the Workflows do not exist an error is returned.
+        """
+        pass
+
+    def test_workflow_service_create_workflow(self) -> None:
+        """Test case for workflow_service_create_workflow
+
+        CreateWorkflow creates a Workflow.
+        """
+        pass
+
+    def test_workflow_service_delete_workflow(self) -> None:
+        """Test case for workflow_service_delete_workflow
+
+        DeleteWorkflow deletes a Workflow by name. If the Workflow does not exist an error is returned.
+        """
+        pass
+
+    def test_workflow_service_get_workflow(self) -> None:
+        """Test case for workflow_service_get_workflow
+
+        GetWorkflow retrieves a Workflow by name. Deleted Workflow is returned without error, it has a delete_time and deleter fields set.
+        """
+        pass
+
+    def test_workflow_service_list_workflows(self) -> None:
+        """Test case for workflow_service_list_workflows
+
+        ListWorkflows lists Workflows.
+        """
+        pass
+
+    def test_workflow_service_update_workflow(self) -> None:
+        """Test case for workflow_service_update_workflow
+
+        UpdateWorkflow updates a Workflow. The update_mask is used to specify the fields to be updated.
+        """
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/client.py

@@ -7,6 +7,7 @@ from eval_studio_client import api
 from eval_studio_client import dashboards
 from eval_studio_client import documents
 from eval_studio_client import evaluators
+from eval_studio_client import leaderboards
 from eval_studio_client import models
 from eval_studio_client import perturbators
 from eval_studio_client import test_labs
@@ -33,6 +34,7 @@ class Client:
             self._api_client = _TokenApiClient(_api_config, token_provider.token)
 
         self._dashboards = dashboards._Dashboards(self._api_client)
+        self._leaderboards = leaderboards._Leaderboards(self._api_client)
         self._documents = documents._Documents(self._api_client)
         self._evaluators = evaluators._Evaluators(self._api_client)
         self._perturbators = perturbators._Perturbators(self._api_client)
@@ -55,6 +57,11 @@ class Client:
         """API for getting information about evaluation techniques in the Eval Studio."""
         return self._evaluators
 
+    @property
+    def leaderboards(self) -> leaderboards._Leaderboards:
+        """API for managing the collection of leaderboards."""
+        return self._leaderboards
+
     @property
     def perturbators(self) -> perturbators._Perturbators:
         """API for getting information about perturbation techniques in the Eval Studio."""

eval_studio_client/dashboards.py

@@ -9,6 +9,7 @@ from eval_studio_client import api
 from eval_studio_client import insights as i6s
 from eval_studio_client import leaderboards as l10s
 from eval_studio_client import problems as p6s
+from eval_studio_client import utils
 from eval_studio_client.api import models
 
 
@@ -41,6 +42,7 @@ class Dashboard:
             self._dashboard_api = api.DashboardServiceApi(self._client)
             self._leaderboard_api = api.LeaderboardServiceApi(self._client)
             self._info_api = api.InfoServiceApi(self._client)
+            self._operation_api = api.OperationServiceApi(self._client)
 
     @property
     def leaderboards(self) -> Optional[List[l10s.Leaderboard]]:
@@ -118,36 +120,56 @@ class Dashboard:
         else:
             raise ValueError("Cannot establish connection to Eval Studio host.")
 
-    def wait_to_finish(self, timeout: Optional[float] = None):
+    def wait_to_finish(self, timeout: Optional[float] = None, verbose: bool = False):
         """Waits for the dashboard to finish.
 
         Args:
             timeout: The maximum time to wait in seconds.
+            verbose (bool): If True, prints the status of the evaluation while waiting.
         """
         timeout = timeout or float("inf")
+        progress_bar = utils.ProgressBar()
         if self.finished:
             return
 
+        if not self._create_operation:
+            # This means that the evaluation has no assigned operation, thus cannot poll.
+            raise RuntimeError("Failed to retrieve running evaluation info.")
+
         if self._client:
             ctr = 0
             while ctr < timeout:
-                lbs = self.leaderboards
-                if lbs:
-                    if all(lb.finished for lb in lbs):
-                        return
-
-                    ctr += 1
-                    time.sleep(1)
+                op = self._operation_api.operation_service_get_operation(
+                    self._create_operation
+                )
+                if not op or not op.operation:
+                    raise RuntimeError(
+                        "Failed to retrieve running evaluation progress."
+                    )
+
+                if verbose:
+                    if not op.operation.metadata:
+                        raise RuntimeError(
+                            "Failed to retrieve running evaluation progress details."
+                        )
+
+                    op_meta = op.operation.metadata.to_dict()
+                    progress = op_meta.get("progress", 0)
+                    progress_msg = op_meta.get("progressMessage", "Running")
+                    progress_bar.update(progress, progress_msg)
+
+                if op.operation.done:
+                    return
+
+                ctr += 1
+                time.sleep(1)
         else:
             raise ValueError("Cannot establish connection to Eval Studio host.")
 
         raise TimeoutError("Waiting timeout has been reached.")
 
-    def show(self):
-        """Opens the evaluation in the default web browser.
-
-        NOTE: This functionality is primarily for interactive use in Jupyter notebooks.
-        """
+    def show(self) -> str:
+        """Prints the endpoint URL of the evaluation dashboard."""
         if self._client:
             info_res = self._info_api.info_service_get_info()
             if not info_res or not info_res.info:
@@ -155,11 +177,8 @@ class Dashboard:
 
             host = info_res.info.base_url
             url = urllib.parse.urljoin(host, self.key)
-
-            # NOTE: Local import is used to avoid problems for users outside Jupyter environment.
-            import webbrowser
-
-            webbrowser.open(url)
+            print(f"Open following url to access evaluation dashboard: \n\n{url}")
+            return url
         else:
             raise ValueError("Cannot establish connection to Eval Studio host.")
 
@@ -179,6 +198,35 @@ class Dashboard:
             _client=client,
         )
 
+    @staticmethod
+    def from_operation(
+        operation: models.V1Operation, client: Optional[api.ApiClient]
+    ) -> Optional["Dashboard"]:
+        """Retrieves the dashboard from the operation, which created it.
+
+        Args:
+            operation: The operation that created the dashboard.
+            client: The API client to use for the dashboard retrieval.
+
+        Returns:
+            Dashboard: The dashboard instance created by the operation.
+        """
+        if not client:
+            raise RuntimeError("API Client is not provided")
+
+        if not operation.metadata:
+            raise RuntimeError(
+                "Operation metadata missing, it's not possible to retrieve dashboard from operation"
+            )
+
+        dashboard_api = api.DashboardServiceApi(client)
+        dashboard_id = operation.metadata.to_dict().get("dashboard", "")
+        res = dashboard_api.dashboard_service_get_dashboard(str(dashboard_id))
+        if res and res.dashboard:
+            return Dashboard._from_api_dashboard(res.dashboard, client)
+
+        return None
+
 
 class _Dashboards:
     def __init__(self, client: api.ApiClient):