eval-studio-client 0.7.0__py3-none-any.whl

eval_studio_client/api/test/test_v1alpha_update_test_response.py

@@ -0,0 +1,67 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1alpha/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1alpha_update_test_response import V1alphaUpdateTestResponse
+
+class TestV1alphaUpdateTestResponse(unittest.TestCase):
+    """V1alphaUpdateTestResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1alphaUpdateTestResponse:
+        """Test V1alphaUpdateTestResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1alphaUpdateTestResponse`
+        """
+        model = V1alphaUpdateTestResponse()
+        if include_optional:
+            return V1alphaUpdateTestResponse(
+                test = eval_studio_client.api.models.v1alpha_test.v1alphaTest(
+                    name = '', 
+                    create_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    creator = '', 
+                    update_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    updater = '', 
+                    delete_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    deleter = '', 
+                    display_name = '', 
+                    description = '', 
+                    documents = [
+                        ''
+                        ], 
+                    tags = [
+                        ''
+                        ], 
+                    demo = True, )
+            )
+        else:
+            return V1alphaUpdateTestResponse(
+        )
+        """
+
+    def testV1alphaUpdateTestResponse(self):
+        """Test V1alphaUpdateTestResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1alpha_who_am_i_response.py

@@ -0,0 +1,53 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1alpha/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1alpha_who_am_i_response import V1alphaWhoAmIResponse
+
+class TestV1alphaWhoAmIResponse(unittest.TestCase):
+    """V1alphaWhoAmIResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1alphaWhoAmIResponse:
+        """Test V1alphaWhoAmIResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1alphaWhoAmIResponse`
+        """
+        model = V1alphaWhoAmIResponse()
+        if include_optional:
+            return V1alphaWhoAmIResponse(
+                sub = '',
+                preferred_username = '',
+                email = ''
+            )
+        else:
+            return V1alphaWhoAmIResponse(
+        )
+        """
+
+    def testV1alphaWhoAmIResponse(self):
+        """Test V1alphaWhoAmIResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_who_am_i_service_api.py

@@ -0,0 +1,38 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1alpha/collection.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.api.who_am_i_service_api import WhoAmIServiceApi
+
+
+class TestWhoAmIServiceApi(unittest.TestCase):
+    """WhoAmIServiceApi unit test stubs"""
+
+    def setUp(self) -> None:
+        self.api = WhoAmIServiceApi()
+
+    def tearDown(self) -> None:
+        pass
+
+    def test_who_am_i_service_who_am_i(self) -> None:
+        """Test case for who_am_i_service_who_am_i
+
+        WhoAmI is used to retrieve the caller's identity.
+        """
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/client.py

@@ -0,0 +1,98 @@
+from typing import Callable
+from typing import Optional
+
+import h2o_authn
+
+from eval_studio_client import api
+from eval_studio_client import dashboards
+from eval_studio_client import documents
+from eval_studio_client import evaluators
+from eval_studio_client import models
+from eval_studio_client import perturbators
+from eval_studio_client import test_labs
+from eval_studio_client import tests
+
+
+class Client:
+    """Main Eval Studio client class."""
+
+    def __init__(
+        self, host: str, token_provider: Optional[h2o_authn.TokenProvider] = None
+    ):
+        """Initialize the client connection to Eval Studio host.
+
+        Args:
+            host: The Eval Studio host URL.
+            token_provider: Optional TokenProvider used for authentication. If not
+                provided, the client will not set any authentication headers.
+        """
+        _host = host.rstrip("/")
+        _api_config = api.Configuration(host=_host)
+        self._api_client = api.ApiClient(configuration=_api_config)
+        if token_provider is not None:
+            self._api_client = _TokenApiClient(_api_config, token_provider.token)
+
+        self._dashboards = dashboards._Dashboards(self._api_client)
+        self._documents = documents._Documents(self._api_client)
+        self._evaluators = evaluators._Evaluators(self._api_client)
+        self._perturbators = perturbators._Perturbators(self._api_client)
+        self._models = models._Models(self._api_client)
+        self._tests = tests._Tests(self._api_client)
+        self._test_labs = test_labs._TestLabs(self._api_client)
+
+    @property
+    def dashboards(self) -> dashboards._Dashboards:
+        """API for managing the collection of dashboards."""
+        return self._dashboards
+
+    @property
+    def documents(self) -> documents._Documents:
+        """API for managing the collection of documents."""
+        return self._documents
+
+    @property
+    def evaluators(self) -> evaluators._Evaluators:
+        """API for getting information about evaluation techniques in the Eval Studio."""
+        return self._evaluators
+
+    @property
+    def perturbators(self) -> perturbators._Perturbators:
+        """API for getting information about perturbation techniques in the Eval Studio."""
+        return self._perturbators
+
+    @property
+    def models(self) -> models._Models:
+        """API for managing models and connections to RAG/LLM system."""
+        return self._models
+
+    @property
+    def tests(self) -> tests._Tests:
+        """API for managing the collection of tests."""
+        return self._tests
+
+    @property
+    def test_labs(self) -> test_labs._TestLabs:
+        """API for building and evaluating test labs."""
+        return self._test_labs
+
+
+class _TokenApiClient(api.ApiClient):
+    """Custom API client class to handle bearer token authentication."""
+
+    def __init__(
+        self, configuration: api.Configuration, token_provider: Callable[[], str]
+    ):
+        self._token_provider = token_provider
+        super().__init__(configuration=configuration)
+
+    def update_params_for_auth(
+        self,
+        headers,
+        queries,
+        auth_settings,
+        resource_path,
+        method,
+        body,
+        request_auth=None,
+    ):
+        headers["Authorization"] = f"Bearer {self._token_provider()}"

eval_studio_client/dashboards.py

@@ -0,0 +1,187 @@
+import dataclasses
+import datetime
+import time
+from typing import List
+from typing import Optional
+
+from eval_studio_client import api
+from eval_studio_client import insights as i6s
+from eval_studio_client import leaderboards as l10s
+from eval_studio_client import problems as p6s
+from eval_studio_client.api import models
+
+
+@dataclasses.dataclass
+class Dashboard:
+    """Dashboard groups score of Model on given Tests.
+
+    Attributes:
+        key (str): Generated ID of the dashboard.
+        name (str): Name of the dashboard.
+        description (str): Description of the dashboard.
+        url (str): URL of the dashboard host system.
+        api_key (str): API key for the dashboard host system.
+        is_rag (bool): Whether the dashboard is a RAG or LLM-only system.
+        create_time (datetime): Timestamp of the dashboard creation.
+        update_time (datetime): Timestamp of the last dashboard update.
+    """
+
+    key: str
+    name: str
+    description: str
+    _leaderboards: List[str]
+    create_time: Optional[datetime.datetime] = None
+    update_time: Optional[datetime.datetime] = None
+    _create_operation: Optional[str] = None
+    _client: Optional[api.ApiClient] = None
+
+    def __post_init__(self):
+        if self._client:
+            self._dashboard_api = api.DashboardServiceApi(self._client)
+            self._leaderboard_api = api.LeaderboardServiceApi(self._client)
+
+    @property
+    def leaderboards(self) -> Optional[List[l10s.Leaderboard]]:
+        """Returns the leaderboards in the dashboard."""
+        if self._client and self._leaderboards:
+            get_lbs_resp = (
+                self._leaderboard_api.leaderboard_service_batch_get_leaderboards(
+                    self._leaderboards
+                )
+            )
+            if get_lbs_resp:
+                return [
+                    l10s.Leaderboard._from_api_leaderboard(lb, self._client)
+                    for lb in get_lbs_resp.leaderboards
+                ]
+        return None
+
+    @property
+    def finished(self) -> bool:
+        """Indicates whether all leaderboards in the dashboard have finished."""
+        if self.leaderboards:
+            return all(lb.finished for lb in self.leaderboards)
+        return False
+
+    @property
+    def successful(self) -> bool:
+        """Indicates whether all leaderboards in the dashboard have finished successfully."""
+        if self.leaderboards:
+            return all(lb.successful for lb in self.leaderboards)
+        return False
+
+    @property
+    def problems(self) -> List[p6s.Problem]:
+        """Retrieves the problems detected in the dashboard.
+        The problems are aggregated across all leaderboards in the Dashboard.
+        """
+        lbs = self.leaderboards or []
+        problems_agg = []
+        for lb in lbs:
+            problems_agg.extend(lb.problems or [])
+        return problems_agg
+
+    @property
+    def insights(self) -> List[i6s.Insight]:
+        """Retrieves the insights provided by the evaluation.
+        The insights are aggregated across all leaderboards in the Dashboard.
+        """
+        lbs = self.leaderboards or []
+        insights_agg = []
+        for lb in lbs:
+            insights_agg.extend(lb.insights or [])
+        return insights_agg
+
+    def download_report(self, dest: str):
+        """Downloads the dashboard report to the given destination."""
+
+        # All leaderboards in the dashboard have the same report. Download the first successful one.
+        lbs = self.leaderboards
+        if lbs:
+            for lb in lbs:
+                try:
+                    return lb.download_report(dest)
+                except (ValueError, RuntimeError):
+                    continue
+        raise ValueError("No successful leaderboard found.")
+
+    def delete(self, delete_leaderboards: bool = False):
+        """Deletes the dashboard."""
+        if self._client:
+            if delete_leaderboards:
+                if self.leaderboards:
+                    for lb in self.leaderboards:
+                        lb.delete()
+            self._dashboard_api.dashboard_service_delete_dashboard(self.key)
+        else:
+            raise ValueError("Cannot establish connection to Eval Studio host.")
+
+    def wait_to_finish(self, timeout: Optional[float] = None):
+        """Waits for the dashboard to finish.
+
+        Args:
+            timeout: The maximum time to wait in seconds.
+        """
+        timeout = timeout or float("inf")
+        if self.finished:
+            return
+
+        if self._client:
+            ctr = 0
+            while ctr < timeout:
+                lbs = self.leaderboards
+                if lbs:
+                    if all(lb.finished for lb in lbs):
+                        return
+
+                    ctr += 1
+                    time.sleep(1)
+        else:
+            raise ValueError("Cannot establish connection to Eval Studio host.")
+
+        raise TimeoutError("Waiting timeout has been reached.")
+
+    @staticmethod
+    def _from_api_dashboard(
+        api_dashboard: models.V1alphaDashboard, client: api.ApiClient
+    ) -> "Dashboard":
+        """Converts the API dashboard to the client dashboard."""
+        return Dashboard(
+            key=api_dashboard.name or "",
+            name=api_dashboard.display_name or "",
+            description=api_dashboard.description or "",
+            _leaderboards=api_dashboard.leaderboards or [],
+            create_time=api_dashboard.create_time,
+            update_time=api_dashboard.update_time,
+            _create_operation=api_dashboard.create_operation,
+            _client=client,
+        )
+
+
+class _Dashboards:
+    def __init__(self, client: api.ApiClient):
+        self._client = client
+        self._api = api.DashboardServiceApi(client)
+
+    def get(self, key: str) -> Dashboard:
+        """Gets a dashboard with given key from Eval Studio.
+
+        Args:
+            key: The dashboard resource name to retrieve.
+        """
+        res = self._api.dashboard_service_get_dashboard(key)
+        if res and res.dashboard:
+            return Dashboard._from_api_dashboard(res.dashboard, self._client)
+
+        raise KeyError("Dashboard not found.")
+
+    def list(self) -> List[Dashboard]:
+        """Lists all user dashboards in Eval Studio."""
+        res = self._api.dashboard_service_list_dashboards()
+        if res:
+            res_dashboards = res.dashboards or []
+            return [
+                Dashboard._from_api_dashboard(m, self._client) for m in res_dashboards
+            ]
+
+        return []

eval_studio_client/documents.py

@@ -0,0 +1,95 @@
+import dataclasses
+import datetime
+from typing import List
+from typing import Optional
+
+from eval_studio_client import api
+from eval_studio_client.api import models
+
+
+@dataclasses.dataclass
+class Document:
+    """Represents a document in Eval Studio, that can be uploaded to RAG system.
+
+    Attributes:
+        key (str): Generated ID of the document.
+        name (str): Display name of the document.
+        description (str): Description of the document.
+        url (str): URL of the document.
+        create_time (datetime): Timestamp of the document creation.
+        update_time (datetime): Timestamp of the last document update.
+    """
+
+    key: str
+    name: str
+    description: str
+    url: str
+    create_time: Optional[datetime.datetime] = None
+    update_time: Optional[datetime.datetime] = None
+    _client: Optional[api.ApiClient] = None
+
+    def __post_init__(self):
+        if self._client:
+            self._document_api = api.DocumentServiceApi(self._client)
+
+    def delete(self):
+        """Deletes the document."""
+        self._document_api.document_service_delete_document(self.key)
+
+    def to_api_proto(self) -> models.V1alphaDocument:
+        """Converts the client Document to an API Document."""
+        return models.V1alphaDocument(
+            display_name=self.name, description=self.description, url=self.url
+        )
+
+    @staticmethod
+    def _from_api_document(
+        api_document: models.V1alphaDocument, client: Optional[api.ApiClient]
+    ) -> "Document":
+        """Converts an API Document to a client Document."""
+        return Document(
+            key=api_document.name or "",
+            name=api_document.display_name or "",
+            description=api_document.description or "",
+            url=api_document.url or "",
+            create_time=api_document.create_time or None,
+            update_time=api_document.update_time or None,
+            _client=client,
+        )
+
+
+class _Documents:
+    def __init__(self, client: api.ApiClient):
+        self._client = client
+        self._api = api.DocumentServiceApi(client)
+
+    def get(self, key: str) -> Document:
+        """Retrieves a document by its resource name.
+
+        Args:
+            key: The document resource name to retrieve.
+
+        Returns:
+            The document with the given resource name.
+        """
+        res = self._api.document_service_get_document(key)
+        if res and res.document:
+            return Document._from_api_document(res.document, self._client)
+
+        raise KeyError("Document not found.")
+
+    def list(self) -> List[Document]:
+        """List all user documents in Eval Studio."""
+        res = self._api.document_service_list_documents()
+        if res and res.documents:
+            return [Document._from_api_document(d, self._client) for d in res.documents]
+
+        return []
+
+    def delete(self, key: str):
+        """Deletes the document with the given resource name from the Eval Studio.
+
+        Args:
+            key: The document resource name to delete.
+        """
+        self._api.document_service_delete_document(key)

eval_studio_client/evaluators.py

@@ -0,0 +1,65 @@
+import dataclasses
+from typing import List
+
+from eval_studio_client import api
+from eval_studio_client.api import models
+
+
+@dataclasses.dataclass
+class Evaluator:
+    """Represents an evaluation method in Eval Studio.
+
+    Attributes:
+        key (str): Generated ID of the evaluator.
+        name (str): Display name of the evaluator.
+        description (str): Description of the evaluator.
+        keywords (List[str]): Keywords associated with the evaluator.
+            These include tags specifying whether evaluators can be used for
+            RAG evaluation, LLM evaluation, or both.
+        enabled (bool): Whether this evaluator is enabled and can be used for evaluation.
+    """
+
+    key: str
+    name: str
+    description: str
+    keywords: List[str]
+    enabled: bool
+
+    @staticmethod
+    def _from_api_evaluator(api_evaluator: models.V1alphaEvaluator) -> "Evaluator":
+        """Converts an API Evaluator to a client Evaluator."""
+        return Evaluator(
+            key=api_evaluator.name or "",
+            name=api_evaluator.display_name or "",
+            description=(
+                api_evaluator.description or api_evaluator.brief_description or ""
+            ),
+            keywords=api_evaluator.tags or [],
+            enabled=api_evaluator.enabled or False,
+        )
+
+
+class _Evaluators:
+    def __init__(self, client: api.ApiClient):
+        self._client = client
+        self._api = api.EvaluatorServiceApi(client)
+
+    def get(self, key: str) -> Evaluator:
+        """Retrieves an evaluator by key.
+
+        Args:
+            key (str): ID of the evaluator.
+        """
+        res = self._api.evaluator_service_get_evaluator(key)
+        if res and res.evaluator:
+            return Evaluator._from_api_evaluator(res.evaluator)
+
+        raise KeyError("Evaluator not found")
+
+    def list(self) -> List[Evaluator]:
+        """Lists all available evaluators in Eval Studio."""
+        res = self._api.evaluator_service_list_evaluators()
+        if res and res.evaluators:
+            return [Evaluator._from_api_evaluator(e) for e in res.evaluators]
+
+        return []