trismik 0.9.1__py3-none-any.whl → 0.9.4__py3-none-any.whl

trismik/client_async.py

@@ -1,338 +1,362 @@
-from typing import List, Any, Optional
+"""
+Trismik async client for interacting with the Trismik API.
+
+This module provides an asynchronous client for interacting with the Trismik
+API. It uses httpx for making HTTP requests.
+"""
+
+from typing import List, Optional
 
 import httpx
 
-from ._mapper import TrismikResponseMapper
-from ._utils import TrismikUtils
-from .exceptions import TrismikApiError
-from .types import (
-    TrismikSessionMetadata,
-    TrismikTest,
-    TrismikAuth,
-    TrismikSession,
-    TrismikItem,
-    TrismikResult,
-    TrismikResponse,
+from trismik._mapper import TrismikResponseMapper
+from trismik._utils import TrismikUtils
+from trismik.exceptions import (
+    TrismikApiError,
+    TrismikPayloadTooLargeError,
+    TrismikValidationError,
+)
+from trismik.settings import client_settings, environment_settings
+from trismik.types import (
+    TrismikClassicEvalRequest,
+    TrismikClassicEvalResponse,
+    TrismikDataset,
+    TrismikMeResponse,
+    TrismikReplayRequest,
+    TrismikReplayResponse,
+    TrismikRunMetadata,
+    TrismikRunResponse,
+    TrismikRunSummary,
 )
 
 
 class TrismikAsyncClient:
-    _serviceUrl: str = "https://trismik.e-psychometrics.com/api"
+    """
+    Asynchronous client for the Trismik API.
+
+    This class provides an asynchronous interface to interact with the Trismik
+    API, handling authentication, dataset runs, and responses.
+    """
 
     def __init__(
-            self,
-            service_url: Optional[str] = None,
-            api_key: Optional[str] = None,
-            http_client: Optional[httpx.AsyncClient] | None = None,
+        self,
+        service_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        http_client: Optional[httpx.AsyncClient] = None,
     ) -> None:
         """
-        Initializes a new Trismik client (async version).
+        Initialize the Trismik async client.
 
         Args:
             service_url (Optional[str]): URL of the Trismik service.
             api_key (Optional[str]): API key for the Trismik service.
-            http_client (Optional[httpx.Client]): HTTP client to use for requests.
+            http_client (Optional[httpx.AsyncClient]): HTTP client to use for
+                requests.
 
         Raises:
-            TrismikError: If service_url or api_key are not provided and not found in environment.
+            TrismikError: If service_url or api_key are not provided and not
+                found in environment.
             TrismikApiError: If API request fails.
         """
         self._service_url = TrismikUtils.option(
-                service_url, self._serviceUrl, "TRISMIK_SERVICE_URL"
+            service_url,
+            client_settings["endpoint"],
+            environment_settings["trismik_service_url"],
         )
         self._api_key = TrismikUtils.required_option(
-                api_key, "api_key", "TRISMIK_API_KEY"
+            api_key, "api_key", environment_settings["trismik_api_key"]
         )
-        self._http_client = http_client or httpx.AsyncClient(
-                base_url=self._service_url)
-
-    async def authenticate(self) -> TrismikAuth:
-        """
-        Authenticates with the Trismik service.
 
-        Returns:
-            TrismikAuth: Authentication token.
+        # Set default headers with API key
+        default_headers = {"x-api-key": self._api_key}
 
-        Raises:
-            TrismikApiError: If API request fails.
-        """
-        try:
-            url = "/client/auth"
-            body = {"apiKey": self._api_key}
-            response = await self._http_client.post(url, json=body)
-            response.raise_for_status()
-            json = response.json()
-            return TrismikResponseMapper.to_auth(json)
-        except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
-        except httpx.HTTPError as e:
-            raise TrismikApiError(str(e)) from e
+        self._http_client = http_client or httpx.AsyncClient(
+            base_url=self._service_url, headers=default_headers
+        )
 
-    async def refresh_token(self, token: str) -> TrismikAuth:
+    def _handle_http_error(self, e: httpx.HTTPStatusError) -> Exception:
         """
-        Refreshes the authentication token.
+        Handle HTTP errors and return appropriate Trismik exceptions.
 
         Args:
-            token (str): Current authentication token.
+            e (httpx.HTTPStatusError): The HTTP status error to handle.
 
         Returns:
-            TrismikAuth: New authentication token.
-
-        Raises:
-            TrismikApiError: If API request fails.
+            Exception: The appropriate Trismik exception to raise.
         """
-        try:
-            url = "/client/token"
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.get(url, headers=headers)
-            response.raise_for_status()
-            json = response.json()
-            return TrismikResponseMapper.to_auth(json)
-        except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
-        except httpx.HTTPError as e:
-            raise TrismikApiError(str(e)) from e
-
-    async def available_tests(self, token: str) -> List[TrismikTest]:
+        if e.response.status_code == 413:
+            # Handle payload too large error specifically
+            try:
+                backend_message = e.response.json().get(
+                    "detail", "Payload too large."
+                )
+            except Exception:
+                backend_message = "Payload too large."
+            return TrismikPayloadTooLargeError(backend_message)
+        elif e.response.status_code == 422:
+            # Handle validation error specifically
+            try:
+                backend_message = e.response.json().get(
+                    "detail", "Validation failed."
+                )
+            except Exception:
+                backend_message = "Validation failed."
+            return TrismikValidationError(backend_message)
+        else:
+            return TrismikApiError(TrismikUtils.get_error_message(e.response))
+
+    async def list_datasets(self) -> List[TrismikDataset]:
         """
-        Retrieves a list of available tests.
-
-        Args:
-            token (str): Authentication token.
+        Get a list of available datasets.
 
         Returns:
-            List[TrismikTest]: List of available tests.
+            List[TrismikDataset]: List of available datasets.
 
         Raises:
             TrismikApiError: If API request fails.
         """
         try:
-            url = "/client/tests"
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.get(url, headers=headers)
+            url = "/datasets"
+            response = await self._http_client.get(url)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_tests(json)
+            return TrismikResponseMapper.to_datasets(json)
         except httpx.HTTPStatusError as e:
             raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+                TrismikUtils.get_error_message(e.response)
+            ) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
 
-    async def create_session(self, test_id: str, metadata: TrismikSessionMetadata, token: str) -> TrismikSession:
+    async def start_run(
+        self,
+        dataset_id: str,
+        project_id: str,
+        experiment: str,
+        metadata: Optional[TrismikRunMetadata] = None,
+    ) -> TrismikRunResponse:
         """
-        Creates a new session for a test.
+        Start a new run for a dataset and get the first item.
 
         Args:
-            test_id (str): ID of the test.
-            token (str): Authentication token.
+            dataset_id (str): ID of the dataset.
+            project_id (str): ID of the project.
+            experiment (str): Name of the experiment.
+            metadata (Optional[TrismikRunMetadata]): Run metadata.
 
         Returns:
-            TrismikSession: New session
+            TrismikRunResponse: Run response.
 
         Raises:
+            TrismikPayloadTooLargeError: If the request payload exceeds the
+            server's size limit.
             TrismikApiError: If API request fails.
         """
         try:
-            url = "/client/sessions"
-            headers = {"Authorization": f"Bearer {token}"}
-            body = {"testId": test_id, "metadata": metadata.toDict()}
-            response = await self._http_client.post(url, headers=headers,
-                                                    json=body)
+            url = "/runs/start"
+            body = {
+                "datasetId": dataset_id,
+                "projectId": project_id,
+                "experiment": experiment,
+                "metadata": metadata.toDict() if metadata else {},
+            }
+            response = await self._http_client.post(url, json=body)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_session(json)
+            return TrismikResponseMapper.to_run_response(json)
         except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+            raise self._handle_http_error(e) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
-        
-    async def create_replay_session(self, previous_session_id: str, metadata: TrismikSessionMetadata, token: str) -> TrismikSession:
+
+    async def continue_run(
+        self, run_id: str, item_choice_id: str
+    ) -> TrismikRunResponse:
         """
-        Creates a new session that replays exactly the question sequence of a previous session
+        Continue a run: respond to the current item and get the next one.
 
         Args:
-            previous_session_id (str): Session id of the session to replay.
-            token (str): Authentication token.
+            run_id (str): ID of the run.
+            item_choice_id (str): ID of the chosen item response.
 
         Returns:
-            TrismikSession: New session
+            TrismikRunResponse: Run response.
 
         Raises:
             TrismikApiError: If API request fails.
         """
         try:
-            url = "/client/sessions/replay"
-            headers = {"Authorization": f"Bearer {token}"}
-            body = {"previousSessionToken": previous_session_id, "metadata": metadata.toDict(), }
-            response = await self._http_client.post(url, headers=headers,
-                                                    json=body)
+            url = "/runs/continue"
+            body = {"itemChoiceId": item_choice_id, "runId": run_id}
+            response = await self._http_client.post(url, json=body)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_session(json)
-        except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
-        except httpx.HTTPError as e:
-            raise TrismikApiError(str(e)) from e
-        
-    async def add_metadata(self, session_id: str, metadata: TrismikSessionMetadata, token: str) -> None:
-        """
-        Adds metadata to the session, merging it with any already stored
-
-        Args:
-            session_id (str): id of the session object
-            metadata: object cotaining the metadata to add
-            token (str): Authentication token.
-
-        Returns:
-            None
-
-        Raises:
-            TrismikApiError: If API request fails.
-        """ 
-        try:
-            url = f"/client/sessions/{session_id}/metadata"
-            headers = {"Authorization": f"Bearer {token}"}
-            body = metadata.toDict()
-            response = await self._http_client.post(url, headers=headers,
-                                                    json=body)
-            response.raise_for_status()
+            return TrismikResponseMapper.to_run_response(json)
         except httpx.HTTPStatusError as e:
             raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+                TrismikUtils.get_error_message(e.response)
+            ) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
 
-    async def current_item(
-            self,
-            session_url: str,
-            token: str
-    ) -> TrismikItem:
+    async def run_summary(self, run_id: str) -> TrismikRunSummary:
         """
-        Retrieves the current test item.
+        Get run summary including responses, dataset, and state.
 
         Args:
-            session_url (str): URL of the session.
-            token (str): Authentication token.
+            run_id (str): ID of the run.
 
         Returns:
-            TrismikItem: Current test item.
+            TrismikRunSummary: Complete run summary with responses,
+                dataset, state, and metadata.
 
         Raises:
             TrismikApiError: If API request fails.
         """
         try:
-            url = f"{session_url}/item"
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.get(url, headers=headers)
+            url = f"/runs/{run_id}"
+            response = await self._http_client.get(url)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_item(json)
+            return TrismikResponseMapper.to_run_summary(json)
         except httpx.HTTPStatusError as e:
             raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+                TrismikUtils.get_error_message(e.response)
+            ) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
 
-    async def respond_to_current_item(
-            self,
-            session_url: str,
-            value: Any,
-            token: str
-    ) -> TrismikItem | None:
+    async def submit_replay(
+        self,
+        run_id: str,
+        replay_request: TrismikReplayRequest,
+        metadata: Optional[TrismikRunMetadata] = None,
+    ) -> TrismikReplayResponse:
         """
-        Responds to the current test item.
+        Submit a replay of a run with specific responses.
 
         Args:
-            session_url (str): URL of the session.
-            value (Any): Response value.
-            token (str): Authentication token.
+            run_id (str): ID of the run to replay.
+            replay_request (TrismikReplayRequest): Request containing responses
+                to submit.
+            metadata (Optional[TrismikRunMetadata]): Run metadata.
 
         Returns:
-            TrismikItem | None: Next test item or None if session is finished.
+            TrismikReplayResponse: Response from the replay endpoint.
 
         Raises:
+            TrismikPayloadTooLargeError: If the request payload exceeds the
+                server's size limit.
+            TrismikValidationError: If the request fails validation (e.g.,
+                duplicate item IDs, unknown item IDs).
             TrismikApiError: If API request fails.
         """
         try:
-            url = f"{session_url}/item"
-            body = {"value": value}
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.post(
-                    url, headers=headers, json=body
-            )
+            url = f"runs/{run_id}/replay"
+
+            # Convert TrismikReplayRequestItem objects to dictionaries
+            responses_dict = [
+                {"itemId": item.itemId, "itemChoiceId": item.itemChoiceId}
+                for item in replay_request.responses
+            ]
+
+            body = {
+                "responses": responses_dict,
+                "metadata": metadata.toDict() if metadata else {},
+            }
+            response = await self._http_client.post(url, json=body)
             response.raise_for_status()
-            if response.status_code == 204:
-                return None
-            else:
-                json = response.json()
-                return TrismikResponseMapper.to_item(json)
+            json = response.json()
+            return TrismikResponseMapper.to_replay_response(json)
         except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+            raise self._handle_http_error(e) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
 
-    async def results(self,
-            session_url: str,
-            token: str
-    ) -> List[TrismikResult]:
+    async def me(self) -> TrismikMeResponse:
         """
-        Retrieves the results of a session.
-
-        Args:
-            session_url (str): URL of the session.
-            token (str): Authentication token.
+        Get current user information.
 
         Returns:
-            List[TrismikResult]: Results of the session.
+            TrismikMeResponse: User information including validity and payload.
 
         Raises:
             TrismikApiError: If API request fails.
         """
         try:
-            url = f"{session_url}/results"
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.get(url, headers=headers)
+            url = "../admin/api-keys/me"
+            response = await self._http_client.get(url)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_results(json)
+            return TrismikResponseMapper.to_me_response(json)
         except httpx.HTTPStatusError as e:
             raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+                TrismikUtils.get_error_message(e.response)
+            ) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e
 
-    async def responses(self,
-            session_url: str,
-            token: str
-    ) -> List[TrismikResponse]:
+    async def submit_classic_eval(
+        self, classic_eval_request: TrismikClassicEvalRequest
+    ) -> TrismikClassicEvalResponse:
         """
-        Retrieves responses to session items.
+        Submit a classic evaluation run with pre-computed results.
 
         Args:
-            session_url (str): URL of the session.
-            token (str): Authentication token.
+            classic_eval_request (TrismikClassicEvalRequest): Request containing
+                project info, dataset, model outputs, and metrics.
 
         Returns:
-            List[TrismikResponse]: Responses of the session.
+            TrismikClassicEvalResponse: Response from the classic evaluation
+                endpoint.
 
         Raises:
+            TrismikPayloadTooLargeError: If the request payload exceeds the
+                server's size limit.
+            TrismikValidationError: If the request fails validation.
             TrismikApiError: If API request fails.
         """
         try:
-            url = f"{session_url}/responses"
-            headers = {"Authorization": f"Bearer {token}"}
-            response = await self._http_client.get(url, headers=headers)
+            url = "/runs/classic"
+
+            # Convert request object to dictionary
+            items_dict = [
+                {
+                    "datasetItemId": item.datasetItemId,
+                    "modelInput": item.modelInput,
+                    "modelOutput": item.modelOutput,
+                    "goldOutput": item.goldOutput,
+                    "metrics": item.metrics,
+                }
+                for item in classic_eval_request.items
+            ]
+
+            metrics_dict = [
+                {
+                    "metricId": metric.metricId,
+                    "valueType": TrismikUtils.metric_value_to_type(
+                        metric.value
+                    ),
+                    "value": metric.value,
+                }
+                for metric in classic_eval_request.metrics
+            ]
+
+            body = {
+                "projectId": classic_eval_request.projectId,
+                "experimentName": classic_eval_request.experimentName,
+                "datasetId": classic_eval_request.datasetId,
+                "modelName": classic_eval_request.modelName,
+                "hyperparameters": classic_eval_request.hyperparameters,
+                "items": items_dict,
+                "metrics": metrics_dict,
+            }
+
+            response = await self._http_client.post(url, json=body)
             response.raise_for_status()
             json = response.json()
-            return TrismikResponseMapper.to_responses(json)
+            return TrismikResponseMapper.to_classic_eval_response(json)
         except httpx.HTTPStatusError as e:
-            raise TrismikApiError(
-                    TrismikUtils.get_error_message(e.response)) from e
+            raise self._handle_http_error(e) from e
         except httpx.HTTPError as e:
             raise TrismikApiError(str(e)) from e

trismik/exceptions.py

@@ -1,13 +1,64 @@
+"""
+Exception classes for the Trismik client.
+
+This module defines custom exceptions used throughout the Trismik client
+library.
+"""
+
+
 class TrismikError(Exception):
+    """Base class for all exceptions raised by the Trismik package."""
+
+
+class TrismikApiError(TrismikError):
     """
-    Base class for all exceptions raised.
-    Raised when an error occurs in the Trismik package, i.e. configuration.
+    Exception raised when an error occurs during API interaction.
+
+    This exception is raised when there is an error during API communication.
     """
-    pass
 
 
-class TrismikApiError(TrismikError):
+class TrismikPayloadTooLargeError(TrismikApiError):
+    """
+    Exception raised when the request payload exceeds the server's size limit.
+
+    This exception is raised when a 413 "Content Too Large" error is received
+    from the API, indicating that the request payload (typically metadata)
+    exceeds the server's size limit.
     """
-    Raised when an error occurs while interacting with the Trismik API
+
+    def __init__(self, message: str):
+        """
+        Initialize the TrismikPayloadTooLargeError.
+
+        Args:
+            message (str): The error message from the server.
+        """
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        """Return a human-readable string representation of the exception."""
+        return f"Payload too large: {self.args[0]}"
+
+
+class TrismikValidationError(TrismikApiError):
     """
-    pass
+    Exception raised when the request fails validation.
+
+    This exception is raised when a 422 "Unprocessable Entity" error is received
+    from the API, indicating that the request failed validation (e.g., duplicate
+    item IDs, unknown item IDs in replay requests).
+    """
+
+    def __init__(self, message: str):
+        """
+        Initialize the TrismikValidationError.
+
+        Args:
+            message (str): The error message from the server.
+        """
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        """Return a human-readable string representation of the exception."""
+        return f"Validation error: {self.args[0]}"

trismik/settings.py

@@ -0,0 +1,15 @@
+"""Default settings for the Trismik client."""
+
+evaluation_settings = {
+    "max_iterations": 150,
+}
+
+client_settings = {"endpoint": "https://dashboard.trismik.com/api"}
+
+# Environment variable names used by the Trismik client
+environment_settings = {
+    # URL of the Trismik service
+    "trismik_service_url": "TRISMIK_SERVICE_URL",
+    # API key for authentication
+    "trismik_api_key": "TRISMIK_API_KEY",
+}