trismik 0.9.0__py3-none-any.whl → 0.9.4__py3-none-any.whl

trismik/exceptions.py

@@ -1,13 +1,64 @@
+"""
+Exception classes for the Trismik client.
+
+This module defines custom exceptions used throughout the Trismik client
+library.
+"""
+
+
 class TrismikError(Exception):
+    """Base class for all exceptions raised by the Trismik package."""
+
+
+class TrismikApiError(TrismikError):
     """
-    Base class for all exceptions raised.
-    Raised when an error occurs in the Trismik package, i.e. configuration.
+    Exception raised when an error occurs during API interaction.
+
+    This exception is raised when there is an error during API communication.
     """
-    pass
 
 
-class TrismikApiError(TrismikError):
+class TrismikPayloadTooLargeError(TrismikApiError):
+    """
+    Exception raised when the request payload exceeds the server's size limit.
+
+    This exception is raised when a 413 "Content Too Large" error is received
+    from the API, indicating that the request payload (typically metadata)
+    exceeds the server's size limit.
     """
-    Raised when an error occurs while interacting with the Trismik API
+
+    def __init__(self, message: str):
+        """
+        Initialize the TrismikPayloadTooLargeError.
+
+        Args:
+            message (str): The error message from the server.
+        """
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        """Return a human-readable string representation of the exception."""
+        return f"Payload too large: {self.args[0]}"
+
+
+class TrismikValidationError(TrismikApiError):
     """
-    pass
+    Exception raised when the request fails validation.
+
+    This exception is raised when a 422 "Unprocessable Entity" error is received
+    from the API, indicating that the request failed validation (e.g., duplicate
+    item IDs, unknown item IDs in replay requests).
+    """
+
+    def __init__(self, message: str):
+        """
+        Initialize the TrismikValidationError.
+
+        Args:
+            message (str): The error message from the server.
+        """
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        """Return a human-readable string representation of the exception."""
+        return f"Validation error: {self.args[0]}"

trismik/settings.py

@@ -0,0 +1,15 @@
+"""Default settings for the Trismik client."""
+
+evaluation_settings = {
+    "max_iterations": 150,
+}
+
+client_settings = {"endpoint": "https://dashboard.trismik.com/api"}
+
+# Environment variable names used by the Trismik client
+environment_settings = {
+    # URL of the Trismik service
+    "trismik_service_url": "TRISMIK_SERVICE_URL",
+    # API key for authentication
+    "trismik_api_key": "TRISMIK_API_KEY",
+}

trismik/types.py

@@ -1,133 +1,301 @@
-from dataclasses import dataclass
-from datetime import datetime
-from typing import List, Any
-
-
-@dataclass
-class TrismikAuth:
-    """
-    Authentication token.
-
-    Attributes:
-        token (str): Authentication token value.
-        expires (datetime): Expiration date.
-    """
-    token: str
-    expires: datetime
-
-
-@dataclass
-class TrismikTest:
-    """
-    Available test.
-
-    Attributes:
-        id (str): Test ID.
-        name (str): Test name.
-    """
-    id: str
-    name: str
-
-
-@dataclass
-class TrismikSession:
-    """
-    Test session.
-
-    Attributes:
-        id (str): Session ID.
-        url (str): Session URL.
-        status (str): Session status
-    """
-    id: str
-    url: str
-    status: str
-
-
-@dataclass
-class TrismikItem:
-    """
-    Base class for test items.
-
-    Attributes:
-        id (str): Item ID.
-    """
-    id: str
-
-
-@dataclass
-class TrismikChoice:
-    """
-    Base class for choices in items that use them.
-
-    Attributes:
-        id (str): Choice ID.
-    """
-    id: str
-
-
-@dataclass
-class TrismikTextChoice(TrismikChoice):
-    """
-    Text choice.
-
-    Attributes:
-        text (str): Choice text.
-    """
-    text: str
-
-
-@dataclass
-class TrismikMultipleChoiceTextItem(TrismikItem):
-    """
-    Multiple choice text item.
-
-    Attributes:
-        question (str): Question text.
-        choices (List[TrismikTextChoice]): List of choices.
-    """
-    question: str
-    choices: List[TrismikTextChoice]
-
-
-@dataclass
-class TrismikResult:
-    """
-    Test result.
-
-    Attributes:
-        trait (str): Trait name.
-        name (str): Result name/type.
-        value (Any): Result value.
-    """
-    trait: str
-    name: str
-    value: Any
-
-
-@dataclass
-class TrismikResponse:
-    """
-    Test result.
-
-    Attributes:
-        item_id (str): Item ID.
-        value (Any): Result value.
-        score (float): Score.
-    """
-    item_id: str
-    value: Any
-    score: float
-
-
-@dataclass
-class TrismikResultsAndResponses:
-    """
-    Test results and responses.
-
-    Attributes:
-        results (List[TrismikResult]): Results.
-        responses (List[TrismikResponse]): Responses.
-    """
-    results: List[TrismikResult]
-    responses: List[TrismikResponse]
+"""
+Type definitions for the Trismik client.
+
+This module defines the data structures used throughout the Trismik client
+library.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+
+@dataclass
+class TrismikDataset:
+    """Test metadata including ID and name."""
+
+    id: str
+    name: str
+
+
+@dataclass
+class TrismikRun:
+    """Run metadata including ID, URL, and status."""
+
+    id: str
+    url: str
+    status: str
+
+
+@dataclass
+class TrismikRunInfo:
+    """Run info from new API endpoints."""
+
+    id: str
+
+
+@dataclass
+class TrismikRunState:
+    """Run state including responses, thetas, and other metrics."""
+
+    responses: List[str]
+    thetas: List[float]
+    std_error_history: List[float]
+    kl_info_history: List[float]
+    effective_difficulties: List[float]
+
+
+@dataclass
+class TrismikRunResponse:
+    """Response from run endpoints (start and continue)."""
+
+    run_info: TrismikRunInfo
+    state: TrismikRunState
+    next_item: Optional["TrismikItem"]
+    completed: bool
+
+
+@dataclass
+class TrismikAdaptiveTestState:
+    """State tracking for adaptive tests."""
+
+    run_id: str
+    state: TrismikRunState
+    completed: bool
+
+
+@dataclass
+class AdaptiveTestScore:
+    """Final scores of an adaptive test run."""
+
+    theta: float
+    std_error: float
+
+
+@dataclass
+class TrismikItem:
+    """Base class for test items."""
+
+    id: str
+
+
+@dataclass
+class TrismikChoice:
+    """Base class for choices in items that use them."""
+
+    id: str
+
+
+@dataclass
+class TrismikTextChoice(TrismikChoice):
+    """Text choice for multiple choice questions."""
+
+    text: str
+
+
+@dataclass
+class TrismikMultipleChoiceTextItem(TrismikItem):
+    """Multiple choice text question."""
+
+    question: str
+    choices: List[TrismikTextChoice]
+
+
+@dataclass
+class TrismikResult:
+    """Test result for a specific trait."""
+
+    trait: str
+    name: str
+    value: Any
+
+
+@dataclass
+class TrismikResponse:
+    """Response to a test item."""
+
+    dataset_item_id: str
+    value: Any
+    correct: bool
+
+
+@dataclass
+class TrismikRunResults:
+    """Test results and responses."""
+
+    run_id: str
+    score: Optional[AdaptiveTestScore] = None
+    responses: Optional[List[TrismikResponse]] = None
+
+
+@dataclass
+class TrismikRunSummary:
+    """Complete run summary."""
+
+    id: str
+    dataset_id: str
+    state: TrismikRunState
+    dataset: List[TrismikItem]
+    responses: List[TrismikResponse]
+    metadata: dict
+
+    @property
+    def theta(self) -> float:
+        """Get the theta of the run."""
+        return self.state.thetas[-1]
+
+    @property
+    def std_error(self) -> float:
+        """Get the standard error of the run."""
+        return self.state.std_error_history[-1]
+
+    @property
+    def total_responses(self) -> int:
+        """Get the total number of responses in the run."""
+        return len(self.responses)
+
+    @property
+    def correct_responses(self) -> int:
+        """Get the number of correct responses in the run."""
+        return sum(response.correct for response in self.responses)
+
+    @property
+    def wrong_responses(self) -> int:
+        """Get the number of wrong responses in the run."""
+        return self.total_responses - self.correct_responses
+
+
+@dataclass
+class TrismikRunMetadata:
+    """Metadata for a test run."""
+
+    class ModelMetadata:
+        """Model metadata for a test run."""
+
+        def __init__(self, name: str, **kwargs: Any):
+            """Initialize ModelMetadata with a name and optional attributes."""
+            self.name = name
+            for key, value in kwargs.items():
+                setattr(self, key, value)
+
+    model_metadata: ModelMetadata
+    test_configuration: Dict[str, Any]
+    inference_setup: Dict[str, Any]
+
+    def toDict(self) -> Dict[str, Any]:
+        """Convert run metadata to a dictionary."""
+        return {
+            "model_metadata": vars(self.model_metadata),
+            "test_configuration": self.test_configuration,
+            "inference_setup": self.inference_setup,
+        }
+
+
+@dataclass
+class TrismikReplayRequestItem:
+    """Item in a replay request."""
+
+    itemId: str
+    itemChoiceId: str
+
+
+@dataclass
+class TrismikReplayRequest:
+    """Request to replay a run with specific responses."""
+
+    responses: List[TrismikReplayRequestItem]
+
+
+@dataclass
+class TrismikReplayResponse:
+    """Response from a replay run request."""
+
+    id: str
+    datasetId: str
+    state: TrismikRunState
+    replay_of_run: str
+    completedAt: Optional[datetime] = None
+    createdAt: Optional[datetime] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    dataset: List[TrismikItem] = field(default_factory=list)
+    responses: List[TrismikResponse] = field(default_factory=list)
+
+
+@dataclass
+class TrismikOrganization:
+    """Organization information."""
+
+    id: str
+    name: str
+    type: str
+    role: str
+
+
+@dataclass
+class TrismikUserInfo:
+    """User information."""
+
+    id: str
+    email: str
+    firstname: str
+    lastname: str
+    createdAt: Optional[str] = None
+
+
+@dataclass
+class TrismikMeResponse:
+    """Response from the /admin/api-keys/me endpoint."""
+
+    user: TrismikUserInfo
+    organization: TrismikOrganization
+
+
+@dataclass
+class TrismikClassicEvalItem:
+    """Item in a classic evaluation request."""
+
+    datasetItemId: str
+    modelInput: str
+    modelOutput: str
+    goldOutput: str
+    metrics: Dict[str, Any]
+
+
+@dataclass
+class TrismikClassicEvalMetric:
+    """Metric in a classic evaluation request."""
+
+    metricId: str
+    value: Union[str, float, int, bool]
+
+
+@dataclass
+class TrismikClassicEvalRequest:
+    """Request to submit a classic evaluation."""
+
+    projectId: str
+    experimentName: str
+    datasetId: str
+    modelName: str
+    hyperparameters: Dict[str, Any]
+    items: List[TrismikClassicEvalItem]
+    metrics: List[TrismikClassicEvalMetric]
+
+
+@dataclass
+class TrismikClassicEvalResponse:
+    """Response from a classic evaluation submission."""
+
+    id: str
+    organizationId: str
+    projectId: str
+    experimentId: str
+    experimentName: str
+    datasetId: str
+    userId: str
+    type: str
+    modelName: str
+    hyperparameters: Dict[str, Any]
+    createdAt: str
+    user: TrismikUserInfo
+    responseCount: int

{trismik-0.9.0.dist-info → trismik-0.9.4.dist-info}/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2024 Cambridge Enterprise
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2024 Cambridge Enterprise
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.