trismik 0.9.1__py3-none-any.whl → 0.9.5__py3-none-any.whl

trismik/settings.py

@@ -0,0 +1,15 @@
+"""Default settings for the Trismik client."""
+
+evaluation_settings = {
+    "max_iterations": 150,
+}
+
+client_settings = {"endpoint": "https://dashboard.trismik.com/api"}
+
+# Environment variable names used by the Trismik client
+environment_settings = {
+    # URL of the Trismik service
+    "trismik_service_url": "TRISMIK_SERVICE_URL",
+    # API key for authentication
+    "trismik_api_key": "TRISMIK_API_KEY",
+}

trismik/types.py

@@ -1,105 +1,110 @@
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Dict, List, Any, Optional
-
+"""
+Type definitions for the Trismik client.
 
-@dataclass
-class TrismikAuth:
-    """
-    Authentication token.
+This module defines the data structures used throughout the Trismik client
+library.
+"""
 
-    Attributes:
-        token (str): Authentication token value.
-        expires (datetime): Expiration date.
-    """
-    token: str
-    expires: datetime
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
 
 
 @dataclass
-class TrismikTest:
-    """
-    Available test.
+class TrismikDataset:
+    """Test metadata including ID and name."""
 
-    Attributes:
-        id (str): Test ID.
-        name (str): Test name.
-    """
     id: str
     name: str
 
 
 @dataclass
-class TrismikSession:
-    """
-    Test session.
+class TrismikRun:
+    """Run metadata including ID, URL, and status."""
 
-    Attributes:
-        id (str): Session ID.
-        url (str): Session URL.
-        status (str): Session status
-    """
     id: str
     url: str
     status: str
 
 
+@dataclass
+class TrismikRunInfo:
+    """Run info from new API endpoints."""
+
+    id: str
+
+
+@dataclass
+class TrismikRunState:
+    """Run state including responses, thetas, and other metrics."""
+
+    responses: List[str]
+    thetas: List[float]
+    std_error_history: List[float]
+    kl_info_history: List[float]
+    effective_difficulties: List[float]
+
+
+@dataclass
+class TrismikRunResponse:
+    """Response from run endpoints (start and continue)."""
+
+    run_info: TrismikRunInfo
+    state: TrismikRunState
+    next_item: Optional["TrismikItem"]
+    completed: bool
+
+
+@dataclass
+class TrismikAdaptiveTestState:
+    """State tracking for adaptive tests."""
+
+    run_id: str
+    state: TrismikRunState
+    completed: bool
+
+
+@dataclass
+class AdaptiveTestScore:
+    """Final scores of an adaptive test run."""
+
+    theta: float
+    std_error: float
+
+
 @dataclass
 class TrismikItem:
-    """
-    Base class for test items.
+    """Base class for test items."""
 
-    Attributes:
-        id (str): Item ID.
-    """
     id: str
 
 
 @dataclass
 class TrismikChoice:
-    """
-    Base class for choices in items that use them.
+    """Base class for choices in items that use them."""
 
-    Attributes:
-        id (str): Choice ID.
-    """
     id: str
 
 
 @dataclass
 class TrismikTextChoice(TrismikChoice):
-    """
-    Text choice.
+    """Text choice for multiple choice questions."""
 
-    Attributes:
-        text (str): Choice text.
-    """
     text: str
 
 
 @dataclass
 class TrismikMultipleChoiceTextItem(TrismikItem):
-    """
-    Multiple choice text item.
+    """Multiple choice text question."""
 
-    Attributes:
-        question (str): Question text.
-        choices (List[TrismikTextChoice]): List of choices.
-    """
     question: str
     choices: List[TrismikTextChoice]
 
 
 @dataclass
 class TrismikResult:
-    """
-    Test result.
-
-    Attributes:
-        trait (str): Trait name.
-        name (str): Result name/type.
-        value (Any): Result value.
-    """
+    """Test result for a specific trait."""
+
     trait: str
     name: str
     value: Any
@@ -107,57 +112,210 @@ class TrismikResult:
 
 @dataclass
 class TrismikResponse:
-    """
-    Test result.
-
-    Attributes:
-        item_id (str): Item ID.
-        value (Any): Result value.
-        score (float): Score.
-    """
-    item_id: str
+    """Response to a test item."""
+
+    dataset_item_id: str
     value: Any
-    score: float
+    correct: bool
 
 
 @dataclass
 class TrismikRunResults:
-    """
-    Test results and responses.
-
-    Attributes:
-        results (List[TrismikResult]): Results.
-        responses (List[TrismikResponse]): Responses.
-    """
-    session_id: str
-    results: List[TrismikResult]
+    """Test results and responses."""
+
+    run_id: str
+    score: Optional[AdaptiveTestScore] = None
     responses: Optional[List[TrismikResponse]] = None
 
+
 @dataclass
-class TrismikSessionMetadata:
-    """
-    Metadata associated to a session
+class TrismikRunSummary:
+    """Complete run summary."""
+
+    id: str
+    dataset_id: str
+    state: TrismikRunState
+    dataset: List[TrismikItem]
+    responses: List[TrismikResponse]
+    metadata: dict
+
+    @property
+    def theta(self) -> float:
+        """Get the theta of the run."""
+        return self.state.thetas[-1]
+
+    @property
+    def std_error(self) -> float:
+        """Get the standard error of the run."""
+        return self.state.std_error_history[-1]
+
+    @property
+    def total_responses(self) -> int:
+        """Get the total number of responses in the run."""
+        return len(self.responses)
+
+    @property
+    def correct_responses(self) -> int:
+        """Get the number of correct responses in the run."""
+        return sum(response.correct for response in self.responses)
+
+    @property
+    def wrong_responses(self) -> int:
+        """Get the number of wrong responses in the run."""
+        return self.total_responses - self.correct_responses
 
-    Attributes:
-        model_metadata (dict[str, Any]): Metadata about the model.
-        test_configuration (dict[str, Any]): Metadata about the test.
-        inference_setup (dict[str, Any]): Metadata about the inference setup.
-    """   
+
+@dataclass
+class TrismikRunMetadata:
+    """Metadata for a test run."""
 
     class ModelMetadata:
+        """Model metadata for a test run."""
+
         def __init__(self, name: str, **kwargs: Any):
+            """Initialize ModelMetadata with a name and optional attributes."""
             self.name = name
             for key, value in kwargs.items():
-                setattr(self, key, value)                
+                setattr(self, key, value)
 
     model_metadata: ModelMetadata
-    test_configuration: dict[str, Any]
-    inference_setup: dict[str, Any]
+    test_configuration: Dict[str, Any]
+    inference_setup: Dict[str, Any]
 
     def toDict(self) -> Dict[str, Any]:
+        """Convert run metadata to a dictionary."""
         return {
             "model_metadata": vars(self.model_metadata),
             "test_configuration": self.test_configuration,
             "inference_setup": self.inference_setup,
         }
 
+
+@dataclass
+class TrismikReplayRequestItem:
+    """Item in a replay request."""
+
+    itemId: str
+    itemChoiceId: str
+
+
+@dataclass
+class TrismikReplayRequest:
+    """Request to replay a run with specific responses."""
+
+    responses: List[TrismikReplayRequestItem]
+
+
+@dataclass
+class TrismikReplayResponse:
+    """Response from a replay run request."""
+
+    id: str
+    datasetId: str
+    state: TrismikRunState
+    replay_of_run: str
+    completedAt: Optional[datetime] = None
+    createdAt: Optional[datetime] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    dataset: List[TrismikItem] = field(default_factory=list)
+    responses: List[TrismikResponse] = field(default_factory=list)
+
+
+@dataclass
+class TrismikOrganization:
+    """Organization information."""
+
+    id: str
+    name: str
+    type: str
+    role: str
+
+
+@dataclass
+class TrismikUserInfo:
+    """User information."""
+
+    id: str
+    email: str
+    firstname: str
+    lastname: str
+    createdAt: Optional[str] = None
+
+
+@dataclass
+class TrismikMeResponse:
+    """Response from the /admin/api-keys/me endpoint."""
+
+    user: TrismikUserInfo
+    organizations: List[TrismikOrganization]
+
+
+@dataclass
+class TrismikClassicEvalItem:
+    """Item in a classic evaluation request."""
+
+    datasetItemId: str
+    modelInput: str
+    modelOutput: str
+    goldOutput: str
+    metrics: Dict[str, Any]
+
+
+@dataclass
+class TrismikClassicEvalMetric:
+    """Metric in a classic evaluation request."""
+
+    metricId: str
+    value: Union[str, float, int, bool]
+
+
+@dataclass
+class TrismikClassicEvalRequest:
+    """Request to submit a classic evaluation."""
+
+    projectId: str
+    experimentName: str
+    datasetId: str
+    modelName: str
+    hyperparameters: Dict[str, Any]
+    items: List[TrismikClassicEvalItem]
+    metrics: List[TrismikClassicEvalMetric]
+
+
+@dataclass
+class TrismikClassicEvalResponse:
+    """Response from a classic evaluation submission."""
+
+    id: str
+    organizationId: str
+    projectId: str
+    experimentId: str
+    experimentName: str
+    datasetId: str
+    userId: str
+    type: str
+    modelName: str
+    hyperparameters: Dict[str, Any]
+    createdAt: str
+    user: TrismikUserInfo
+    responseCount: int
+
+
+@dataclass
+class TrismikProjectRequest:
+    """Request to create a new project."""
+
+    name: str
+    description: Optional[str] = None
+
+
+@dataclass
+class TrismikProject:
+    """Project information."""
+
+    id: str
+    name: str
+    description: Optional[str]
+    organizationId: str
+    createdAt: str
+    updatedAt: str

trismik-0.9.5.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: trismik
+Version: 0.9.5
+Summary: 
+License-File: LICENSE
+Author: Bartosz Kielczewski
+Author-email: bk352@cam.ac.uk
+Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: examples
+Requires-Dist: accelerate (>=1.7.0,<2.0.0) ; extra == "examples"
+Requires-Dist: notebook (>=7.4.4,<8.0.0) ; extra == "examples"
+Requires-Dist: openai (>=1.81.0,<2.0.0) ; extra == "examples"
+Requires-Dist: torch (>=2.7.0,<3.0.0) ; extra == "examples"
+Requires-Dist: torchaudio (>=2.7.0,<3.0.0) ; extra == "examples"
+Requires-Dist: torchvision (>=0.22.0,<1.0.0) ; extra == "examples"
+Requires-Dist: transformers (>=4.51.3,<5.0.0) ; extra == "examples"
+Description-Content-Type: text/markdown
+
+<h1 align="center"> Trismik SDK</h1>
+
+<p align="center">
+  <img alt="PyPI - Version" src="https://img.shields.io/pypi/v/trismik">
+  <img alt="Python Version" src="https://img.shields.io/badge/python-3.9%2B-blue">
+  <img alt="License" src="https://img.shields.io/badge/license-MIT-green">
+</p>
+
+## Table of Contents
+- [Overview](#overview)
+- [Quick Start](#quick-start)
+  - [Installation](#installation)
+  - [API Key Setup](#api-key-setup)
+  - [Basic Usage](#basic-usage)
+- [Interpreting Results](#interpreting-results)
+  - [Theta (θ)](#theta-θ)
+  - [Other Metrics](#other-metrics)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Overview
+
+[**Trismik**](https://trismik.com) is a Cambridge, UK based startup offering adversarial testing for LLMs. The APIs we provide through this library allow you to call our adaptive test engine and evaluate LLMs up to 95% faster (and cheaper!) than traditional evaluation techniques.
+
+Our **adaptive testing** algorithm allows to estimate the precision of the model by looking only at a small portion of a dataset. Through this library, we provide access to a number of open source datasets over several dimensions (reasoning, toxicity, tool use...) to speed up model testing in several scenarios, like foundation model training, supervised fine tuning, prompt engineering, and so on.
+
+## Quick Start
+
+### Installation
+
+To use our API, you need to get an API key first. Please register on [dashboard.trismik.com](https://dashboard.trismik.com) and obtain an API key.
+
+Trismik is available via [pypi](https://pypi.org/project/trismik/). To install Trismik, run the following in your terminal (in a virtualenv, if you use one):
+
+```bash
+pip install trismik
+```
+
+### API Key Setup
+
+You can provide your API key in one of the following ways:
+
+1. **Environment Variable**:
+   ```bash
+   export TRISMIK_API_KEY="your-api-key"
+   ```
+
+2. **`.env` File**:
+   ```bash
+   # .env
+   TRISMIK_API_KEY=your-api-key
+   ```
+   Then load it with `python-dotenv`:
+   ```python
+   from dotenv import load_dotenv
+   load_dotenv()
+   ```
+
+3. **Direct Initialization**:
+   ```python
+   client = TrismikAsyncClient(api_key="YOUR_API_KEY")
+   ```
+
+### Basic Usage
+
+Running a test is straightforward:
+
+1. Implement a method that wraps model inference over a dataset item
+2. Create an `AdaptiveTest` instance
+3. Run the test!
+
+Here's a basic example:
+
+```python
+def model_inference(item: TrismikItem) -> Any:
+    model_output = ...  # call your model here
+    return model_output
+
+
+# Initialize the test runner
+runner = AdaptiveTest(model_inference)
+
+# Run the test
+results = await runner.run_async(
+    "MMLUPro2025",  # or any dataset we support
+    with_responses=True,
+    run_metadata=sample_metadata,
+)
+
+# Print the test output
+for result in results:
+    print(f"{result.trait} ({result.name}): {result.value}")
+```
+
+### Examples
+
+You can find more examples in the `examples` folder:
+- [`example_transformers.py`](examples/example_transformers.py) - Example using Hugging Face Transformers models
+- [`example_openai.py`](examples/example_openai.py) - Example using OpenAI models
+- [`example_adaptive_test.py`](examples/example_adaptive_test.py) - Example of adaptive testing configuration
+
+To run the examples, you will need to clone this repo, navigate to the
+source folder, and then run:
+
+```bash
+poetry install --with examples
+poetry run python examples/example_adaptive_test.py # or any other example
+```
+
+## Interpreting Results
+
+### Theta (θ)
+
+Our adversarial test returns several values; however, you will be interested mainly in `theta`. Theta ($\theta$) is our metric; it measures the ability of the model on a certain dataset, and it can be used as a proxy to approximate the original metric used on that dataset. For example, on an accuracy-based dataset, a high theta correlates with a high accuracy, and low theta correlates with low accuracy.
+
+To interpret a theta score, consider that $\theta=0$ corresponds to a 50% chance for a model to get an answer right - in other words, to an accuracy of 50%.
+A negative theta means that the model will give more bad answers then good ones, while a positive theta means that the model will give more good answers then bad answers.
+While theta is unbounded in our implementation (i.e. $-\infty < \theta < \infty$), in practice we have that for most cases $\theta$ will take values between -3 and 3.
+
+Compared to classical benchmark testing, the estimated accuracy from adaptive testing uses fewer but more informative items while avoiding noise from overly easy or difficult questions. This makes it a more efficient and stable measure, especially on very large datasets.
+
+### Other Metrics
+
+- **Standard Deviation (`std`)**:
+  - A measure of the uncertainty or error in the theta estimate
+  - A smaller `std` indicates a more precise estimate
+  - You should see a `std` around or below 0.25
+
+- **Correct Responses (`responsesCorrect`)**:
+  - The number of correct answers delivered by the model
+
+  - **Important note**: A higher number of correct answers does not necessarily
+  correlate with a high theta. Our algorithm navigates the dataset to find a
+   balance of “hard” and “easy” items for your model, so by the end of the test,
+  it encounters a representative mix of inputs it can and cannot handle. In
+   practice, expect responsesCorrect to be roughly half of responsesTotal.
+
+- **Total Responses (`responsesTotal`)**:
+  - The number of items processed before reaching a stable theta.
+  - Expected range: 60 ≤ responses_total ≤ 80
+
+## Contributing
+
+See `CONTRIBUTING.md`.
+
+## License
+
+This library is licensed under the MIT license. See `LICENSE` file.
+

trismik-0.9.5.dist-info/RECORD

@@ -0,0 +1,12 @@
+trismik/__init__.py,sha256=20SwXrda9YsgykaoPohwz6foj2FkraniPA-GTQS9m00,197
+trismik/_mapper.py,sha256=9LtEtekXas6tLP_BETl5geDeLKmTGrjdGwkU3bkgmHk,10995
+trismik/_utils.py,sha256=WZ7x0EaG7PdXdFZMIs1wzgwqiPQm59QDDmQzBKYGCEg,3753
+trismik/adaptive_test.py,sha256=Fmc6PJZg_SKQjV_xQF29ZJQwPxpGwfKuvi2VznAmOB0,21646
+trismik/client_async.py,sha256=Z0tmePA86nO3Ul_55eZdT_52LSgRESANYqqrX4hH9Us,13883
+trismik/exceptions.py,sha256=2wb4_K7GdDf00s3xUaiSfw6718ZV3Eaa4M2lYbiEZl4,1945
+trismik/settings.py,sha256=FCP-d8ZEiYWUTWoa9nOVzSwTOLvg8y0pU08dAseiOwY,412
+trismik/types.py,sha256=DQ88f_PcJmvmSAQPFDxEmtJSrQQDTSvhxbdrzzRHd-4,6571
+trismik-0.9.5.dist-info/METADATA,sha256=rMF2N0d9MaoA2uIA9CL9LuoURTXs2ZCPSb4ZNZ2yZHc,6692
+trismik-0.9.5.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+trismik-0.9.5.dist-info/licenses/LICENSE,sha256=tgetRhapGLh7ZxfknW6Mm-WobfziPd64nAK52X5XKaw,1077
+trismik-0.9.5.dist-info/RECORD,,

{trismik-0.9.1.dist-info → trismik-0.9.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.2
+Generator: poetry-core 2.2.0
 Root-Is-Purelib: true
 Tag: py3-none-any