trustmodel 0.2.9__py3-none-any.whl

trustmodel/__init__.py

@@ -0,0 +1,42 @@
+"""
+TrustModel Python SDK
+
+Official Python SDK for the TrustModel AI evaluation platform.
+Provides a simple interface to evaluate AI models and retrieve results.
+"""
+
+try:
+    from importlib.metadata import PackageNotFoundError, version
+except ImportError:
+    # Python < 3.8 fallback
+    from importlib_metadata import (  # type: ignore[import-not-found, no-redef, assignment, unused-ignore]
+        PackageNotFoundError,
+        version,
+    )
+
+from .client import TrustModelClient
+from .exceptions import (
+    APIError,
+    AuthenticationError,
+    ConnectionValidationError,
+    InsufficientCreditsError,
+    RateLimitError,
+    TrustModelError,
+    ValidationError,
+)
+
+try:
+    __version__ = version("trustmodel")
+except PackageNotFoundError:
+    # Package not installed, use fallback
+    __version__ = "0.1.0"
+__all__ = [
+    "TrustModelClient",
+    "TrustModelError",
+    "AuthenticationError",
+    "APIError",
+    "RateLimitError",
+    "ValidationError",
+    "InsufficientCreditsError",
+    "ConnectionValidationError",
+]

trustmodel/client.py

@@ -0,0 +1,259 @@
+"""
+Main client class for the TrustModel SDK.
+"""
+
+import json
+import logging
+from typing import Any, Optional
+from urllib.parse import urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from .endpoints.config import ConfigEndpoint
+from .endpoints.credits import CreditsEndpoint
+from .endpoints.evaluations import EvaluationsEndpoint
+from .endpoints.models import ModelsEndpoint
+from .exceptions import (
+    APIError,
+    AuthenticationError,
+    ConnectionValidationError,
+    InsufficientCreditsError,
+    RateLimitError,
+    TrustModelError,
+)
+from .utils.validation import validate_api_key
+from .utils.version import get_user_agent, get_version
+
+logger = logging.getLogger(__name__)
+
+
+class TrustModelClient:
+    """
+    Main client for interacting with the TrustModel API.
+
+    This client provides access to all TrustModel SDK endpoints for evaluating
+    AI models and retrieving results.
+
+    Args:
+        api_key: Your TrustModel API key (starts with 'tm-')
+        base_url: Base URL for the TrustModel API (default: production URL)
+        timeout: Request timeout in seconds (default: 60)
+        max_retries: Maximum number of retry attempts (default: 3)
+
+    Example:
+        >>> import trustmodel
+        >>> client = trustmodel.TrustModelClient(api_key="tm-your_key_here")
+        >>> models = client.models.list()
+        >>> evaluation = client.evaluations.create(
+        ...     model_identifier="gpt-4",
+        ...     vendor_identifier="openai"
+        ... )
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "https://api.trustmodel.ai",
+        timeout: int = 60,
+        max_retries: int = 3,
+        user_agent: Optional[str] = None,
+    ) -> None:
+        # Validate API key
+        validate_api_key(api_key)
+
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        self._timeout = timeout
+
+        # Set up session with retries
+        self._session = requests.Session()
+
+        # Configure retry strategy
+        retry_strategy = Retry(
+            total=max_retries,
+            backoff_factor=1,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS", "POST"],
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self._session.mount("http://", adapter)
+        self._session.mount("https://", adapter)
+
+        # Set default headers
+        self._session.headers.update(
+            {
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json",
+                "User-Agent": get_user_agent(user_agent),
+                "Accept": "application/json",
+                "X-SDK-Version": get_version(),
+            }
+        )
+
+        # Initialize endpoint handlers
+        self.models = ModelsEndpoint(self)
+        self.evaluations = EvaluationsEndpoint(self)
+        self.credits = CreditsEndpoint(self)
+        self.config = ConfigEndpoint(self)
+
+    def _request(
+        self,
+        method: str,
+        endpoint: str,
+        data: Optional[dict[str, Any]] = None,
+        params: Optional[dict[str, Any]] = None,
+    ) -> dict[str, Any]:
+        """
+        Make an HTTP request to the TrustModel API.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            endpoint: API endpoint (without base URL)
+            data: Request body data
+            params: Query parameters
+
+        Returns:
+            Response data as dictionary
+
+        Raises:
+            Various TrustModelError subclasses based on the error type
+        """
+        url = urljoin(f"{self._base_url}/", endpoint.lstrip("/"))
+
+        logger.debug(f"Making {method} request to {url}")
+
+        try:
+            response = self._session.request(
+                method=method,
+                url=url,
+                json=data,
+                params=params,
+                timeout=self._timeout,
+            )
+        except requests.exceptions.Timeout as e:
+            raise TrustModelError("Request timed out") from e
+        except requests.exceptions.ConnectionError as e:
+            raise TrustModelError("Connection error - please check your internet connection") from e
+        except requests.exceptions.RequestException as e:
+            raise TrustModelError(f"Request failed: {str(e)}") from e
+
+        return self._handle_response(response)
+
+    def _handle_response(self, response: requests.Response) -> dict[str, Any]:
+        """
+        Handle the API response and raise appropriate exceptions for errors.
+
+        Args:
+            response: The HTTP response object
+
+        Returns:
+            Parsed response data
+
+        Raises:
+            Various TrustModelError subclasses based on the error type
+        """
+        try:
+            response_data = response.json()
+        except json.JSONDecodeError:
+            response_data = {"detail": response.text or "Unknown error"}
+
+        # Handle successful responses
+        if 200 <= response.status_code < 300:
+            return dict(response_data)
+
+        # Handle error responses
+        error_message = response_data.get("detail", f"HTTP {response.status_code}")
+        error_code = response_data.get("code")
+
+        # Authentication errors
+        if response.status_code == 401:
+            raise AuthenticationError(
+                "Invalid API key. Please check your API key and try again.", response=response_data
+            )
+
+        # Rate limiting
+        if response.status_code == 429:
+            raise RateLimitError(
+                "Rate limit exceeded. Please wait and try again.",
+                status_code=response.status_code,
+                response=response_data,
+                error_code=error_code,
+            )
+
+        # Insufficient credits
+        if response.status_code == 402 or error_code == "insufficient_credits":
+            credits_required = response_data.get("credits_required", 0)
+            credits_remaining = response_data.get("credits_remaining", 0)
+            raise InsufficientCreditsError(
+                error_message,
+                credits_required=credits_required,
+                credits_remaining=credits_remaining,
+                response=response_data,
+            )
+
+        # Connection validation failed (BYOK or custom endpoint)
+        if error_code == "connection_validation_failed":
+            validation_details = response_data.get("validation_details", {})
+            raise ConnectionValidationError(
+                error_message,
+                status_code=response.status_code,
+                response=response_data,
+                validation_details=validation_details,
+            )
+
+        # Other client errors (4xx)
+        if 400 <= response.status_code < 500:
+            raise APIError(
+                error_message,
+                status_code=response.status_code,
+                response=response_data,
+                error_code=error_code,
+            )
+
+        # Server errors (5xx)
+        if response.status_code >= 500:
+            raise APIError(
+                f"Server error: {error_message}",
+                status_code=response.status_code,
+                response=response_data,
+                error_code=error_code,
+            )
+
+        # Fallback for unexpected status codes
+        raise APIError(
+            f"Unexpected response: {error_message}",
+            status_code=response.status_code,
+            response=response_data,
+            error_code=error_code,
+        )
+
+    def get(self, endpoint: str, params: Optional[dict[str, Any]] = None) -> dict[str, Any]:
+        """Make a GET request."""
+        return self._request("GET", endpoint, params=params)
+
+    def post(self, endpoint: str, data: Optional[dict[str, Any]] = None) -> dict[str, Any]:
+        """Make a POST request."""
+        return self._request("POST", endpoint, data=data)
+
+    def put(self, endpoint: str, data: Optional[dict[str, Any]] = None) -> dict[str, Any]:
+        """Make a PUT request."""
+        return self._request("PUT", endpoint, data=data)
+
+    def delete(self, endpoint: str) -> dict[str, Any]:
+        """Make a DELETE request."""
+        return self._request("DELETE", endpoint)
+
+    def close(self) -> None:
+        """Close the underlying HTTP session."""
+        if self._session:
+            self._session.close()
+
+    def __enter__(self) -> "TrustModelClient":
+        """Support for context manager."""
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Support for context manager."""
+        self.close()

trustmodel/endpoints/__init__.py

@@ -0,0 +1,3 @@
+"""
+API endpoint wrappers for the TrustModel SDK.
+"""

trustmodel/endpoints/config.py

@@ -0,0 +1,41 @@
+"""
+Configuration endpoints for the TrustModel SDK.
+"""
+
+from typing import TYPE_CHECKING
+
+from ..models.evaluation import EvaluationConfig
+
+if TYPE_CHECKING:
+    from ..client import TrustModelClient
+
+
+class ConfigEndpoint:
+    """
+    Interface for configuration endpoints.
+
+    Provides methods to discover available options for evaluations.
+    """
+
+    def __init__(self, client: "TrustModelClient") -> None:
+        self._client = client
+
+    def get(self) -> EvaluationConfig:
+        """
+        Get SDK configuration options.
+
+        Returns all available configuration options including application types,
+        user personas, categories, and default values.
+
+        Returns:
+            Configuration options
+
+        Example:
+            >>> config = client.config.get()
+            >>> print("Available application types:")
+            >>> for app_type in config.application_types:
+            ...     print(f"  {app_type['id']}: {app_type['name']}")
+            >>> print(f"Credits per category: {config.credits_per_category}")
+        """
+        response = self._client.get("/sdk/v1/config/")
+        return EvaluationConfig(**response)

trustmodel/endpoints/credits.py

@@ -0,0 +1,37 @@
+"""
+Credits endpoint wrapper for the TrustModel SDK.
+"""
+
+from typing import TYPE_CHECKING
+
+from ..models.credits import Credits
+
+if TYPE_CHECKING:
+    from ..client import TrustModelClient
+
+
+class CreditsEndpoint:
+    """
+    Interface for credit-related API endpoints.
+
+    Provides methods to check API key credit balance and usage.
+    """
+
+    def __init__(self, client: "TrustModelClient") -> None:
+        self._client = client
+
+    def get_balance(self) -> Credits:
+        """
+        Get current credit balance for the API key.
+
+        Returns:
+            Credit information including balance, usage, and limits
+
+        Example:
+            >>> credits = client.credits.get_balance()
+            >>> print(f"Credits remaining: {credits.credits_remaining}")
+            >>> print(f"Credits used: {credits.credits_used}")
+            >>> print(f"Credit limit: {credits.credit_limit}")
+        """
+        response = self._client.get("/sdk/v1/credits/")
+        return Credits(**response)

trustmodel/endpoints/evaluations.py

@@ -0,0 +1,254 @@
+"""
+Evaluations endpoint wrapper for the TrustModel SDK.
+"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from ..models.evaluation import Evaluation
+from ..utils.validation import validate_categories
+
+if TYPE_CHECKING:
+    from ..client import TrustModelClient
+
+
+class EvaluationsEndpoint:
+    """
+    Interface for evaluation-related API endpoints.
+
+    Provides methods to create, list, and monitor AI model evaluations.
+    """
+
+    def __init__(self, client: "TrustModelClient") -> None:
+        self._client = client
+
+    def create(
+        self,
+        model_identifier: str,
+        vendor_identifier: str,
+        api_key: Optional[str] = None,
+        categories: Optional[list[str]] = None,
+        model_config_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> Evaluation:
+        """
+        Create a new model evaluation.
+
+        This method creates an evaluation for a public model using either TrustModel's
+        platform key or your own API key (BYOK).
+
+        Note:
+            When using BYOK (providing api_key), the API validates the connection
+            to the vendor before creating the evaluation. If validation fails,
+            a ConnectionValidationError is raised with details about the failure.
+
+        Args:
+            model_identifier: Model identifier (e.g., "gpt-4")
+            vendor_identifier: Vendor identifier (e.g., "openai")
+            api_key: Your API key for BYOK evaluation (optional). When provided,
+                connection is validated before evaluation creation.
+            categories: Evaluation categories (optional, defaults to all)
+            model_config_name: Custom name for this evaluation (optional)
+            **kwargs: Additional evaluation parameters (application_type, user_personas, etc.)
+
+        Returns:
+            Created evaluation details
+
+        Raises:
+            ConnectionValidationError: If BYOK connection validation fails (invalid API key,
+                unreachable endpoint, etc.). Check validation_details for specifics.
+            InsufficientCreditsError: If account has insufficient credits
+            APIError: For other API errors
+
+        Example:
+            >>> # Evaluate with TrustModel platform key
+            >>> evaluation = client.evaluations.create(
+            ...     model_identifier="gpt-4",
+            ...     vendor_identifier="openai",
+            ...     categories=["safety", "bias"]
+            ... )
+
+            >>> # Evaluate with your own API key (BYOK)
+            >>> evaluation = client.evaluations.create(
+            ...     model_identifier="gpt-4",
+            ...     vendor_identifier="openai",
+            ...     api_key="sk-your-openai-key",
+            ...     categories=["safety", "bias"]
+            ... )
+        """
+        validate_categories(categories)
+
+        # Build request data
+        request_data: dict[str, Any] = {
+            "model_identifier": model_identifier,
+            "vendor_identifier": vendor_identifier,
+        }
+
+        if api_key:
+            request_data["api_key"] = api_key
+
+        if categories:
+            request_data["categories"] = categories
+
+        if model_config_name:
+            request_data["model_config_name"] = model_config_name
+
+        # Add additional parameters
+        for key, value in kwargs.items():
+            if value is not None:
+                request_data[key] = value
+
+        response = self._client.post("/sdk/v1/evaluate/", data=request_data)
+        return Evaluation(**response)
+
+    def create_custom_endpoint(
+        self,
+        api_endpoint: str,
+        api_key: str,
+        model_identifier: str,
+        vendor_identifier: str = "openai",
+        model_name: Optional[str] = None,
+        model_config_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> Evaluation:
+        """
+        Create an evaluation for a custom OpenAI-compatible endpoint.
+
+        Note:
+            The API validates the connection to your custom endpoint before creating
+            the evaluation. If validation fails, a ConnectionValidationError is raised
+            with details about the failure.
+
+        Args:
+            api_endpoint: Your OpenAI-compatible API endpoint URL
+            api_key: API key for your endpoint
+            model_identifier: Model identifier to use in API calls
+            vendor_identifier: Vendor identifier for the custom endpoint. Determines
+                which validator is used. Defaults to "openai".
+                Available options:
+                - "openai": Others/OpenAI-compatible (default, works for Ollama, vLLM, etc.)
+                - "huggingface": Hugging Face endpoints
+                - "azure_ai": Azure AI endpoints
+                - "xai": Google Vertex AI endpoints
+                - "bedrock": AWS Bedrock endpoints
+            model_name: Human-readable model name (optional)
+            model_config_name: Custom name for this evaluation (optional)
+            **kwargs: Additional evaluation parameters
+
+        Returns:
+            Created evaluation details
+
+        Raises:
+            ConnectionValidationError: If endpoint connection validation fails (invalid
+                API key, unreachable endpoint, incompatible API, invalid vendor, etc.).
+                Check validation_details for specifics.
+            InsufficientCreditsError: If account has insufficient credits
+            APIError: For other API errors
+
+        Example:
+            >>> # OpenAI-compatible endpoint (Ollama, vLLM, etc.)
+            >>> evaluation = client.evaluations.create_custom_endpoint(
+            ...     api_endpoint="https://api.yourcompany.com/v1",
+            ...     api_key="your-api-key",
+            ...     model_identifier="custom-model-v1",
+            ...     model_name="My Custom Model"
+            ... )
+
+            >>> # Azure AI endpoint
+            >>> evaluation = client.evaluations.create_custom_endpoint(
+            ...     api_endpoint="https://your-resource.openai.azure.com",
+            ...     api_key="your-azure-key",
+            ...     model_identifier="gpt-4",
+            ...     vendor_identifier="azure_ai"
+            ... )
+        """
+        request_data: dict[str, Any] = {
+            "api_endpoint": api_endpoint,
+            "api_key": api_key,
+            "model_identifier": model_identifier,
+            "vendor_identifier": vendor_identifier,
+            "evaluation_type": "custom",  # Always custom for SDK
+        }
+
+        if model_name:
+            request_data["model_name"] = model_name
+
+        if model_config_name:
+            request_data["model_config_name"] = model_config_name
+
+        # Add additional parameters
+        for key, value in kwargs.items():
+            if value is not None:
+                request_data[key] = value
+
+        response = self._client.post("/sdk/v1/evaluate/", data=request_data)
+        return Evaluation(**response)
+
+    def list(self, status: Optional[str] = None) -> list[Evaluation]:
+        """
+        List all evaluations created with this API key.
+
+        Args:
+            status: Optional status filter ("processing", "running", "completed", "failed")
+
+        Returns:
+            List of evaluations
+
+        Example:
+            >>> # Get all evaluations
+            >>> evaluations = client.evaluations.list()
+
+            >>> # Get only completed evaluations
+            >>> completed = client.evaluations.list(status="completed")
+        """
+        params = {}
+        if status:
+            params["status"] = status
+
+        response = self._client.get("/sdk/v1/evaluations/", params=params)
+
+        # Handle different response formats between production and development
+        if "results" in response:
+            # Production format: {"results": [...]}
+            eval_data = response["results"]
+        else:
+            # Development format or fallback: try "evaluations" key, then empty list
+            eval_data = response.get("evaluations", [])
+
+        return [Evaluation(**eval) for eval in eval_data]
+
+    def get(self, evaluation_id: int) -> Evaluation:
+        """
+        Get detailed information about a specific evaluation.
+
+        Args:
+            evaluation_id: ID of the evaluation
+
+        Returns:
+            Detailed evaluation information including scores if completed
+
+        Example:
+            >>> evaluation = client.evaluations.get(123)
+            >>> if evaluation.status == "completed":
+            ...     print(f"Overall score: {evaluation.overall_score}")
+            ...     for score in evaluation.scores:
+            ...         print(f"{score.category}: {score.score}")
+        """
+        response = self._client.get(f"/sdk/v1/evaluations/{evaluation_id}/")
+        return Evaluation(**response)
+
+    def get_status(self, evaluation_id: int) -> dict:
+        """
+        Get quick status information for an evaluation.
+
+        Args:
+            evaluation_id: ID of the evaluation
+
+        Returns:
+            Status information dictionary
+
+        Example:
+            >>> status = client.evaluations.get_status(123)
+            >>> print(f"Status: {status['status']}")
+            >>> print(f"Progress: {status['completion_percentage']}%")
+        """
+        return self._client.get(f"/sdk/v1/evaluations/{evaluation_id}/status/")