switch-trust-cli 1.7.0__py3-none-any.whl

switch_trust/eval/db/base/eval/model_eval_types.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from dataclasses_json import dataclass_json
+
+from switch_trust.eval.common.reference import Reference, ReferenceType
+from switch_trust.eval.common.utils import datetime_config, generate_id, now_utc
+from switch_trust.eval.db.base.common_types import SortDirection
+
+
+@dataclass_json
+@dataclass
+class DbModelEvaluation:
+    """Links a Model to an Evaluation with a weight."""
+
+    model_id: str
+    evaluation_id: str
+    name: str
+    id: str = field(default_factory=generate_id)
+    created: datetime = field(
+        default_factory=now_utc,
+        metadata=datetime_config,
+    )
+    description: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
+    weight: float = 0.5
+
+    def get_ref(self) -> Reference:
+        return Reference(
+            self.id,
+            self.name,
+            ReferenceType.MODEL_EVALUATION_SUITE,
+            self.description,
+        )
+
+
+class ModelEvaluationSortField(str, Enum):
+    NAME = "name"
+    CREATED = "created"
+    WEIGHT = "weight"
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationSortOrder:
+    field: ModelEvaluationSortField = ModelEvaluationSortField.NAME
+    direction: SortDirection = SortDirection.ASC
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationListItem:
+    """A model-evaluation assignment with resolved references."""
+
+    config: DbModelEvaluation
+    model_ref: Reference | None = None
+    evaluation_ref: Reference | None = None
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationListView:
+    """Paginated list of model-evaluation assignments."""
+
+    items: list[ModelEvaluationListItem] = field(
+        default_factory=list,
+    )
+    total: int = 0

switch_trust/eval/db/base/message/message_collection_helpers.py

@@ -0,0 +1,50 @@
+from switch_trust.eval.common.schema import Message
+from switch_trust.eval.core.message.message_collection import MessageCollection
+from switch_trust.eval.db.base.message.message_collection_types import (
+    DbMessageCollection,
+    MessageCollectionType,
+)
+
+
+def create_message_collection(db_collection: DbMessageCollection) -> MessageCollection:
+    """Create a MessageCollection instance from a DbMessageCollection."""
+    if db_collection.type == MessageCollectionType.IN_MEMORY:
+        from switch_trust.eval.core.message.message_collection_memory import (  # noqa: PLC0415 - mirrors the eval/db original
+            InMemoryMessageCollection,
+        )
+
+        if db_collection.prompts:
+            return InMemoryMessageCollection().from_strings(
+                db_collection.prompts,
+            )
+
+        messages = None
+        if db_collection.messages:
+            messages = [Message.from_dict(m) for m in db_collection.messages]
+        return InMemoryMessageCollection(messages=messages)
+
+    elif db_collection.type == MessageCollectionType.CSV:
+        from switch_trust.eval.core.message.message_collection_csv import (  # noqa: PLC0415 - mirrors the eval/db original
+            CsvMessageCollection,
+        )
+
+        if not db_collection.filename:
+            raise ValueError("filename is required for csv collections")
+        return CsvMessageCollection(
+            filename=db_collection.filename,
+            column=db_collection.column or "prompt",
+        )
+
+    elif db_collection.type == MessageCollectionType.GARAK:
+        from switch_trust.eval.core.message.message_collection_garak import (  # noqa: PLC0415 - mirrors the eval/db original
+            GarakMessageCollection,
+        )
+
+        if not db_collection.probe_name:
+            raise ValueError("probe_name is required for garak collections")
+        return GarakMessageCollection(
+            probe_name=db_collection.probe_name,
+        )
+
+    else:
+        raise ValueError(f"unknown collection type: {db_collection.type}")

switch_trust/eval/db/base/message/message_collection_repository.py

@@ -0,0 +1,95 @@
+"""
+Abstract interface for a repository of MessageCollections.
+
+A MessageCollectionRepository manages multiple collections,
+providing CRUD operations over them.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.core.message.message_collection import MessageCollection
+from switch_trust.eval.db.base.message.message_collection_types import (
+    DbMessageCollection,
+    MessageCollectionListView,
+    MessageCollectionSortOrder,
+)
+
+
+class MessageCollectionRepository(ABC):
+    """Repository that manages multiple MessageCollections."""
+
+    @abstractmethod
+    def list(self) -> list[DbMessageCollection]:
+        """Return all collections in the repository."""
+        pass
+
+    @abstractmethod
+    def search(
+        self,
+        query: str | None = None,
+        types: list[str] | None = None,
+        order: MessageCollectionSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> MessageCollectionListView:
+        """Search collections with pagination and sorting."""
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> DbMessageCollection:
+        """Return the collection descriptor with the given id.
+
+        Raises KeyError if the collection does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def get_message_collection(
+        self,
+        id: str,
+    ) -> MessageCollection:
+        """Return a MessageCollection instance for the given id.
+
+        Raises KeyError if the collection does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def create(
+        self,
+        db_collection: DbMessageCollection,
+    ) -> DbMessageCollection:
+        """Persist a new collection.
+
+        Raises ValueError if a collection with that name already
+        exists.
+        """
+        pass
+
+    @abstractmethod
+    def update(
+        self,
+        db_collection: DbMessageCollection,
+    ) -> DbMessageCollection:
+        """Replace an existing collection.
+
+        The collection is identified by ``db_collection.id``.
+
+        Raises KeyError if the collection does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        """Delete the collection with the given id.
+
+        Raises KeyError if the collection does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all collections from the repository."""
+        pass

switch_trust/eval/db/base/message/message_collection_types.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from dataclasses_json import dataclass_json
+
+from switch_trust.eval.common.reference import Reference, ReferenceType
+from switch_trust.eval.common.utils import datetime_config, generate_id, now_utc
+from switch_trust.eval.db.base.common_types import SortDirection
+
+
+class MessageCollectionType(str, Enum):
+    """Discriminator for the kind of MessageCollection."""
+
+    IN_MEMORY = "in-memory"
+    POSTGRES = "postgres"
+    GARAK = "garak"
+    CSV = "csv"
+
+
+@dataclass_json
+@dataclass
+class DbMessageCollection:
+    """JSON-serialisable description of a MessageCollection."""
+
+    type: MessageCollectionType
+    name: str
+    id: str = field(default_factory=generate_id)
+    created: datetime = field(
+        default_factory=now_utc,
+        metadata=datetime_config,
+    )
+    description: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
+
+    # -- in-memory type --
+    messages: list[dict] | None = None
+    prompts: list[str] | None = None
+
+    # -- garak type --
+    probe_name: str | None = None
+
+    # -- csv type --
+    filename: str | None = None
+    column: str | None = None
+
+    def get_ref(self) -> Reference:
+        return Reference(
+            self.id,
+            self.name,
+            ReferenceType.MESSAGE_COLLECTION,
+            self.description,
+        )
+
+
+class MessageCollectionSortField(str, Enum):
+    NAME = "name"
+    CREATED = "created"
+    TYPE = "type"
+
+
+@dataclass_json
+@dataclass
+class MessageCollectionSortOrder:
+    field: MessageCollectionSortField = MessageCollectionSortField.NAME
+    direction: SortDirection = SortDirection.ASC
+
+
+@dataclass_json
+@dataclass
+class MessageCollectionListView:
+    """Paginated list of message collections."""
+
+    items: list[DbMessageCollection] = field(default_factory=list)
+    total: int = 0
+
+
+@dataclass_json
+@dataclass
+class MessageCollectionDetailView:
+    """A message collection with its message count."""
+
+    id: str = ""
+    name: str = ""
+    description: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
+    message_count: int = 0

switch_trust/eval/db/base/models/model_helpers.py

@@ -0,0 +1,253 @@
+from collections.abc import Callable
+
+import aiohttp
+
+from switch_trust.eval.common.utils import resolve_env, resolve_env_dict
+from switch_trust.eval.core.models.model import Model
+from switch_trust.eval.core.models.model_per_endpoint_limiter import (
+    PerEndpointLimiter,
+)
+from switch_trust.eval.core.models.model_retry import RetryModel
+from switch_trust.eval.db.base.models.model_types import DbModel, ModelType
+
+# HTTP-based agent types. SDK-based providers (Anthropic, OpenAI, Gemini,
+# LiteLLM, HuggingFace, Ollama, OpenAI-compatible) own their own rate limiting
+# and don't need our per-endpoint semaphore.
+_LIMITED_HTTP_TYPES = {
+    ModelType.ADK,
+    ModelType.OPENAI_AGENT,
+    ModelType.ANTHROPIC_AGENT,
+    ModelType.GENERIC_HTTP,
+    ModelType.LANGSERVE,
+    ModelType.VERTEX_AGENT_RUNTIME,
+}
+
+
+def create_model(
+    db_model: DbModel,
+    max_retries: int = 5,
+    resolve_env_vars: bool = True,
+    connector_factory: Callable[[], aiohttp.BaseConnector] | None = None,
+    per_endpoint_limit: int = 8,
+    initial_jitter_max: float = 1.0,
+) -> Model:
+    """Create a Model instance from a DbModel, wrapped in a RetryModel for
+    transient error handling. HTTP-based types are additionally wrapped in a
+    PerEndpointLimiter so bursts against one tenant endpoint don't blow past
+    its QPS quota.
+
+    connector_factory, when set, is passed to the HTTP-based models so their
+    aiohttp sessions enforce the eval worker's request-time SSRF guard
+    (platform/ssrf.py). None keeps the default connector for CLI/local use.
+    """
+    inner = _create_inner_model(
+        db_model,
+        resolve_env_vars=resolve_env_vars,
+        connector_factory=connector_factory,
+    )
+    if db_model.type in _LIMITED_HTTP_TYPES and db_model.host:
+        inner = PerEndpointLimiter(
+            inner,
+            endpoint_url=db_model.host,
+            limit=per_endpoint_limit,
+            jitter_max=initial_jitter_max,
+        )
+    return RetryModel(inner, max_retries=max_retries)
+
+
+def _create_inner_model(
+    db_model: DbModel,
+    resolve_env_vars: bool = True,
+    connector_factory: Callable[[], aiohttp.BaseConnector] | None = None,
+) -> Model:
+    temp = db_model.temperature
+    if resolve_env_vars:
+        key = resolve_env(db_model.key)
+        headers = resolve_env_dict(db_model.headers)
+    else:
+        key = db_model.key
+        headers = dict(db_model.headers)
+
+    if db_model.type == ModelType.ANTHROPIC:
+        from anthropic import (  # noqa: PLC0415 - patched at source in tests
+            AsyncAnthropic,
+        )
+
+        from switch_trust.eval.core.models.model_anthropic import (  # noqa: PLC0415 - patched at source in tests
+            AnthropicModel,
+        )
+
+        client_kwargs = {}
+        if key:
+            client_kwargs["api_key"] = key
+
+        return AnthropicModel(
+            AsyncAnthropic(**client_kwargs),
+            db_model.model_name,
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.OPENAI:
+        from openai import AsyncOpenAI  # noqa: PLC0415 - patched at source in tests
+
+        from switch_trust.eval.core.models.model_openai import (  # noqa: PLC0415 - patched at source in tests
+            OpenAIModel,
+        )
+
+        client_kwargs = {}
+        if key:
+            client_kwargs["api_key"] = key
+
+        return OpenAIModel(
+            AsyncOpenAI(**client_kwargs),
+            db_model.model_name,
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.GEMINI:
+        from google.genai import Client  # noqa: PLC0415 - patched at source in tests
+
+        from switch_trust.eval.core.models.model_gemini import (  # noqa: PLC0415 - patched at source in tests
+            GeminiModel,
+        )
+
+        client_kwargs = {}
+        if key:
+            client_kwargs["api_key"] = key
+
+        return GeminiModel(
+            Client(**client_kwargs),
+            db_model.model_name,
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.LITELLM:
+        from switch_trust.eval.core.models.model_litellm import (  # noqa: PLC0415 - patched at source in tests
+            LiteLLMModel,
+        )
+
+        return LiteLLMModel(
+            db_model.model_name,
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.HUGGINGFACE:
+        from switch_trust.eval.core.models.model_huggingface import (  # noqa: PLC0415 - patched at source in tests
+            HuggingFaceModel,
+        )
+
+        return HuggingFaceModel(
+            db_model.model_name,
+            token=key or None,
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.OLLAMA:
+        from switch_trust.eval.core.models.model_ollama import (  # noqa: PLC0415 - patched at source in tests
+            OllamaModel,
+        )
+
+        return OllamaModel(
+            db_model.model_name,
+            host=db_model.host or "http://localhost:11434",
+            temperature=temp,
+        )
+
+    elif db_model.type == ModelType.ADK:
+        from switch_trust.eval.core.models.model_adk import (  # noqa: PLC0415 - patched at source in tests
+            ADKModel,
+        )
+
+        return ADKModel(
+            app_name=db_model.model_name,
+            host=db_model.host or "http://localhost:8000",
+            immediate_result=db_model.immediate_result,
+            headers=headers or None,
+            connector_factory=connector_factory,
+        )
+
+    elif db_model.type == ModelType.OPENAI_AGENT:
+        from switch_trust.eval.core.models.model_openai_agent import (  # noqa: PLC0415 - patched at source in tests
+            OpenAIAgentModel,
+        )
+
+        return OpenAIAgentModel(
+            host=db_model.host or "http://localhost:8000",
+            endpoint=db_model.endpoint or "/run",
+            connector_factory=connector_factory,
+        )
+
+    elif db_model.type == ModelType.ANTHROPIC_AGENT:
+        from switch_trust.eval.core.models.model_anthropic_agent import (  # noqa: PLC0415 - patched at source in tests
+            AnthropicAgentModel,
+        )
+
+        return AnthropicAgentModel(
+            host=db_model.host or "http://localhost:8000",
+            endpoint=db_model.endpoint or "/run",
+            connector_factory=connector_factory,
+        )
+
+    elif db_model.type == ModelType.OPENAI_COMPATIBLE:
+        from switch_trust.eval.core.models.model_openai_compatible import (  # noqa: PLC0415 - patched at source in tests
+            OpenAICompatibleModel,
+        )
+
+        return OpenAICompatibleModel(
+            base_url=db_model.host or "http://localhost:8000/v1",
+            model=db_model.model_name,
+            api_key=key or "EMPTY",
+            temperature=temp,
+            headers=headers or None,
+        )
+
+    elif db_model.type == ModelType.GENERIC_HTTP:
+        from switch_trust.eval.core.models.model_generic_http import (  # noqa: PLC0415 - patched at source in tests
+            GenericHttpModel,
+        )
+
+        url = db_model.host or "http://localhost:8000"
+        if db_model.endpoint:
+            url = f"{url.rstrip('/')}{db_model.endpoint}"
+
+        return GenericHttpModel(
+            url=url,
+            headers=headers or None,
+            input_path=db_model.input_path or "input",
+            output_path=db_model.output_path or "output",
+            connector_factory=connector_factory,
+        )
+
+    elif db_model.type == ModelType.VERTEX_AGENT_RUNTIME:
+        from switch_trust.eval.core.models.model_vertex_agent_runtime import (  # noqa: PLC0415 - patched at source in tests
+            DEFAULT_CLASS_METHOD,
+            DEFAULT_USER_ID,
+            VertexAgentRuntimeModel,
+        )
+
+        # The engine URL already carries the :streamQuery suffix, so `endpoint`
+        # is not appended here the way it is for the generic HTTP types.
+        return VertexAgentRuntimeModel(
+            url=db_model.host or "",
+            credential=key or None,
+            headers=headers or None,
+            user_id=db_model.user_id or DEFAULT_USER_ID,
+            class_method=db_model.class_method or DEFAULT_CLASS_METHOD,
+            immediate_result=db_model.immediate_result,
+            connector_factory=connector_factory,
+        )
+
+    elif db_model.type == ModelType.LANGSERVE:
+        from switch_trust.eval.core.models.model_langserve import (  # noqa: PLC0415 - patched at source in tests
+            LangServeModel,
+        )
+
+        return LangServeModel(
+            base_url=db_model.host or "http://localhost:8000",
+            chain_path=db_model.endpoint or "",
+            headers=headers or None,
+            connector_factory=connector_factory,
+        )
+
+    else:
+        raise ValueError(f"unknown model type: {db_model.type}")

switch_trust/eval/db/base/models/model_repository.py

@@ -0,0 +1,104 @@
+"""
+Abstract interface for a repository of Models.
+
+A ModelRepository manages model configurations,
+providing CRUD operations over them.  Models are
+always referenced by ID.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.core.models.model import Model
+from switch_trust.eval.db.base.models.model_types import (
+    DbModel,
+    DbModelListView,
+    ModelSortOrder,
+)
+
+
+class ModelRepository(ABC):
+    """Repository that manages Model configurations."""
+
+    @abstractmethod
+    def list(self) -> list[DbModel]:
+        """Return all model configurations in the repository."""
+        pass
+
+    @abstractmethod
+    def search(
+        self,
+        query: str | None = None,
+        types: list[str] | None = None,
+        order: ModelSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> DbModelListView:
+        """Search models by name with offset-based pagination.
+
+        Args:
+            query: Text to match against the model name
+                (case-insensitive substring match).
+            order: Sort field and direction.
+            offset: Number of items to skip.
+            limit: Maximum number of items to return.
+        """
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> DbModel:
+        """Return the model configuration with the given id.
+
+        Raises KeyError if the model does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def create(
+        self,
+        config: DbModel,
+    ) -> DbModel:
+        """Persist a new model configuration.
+
+        Raises ValueError if a model with that name already
+        exists.
+        """
+        pass
+
+    @abstractmethod
+    def update(
+        self,
+        config: DbModel,
+    ) -> DbModel:
+        """Replace an existing model configuration.
+
+        The model is identified by ``config.id``.
+
+        Raises KeyError if the model does not exist.
+        """
+        pass
+
+    def get_model(self, id: str) -> Model:
+        """Return a Model instance for the given id.
+
+        Raises KeyError if the model does not exist.
+        """
+        from switch_trust.eval.db.base.models.model_helpers import (  # noqa: PLC0415 - deferred import cost
+            create_model,
+        )
+
+        return create_model(self.get(id))
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        """Delete the model with the given id.
+
+        Raises KeyError if the model does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all models from the repository."""
+        pass