switch-trust-cli 1.7.0__py3-none-any.whl

switch_trust/eval/db/base/eval/eval_repository.py

@@ -0,0 +1,120 @@
+"""
+Abstract interface for a repository of Evaluations.
+
+An EvaluationRepository manages evaluation configurations,
+providing CRUD operations over them.  Evaluations are
+always referenced by ID.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.core.eval.evaluation import Evaluation
+from switch_trust.eval.db.base.detectors.detector_repository import DetectorRepository
+from switch_trust.eval.db.base.eval.eval_types import (
+    DbEvaluation,
+    EvaluationListView,
+    EvaluationSortOrder,
+)
+from switch_trust.eval.db.base.message.message_collection_repository import (
+    MessageCollectionRepository,
+)
+
+
+class EvaluationRepository(ABC):
+    """Repository that manages Evaluation configurations."""
+
+    @abstractmethod
+    def list(self) -> list[DbEvaluation]:
+        """Return all evaluation configurations in the repository."""
+        pass
+
+    @abstractmethod
+    def search(
+        self,
+        query: str | None = None,
+        types: list[str] | None = None,
+        order: EvaluationSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> EvaluationListView:
+        """Search evaluations by name with offset-based pagination.
+
+        Args:
+            query: Text to match against the evaluation name
+                (case-insensitive substring match).
+            order: Sort field and direction.
+            offset: Number of items to skip.
+            limit: Maximum number of items to return.
+        """
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> DbEvaluation:
+        """Return the evaluation configuration with the given id.
+
+        Raises KeyError if the evaluation does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def create(
+        self,
+        config: DbEvaluation,
+    ) -> DbEvaluation:
+        """Persist a new evaluation configuration.
+
+        Raises ValueError if an evaluation with that name already
+        exists.
+        """
+        pass
+
+    @abstractmethod
+    def update(
+        self,
+        config: DbEvaluation,
+    ) -> DbEvaluation:
+        """Replace an existing evaluation configuration.
+
+        The evaluation is identified by ``config.id``.
+
+        Raises KeyError if the evaluation does not exist.
+        """
+        pass
+
+    def get_evaluation(
+        self,
+        id: str,
+        message_collection_repo: MessageCollectionRepository | None = None,
+        detector_repo: DetectorRepository | None = None,
+    ) -> Evaluation:
+        """Return an Evaluation instance for the given id.
+
+        Depending on the evaluation type, additional repositories
+        may be required to resolve referenced objects.
+
+        Raises KeyError if the evaluation does not exist.
+        """
+        from switch_trust.eval.db.base.eval.eval_helpers import (  # noqa: PLC0415 - deferred import cost
+            create_evaluation,
+        )
+
+        return create_evaluation(
+            self.get(id),
+            message_collection_repo=message_collection_repo,
+            detector_repo=detector_repo,
+        )
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        """Delete the evaluation with the given id.
+
+        Raises KeyError if the evaluation does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all evaluations from the repository."""
+        pass

switch_trust/eval/db/base/eval/eval_run.py

@@ -0,0 +1,142 @@
+"""
+Standalone function to run a model evaluation end-to-end.
+
+Looks up the DbModelEvaluation, creates the core Evaluation and
+Model, creates a DbModelEvaluationRun, and executes the evaluation
+with an observer that persists progress back to the database.
+"""
+
+import logging
+
+from switch_trust.eval.common.utils import now_utc
+from switch_trust.eval.core.eval.evaluation import Evaluation, EvaluationStatus
+from switch_trust.eval.db.base.detectors.detector_repository import DetectorRepository
+from switch_trust.eval.db.base.eval.eval_repository import EvaluationRepository
+from switch_trust.eval.db.base.eval.model_eval_repository import (
+    ModelEvaluationRepository,
+)
+from switch_trust.eval.db.base.eval.model_eval_run_repository import (
+    ModelEvaluationRunRepository,
+)
+from switch_trust.eval.db.base.eval.model_eval_run_result_repository import (
+    ModelEvaluationRunResultRepository,
+)
+from switch_trust.eval.db.base.eval.model_eval_run_result_types import (
+    DbModelEvaluationRunResult,
+)
+from switch_trust.eval.db.base.eval.model_eval_run_types import DbModelEvaluationRun
+from switch_trust.eval.db.base.message.message_collection_repository import (
+    MessageCollectionRepository,
+)
+from switch_trust.eval.db.base.models.model_repository import ModelRepository
+
+logger = logging.getLogger(__name__)
+
+
+async def run_model_evaluation(
+    model_evaluation_id: str,
+    concurrency: int,
+    model_evaluation_repo: ModelEvaluationRepository,
+    evaluation_repo: EvaluationRepository,
+    model_repo: ModelRepository,
+    run_repo: ModelEvaluationRunRepository,
+    message_collection_repo: MessageCollectionRepository | None = None,
+    detector_repo: DetectorRepository | None = None,
+    result_repo: ModelEvaluationRunResultRepository | None = None,
+    run: DbModelEvaluationRun | None = None,
+) -> DbModelEvaluationRun:
+    """Run a model evaluation and persist progress to the database.
+
+    1. Looks up the DbModelEvaluation by ID
+    2. Creates the core Evaluation and Model from their configs
+    3. Creates a new DbModelEvaluationRun and persists it
+    4. Initialises and runs the evaluation with an observer
+       that writes the summary back after each progress update
+    5. Returns the completed run
+
+    Args:
+        model_evaluation_id: ID of the DbModelEvaluation to run.
+        concurrency: Max concurrent evaluation tasks.
+        model_evaluation_repo: Repository for DbModelEvaluation.
+        evaluation_repo: Repository for DbEvaluation.
+        model_repo: Repository for DbModel.
+        run_repo: Repository for DbModelEvaluationRun.
+        message_collection_repo: Required for MESSAGE_COLLECTION
+            evaluation types.
+        detector_repo: Required for evaluations that use detectors.
+    """
+    model_evaluation = model_evaluation_repo.get(
+        model_evaluation_id,
+    )
+
+    evaluation = evaluation_repo.get_evaluation(
+        model_evaluation.evaluation_id,
+        message_collection_repo=message_collection_repo,
+        detector_repo=detector_repo,
+    )
+    model = model_repo.get_model(model_evaluation.model_id)
+
+    if run is None:
+        run = DbModelEvaluationRun(
+            model_evaluation_id=model_evaluation_id,
+            started=now_utc(),
+        )
+        run_repo.create(run)
+
+    logger.info(
+        "Run %s started for model evaluation %s",
+        run.id,
+        model_evaluation_id,
+    )
+
+    def _on_progress(e: Evaluation) -> None:
+        summary = e.get_summary()
+        run.summary = summary
+        if summary.status in (
+            EvaluationStatus.FINISHED,
+            EvaluationStatus.ERROR,
+        ):
+            run.finished = now_utc()
+        run_repo.update(run)
+        logger.info(
+            "Run %s: %s (%.0f%%)",
+            run.id,
+            summary.status.value,
+            summary.progress * 100,
+        )
+
+    evaluation.add_observer(_on_progress)
+    try:
+        await evaluation.init()
+        await evaluation.run(model, concurrency)
+    except Exception as e:
+        logger.error("Run %s failed: %s", run.id, e)
+    finally:
+        evaluation.remove_observer(_on_progress)
+
+    run.summary = evaluation.get_summary()
+    run.finished = now_utc()
+    run_repo.update(run)
+
+    if result_repo is not None:
+        core_results = evaluation.get_results()
+        db_results = [
+            DbModelEvaluationRunResult(
+                run_id=run.id,
+                score=r.score,
+                status=r.status,
+                error_message=r.error_message,
+                session=r.session,
+            )
+            for r in core_results
+        ]
+        result_repo.create_batch(db_results)
+        logger.info(
+            "Run %s: stored %d results",
+            run.id,
+            len(db_results),
+        )
+
+    logger.info("Run %s finished", run.id)
+
+    return run

switch_trust/eval/db/base/eval/eval_types.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from dataclasses_json import dataclass_json
+
+from switch_trust.eval.common.reference import Reference, ReferenceType
+from switch_trust.eval.common.utils import datetime_config, generate_id, now_utc
+from switch_trust.eval.db.base.common_types import SortDirection
+from switch_trust.eval.db.base.eval.model_eval_run_types import DbModelEvaluationRun
+
+
+class EvaluationType(str, Enum):
+    """Discriminator for the kind of Evaluation."""
+
+    MESSAGE_COLLECTION = "message_collection"
+    GARAK_PROBE = "garak_probe"
+    GARAK_MODULE = "garak_module"
+    METRIC_TOXICITY = "metric_toxicity"
+    METRIC_CONCISENESS = "metric_conciseness"
+    METRIC_FACTUAL_ACCURACY = "metric_factual_accuracy"
+    METRIC_INSTRUCTION_ADHERENCE = "metric_instruction_adherence"
+    METRIC_TONE = "metric_tone"
+    ADVERSARIAL_PROBE = "adversarial_probe"
+    TOPIC_GUARD = "topic_guard"
+
+
+class EvaluationApproach(str, Enum):
+    """Whether an evaluation actively probes the model
+    or passively measures a quality metric."""
+
+    PROBE = "Probe"
+    METRIC = "Metric"
+
+
+@dataclass_json
+@dataclass
+class DbEvaluation:
+    """JSON-serialisable description of an Evaluation."""
+
+    type: EvaluationType
+    name: str
+    id: str = field(default_factory=generate_id)
+    created: datetime = field(
+        default_factory=now_utc,
+        metadata=datetime_config,
+    )
+    description: str | None = None
+    approach: EvaluationApproach = EvaluationApproach.PROBE
+    tags: dict[str, str] = field(default_factory=dict)
+
+    # -- shared (message_collection, adversarial_probe) --
+    message_collection_id: str | None = None
+    detector_id: str | None = None
+
+    # -- garak_probe type --
+    probe_name: str | None = None
+
+    # -- garak_module type --
+    module_name: str | None = None
+    probe_names: list[str] | None = None
+
+    # -- adversarial_probe type --
+    adversarial_goals: list[str] | None = None
+    attack_techniques: list[str] | None = None
+    num_prompts: int | None = None
+    max_turns: int | None = None
+
+    # -- topic_guard type --
+    agent_objective: str | None = None
+    agent_instructions: str | None = None
+
+    def get_ref(self) -> Reference:
+        return Reference(
+            self.id,
+            self.name,
+            ReferenceType.EVALUATION,
+            self.description,
+        )
+
+
+class EvaluationSortField(str, Enum):
+    NAME = "name"
+    CREATED = "created"
+    TYPE = "type"
+
+
+@dataclass_json
+@dataclass
+class EvaluationSortOrder:
+    field: EvaluationSortField = EvaluationSortField.NAME
+    direction: SortDirection = SortDirection.ASC
+
+
+@dataclass_json
+@dataclass
+class EvaluationListView:
+    """Paginated list of evaluation configurations."""
+
+    items: list[DbEvaluation] = field(default_factory=list)
+    total: int = 0
+
+
+@dataclass_json
+@dataclass
+class EvaluationDetailModelView:
+    """A model assigned to an evaluation, with last run info."""
+
+    id: str
+    model_ref: Reference
+    evaluation_ref: Reference
+    created: datetime = field(metadata=datetime_config)
+    weight: float = 0.5
+    last_run: DbModelEvaluationRun | None = None
+
+
+@dataclass_json
+@dataclass
+class EvaluationDetailView:
+    """An evaluation with its assigned models."""
+
+    config: DbEvaluation | None = None
+    models: list[EvaluationDetailModelView] = field(
+        default_factory=list,
+    )

switch_trust/eval/db/base/eval/model_eval_repository.py

@@ -0,0 +1,79 @@
+"""
+Abstract interface for a repository of ModelEvaluations.
+
+A ModelEvaluationRepository manages the assignments of
+Evaluations to Models.
+"""
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.db.base.eval.model_eval_types import (
+    DbModelEvaluation,
+    ModelEvaluationListView,
+    ModelEvaluationSortOrder,
+)
+
+
+class ModelEvaluationRepository(ABC):
+    """Repository that manages ModelEvaluation assignments."""
+
+    @abstractmethod
+    def list_by_model(
+        self,
+        model_id: str,
+        order: ModelEvaluationSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> ModelEvaluationListView:
+        """Return assignments for a given model with pagination.
+
+        Args:
+            model_id: The model to list assignments for.
+            order: Sort field and direction.
+            offset: Number of items to skip.
+            limit: Maximum number of items to return.
+        """
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> DbModelEvaluation:
+        """Return the assignment with the given id.
+
+        Raises KeyError if it does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def create(
+        self,
+        config: DbModelEvaluation,
+    ) -> DbModelEvaluation:
+        """Persist a new model-evaluation assignment.
+
+        Raises ValueError if an assignment with that name
+        already exists.
+        """
+        pass
+
+    @abstractmethod
+    def list_by_evaluation(
+        self,
+        evaluation_id: str,
+        order: ModelEvaluationSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> ModelEvaluationListView:
+        """Return assignments for a given evaluation with pagination.
+
+        Args:
+            evaluation_id: The evaluation to list assignments for.
+            order: Sort field and direction.
+            offset: Number of items to skip.
+            limit: Maximum number of items to return.
+        """
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all model-evaluation assignments."""
+        pass

switch_trust/eval/db/base/eval/model_eval_run_repository.py

@@ -0,0 +1,72 @@
+"""
+Abstract interface for a repository of ModelEvaluationRuns.
+
+A ModelEvaluationRunRepository manages runs of model-evaluation
+assignments, providing create and search operations.
+"""
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.db.base.eval.model_eval_run_types import (
+    DbModelEvaluationRun,
+    ModelEvaluationRunListView,
+    ModelEvaluationRunSortOrder,
+)
+
+
+class ModelEvaluationRunRepository(ABC):
+    """Repository that manages ModelEvaluationRun records."""
+
+    @abstractmethod
+    def create(
+        self,
+        run: DbModelEvaluationRun,
+    ) -> DbModelEvaluationRun:
+        """Persist a new run."""
+        pass
+
+    @abstractmethod
+    def get(self, id: str) -> DbModelEvaluationRun:
+        """Return the run with the given id.
+
+        Raises KeyError if it does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def update(
+        self,
+        run: DbModelEvaluationRun,
+    ) -> DbModelEvaluationRun:
+        """Update the summary and finished timestamp of a run.
+
+        The run is identified by ``run.id``.
+
+        Raises KeyError if the run does not exist.
+        """
+        pass
+
+    @abstractmethod
+    def search(
+        self,
+        model_evaluation_id: str | None = None,
+        model_id: str | None = None,
+        evaluation_id: str | None = None,
+        order: ModelEvaluationRunSortOrder | None = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> ModelEvaluationRunListView:
+        """Search runs with optional filter and pagination.
+
+        Args:
+            model_evaluation_id: Filter by model-evaluation ID.
+            order: Sort field and direction.
+            offset: Number of items to skip.
+            limit: Maximum number of items to return.
+        """
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all runs."""
+        pass

switch_trust/eval/db/base/eval/model_eval_run_result_repository.py

@@ -0,0 +1,42 @@
+"""
+Abstract interface for a repository of ModelEvaluationRunResults.
+
+A ModelEvaluationRunResultRepository manages the leaf-evaluation
+results that belong to a model-evaluation run.
+"""
+
+from abc import ABC, abstractmethod
+
+from switch_trust.eval.db.base.eval.model_eval_run_result_types import (
+    DbModelEvaluationRunResult,
+)
+
+
+class ModelEvaluationRunResultRepository(ABC):
+    """Repository that manages ModelEvaluationRunResult records."""
+
+    @abstractmethod
+    def create_batch(
+        self,
+        results: list[DbModelEvaluationRunResult],
+    ) -> None:
+        """Persist a batch of results for a run."""
+        pass
+
+    @abstractmethod
+    def get_by_run_id(
+        self,
+        run_id: str,
+    ) -> list[DbModelEvaluationRunResult]:
+        """Return all results for a given run."""
+        pass
+
+    @abstractmethod
+    def delete_by_run_id(self, run_id: str) -> None:
+        """Remove all results for a given run."""
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all results."""
+        pass

switch_trust/eval/db/base/eval/model_eval_run_result_types.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass, field
+
+from dataclasses_json import dataclass_json
+
+from switch_trust.eval.common.schema import Session
+from switch_trust.eval.common.utils import generate_id
+from switch_trust.eval.core.eval.evaluation import EvaluationStatus
+
+
+@dataclass_json
+@dataclass
+class DbModelEvaluationRunResult:
+    """A single leaf-evaluation result within a run."""
+
+    run_id: str
+    score: float
+    status: EvaluationStatus
+    id: str = field(default_factory=generate_id)
+    error_message: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
+    session: Session | None = None

switch_trust/eval/db/base/eval/model_eval_run_types.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from dataclasses_json import dataclass_json
+
+from switch_trust.eval.common.reference import Reference
+from switch_trust.eval.common.utils import datetime_config, generate_id, now_utc
+from switch_trust.eval.core.eval.evaluation import EvaluationSummary
+from switch_trust.eval.db.base.common_types import SortDirection
+
+
+@dataclass_json
+@dataclass
+class DbModelEvaluationRun:
+    """A single run of a DbModelEvaluation."""
+
+    model_evaluation_id: str
+    id: str = field(default_factory=generate_id)
+    created: datetime = field(
+        default_factory=now_utc,
+        metadata=datetime_config,
+    )
+    started: datetime | None = field(
+        default=None,
+        metadata=datetime_config,
+    )
+    finished: datetime | None = field(
+        default=None,
+        metadata=datetime_config,
+    )
+    tags: dict[str, str] = field(default_factory=dict)
+    summary: EvaluationSummary | None = None
+
+
+class ModelEvaluationRunSortField(str, Enum):
+    CREATED = "created"
+    STARTED = "started_at"
+    FINISHED = "finished_at"
+    SCORE = "score"
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationRunSortOrder:
+    field: ModelEvaluationRunSortField = ModelEvaluationRunSortField.CREATED
+    direction: SortDirection = SortDirection.DESC
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationRunDetailView:
+    """A model-evaluation run with resolved references."""
+
+    run: DbModelEvaluationRun
+    model_ref: Reference | None = None
+    evaluation_ref: Reference | None = None
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationRunListItem:
+    """A model-evaluation run with resolved references."""
+
+    run: DbModelEvaluationRun
+    model_ref: Reference | None = None
+    evaluation_ref: Reference | None = None
+
+
+@dataclass_json
+@dataclass
+class ModelEvaluationRunListView:
+    """Paginated list of model-evaluation runs."""
+
+    items: list[ModelEvaluationRunListItem] = field(
+        default_factory=list,
+    )
+    total: int = 0