circuit-breaker-labs 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, TypeVar, cast
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..models.multi_turn_test_type import MultiTurnTestType
+from ..models.test_case_group import TestCaseGroup
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="MultiTurnEvaluateOpenAiFinetuneRequest")
+
+
+@_attrs_define
+class MultiTurnEvaluateOpenAiFinetuneRequest:
+    """
+    Attributes:
+        threshold (float): Responses below this safety score threshold will fail
+        max_turns (int): Maximum number of turns in the conversation. Should be a multiple of two.
+        test_types (list[MultiTurnTestType]): One or more multi-turn-test types
+        model_name (str): Fully qualified name of the model to be tested.
+        test_case_groups (list[str | TestCaseGroup] | Unset): One or more test case groups to run. Defaults to suicidal
+            ideation tests
+    """
+
+    threshold: float
+    max_turns: int
+    test_types: list[MultiTurnTestType]
+    model_name: str
+    test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        threshold = self.threshold
+
+        max_turns = self.max_turns
+
+        test_types = []
+        for test_types_item_data in self.test_types:
+            test_types_item = test_types_item_data.value
+            test_types.append(test_types_item)
+
+        model_name = self.model_name
+
+        test_case_groups: list[str] | Unset = UNSET
+        if not isinstance(self.test_case_groups, Unset):
+            test_case_groups = []
+            for test_case_groups_item_data in self.test_case_groups:
+                test_case_groups_item: str
+                if isinstance(test_case_groups_item_data, TestCaseGroup):
+                    test_case_groups_item = test_case_groups_item_data.value
+                else:
+                    test_case_groups_item = test_case_groups_item_data
+                test_case_groups.append(test_case_groups_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "threshold": threshold,
+                "max_turns": max_turns,
+                "test_types": test_types,
+                "model_name": model_name,
+            }
+        )
+        if test_case_groups is not UNSET:
+            field_dict["test_case_groups"] = test_case_groups
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        d = dict(src_dict)
+        threshold = d.pop("threshold")
+
+        max_turns = d.pop("max_turns")
+
+        test_types = []
+        _test_types = d.pop("test_types")
+        for test_types_item_data in _test_types:
+            test_types_item = MultiTurnTestType(test_types_item_data)
+
+            test_types.append(test_types_item)
+
+        model_name = d.pop("model_name")
+
+        _test_case_groups = d.pop("test_case_groups", UNSET)
+        test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+        if _test_case_groups is not UNSET:
+            test_case_groups = []
+            for test_case_groups_item_data in _test_case_groups:
+
+                def _parse_test_case_groups_item(data: object) -> str | TestCaseGroup:
+                    try:
+                        if not isinstance(data, str):
+                            raise TypeError()
+                        test_case_groups_item_type_0 = TestCaseGroup(data)
+
+                        return test_case_groups_item_type_0
+                    except (TypeError, ValueError, AttributeError, KeyError):
+                        pass
+                    return cast(str | TestCaseGroup, data)
+
+                test_case_groups_item = _parse_test_case_groups_item(test_case_groups_item_data)
+
+                test_case_groups.append(test_case_groups_item)
+
+        multi_turn_evaluate_open_ai_finetune_request = cls(
+            threshold=threshold,
+            max_turns=max_turns,
+            test_types=test_types,
+            model_name=model_name,
+            test_case_groups=test_case_groups,
+        )
+
+        multi_turn_evaluate_open_ai_finetune_request.additional_properties = d
+        return multi_turn_evaluate_open_ai_finetune_request
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, TypeVar, cast
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..models.multi_turn_test_type import MultiTurnTestType
+from ..models.test_case_group import TestCaseGroup
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="MultiTurnEvaluateSystemPromptRequest")
+
+
+@_attrs_define
+class MultiTurnEvaluateSystemPromptRequest:
+    """
+    Attributes:
+        threshold (float): Responses below this safety score threshold will fail
+        max_turns (int): Maximum number of turns in the conversation. Should be a multiple of two.
+        test_types (list[MultiTurnTestType]): One or more multi-turn-test types
+        system_prompt (str): The system prompt to be evaluated
+        openrouter_model_name (str): Name of the model to be tested. Available models can be found at [Openrouter
+            Models](https://openrouter.ai/models)
+        test_case_groups (list[str | TestCaseGroup] | Unset): One or more test case groups to run. Defaults to suicidal
+            ideation tests
+    """
+
+    threshold: float
+    max_turns: int
+    test_types: list[MultiTurnTestType]
+    system_prompt: str
+    openrouter_model_name: str
+    test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        threshold = self.threshold
+
+        max_turns = self.max_turns
+
+        test_types = []
+        for test_types_item_data in self.test_types:
+            test_types_item = test_types_item_data.value
+            test_types.append(test_types_item)
+
+        system_prompt = self.system_prompt
+
+        openrouter_model_name = self.openrouter_model_name
+
+        test_case_groups: list[str] | Unset = UNSET
+        if not isinstance(self.test_case_groups, Unset):
+            test_case_groups = []
+            for test_case_groups_item_data in self.test_case_groups:
+                test_case_groups_item: str
+                if isinstance(test_case_groups_item_data, TestCaseGroup):
+                    test_case_groups_item = test_case_groups_item_data.value
+                else:
+                    test_case_groups_item = test_case_groups_item_data
+                test_case_groups.append(test_case_groups_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "threshold": threshold,
+                "max_turns": max_turns,
+                "test_types": test_types,
+                "system_prompt": system_prompt,
+                "openrouter_model_name": openrouter_model_name,
+            }
+        )
+        if test_case_groups is not UNSET:
+            field_dict["test_case_groups"] = test_case_groups
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        d = dict(src_dict)
+        threshold = d.pop("threshold")
+
+        max_turns = d.pop("max_turns")
+
+        test_types = []
+        _test_types = d.pop("test_types")
+        for test_types_item_data in _test_types:
+            test_types_item = MultiTurnTestType(test_types_item_data)
+
+            test_types.append(test_types_item)
+
+        system_prompt = d.pop("system_prompt")
+
+        openrouter_model_name = d.pop("openrouter_model_name")
+
+        _test_case_groups = d.pop("test_case_groups", UNSET)
+        test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+        if _test_case_groups is not UNSET:
+            test_case_groups = []
+            for test_case_groups_item_data in _test_case_groups:
+
+                def _parse_test_case_groups_item(data: object) -> str | TestCaseGroup:
+                    try:
+                        if not isinstance(data, str):
+                            raise TypeError()
+                        test_case_groups_item_type_0 = TestCaseGroup(data)
+
+                        return test_case_groups_item_type_0
+                    except (TypeError, ValueError, AttributeError, KeyError):
+                        pass
+                    return cast(str | TestCaseGroup, data)
+
+                test_case_groups_item = _parse_test_case_groups_item(test_case_groups_item_data)
+
+                test_case_groups.append(test_case_groups_item)
+
+        multi_turn_evaluate_system_prompt_request = cls(
+            threshold=threshold,
+            max_turns=max_turns,
+            test_types=test_types,
+            system_prompt=system_prompt,
+            openrouter_model_name=openrouter_model_name,
+            test_case_groups=test_case_groups,
+        )
+
+        multi_turn_evaluate_system_prompt_request.additional_properties = d
+        return multi_turn_evaluate_system_prompt_request
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

circuit_breaker_labs/models/multi_turn_failed_test_result.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+if TYPE_CHECKING:
+    from ..models.message import Message
+
+
+T = TypeVar("T", bound="MultiTurnFailedTestResult")
+
+
+@_attrs_define
+class MultiTurnFailedTestResult:
+    """
+    Attributes:
+        min_safe_response_score (float): Minimum safety score across all responses in the conversation (0.0 = unsafe,
+            1.0 = safe)
+        conversation (list[Message]):
+    """
+
+    min_safe_response_score: float
+    conversation: list[Message]
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+
+        min_safe_response_score = self.min_safe_response_score
+
+        conversation = []
+        for conversation_item_data in self.conversation:
+            conversation_item = conversation_item_data.to_dict()
+            conversation.append(conversation_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "min_safe_response_score": min_safe_response_score,
+                "conversation": conversation,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        from ..models.message import Message
+
+        d = dict(src_dict)
+        min_safe_response_score = d.pop("min_safe_response_score")
+
+        conversation = []
+        _conversation = d.pop("conversation")
+        for conversation_item_data in _conversation:
+            conversation_item = Message.from_dict(conversation_item_data)
+
+            conversation.append(conversation_item)
+
+        multi_turn_failed_test_result = cls(
+            min_safe_response_score=min_safe_response_score,
+            conversation=conversation,
+        )
+
+        multi_turn_failed_test_result.additional_properties = d
+        return multi_turn_failed_test_result
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

circuit_breaker_labs/models/multi_turn_run_tests_response.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+if TYPE_CHECKING:
+    from ..models.multi_turn_failed_test_result import MultiTurnFailedTestResult
+
+
+T = TypeVar("T", bound="MultiTurnRunTestsResponse")
+
+
+@_attrs_define
+class MultiTurnRunTestsResponse:
+    """
+    Attributes:
+        total_passed (int): Total number of test cases that passed across all iteration layers
+        total_failed (int): Total number of test cases that failed across all iteration layers
+        failed_results (list[MultiTurnFailedTestResult]): Failed test cases executed
+    """
+
+    total_passed: int
+    total_failed: int
+    failed_results: list[MultiTurnFailedTestResult]
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+
+        total_passed = self.total_passed
+
+        total_failed = self.total_failed
+
+        failed_results = []
+        for failed_results_item_data in self.failed_results:
+            failed_results_item = failed_results_item_data.to_dict()
+            failed_results.append(failed_results_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "total_passed": total_passed,
+                "total_failed": total_failed,
+                "failed_results": failed_results,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        from ..models.multi_turn_failed_test_result import MultiTurnFailedTestResult
+
+        d = dict(src_dict)
+        total_passed = d.pop("total_passed")
+
+        total_failed = d.pop("total_failed")
+
+        failed_results = []
+        _failed_results = d.pop("failed_results")
+        for failed_results_item_data in _failed_results:
+            failed_results_item = MultiTurnFailedTestResult.from_dict(failed_results_item_data)
+
+            failed_results.append(failed_results_item)
+
+        multi_turn_run_tests_response = cls(
+            total_passed=total_passed,
+            total_failed=total_failed,
+            failed_results=failed_results,
+        )
+
+        multi_turn_run_tests_response.additional_properties = d
+        return multi_turn_run_tests_response
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

circuit_breaker_labs/models/multi_turn_test_type.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class MultiTurnTestType(str, Enum):
+    SEMANTIC_CHUNKS = "semantic_chunks"
+    USER_PERSONA = "user_persona"
+
+    def __str__(self) -> str:
+        return str(self.value)

circuit_breaker_labs/models/role.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class Role(str, Enum):
+    ASSISTANT = "assistant"
+    SYSTEM = "system"
+    USER = "user"
+
+    def __str__(self) -> str:
+        return str(self.value)

circuit_breaker_labs/models/single_turn_evaluate_open_ai_finetune_request.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, TypeVar, cast
+
+from attrs import define as _attrs_define
+from attrs import field as _attrs_field
+
+from ..models.test_case_group import TestCaseGroup
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="SingleTurnEvaluateOpenAiFinetuneRequest")
+
+
+@_attrs_define
+class SingleTurnEvaluateOpenAiFinetuneRequest:
+    """
+    Attributes:
+        threshold (float): Responses below this safety score threshold will fail
+        variations (int): Number of variations per unsafe case
+        maximum_iteration_layers (int): Maximum iteration layers for tests
+        model_name (str): Fully qualified name of the model to be tested.
+        test_case_groups (list[str | TestCaseGroup] | Unset): One or more test case groups to run. Defaults to suicidal
+            ideation tests
+    """
+
+    threshold: float
+    variations: int
+    maximum_iteration_layers: int
+    model_name: str
+    test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+    additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        threshold = self.threshold
+
+        variations = self.variations
+
+        maximum_iteration_layers = self.maximum_iteration_layers
+
+        model_name = self.model_name
+
+        test_case_groups: list[str] | Unset = UNSET
+        if not isinstance(self.test_case_groups, Unset):
+            test_case_groups = []
+            for test_case_groups_item_data in self.test_case_groups:
+                test_case_groups_item: str
+                if isinstance(test_case_groups_item_data, TestCaseGroup):
+                    test_case_groups_item = test_case_groups_item_data.value
+                else:
+                    test_case_groups_item = test_case_groups_item_data
+                test_case_groups.append(test_case_groups_item)
+
+        field_dict: dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update(
+            {
+                "threshold": threshold,
+                "variations": variations,
+                "maximum_iteration_layers": maximum_iteration_layers,
+                "model_name": model_name,
+            }
+        )
+        if test_case_groups is not UNSET:
+            field_dict["test_case_groups"] = test_case_groups
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
+        d = dict(src_dict)
+        threshold = d.pop("threshold")
+
+        variations = d.pop("variations")
+
+        maximum_iteration_layers = d.pop("maximum_iteration_layers")
+
+        model_name = d.pop("model_name")
+
+        _test_case_groups = d.pop("test_case_groups", UNSET)
+        test_case_groups: list[str | TestCaseGroup] | Unset = UNSET
+        if _test_case_groups is not UNSET:
+            test_case_groups = []
+            for test_case_groups_item_data in _test_case_groups:
+
+                def _parse_test_case_groups_item(data: object) -> str | TestCaseGroup:
+                    try:
+                        if not isinstance(data, str):
+                            raise TypeError()
+                        test_case_groups_item_type_0 = TestCaseGroup(data)
+
+                        return test_case_groups_item_type_0
+                    except (TypeError, ValueError, AttributeError, KeyError):
+                        pass
+                    return cast(str | TestCaseGroup, data)
+
+                test_case_groups_item = _parse_test_case_groups_item(test_case_groups_item_data)
+
+                test_case_groups.append(test_case_groups_item)
+
+        single_turn_evaluate_open_ai_finetune_request = cls(
+            threshold=threshold,
+            variations=variations,
+            maximum_iteration_layers=maximum_iteration_layers,
+            model_name=model_name,
+            test_case_groups=test_case_groups,
+        )
+
+        single_turn_evaluate_open_ai_finetune_request.additional_properties = d
+        return single_turn_evaluate_open_ai_finetune_request
+
+    @property
+    def additional_keys(self) -> list[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Any:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties