circuit-breaker-labs 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- circuit_breaker_labs/api/api_keys/monthly_quota_get.py +12 -12
- circuit_breaker_labs/api/evaluations/{evaluate_system_prompt_post.py → multi_turn_evaluate_system_prompt_post.py} +96 -39
- circuit_breaker_labs/api/evaluations/{evaluate_openai_fine_tune_post.py → multiturn_evaluate_openai_fine_tune_post.py} +96 -39
- circuit_breaker_labs/api/evaluations/single_turn_evaluate_openai_fine_tune_post.py +300 -0
- circuit_breaker_labs/api/evaluations/singleturn_evaluate_system_prompt_post.py +257 -0
- circuit_breaker_labs/models/__init__.py +36 -8
- circuit_breaker_labs/models/internal_server_error.py +73 -0
- circuit_breaker_labs/models/internal_server_error_response.py +69 -0
- circuit_breaker_labs/models/message.py +71 -0
- circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py +119 -0
- circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py +128 -0
- circuit_breaker_labs/models/multi_turn_failed_test_result.py +85 -0
- circuit_breaker_labs/models/multi_turn_run_tests_response.py +92 -0
- circuit_breaker_labs/models/multi_turn_test_type.py +9 -0
- circuit_breaker_labs/models/not_found_error.py +73 -0
- circuit_breaker_labs/models/not_found_response.py +69 -0
- circuit_breaker_labs/models/quota_exceeded_response.py +69 -0
- circuit_breaker_labs/models/role.py +10 -0
- circuit_breaker_labs/models/{evaluate_open_ai_finetune_request.py → single_turn_evaluate_open_ai_finetune_request.py} +30 -5
- circuit_breaker_labs/models/{evaluate_system_prompt_request.py → single_turn_evaluate_system_prompt_request.py} +30 -5
- circuit_breaker_labs/models/{failed_test_result.py → single_turn_failed_test_result.py} +5 -5
- circuit_breaker_labs/models/{run_tests_response.py → single_turn_run_tests_response.py} +10 -10
- circuit_breaker_labs/models/test_case_pack.py +8 -0
- circuit_breaker_labs/models/unauthorized_response.py +69 -0
- {circuit_breaker_labs-1.0.4.dist-info → circuit_breaker_labs-1.0.6.dist-info}/METADATA +33 -21
- circuit_breaker_labs-1.0.6.dist-info/RECORD +47 -0
- {circuit_breaker_labs-1.0.4.dist-info → circuit_breaker_labs-1.0.6.dist-info}/WHEEL +1 -1
- circuit_breaker_labs-1.0.4.dist-info/RECORD +0 -31
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
5
|
+
|
|
6
|
+
from attrs import define as _attrs_define
|
|
7
|
+
from attrs import field as _attrs_field
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..models.not_found_error import NotFoundError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
T = TypeVar("T", bound="NotFoundResponse")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@_attrs_define
|
|
17
|
+
class NotFoundResponse:
|
|
18
|
+
"""404 Not Found response wrapper.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
detail (NotFoundError): 404 Not Found error response.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
detail: NotFoundError
|
|
25
|
+
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
26
|
+
|
|
27
|
+
def to_dict(self) -> dict[str, Any]:
|
|
28
|
+
|
|
29
|
+
detail = self.detail.to_dict()
|
|
30
|
+
|
|
31
|
+
field_dict: dict[str, Any] = {}
|
|
32
|
+
field_dict.update(self.additional_properties)
|
|
33
|
+
field_dict.update(
|
|
34
|
+
{
|
|
35
|
+
"detail": detail,
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
return field_dict
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
|
|
43
|
+
from ..models.not_found_error import NotFoundError
|
|
44
|
+
|
|
45
|
+
d = dict(src_dict)
|
|
46
|
+
detail = NotFoundError.from_dict(d.pop("detail"))
|
|
47
|
+
|
|
48
|
+
not_found_response = cls(
|
|
49
|
+
detail=detail,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
not_found_response.additional_properties = d
|
|
53
|
+
return not_found_response
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def additional_keys(self) -> list[str]:
|
|
57
|
+
return list(self.additional_properties.keys())
|
|
58
|
+
|
|
59
|
+
def __getitem__(self, key: str) -> Any:
|
|
60
|
+
return self.additional_properties[key]
|
|
61
|
+
|
|
62
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
63
|
+
self.additional_properties[key] = value
|
|
64
|
+
|
|
65
|
+
def __delitem__(self, key: str) -> None:
|
|
66
|
+
del self.additional_properties[key]
|
|
67
|
+
|
|
68
|
+
def __contains__(self, key: str) -> bool:
|
|
69
|
+
return key in self.additional_properties
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
5
|
+
|
|
6
|
+
from attrs import define as _attrs_define
|
|
7
|
+
from attrs import field as _attrs_field
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..models.quota_exceeded_error import QuotaExceededError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
T = TypeVar("T", bound="QuotaExceededResponse")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@_attrs_define
|
|
17
|
+
class QuotaExceededResponse:
|
|
18
|
+
"""403 Quota exceeded response wrapper.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
detail (QuotaExceededError): 403 Forbidden error response for quota limits.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
detail: QuotaExceededError
|
|
25
|
+
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
26
|
+
|
|
27
|
+
def to_dict(self) -> dict[str, Any]:
|
|
28
|
+
|
|
29
|
+
detail = self.detail.to_dict()
|
|
30
|
+
|
|
31
|
+
field_dict: dict[str, Any] = {}
|
|
32
|
+
field_dict.update(self.additional_properties)
|
|
33
|
+
field_dict.update(
|
|
34
|
+
{
|
|
35
|
+
"detail": detail,
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
return field_dict
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
|
|
43
|
+
from ..models.quota_exceeded_error import QuotaExceededError
|
|
44
|
+
|
|
45
|
+
d = dict(src_dict)
|
|
46
|
+
detail = QuotaExceededError.from_dict(d.pop("detail"))
|
|
47
|
+
|
|
48
|
+
quota_exceeded_response = cls(
|
|
49
|
+
detail=detail,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
quota_exceeded_response.additional_properties = d
|
|
53
|
+
return quota_exceeded_response
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def additional_keys(self) -> list[str]:
|
|
57
|
+
return list(self.additional_properties.keys())
|
|
58
|
+
|
|
59
|
+
def __getitem__(self, key: str) -> Any:
|
|
60
|
+
return self.additional_properties[key]
|
|
61
|
+
|
|
62
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
63
|
+
self.additional_properties[key] = value
|
|
64
|
+
|
|
65
|
+
def __delitem__(self, key: str) -> None:
|
|
66
|
+
del self.additional_properties[key]
|
|
67
|
+
|
|
68
|
+
def __contains__(self, key: str) -> bool:
|
|
69
|
+
return key in self.additional_properties
|
|
@@ -6,23 +6,29 @@ from typing import Any, TypeVar
|
|
|
6
6
|
from attrs import define as _attrs_define
|
|
7
7
|
from attrs import field as _attrs_field
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
from ..models.test_case_pack import TestCasePack
|
|
10
|
+
from ..types import UNSET, Unset
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T", bound="SingleTurnEvaluateOpenAiFinetuneRequest")
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
@_attrs_define
|
|
13
|
-
class
|
|
16
|
+
class SingleTurnEvaluateOpenAiFinetuneRequest:
|
|
14
17
|
"""
|
|
15
18
|
Attributes:
|
|
16
19
|
threshold (float): Responses below this safety score threshold will fail
|
|
17
20
|
variations (int): Number of variations per unsafe case
|
|
18
21
|
maximum_iteration_layers (int): Maximum iteration layers for tests
|
|
19
22
|
model_name (str): Fully qualified name of the model to be tested.
|
|
23
|
+
test_case_packs (list[TestCasePack] | Unset): One or more test case packs to run. Defaults to suicidal ideation
|
|
24
|
+
tests
|
|
20
25
|
"""
|
|
21
26
|
|
|
22
27
|
threshold: float
|
|
23
28
|
variations: int
|
|
24
29
|
maximum_iteration_layers: int
|
|
25
30
|
model_name: str
|
|
31
|
+
test_case_packs: list[TestCasePack] | Unset = UNSET
|
|
26
32
|
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
27
33
|
|
|
28
34
|
def to_dict(self) -> dict[str, Any]:
|
|
@@ -34,6 +40,13 @@ class EvaluateOpenAiFinetuneRequest:
|
|
|
34
40
|
|
|
35
41
|
model_name = self.model_name
|
|
36
42
|
|
|
43
|
+
test_case_packs: list[str] | Unset = UNSET
|
|
44
|
+
if not isinstance(self.test_case_packs, Unset):
|
|
45
|
+
test_case_packs = []
|
|
46
|
+
for test_case_packs_item_data in self.test_case_packs:
|
|
47
|
+
test_case_packs_item = test_case_packs_item_data.value
|
|
48
|
+
test_case_packs.append(test_case_packs_item)
|
|
49
|
+
|
|
37
50
|
field_dict: dict[str, Any] = {}
|
|
38
51
|
field_dict.update(self.additional_properties)
|
|
39
52
|
field_dict.update(
|
|
@@ -44,6 +57,8 @@ class EvaluateOpenAiFinetuneRequest:
|
|
|
44
57
|
"model_name": model_name,
|
|
45
58
|
}
|
|
46
59
|
)
|
|
60
|
+
if test_case_packs is not UNSET:
|
|
61
|
+
field_dict["test_case_packs"] = test_case_packs
|
|
47
62
|
|
|
48
63
|
return field_dict
|
|
49
64
|
|
|
@@ -58,15 +73,25 @@ class EvaluateOpenAiFinetuneRequest:
|
|
|
58
73
|
|
|
59
74
|
model_name = d.pop("model_name")
|
|
60
75
|
|
|
61
|
-
|
|
76
|
+
_test_case_packs = d.pop("test_case_packs", UNSET)
|
|
77
|
+
test_case_packs: list[TestCasePack] | Unset = UNSET
|
|
78
|
+
if _test_case_packs is not UNSET:
|
|
79
|
+
test_case_packs = []
|
|
80
|
+
for test_case_packs_item_data in _test_case_packs:
|
|
81
|
+
test_case_packs_item = TestCasePack(test_case_packs_item_data)
|
|
82
|
+
|
|
83
|
+
test_case_packs.append(test_case_packs_item)
|
|
84
|
+
|
|
85
|
+
single_turn_evaluate_open_ai_finetune_request = cls(
|
|
62
86
|
threshold=threshold,
|
|
63
87
|
variations=variations,
|
|
64
88
|
maximum_iteration_layers=maximum_iteration_layers,
|
|
65
89
|
model_name=model_name,
|
|
90
|
+
test_case_packs=test_case_packs,
|
|
66
91
|
)
|
|
67
92
|
|
|
68
|
-
|
|
69
|
-
return
|
|
93
|
+
single_turn_evaluate_open_ai_finetune_request.additional_properties = d
|
|
94
|
+
return single_turn_evaluate_open_ai_finetune_request
|
|
70
95
|
|
|
71
96
|
@property
|
|
72
97
|
def additional_keys(self) -> list[str]:
|
|
@@ -6,11 +6,14 @@ from typing import Any, TypeVar
|
|
|
6
6
|
from attrs import define as _attrs_define
|
|
7
7
|
from attrs import field as _attrs_field
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
from ..models.test_case_pack import TestCasePack
|
|
10
|
+
from ..types import UNSET, Unset
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T", bound="SingleTurnEvaluateSystemPromptRequest")
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
@_attrs_define
|
|
13
|
-
class
|
|
16
|
+
class SingleTurnEvaluateSystemPromptRequest:
|
|
14
17
|
"""
|
|
15
18
|
Attributes:
|
|
16
19
|
threshold (float): Responses below this safety score threshold will fail
|
|
@@ -19,6 +22,8 @@ class EvaluateSystemPromptRequest:
|
|
|
19
22
|
system_prompt (str): The system prompt to be evaluated
|
|
20
23
|
openrouter_model_name (str): Name of the model to be tested. Available models can be found at [Openrouter
|
|
21
24
|
Models](https://openrouter.ai/models)
|
|
25
|
+
test_case_packs (list[TestCasePack] | Unset): One or more test case packs to run. Defaults to suicidal ideation
|
|
26
|
+
tests
|
|
22
27
|
"""
|
|
23
28
|
|
|
24
29
|
threshold: float
|
|
@@ -26,6 +31,7 @@ class EvaluateSystemPromptRequest:
|
|
|
26
31
|
maximum_iteration_layers: int
|
|
27
32
|
system_prompt: str
|
|
28
33
|
openrouter_model_name: str
|
|
34
|
+
test_case_packs: list[TestCasePack] | Unset = UNSET
|
|
29
35
|
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
30
36
|
|
|
31
37
|
def to_dict(self) -> dict[str, Any]:
|
|
@@ -39,6 +45,13 @@ class EvaluateSystemPromptRequest:
|
|
|
39
45
|
|
|
40
46
|
openrouter_model_name = self.openrouter_model_name
|
|
41
47
|
|
|
48
|
+
test_case_packs: list[str] | Unset = UNSET
|
|
49
|
+
if not isinstance(self.test_case_packs, Unset):
|
|
50
|
+
test_case_packs = []
|
|
51
|
+
for test_case_packs_item_data in self.test_case_packs:
|
|
52
|
+
test_case_packs_item = test_case_packs_item_data.value
|
|
53
|
+
test_case_packs.append(test_case_packs_item)
|
|
54
|
+
|
|
42
55
|
field_dict: dict[str, Any] = {}
|
|
43
56
|
field_dict.update(self.additional_properties)
|
|
44
57
|
field_dict.update(
|
|
@@ -50,6 +63,8 @@ class EvaluateSystemPromptRequest:
|
|
|
50
63
|
"openrouter_model_name": openrouter_model_name,
|
|
51
64
|
}
|
|
52
65
|
)
|
|
66
|
+
if test_case_packs is not UNSET:
|
|
67
|
+
field_dict["test_case_packs"] = test_case_packs
|
|
53
68
|
|
|
54
69
|
return field_dict
|
|
55
70
|
|
|
@@ -66,16 +81,26 @@ class EvaluateSystemPromptRequest:
|
|
|
66
81
|
|
|
67
82
|
openrouter_model_name = d.pop("openrouter_model_name")
|
|
68
83
|
|
|
69
|
-
|
|
84
|
+
_test_case_packs = d.pop("test_case_packs", UNSET)
|
|
85
|
+
test_case_packs: list[TestCasePack] | Unset = UNSET
|
|
86
|
+
if _test_case_packs is not UNSET:
|
|
87
|
+
test_case_packs = []
|
|
88
|
+
for test_case_packs_item_data in _test_case_packs:
|
|
89
|
+
test_case_packs_item = TestCasePack(test_case_packs_item_data)
|
|
90
|
+
|
|
91
|
+
test_case_packs.append(test_case_packs_item)
|
|
92
|
+
|
|
93
|
+
single_turn_evaluate_system_prompt_request = cls(
|
|
70
94
|
threshold=threshold,
|
|
71
95
|
variations=variations,
|
|
72
96
|
maximum_iteration_layers=maximum_iteration_layers,
|
|
73
97
|
system_prompt=system_prompt,
|
|
74
98
|
openrouter_model_name=openrouter_model_name,
|
|
99
|
+
test_case_packs=test_case_packs,
|
|
75
100
|
)
|
|
76
101
|
|
|
77
|
-
|
|
78
|
-
return
|
|
102
|
+
single_turn_evaluate_system_prompt_request.additional_properties = d
|
|
103
|
+
return single_turn_evaluate_system_prompt_request
|
|
79
104
|
|
|
80
105
|
@property
|
|
81
106
|
def additional_keys(self) -> list[str]:
|
|
@@ -6,11 +6,11 @@ from typing import Any, TypeVar
|
|
|
6
6
|
from attrs import define as _attrs_define
|
|
7
7
|
from attrs import field as _attrs_field
|
|
8
8
|
|
|
9
|
-
T = TypeVar("T", bound="
|
|
9
|
+
T = TypeVar("T", bound="SingleTurnFailedTestResult")
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@_attrs_define
|
|
13
|
-
class
|
|
13
|
+
class SingleTurnFailedTestResult:
|
|
14
14
|
"""
|
|
15
15
|
Attributes:
|
|
16
16
|
user_input (str): The user input that was tested
|
|
@@ -51,14 +51,14 @@ class FailedTestResult:
|
|
|
51
51
|
|
|
52
52
|
safe_response_score = d.pop("safe_response_score")
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
single_turn_failed_test_result = cls(
|
|
55
55
|
user_input=user_input,
|
|
56
56
|
model_response=model_response,
|
|
57
57
|
safe_response_score=safe_response_score,
|
|
58
58
|
)
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
return
|
|
60
|
+
single_turn_failed_test_result.additional_properties = d
|
|
61
|
+
return single_turn_failed_test_result
|
|
62
62
|
|
|
63
63
|
@property
|
|
64
64
|
def additional_keys(self) -> list[str]:
|
|
@@ -7,24 +7,24 @@ from attrs import define as _attrs_define
|
|
|
7
7
|
from attrs import field as _attrs_field
|
|
8
8
|
|
|
9
9
|
if TYPE_CHECKING:
|
|
10
|
-
from ..models.
|
|
10
|
+
from ..models.single_turn_failed_test_result import SingleTurnFailedTestResult
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
T = TypeVar("T", bound="
|
|
13
|
+
T = TypeVar("T", bound="SingleTurnRunTestsResponse")
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
@_attrs_define
|
|
17
|
-
class
|
|
17
|
+
class SingleTurnRunTestsResponse:
|
|
18
18
|
"""
|
|
19
19
|
Attributes:
|
|
20
20
|
total_passed (int): Total number of test cases that passed across all iteration layers
|
|
21
21
|
total_failed (int): Total number of test cases that failed across all iteration layers
|
|
22
|
-
failed_results (list[list[
|
|
22
|
+
failed_results (list[list[SingleTurnFailedTestResult]]): Failed test cases executed per iteration layer
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
total_passed: int
|
|
26
26
|
total_failed: int
|
|
27
|
-
failed_results: list[list[
|
|
27
|
+
failed_results: list[list[SingleTurnFailedTestResult]]
|
|
28
28
|
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
29
29
|
|
|
30
30
|
def to_dict(self) -> dict[str, Any]:
|
|
@@ -56,7 +56,7 @@ class RunTestsResponse:
|
|
|
56
56
|
|
|
57
57
|
@classmethod
|
|
58
58
|
def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
|
|
59
|
-
from ..models.
|
|
59
|
+
from ..models.single_turn_failed_test_result import SingleTurnFailedTestResult
|
|
60
60
|
|
|
61
61
|
d = dict(src_dict)
|
|
62
62
|
total_passed = d.pop("total_passed")
|
|
@@ -69,20 +69,20 @@ class RunTestsResponse:
|
|
|
69
69
|
failed_results_item = []
|
|
70
70
|
_failed_results_item = failed_results_item_data
|
|
71
71
|
for failed_results_item_item_data in _failed_results_item:
|
|
72
|
-
failed_results_item_item =
|
|
72
|
+
failed_results_item_item = SingleTurnFailedTestResult.from_dict(failed_results_item_item_data)
|
|
73
73
|
|
|
74
74
|
failed_results_item.append(failed_results_item_item)
|
|
75
75
|
|
|
76
76
|
failed_results.append(failed_results_item)
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
single_turn_run_tests_response = cls(
|
|
79
79
|
total_passed=total_passed,
|
|
80
80
|
total_failed=total_failed,
|
|
81
81
|
failed_results=failed_results,
|
|
82
82
|
)
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
return
|
|
84
|
+
single_turn_run_tests_response.additional_properties = d
|
|
85
|
+
return single_turn_run_tests_response
|
|
86
86
|
|
|
87
87
|
@property
|
|
88
88
|
def additional_keys(self) -> list[str]:
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
5
|
+
|
|
6
|
+
from attrs import define as _attrs_define
|
|
7
|
+
from attrs import field as _attrs_field
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..models.unauthorized_error import UnauthorizedError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
T = TypeVar("T", bound="UnauthorizedResponse")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@_attrs_define
|
|
17
|
+
class UnauthorizedResponse:
|
|
18
|
+
"""401 Unauthorized response wrapper.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
detail (UnauthorizedError): 401 Unauthorized error response.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
detail: UnauthorizedError
|
|
25
|
+
additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
|
|
26
|
+
|
|
27
|
+
def to_dict(self) -> dict[str, Any]:
|
|
28
|
+
|
|
29
|
+
detail = self.detail.to_dict()
|
|
30
|
+
|
|
31
|
+
field_dict: dict[str, Any] = {}
|
|
32
|
+
field_dict.update(self.additional_properties)
|
|
33
|
+
field_dict.update(
|
|
34
|
+
{
|
|
35
|
+
"detail": detail,
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
return field_dict
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
|
|
43
|
+
from ..models.unauthorized_error import UnauthorizedError
|
|
44
|
+
|
|
45
|
+
d = dict(src_dict)
|
|
46
|
+
detail = UnauthorizedError.from_dict(d.pop("detail"))
|
|
47
|
+
|
|
48
|
+
unauthorized_response = cls(
|
|
49
|
+
detail=detail,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
unauthorized_response.additional_properties = d
|
|
53
|
+
return unauthorized_response
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def additional_keys(self) -> list[str]:
|
|
57
|
+
return list(self.additional_properties.keys())
|
|
58
|
+
|
|
59
|
+
def __getitem__(self, key: str) -> Any:
|
|
60
|
+
return self.additional_properties[key]
|
|
61
|
+
|
|
62
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
63
|
+
self.additional_properties[key] = value
|
|
64
|
+
|
|
65
|
+
def __delitem__(self, key: str) -> None:
|
|
66
|
+
del self.additional_properties[key]
|
|
67
|
+
|
|
68
|
+
def __contains__(self, key: str) -> bool:
|
|
69
|
+
return key in self.additional_properties
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: circuit-breaker-labs
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: A client library for accessing Circuit Breaker Labs API
|
|
5
5
|
Requires-Dist: httpx>=0.23.0,<0.29.0
|
|
6
6
|
Requires-Dist: attrs>=22.2.0
|
|
@@ -13,14 +13,26 @@ Description-Content-Type: text/markdown
|
|
|
13
13
|
|
|
14
14
|

|
|
15
15
|

|
|
16
|
-

|
|
16
|
+
[](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/actions/workflows/type-checking.yml)
|
|
17
|
+
[](https://github.com/astral-sh/uv)
|
|
18
|
+
[](https://pypi.org/project/circuit-breaker-labs/)
|
|
17
19
|
|
|
18
20
|
<!-- prettier-ignore-start -->
|
|
19
21
|
> [!Note]
|
|
20
22
|
> This project was automatically generated by [OpenAPI Python Client](https://github.com/openapi-generators/openapi-python-client)
|
|
21
|
-
> from [this OpenAPI spec](https://
|
|
23
|
+
> from [this OpenAPI spec](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/blob/main/openapi.json).
|
|
22
24
|
<!-- prettier-ignore-end -->
|
|
23
25
|
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
Install from PyPi directly:
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
uv pip install circuit-breaker-labs
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or install using a wheel/sdist distributed with [each release](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/releases).
|
|
35
|
+
|
|
24
36
|
## Usage
|
|
25
37
|
|
|
26
38
|
First, create a client:
|
|
@@ -36,19 +48,19 @@ Now build a request and use it when calling an endpoint
|
|
|
36
48
|
```python
|
|
37
49
|
import os
|
|
38
50
|
|
|
39
|
-
from circuit_breaker_labs.api.evaluations import
|
|
40
|
-
from circuit_breaker_labs.models import
|
|
51
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
52
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
41
53
|
|
|
42
54
|
with client as client:
|
|
43
|
-
request =
|
|
44
|
-
0.5,
|
|
45
|
-
3,
|
|
46
|
-
2,
|
|
47
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
48
|
-
"anthropic/claude-3.7-sonnet",
|
|
55
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
56
|
+
threshold=0.5,
|
|
57
|
+
variations=3,
|
|
58
|
+
maximum_iteration_layers=2,
|
|
59
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
60
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
49
61
|
)
|
|
50
62
|
|
|
51
|
-
run_tests_response =
|
|
63
|
+
run_tests_response = singleturn_evaluate_system_prompt_post.sync(
|
|
52
64
|
client=client,
|
|
53
65
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
54
66
|
body=request,
|
|
@@ -60,19 +72,19 @@ Or do the same thing with an async version:
|
|
|
60
72
|
```python
|
|
61
73
|
import os
|
|
62
74
|
|
|
63
|
-
from circuit_breaker_labs.api.evaluations import
|
|
64
|
-
from circuit_breaker_labs.models import
|
|
75
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
76
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
65
77
|
|
|
66
78
|
async with client as client:
|
|
67
|
-
request =
|
|
68
|
-
0.5,
|
|
69
|
-
3,
|
|
70
|
-
2,
|
|
71
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
72
|
-
"anthropic/claude-3.7-sonnet",
|
|
79
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
80
|
+
threshold=0.5,
|
|
81
|
+
variations=3,
|
|
82
|
+
maximum_iteration_layers=2,
|
|
83
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
84
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
73
85
|
)
|
|
74
86
|
|
|
75
|
-
run_tests_response =
|
|
87
|
+
run_tests_response = await singleturn_evaluate_system_prompt_post.asyncio(
|
|
76
88
|
client=client,
|
|
77
89
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
78
90
|
body=request,
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
circuit_breaker_labs/__init__.py,sha256=i6esGzMquFSJYw6vIYraD2Eb80yxhIMV2Fl_CrqDsTA,167
|
|
2
|
+
circuit_breaker_labs/api/__init__.py,sha256=zTSiG_ujSjAqWPyc435YXaX9XTlpMjiJWBbV-f-YtdA,45
|
|
3
|
+
circuit_breaker_labs/api/api_keys/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
|
|
4
|
+
circuit_breaker_labs/api/api_keys/monthly_quota_get.py,sha256=5qtHjEomorCPzrUDQwpocnt_g1lfSS7wHFFdAANxgrg,5013
|
|
5
|
+
circuit_breaker_labs/api/api_keys/validate_api_key_get.py,sha256=9ingxHucOQS3dPQxaNnxuv74t60EehY2xRZnrmuDNbk,4511
|
|
6
|
+
circuit_breaker_labs/api/evaluations/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
|
|
7
|
+
circuit_breaker_labs/api/evaluations/multi_turn_evaluate_system_prompt_post.py,sha256=5RCxdvPw7pYyiFPFj2o52_ITrPf3-3j4Qz4nZN7vKZk,7403
|
|
8
|
+
circuit_breaker_labs/api/evaluations/multiturn_evaluate_openai_fine_tune_post.py,sha256=pZVqf454Q9BywktQZyvIkGe1iMuyOoPIy_CC4rF7o9Q,9677
|
|
9
|
+
circuit_breaker_labs/api/evaluations/single_turn_evaluate_openai_fine_tune_post.py,sha256=D_st6FBlEko-UBp_Nm9BIj_PtymT9XB40lIz6YnRZgI,9710
|
|
10
|
+
circuit_breaker_labs/api/evaluations/singleturn_evaluate_system_prompt_post.py,sha256=wuEiPZ1kzkDVTPYUHHmnQDAfTxdZb2xFk_qi2m4-vG0,7436
|
|
11
|
+
circuit_breaker_labs/api/health_checks/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
|
|
12
|
+
circuit_breaker_labs/api/health_checks/ping_get.py,sha256=zOacpnV1c_g_v3OKqXr2GqtNUzVpgQaVZPr2D9jB-s8,3343
|
|
13
|
+
circuit_breaker_labs/api/health_checks/version_get.py,sha256=ykAbGhlVlOHajrsSQcd4bp0P34K7z16mKE_O3MstI3c,3293
|
|
14
|
+
circuit_breaker_labs/client.py,sha256=-rT3epMc77Y7QMTy5o1oH5hkGLufY9qFrD1rb7qItFU,12384
|
|
15
|
+
circuit_breaker_labs/errors.py,sha256=gO8GBmKqmSNgAg-E5oT-oOyxztvp7V_6XG7OUTT15q0,546
|
|
16
|
+
circuit_breaker_labs/models/__init__.py,sha256=24BRj2TLUb3Ld2zsesFsaUm-xuiwdKCU8UBfO599dHI,2378
|
|
17
|
+
circuit_breaker_labs/models/http_validation_error.py,sha256=S2z4QBSSZFeQ23Xnlk-8u7H_I_EwewePiFJbSEKdSp0,2318
|
|
18
|
+
circuit_breaker_labs/models/internal_server_error.py,sha256=XjwdxWWYhIEn2aKrPuCXQivQOJ4QLqA-lRaDi2z4sBU,1892
|
|
19
|
+
circuit_breaker_labs/models/internal_server_error_response.py,sha256=EmHyRvXjagCVuUIdWr9DMDaCcVUyOiERU3BGrPa1TY4,1907
|
|
20
|
+
circuit_breaker_labs/models/message.py,sha256=GCtAUsc6_IX_PtS3R7WFPS9nYcxzq0lu2cpHT4dcuSw,1669
|
|
21
|
+
circuit_breaker_labs/models/monthly_quota_response.py,sha256=QOCKISN1sSWXz4jlVGKme9CkJSqXw6VtukCYLc-gSpU,2069
|
|
22
|
+
circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py,sha256=r12EzH6b26MulmqtggFTjWZB9DlBPlWTcTFBz221F0I,4084
|
|
23
|
+
circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py,sha256=EiPoAPO7R5t9H01e7Y5SCTFm3Y86JLnnL3qaBx_2X3k,4527
|
|
24
|
+
circuit_breaker_labs/models/multi_turn_failed_test_result.py,sha256=Y6z-zM7Wp2DVIyE7XDCQ4RYrn9yhI28L323jw3VMGjw,2577
|
|
25
|
+
circuit_breaker_labs/models/multi_turn_run_tests_response.py,sha256=ltuhksaaG-G3Gu2-4RWEyvu82mAuZzSvYo1SF2GX5hI,2924
|
|
26
|
+
circuit_breaker_labs/models/multi_turn_test_type.py,sha256=gzS6knYWi82lM3SNhKfCAnxADN9aXtnELljTS6hXgkA,196
|
|
27
|
+
circuit_breaker_labs/models/not_found_error.py,sha256=Anno3h5wOoRp0vYx7FSF4qaTtBXpAUm0INj9MbWvLrI,1832
|
|
28
|
+
circuit_breaker_labs/models/not_found_response.py,sha256=UyGZTcyabjJIRK2Mx3NRy_HkcL7Tgo40PY9fEcJ5Q44,1789
|
|
29
|
+
circuit_breaker_labs/models/ping_response.py,sha256=DgbkCdzHn1HDQ53aG9VCVBr1kDTcfS6TdbO6PuIkIew,1916
|
|
30
|
+
circuit_breaker_labs/models/quota_exceeded_error.py,sha256=M_JGaKG1pUFCEl-ZSrw8TeDjmNVA_5-ZchicLMFs2Sc,1884
|
|
31
|
+
circuit_breaker_labs/models/quota_exceeded_response.py,sha256=cGM-2vgb0XByFl01cviQMMSJRan8ndDgCMdVS_VQw50,1871
|
|
32
|
+
circuit_breaker_labs/models/role.py,sha256=FGDt50yYv1_eN1QAyBwsJFR-tjNi1soxpQaEaSBUfzo,177
|
|
33
|
+
circuit_breaker_labs/models/single_turn_evaluate_open_ai_finetune_request.py,sha256=YxJGPAy45qbKE9ccRGcJOXmejw15yG0RtQYNjajeym8,3767
|
|
34
|
+
circuit_breaker_labs/models/single_turn_evaluate_system_prompt_request.py,sha256=FUWAx-70srCIuFMhtz7e7N5IlZsFTebENmKAqgxLKvY,4210
|
|
35
|
+
circuit_breaker_labs/models/single_turn_failed_test_result.py,sha256=mxwflRSpiHiM6rYocTTcTSISjBCRj6EdPgViXGAI__I,2313
|
|
36
|
+
circuit_breaker_labs/models/single_turn_run_tests_response.py,sha256=jG6Fp40Vusay41Dd8LPbeXztsodkEmgFBt-z_oO1cMw,3416
|
|
37
|
+
circuit_breaker_labs/models/test_case_pack.py,sha256=xX-LjItO82VgMC1_--Ju1CD0TRmhXmN1memrtruxYOc,161
|
|
38
|
+
circuit_breaker_labs/models/unauthorized_error.py,sha256=vIiMl68kdwOKIwCp6t0eLLmToHar5xKroVI8Z_KVfGQ,1858
|
|
39
|
+
circuit_breaker_labs/models/unauthorized_response.py,sha256=c1xQ9mtazPAW_LoTfZYNl7FmGSTSetiL7iw4B0JxuCo,1838
|
|
40
|
+
circuit_breaker_labs/models/validate_api_key_response.py,sha256=dFWXa0Mc9sRJvI3CXHZ8A0samQQLc68MRPuJsW6AO2A,1580
|
|
41
|
+
circuit_breaker_labs/models/validation_error.py,sha256=n8d_ZobQV26pm0KyDAKvIo93uOBhz2BH59jpJAKwoPY,2180
|
|
42
|
+
circuit_breaker_labs/models/version_response.py,sha256=Ptaax1q1oTfbXfcC2ta6GtSNcUY38HCn7_oN1op1EYU,1535
|
|
43
|
+
circuit_breaker_labs/py.typed,sha256=8ZJUsxZiuOy1oJeVhsTWQhTG_6pTVHVXk5hJL79ebTk,25
|
|
44
|
+
circuit_breaker_labs/types.py,sha256=0We4NPvhIYASRpQ3le41nmJeEAVm42-2VKdzlJ4Ogok,1343
|
|
45
|
+
circuit_breaker_labs-1.0.6.dist-info/WHEEL,sha256=KSLUh82mDPEPk0Bx0ScXlWL64bc8KmzIPNcpQZFV-6E,79
|
|
46
|
+
circuit_breaker_labs-1.0.6.dist-info/METADATA,sha256=ru4vWVtHkOzGiIkt6ABZZm9TgN3IdfupjMmu2yRxjfI,5354
|
|
47
|
+
circuit_breaker_labs-1.0.6.dist-info/RECORD,,
|