circuit-breaker-labs 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. circuit_breaker_labs/api/api_keys/monthly_quota_get.py +12 -12
  2. circuit_breaker_labs/api/evaluations/{evaluate_system_prompt_post.py → multi_turn_evaluate_system_prompt_post.py} +96 -39
  3. circuit_breaker_labs/api/evaluations/{evaluate_openai_fine_tune_post.py → multiturn_evaluate_openai_fine_tune_post.py} +96 -39
  4. circuit_breaker_labs/api/evaluations/single_turn_evaluate_openai_fine_tune_post.py +300 -0
  5. circuit_breaker_labs/api/evaluations/singleturn_evaluate_system_prompt_post.py +257 -0
  6. circuit_breaker_labs/models/__init__.py +36 -8
  7. circuit_breaker_labs/models/internal_server_error.py +73 -0
  8. circuit_breaker_labs/models/internal_server_error_response.py +69 -0
  9. circuit_breaker_labs/models/message.py +71 -0
  10. circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py +119 -0
  11. circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py +128 -0
  12. circuit_breaker_labs/models/multi_turn_failed_test_result.py +85 -0
  13. circuit_breaker_labs/models/multi_turn_run_tests_response.py +92 -0
  14. circuit_breaker_labs/models/multi_turn_test_type.py +9 -0
  15. circuit_breaker_labs/models/not_found_error.py +73 -0
  16. circuit_breaker_labs/models/not_found_response.py +69 -0
  17. circuit_breaker_labs/models/quota_exceeded_response.py +69 -0
  18. circuit_breaker_labs/models/role.py +10 -0
  19. circuit_breaker_labs/models/{evaluate_open_ai_finetune_request.py → single_turn_evaluate_open_ai_finetune_request.py} +30 -5
  20. circuit_breaker_labs/models/{evaluate_system_prompt_request.py → single_turn_evaluate_system_prompt_request.py} +30 -5
  21. circuit_breaker_labs/models/{failed_test_result.py → single_turn_failed_test_result.py} +5 -5
  22. circuit_breaker_labs/models/{run_tests_response.py → single_turn_run_tests_response.py} +10 -10
  23. circuit_breaker_labs/models/test_case_pack.py +8 -0
  24. circuit_breaker_labs/models/unauthorized_response.py +69 -0
  25. {circuit_breaker_labs-1.0.4.dist-info → circuit_breaker_labs-1.0.6.dist-info}/METADATA +33 -21
  26. circuit_breaker_labs-1.0.6.dist-info/RECORD +47 -0
  27. {circuit_breaker_labs-1.0.4.dist-info → circuit_breaker_labs-1.0.6.dist-info}/WHEEL +1 -1
  28. circuit_breaker_labs-1.0.4.dist-info/RECORD +0 -31
@@ -0,0 +1,69 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Mapping
4
+ from typing import TYPE_CHECKING, Any, TypeVar
5
+
6
+ from attrs import define as _attrs_define
7
+ from attrs import field as _attrs_field
8
+
9
+ if TYPE_CHECKING:
10
+ from ..models.not_found_error import NotFoundError
11
+
12
+
13
+ T = TypeVar("T", bound="NotFoundResponse")
14
+
15
+
16
+ @_attrs_define
17
+ class NotFoundResponse:
18
+ """404 Not Found response wrapper.
19
+
20
+ Attributes:
21
+ detail (NotFoundError): 404 Not Found error response.
22
+ """
23
+
24
+ detail: NotFoundError
25
+ additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
26
+
27
+ def to_dict(self) -> dict[str, Any]:
28
+
29
+ detail = self.detail.to_dict()
30
+
31
+ field_dict: dict[str, Any] = {}
32
+ field_dict.update(self.additional_properties)
33
+ field_dict.update(
34
+ {
35
+ "detail": detail,
36
+ }
37
+ )
38
+
39
+ return field_dict
40
+
41
+ @classmethod
42
+ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
43
+ from ..models.not_found_error import NotFoundError
44
+
45
+ d = dict(src_dict)
46
+ detail = NotFoundError.from_dict(d.pop("detail"))
47
+
48
+ not_found_response = cls(
49
+ detail=detail,
50
+ )
51
+
52
+ not_found_response.additional_properties = d
53
+ return not_found_response
54
+
55
+ @property
56
+ def additional_keys(self) -> list[str]:
57
+ return list(self.additional_properties.keys())
58
+
59
+ def __getitem__(self, key: str) -> Any:
60
+ return self.additional_properties[key]
61
+
62
+ def __setitem__(self, key: str, value: Any) -> None:
63
+ self.additional_properties[key] = value
64
+
65
+ def __delitem__(self, key: str) -> None:
66
+ del self.additional_properties[key]
67
+
68
+ def __contains__(self, key: str) -> bool:
69
+ return key in self.additional_properties
@@ -0,0 +1,69 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Mapping
4
+ from typing import TYPE_CHECKING, Any, TypeVar
5
+
6
+ from attrs import define as _attrs_define
7
+ from attrs import field as _attrs_field
8
+
9
+ if TYPE_CHECKING:
10
+ from ..models.quota_exceeded_error import QuotaExceededError
11
+
12
+
13
+ T = TypeVar("T", bound="QuotaExceededResponse")
14
+
15
+
16
+ @_attrs_define
17
+ class QuotaExceededResponse:
18
+ """403 Quota exceeded response wrapper.
19
+
20
+ Attributes:
21
+ detail (QuotaExceededError): 403 Forbidden error response for quota limits.
22
+ """
23
+
24
+ detail: QuotaExceededError
25
+ additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
26
+
27
+ def to_dict(self) -> dict[str, Any]:
28
+
29
+ detail = self.detail.to_dict()
30
+
31
+ field_dict: dict[str, Any] = {}
32
+ field_dict.update(self.additional_properties)
33
+ field_dict.update(
34
+ {
35
+ "detail": detail,
36
+ }
37
+ )
38
+
39
+ return field_dict
40
+
41
+ @classmethod
42
+ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
43
+ from ..models.quota_exceeded_error import QuotaExceededError
44
+
45
+ d = dict(src_dict)
46
+ detail = QuotaExceededError.from_dict(d.pop("detail"))
47
+
48
+ quota_exceeded_response = cls(
49
+ detail=detail,
50
+ )
51
+
52
+ quota_exceeded_response.additional_properties = d
53
+ return quota_exceeded_response
54
+
55
+ @property
56
+ def additional_keys(self) -> list[str]:
57
+ return list(self.additional_properties.keys())
58
+
59
+ def __getitem__(self, key: str) -> Any:
60
+ return self.additional_properties[key]
61
+
62
+ def __setitem__(self, key: str, value: Any) -> None:
63
+ self.additional_properties[key] = value
64
+
65
+ def __delitem__(self, key: str) -> None:
66
+ del self.additional_properties[key]
67
+
68
+ def __contains__(self, key: str) -> bool:
69
+ return key in self.additional_properties
@@ -0,0 +1,10 @@
1
+ from enum import Enum
2
+
3
+
4
+ class Role(str, Enum):
5
+ ASSISTANT = "assistant"
6
+ SYSTEM = "system"
7
+ USER = "user"
8
+
9
+ def __str__(self) -> str:
10
+ return str(self.value)
@@ -6,23 +6,29 @@ from typing import Any, TypeVar
6
6
  from attrs import define as _attrs_define
7
7
  from attrs import field as _attrs_field
8
8
 
9
- T = TypeVar("T", bound="EvaluateOpenAiFinetuneRequest")
9
+ from ..models.test_case_pack import TestCasePack
10
+ from ..types import UNSET, Unset
11
+
12
+ T = TypeVar("T", bound="SingleTurnEvaluateOpenAiFinetuneRequest")
10
13
 
11
14
 
12
15
  @_attrs_define
13
- class EvaluateOpenAiFinetuneRequest:
16
+ class SingleTurnEvaluateOpenAiFinetuneRequest:
14
17
  """
15
18
  Attributes:
16
19
  threshold (float): Responses below this safety score threshold will fail
17
20
  variations (int): Number of variations per unsafe case
18
21
  maximum_iteration_layers (int): Maximum iteration layers for tests
19
22
  model_name (str): Fully qualified name of the model to be tested.
23
+ test_case_packs (list[TestCasePack] | Unset): One or more test case packs to run. Defaults to suicidal ideation
24
+ tests
20
25
  """
21
26
 
22
27
  threshold: float
23
28
  variations: int
24
29
  maximum_iteration_layers: int
25
30
  model_name: str
31
+ test_case_packs: list[TestCasePack] | Unset = UNSET
26
32
  additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
27
33
 
28
34
  def to_dict(self) -> dict[str, Any]:
@@ -34,6 +40,13 @@ class EvaluateOpenAiFinetuneRequest:
34
40
 
35
41
  model_name = self.model_name
36
42
 
43
+ test_case_packs: list[str] | Unset = UNSET
44
+ if not isinstance(self.test_case_packs, Unset):
45
+ test_case_packs = []
46
+ for test_case_packs_item_data in self.test_case_packs:
47
+ test_case_packs_item = test_case_packs_item_data.value
48
+ test_case_packs.append(test_case_packs_item)
49
+
37
50
  field_dict: dict[str, Any] = {}
38
51
  field_dict.update(self.additional_properties)
39
52
  field_dict.update(
@@ -44,6 +57,8 @@ class EvaluateOpenAiFinetuneRequest:
44
57
  "model_name": model_name,
45
58
  }
46
59
  )
60
+ if test_case_packs is not UNSET:
61
+ field_dict["test_case_packs"] = test_case_packs
47
62
 
48
63
  return field_dict
49
64
 
@@ -58,15 +73,25 @@ class EvaluateOpenAiFinetuneRequest:
58
73
 
59
74
  model_name = d.pop("model_name")
60
75
 
61
- evaluate_open_ai_finetune_request = cls(
76
+ _test_case_packs = d.pop("test_case_packs", UNSET)
77
+ test_case_packs: list[TestCasePack] | Unset = UNSET
78
+ if _test_case_packs is not UNSET:
79
+ test_case_packs = []
80
+ for test_case_packs_item_data in _test_case_packs:
81
+ test_case_packs_item = TestCasePack(test_case_packs_item_data)
82
+
83
+ test_case_packs.append(test_case_packs_item)
84
+
85
+ single_turn_evaluate_open_ai_finetune_request = cls(
62
86
  threshold=threshold,
63
87
  variations=variations,
64
88
  maximum_iteration_layers=maximum_iteration_layers,
65
89
  model_name=model_name,
90
+ test_case_packs=test_case_packs,
66
91
  )
67
92
 
68
- evaluate_open_ai_finetune_request.additional_properties = d
69
- return evaluate_open_ai_finetune_request
93
+ single_turn_evaluate_open_ai_finetune_request.additional_properties = d
94
+ return single_turn_evaluate_open_ai_finetune_request
70
95
 
71
96
  @property
72
97
  def additional_keys(self) -> list[str]:
@@ -6,11 +6,14 @@ from typing import Any, TypeVar
6
6
  from attrs import define as _attrs_define
7
7
  from attrs import field as _attrs_field
8
8
 
9
- T = TypeVar("T", bound="EvaluateSystemPromptRequest")
9
+ from ..models.test_case_pack import TestCasePack
10
+ from ..types import UNSET, Unset
11
+
12
+ T = TypeVar("T", bound="SingleTurnEvaluateSystemPromptRequest")
10
13
 
11
14
 
12
15
  @_attrs_define
13
- class EvaluateSystemPromptRequest:
16
+ class SingleTurnEvaluateSystemPromptRequest:
14
17
  """
15
18
  Attributes:
16
19
  threshold (float): Responses below this safety score threshold will fail
@@ -19,6 +22,8 @@ class EvaluateSystemPromptRequest:
19
22
  system_prompt (str): The system prompt to be evaluated
20
23
  openrouter_model_name (str): Name of the model to be tested. Available models can be found at [Openrouter
21
24
  Models](https://openrouter.ai/models)
25
+ test_case_packs (list[TestCasePack] | Unset): One or more test case packs to run. Defaults to suicidal ideation
26
+ tests
22
27
  """
23
28
 
24
29
  threshold: float
@@ -26,6 +31,7 @@ class EvaluateSystemPromptRequest:
26
31
  maximum_iteration_layers: int
27
32
  system_prompt: str
28
33
  openrouter_model_name: str
34
+ test_case_packs: list[TestCasePack] | Unset = UNSET
29
35
  additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
30
36
 
31
37
  def to_dict(self) -> dict[str, Any]:
@@ -39,6 +45,13 @@ class EvaluateSystemPromptRequest:
39
45
 
40
46
  openrouter_model_name = self.openrouter_model_name
41
47
 
48
+ test_case_packs: list[str] | Unset = UNSET
49
+ if not isinstance(self.test_case_packs, Unset):
50
+ test_case_packs = []
51
+ for test_case_packs_item_data in self.test_case_packs:
52
+ test_case_packs_item = test_case_packs_item_data.value
53
+ test_case_packs.append(test_case_packs_item)
54
+
42
55
  field_dict: dict[str, Any] = {}
43
56
  field_dict.update(self.additional_properties)
44
57
  field_dict.update(
@@ -50,6 +63,8 @@ class EvaluateSystemPromptRequest:
50
63
  "openrouter_model_name": openrouter_model_name,
51
64
  }
52
65
  )
66
+ if test_case_packs is not UNSET:
67
+ field_dict["test_case_packs"] = test_case_packs
53
68
 
54
69
  return field_dict
55
70
 
@@ -66,16 +81,26 @@ class EvaluateSystemPromptRequest:
66
81
 
67
82
  openrouter_model_name = d.pop("openrouter_model_name")
68
83
 
69
- evaluate_system_prompt_request = cls(
84
+ _test_case_packs = d.pop("test_case_packs", UNSET)
85
+ test_case_packs: list[TestCasePack] | Unset = UNSET
86
+ if _test_case_packs is not UNSET:
87
+ test_case_packs = []
88
+ for test_case_packs_item_data in _test_case_packs:
89
+ test_case_packs_item = TestCasePack(test_case_packs_item_data)
90
+
91
+ test_case_packs.append(test_case_packs_item)
92
+
93
+ single_turn_evaluate_system_prompt_request = cls(
70
94
  threshold=threshold,
71
95
  variations=variations,
72
96
  maximum_iteration_layers=maximum_iteration_layers,
73
97
  system_prompt=system_prompt,
74
98
  openrouter_model_name=openrouter_model_name,
99
+ test_case_packs=test_case_packs,
75
100
  )
76
101
 
77
- evaluate_system_prompt_request.additional_properties = d
78
- return evaluate_system_prompt_request
102
+ single_turn_evaluate_system_prompt_request.additional_properties = d
103
+ return single_turn_evaluate_system_prompt_request
79
104
 
80
105
  @property
81
106
  def additional_keys(self) -> list[str]:
@@ -6,11 +6,11 @@ from typing import Any, TypeVar
6
6
  from attrs import define as _attrs_define
7
7
  from attrs import field as _attrs_field
8
8
 
9
- T = TypeVar("T", bound="FailedTestResult")
9
+ T = TypeVar("T", bound="SingleTurnFailedTestResult")
10
10
 
11
11
 
12
12
  @_attrs_define
13
- class FailedTestResult:
13
+ class SingleTurnFailedTestResult:
14
14
  """
15
15
  Attributes:
16
16
  user_input (str): The user input that was tested
@@ -51,14 +51,14 @@ class FailedTestResult:
51
51
 
52
52
  safe_response_score = d.pop("safe_response_score")
53
53
 
54
- failed_test_result = cls(
54
+ single_turn_failed_test_result = cls(
55
55
  user_input=user_input,
56
56
  model_response=model_response,
57
57
  safe_response_score=safe_response_score,
58
58
  )
59
59
 
60
- failed_test_result.additional_properties = d
61
- return failed_test_result
60
+ single_turn_failed_test_result.additional_properties = d
61
+ return single_turn_failed_test_result
62
62
 
63
63
  @property
64
64
  def additional_keys(self) -> list[str]:
@@ -7,24 +7,24 @@ from attrs import define as _attrs_define
7
7
  from attrs import field as _attrs_field
8
8
 
9
9
  if TYPE_CHECKING:
10
- from ..models.failed_test_result import FailedTestResult
10
+ from ..models.single_turn_failed_test_result import SingleTurnFailedTestResult
11
11
 
12
12
 
13
- T = TypeVar("T", bound="RunTestsResponse")
13
+ T = TypeVar("T", bound="SingleTurnRunTestsResponse")
14
14
 
15
15
 
16
16
  @_attrs_define
17
- class RunTestsResponse:
17
+ class SingleTurnRunTestsResponse:
18
18
  """
19
19
  Attributes:
20
20
  total_passed (int): Total number of test cases that passed across all iteration layers
21
21
  total_failed (int): Total number of test cases that failed across all iteration layers
22
- failed_results (list[list[FailedTestResult]]): Failed test cases executed per iteration layer
22
+ failed_results (list[list[SingleTurnFailedTestResult]]): Failed test cases executed per iteration layer
23
23
  """
24
24
 
25
25
  total_passed: int
26
26
  total_failed: int
27
- failed_results: list[list[FailedTestResult]]
27
+ failed_results: list[list[SingleTurnFailedTestResult]]
28
28
  additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
29
29
 
30
30
  def to_dict(self) -> dict[str, Any]:
@@ -56,7 +56,7 @@ class RunTestsResponse:
56
56
 
57
57
  @classmethod
58
58
  def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
59
- from ..models.failed_test_result import FailedTestResult
59
+ from ..models.single_turn_failed_test_result import SingleTurnFailedTestResult
60
60
 
61
61
  d = dict(src_dict)
62
62
  total_passed = d.pop("total_passed")
@@ -69,20 +69,20 @@ class RunTestsResponse:
69
69
  failed_results_item = []
70
70
  _failed_results_item = failed_results_item_data
71
71
  for failed_results_item_item_data in _failed_results_item:
72
- failed_results_item_item = FailedTestResult.from_dict(failed_results_item_item_data)
72
+ failed_results_item_item = SingleTurnFailedTestResult.from_dict(failed_results_item_item_data)
73
73
 
74
74
  failed_results_item.append(failed_results_item_item)
75
75
 
76
76
  failed_results.append(failed_results_item)
77
77
 
78
- run_tests_response = cls(
78
+ single_turn_run_tests_response = cls(
79
79
  total_passed=total_passed,
80
80
  total_failed=total_failed,
81
81
  failed_results=failed_results,
82
82
  )
83
83
 
84
- run_tests_response.additional_properties = d
85
- return run_tests_response
84
+ single_turn_run_tests_response.additional_properties = d
85
+ return single_turn_run_tests_response
86
86
 
87
87
  @property
88
88
  def additional_keys(self) -> list[str]:
@@ -0,0 +1,8 @@
1
+ from enum import Enum
2
+
3
+
4
+ class TestCasePack(str, Enum):
5
+ SUICIDAL_IDEATION = "suicidal_ideation"
6
+
7
+ def __str__(self) -> str:
8
+ return str(self.value)
@@ -0,0 +1,69 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Mapping
4
+ from typing import TYPE_CHECKING, Any, TypeVar
5
+
6
+ from attrs import define as _attrs_define
7
+ from attrs import field as _attrs_field
8
+
9
+ if TYPE_CHECKING:
10
+ from ..models.unauthorized_error import UnauthorizedError
11
+
12
+
13
+ T = TypeVar("T", bound="UnauthorizedResponse")
14
+
15
+
16
+ @_attrs_define
17
+ class UnauthorizedResponse:
18
+ """401 Unauthorized response wrapper.
19
+
20
+ Attributes:
21
+ detail (UnauthorizedError): 401 Unauthorized error response.
22
+ """
23
+
24
+ detail: UnauthorizedError
25
+ additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict)
26
+
27
+ def to_dict(self) -> dict[str, Any]:
28
+
29
+ detail = self.detail.to_dict()
30
+
31
+ field_dict: dict[str, Any] = {}
32
+ field_dict.update(self.additional_properties)
33
+ field_dict.update(
34
+ {
35
+ "detail": detail,
36
+ }
37
+ )
38
+
39
+ return field_dict
40
+
41
+ @classmethod
42
+ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T:
43
+ from ..models.unauthorized_error import UnauthorizedError
44
+
45
+ d = dict(src_dict)
46
+ detail = UnauthorizedError.from_dict(d.pop("detail"))
47
+
48
+ unauthorized_response = cls(
49
+ detail=detail,
50
+ )
51
+
52
+ unauthorized_response.additional_properties = d
53
+ return unauthorized_response
54
+
55
+ @property
56
+ def additional_keys(self) -> list[str]:
57
+ return list(self.additional_properties.keys())
58
+
59
+ def __getitem__(self, key: str) -> Any:
60
+ return self.additional_properties[key]
61
+
62
+ def __setitem__(self, key: str, value: Any) -> None:
63
+ self.additional_properties[key] = value
64
+
65
+ def __delitem__(self, key: str) -> None:
66
+ del self.additional_properties[key]
67
+
68
+ def __contains__(self, key: str) -> bool:
69
+ return key in self.additional_properties
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: circuit-breaker-labs
3
- Version: 1.0.4
3
+ Version: 1.0.6
4
4
  Summary: A client library for accessing Circuit Breaker Labs API
5
5
  Requires-Dist: httpx>=0.23.0,<0.29.0
6
6
  Requires-Dist: attrs>=22.2.0
@@ -13,14 +13,26 @@ Description-Content-Type: text/markdown
13
13
 
14
14
  ![Python Version](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Fcircuitbreakerlabs%2Fcircuitbreakerlabs-python%2Frefs%2Fheads%2Fmain%2Fpyproject.toml&logo=python&logoColor=yellow&label=Python&color=blue)
15
15
  ![Ruff](https://img.shields.io/badge/Ruff-Check-34223D?logo=ruff)
16
- ![MyPy](https://img.shields.io/badge/Mypy-Check-blue?logo=python)
16
+ [![MyPy](https://img.shields.io/badge/Mypy-Check-blue?logo=python)](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/actions/workflows/type-checking.yml)
17
+ [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
18
+ [![PyPI - Version](https://img.shields.io/pypi/v/circuit-breaker-labs?logo=pypi&label=PyPi)](https://pypi.org/project/circuit-breaker-labs/)
17
19
 
18
20
  <!-- prettier-ignore-start -->
19
21
  > [!Note]
20
22
  > This project was automatically generated by [OpenAPI Python Client](https://github.com/openapi-generators/openapi-python-client)
21
- > from [this OpenAPI spec](https://api.circuitbreakerlabs.ai/v1/openapi.json).
23
+ > from [this OpenAPI spec](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/blob/main/openapi.json).
22
24
  <!-- prettier-ignore-end -->
23
25
 
26
+ ## Installation
27
+
28
+ Install from PyPi directly:
29
+
30
+ ```sh
31
+ uv pip install circuit-breaker-labs
32
+ ```
33
+
34
+ Or install using a wheel/sdist distributed with [each release](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/releases).
35
+
24
36
  ## Usage
25
37
 
26
38
  First, create a client:
@@ -36,19 +48,19 @@ Now build a request and use it when calling an endpoint
36
48
  ```python
37
49
  import os
38
50
 
39
- from circuit_breaker_labs.api.evaluations import evaluate_system_prompt_post
40
- from circuit_breaker_labs.models import EvaluateSystemPromptRequest
51
+ from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
52
+ from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
41
53
 
42
54
  with client as client:
43
- request = EvaluateSystemPromptRequest(
44
- 0.5,
45
- 3,
46
- 2,
47
- os.getenv("SYSTEM_PROMPT"),
48
- "anthropic/claude-3.7-sonnet",
55
+ request = SingleTurnEvaluateSystemPromptRequest(
56
+ threshold=0.5,
57
+ variations=3,
58
+ maximum_iteration_layers=2,
59
+ system_prompt=os.getenv("SYSTEM_PROMPT"),
60
+ openrouter_model_name="anthropic/claude-3.7-sonnet",
49
61
  )
50
62
 
51
- run_tests_response = evaluate_system_prompt_post.sync(
63
+ run_tests_response = singleturn_evaluate_system_prompt_post.sync(
52
64
  client=client,
53
65
  cbl_api_key=os.getenv("CBL_API_KEY"),
54
66
  body=request,
@@ -60,19 +72,19 @@ Or do the same thing with an async version:
60
72
  ```python
61
73
  import os
62
74
 
63
- from circuit_breaker_labs.api.evaluations import evaluate_system_prompt_post
64
- from circuit_breaker_labs.models import EvaluateSystemPromptRequest
75
+ from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
76
+ from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
65
77
 
66
78
  async with client as client:
67
- request = EvaluateSystemPromptRequest(
68
- 0.5,
69
- 3,
70
- 2,
71
- os.getenv("SYSTEM_PROMPT"),
72
- "anthropic/claude-3.7-sonnet",
79
+ request = SingleTurnEvaluateSystemPromptRequest(
80
+ threshold=0.5,
81
+ variations=3,
82
+ maximum_iteration_layers=2,
83
+ system_prompt=os.getenv("SYSTEM_PROMPT"),
84
+ openrouter_model_name="anthropic/claude-3.7-sonnet",
73
85
  )
74
86
 
75
- run_tests_response = evaluate_system_prompt_post.asyncio(
87
+ run_tests_response = await singleturn_evaluate_system_prompt_post.asyncio(
76
88
  client=client,
77
89
  cbl_api_key=os.getenv("CBL_API_KEY"),
78
90
  body=request,
@@ -0,0 +1,47 @@
1
+ circuit_breaker_labs/__init__.py,sha256=i6esGzMquFSJYw6vIYraD2Eb80yxhIMV2Fl_CrqDsTA,167
2
+ circuit_breaker_labs/api/__init__.py,sha256=zTSiG_ujSjAqWPyc435YXaX9XTlpMjiJWBbV-f-YtdA,45
3
+ circuit_breaker_labs/api/api_keys/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
4
+ circuit_breaker_labs/api/api_keys/monthly_quota_get.py,sha256=5qtHjEomorCPzrUDQwpocnt_g1lfSS7wHFFdAANxgrg,5013
5
+ circuit_breaker_labs/api/api_keys/validate_api_key_get.py,sha256=9ingxHucOQS3dPQxaNnxuv74t60EehY2xRZnrmuDNbk,4511
6
+ circuit_breaker_labs/api/evaluations/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
7
+ circuit_breaker_labs/api/evaluations/multi_turn_evaluate_system_prompt_post.py,sha256=5RCxdvPw7pYyiFPFj2o52_ITrPf3-3j4Qz4nZN7vKZk,7403
8
+ circuit_breaker_labs/api/evaluations/multiturn_evaluate_openai_fine_tune_post.py,sha256=pZVqf454Q9BywktQZyvIkGe1iMuyOoPIy_CC4rF7o9Q,9677
9
+ circuit_breaker_labs/api/evaluations/single_turn_evaluate_openai_fine_tune_post.py,sha256=D_st6FBlEko-UBp_Nm9BIj_PtymT9XB40lIz6YnRZgI,9710
10
+ circuit_breaker_labs/api/evaluations/singleturn_evaluate_system_prompt_post.py,sha256=wuEiPZ1kzkDVTPYUHHmnQDAfTxdZb2xFk_qi2m4-vG0,7436
11
+ circuit_breaker_labs/api/health_checks/__init__.py,sha256=5vd9uJWAjRqa9xzxzYkLD1yoZ12Ld_bAaNB5WX4fbE8,56
12
+ circuit_breaker_labs/api/health_checks/ping_get.py,sha256=zOacpnV1c_g_v3OKqXr2GqtNUzVpgQaVZPr2D9jB-s8,3343
13
+ circuit_breaker_labs/api/health_checks/version_get.py,sha256=ykAbGhlVlOHajrsSQcd4bp0P34K7z16mKE_O3MstI3c,3293
14
+ circuit_breaker_labs/client.py,sha256=-rT3epMc77Y7QMTy5o1oH5hkGLufY9qFrD1rb7qItFU,12384
15
+ circuit_breaker_labs/errors.py,sha256=gO8GBmKqmSNgAg-E5oT-oOyxztvp7V_6XG7OUTT15q0,546
16
+ circuit_breaker_labs/models/__init__.py,sha256=24BRj2TLUb3Ld2zsesFsaUm-xuiwdKCU8UBfO599dHI,2378
17
+ circuit_breaker_labs/models/http_validation_error.py,sha256=S2z4QBSSZFeQ23Xnlk-8u7H_I_EwewePiFJbSEKdSp0,2318
18
+ circuit_breaker_labs/models/internal_server_error.py,sha256=XjwdxWWYhIEn2aKrPuCXQivQOJ4QLqA-lRaDi2z4sBU,1892
19
+ circuit_breaker_labs/models/internal_server_error_response.py,sha256=EmHyRvXjagCVuUIdWr9DMDaCcVUyOiERU3BGrPa1TY4,1907
20
+ circuit_breaker_labs/models/message.py,sha256=GCtAUsc6_IX_PtS3R7WFPS9nYcxzq0lu2cpHT4dcuSw,1669
21
+ circuit_breaker_labs/models/monthly_quota_response.py,sha256=QOCKISN1sSWXz4jlVGKme9CkJSqXw6VtukCYLc-gSpU,2069
22
+ circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py,sha256=r12EzH6b26MulmqtggFTjWZB9DlBPlWTcTFBz221F0I,4084
23
+ circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py,sha256=EiPoAPO7R5t9H01e7Y5SCTFm3Y86JLnnL3qaBx_2X3k,4527
24
+ circuit_breaker_labs/models/multi_turn_failed_test_result.py,sha256=Y6z-zM7Wp2DVIyE7XDCQ4RYrn9yhI28L323jw3VMGjw,2577
25
+ circuit_breaker_labs/models/multi_turn_run_tests_response.py,sha256=ltuhksaaG-G3Gu2-4RWEyvu82mAuZzSvYo1SF2GX5hI,2924
26
+ circuit_breaker_labs/models/multi_turn_test_type.py,sha256=gzS6knYWi82lM3SNhKfCAnxADN9aXtnELljTS6hXgkA,196
27
+ circuit_breaker_labs/models/not_found_error.py,sha256=Anno3h5wOoRp0vYx7FSF4qaTtBXpAUm0INj9MbWvLrI,1832
28
+ circuit_breaker_labs/models/not_found_response.py,sha256=UyGZTcyabjJIRK2Mx3NRy_HkcL7Tgo40PY9fEcJ5Q44,1789
29
+ circuit_breaker_labs/models/ping_response.py,sha256=DgbkCdzHn1HDQ53aG9VCVBr1kDTcfS6TdbO6PuIkIew,1916
30
+ circuit_breaker_labs/models/quota_exceeded_error.py,sha256=M_JGaKG1pUFCEl-ZSrw8TeDjmNVA_5-ZchicLMFs2Sc,1884
31
+ circuit_breaker_labs/models/quota_exceeded_response.py,sha256=cGM-2vgb0XByFl01cviQMMSJRan8ndDgCMdVS_VQw50,1871
32
+ circuit_breaker_labs/models/role.py,sha256=FGDt50yYv1_eN1QAyBwsJFR-tjNi1soxpQaEaSBUfzo,177
33
+ circuit_breaker_labs/models/single_turn_evaluate_open_ai_finetune_request.py,sha256=YxJGPAy45qbKE9ccRGcJOXmejw15yG0RtQYNjajeym8,3767
34
+ circuit_breaker_labs/models/single_turn_evaluate_system_prompt_request.py,sha256=FUWAx-70srCIuFMhtz7e7N5IlZsFTebENmKAqgxLKvY,4210
35
+ circuit_breaker_labs/models/single_turn_failed_test_result.py,sha256=mxwflRSpiHiM6rYocTTcTSISjBCRj6EdPgViXGAI__I,2313
36
+ circuit_breaker_labs/models/single_turn_run_tests_response.py,sha256=jG6Fp40Vusay41Dd8LPbeXztsodkEmgFBt-z_oO1cMw,3416
37
+ circuit_breaker_labs/models/test_case_pack.py,sha256=xX-LjItO82VgMC1_--Ju1CD0TRmhXmN1memrtruxYOc,161
38
+ circuit_breaker_labs/models/unauthorized_error.py,sha256=vIiMl68kdwOKIwCp6t0eLLmToHar5xKroVI8Z_KVfGQ,1858
39
+ circuit_breaker_labs/models/unauthorized_response.py,sha256=c1xQ9mtazPAW_LoTfZYNl7FmGSTSetiL7iw4B0JxuCo,1838
40
+ circuit_breaker_labs/models/validate_api_key_response.py,sha256=dFWXa0Mc9sRJvI3CXHZ8A0samQQLc68MRPuJsW6AO2A,1580
41
+ circuit_breaker_labs/models/validation_error.py,sha256=n8d_ZobQV26pm0KyDAKvIo93uOBhz2BH59jpJAKwoPY,2180
42
+ circuit_breaker_labs/models/version_response.py,sha256=Ptaax1q1oTfbXfcC2ta6GtSNcUY38HCn7_oN1op1EYU,1535
43
+ circuit_breaker_labs/py.typed,sha256=8ZJUsxZiuOy1oJeVhsTWQhTG_6pTVHVXk5hJL79ebTk,25
44
+ circuit_breaker_labs/types.py,sha256=0We4NPvhIYASRpQ3le41nmJeEAVm42-2VKdzlJ4Ogok,1343
45
+ circuit_breaker_labs-1.0.6.dist-info/WHEEL,sha256=KSLUh82mDPEPk0Bx0ScXlWL64bc8KmzIPNcpQZFV-6E,79
46
+ circuit_breaker_labs-1.0.6.dist-info/METADATA,sha256=ru4vWVtHkOzGiIkt6ABZZm9TgN3IdfupjMmu2yRxjfI,5354
47
+ circuit_breaker_labs-1.0.6.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: uv 0.9.14
2
+ Generator: uv 0.9.22
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any