circuit-breaker-labs 1.0.5__tar.gz → 1.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/PKG-INFO +20 -20
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/README.md +19 -19
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/api/evaluations/multi_turn_evaluate_system_prompt_post.py +257 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/api/evaluations/multiturn_evaluate_openai_fine_tune_post.py +300 -0
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/api/evaluations/evaluate_openai_fine_tune_post.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/api/evaluations/single_turn_evaluate_openai_fine_tune_post.py +31 -31
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/api/evaluations/evaluate_system_prompt_post.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/api/evaluations/singleturn_evaluate_system_prompt_post.py +31 -31
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/__init__.py +57 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/message.py +71 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/multi_turn_evaluate_open_ai_finetune_request.py +119 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/multi_turn_evaluate_system_prompt_request.py +128 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/multi_turn_failed_test_result.py +85 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/multi_turn_run_tests_response.py +92 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/multi_turn_test_type.py +9 -0
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/role.py +10 -0
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/models/evaluate_open_ai_finetune_request.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/single_turn_evaluate_open_ai_finetune_request.py +30 -5
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/models/evaluate_system_prompt_request.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/single_turn_evaluate_system_prompt_request.py +30 -5
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/models/failed_test_result.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/single_turn_failed_test_result.py +5 -5
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/models/run_tests_response.py → circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/single_turn_run_tests_response.py +10 -10
- circuit_breaker_labs-1.0.6/circuit_breaker_labs/models/test_case_pack.py +8 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/pyproject.toml +1 -1
- circuit_breaker_labs-1.0.5/circuit_breaker_labs/models/__init__.py +0 -41
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/__init__.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/__init__.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/api_keys/__init__.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/api_keys/monthly_quota_get.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/api_keys/validate_api_key_get.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/evaluations/__init__.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/health_checks/__init__.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/health_checks/ping_get.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/api/health_checks/version_get.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/client.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/errors.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/http_validation_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/internal_server_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/internal_server_error_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/monthly_quota_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/not_found_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/not_found_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/ping_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/quota_exceeded_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/quota_exceeded_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/unauthorized_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/unauthorized_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/validate_api_key_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/validation_error.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/models/version_response.py +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/py.typed +0 -0
- {circuit_breaker_labs-1.0.5 → circuit_breaker_labs-1.0.6}/circuit_breaker_labs/types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: circuit-breaker-labs
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: A client library for accessing Circuit Breaker Labs API
|
|
5
5
|
Requires-Dist: httpx>=0.23.0,<0.29.0
|
|
6
6
|
Requires-Dist: attrs>=22.2.0
|
|
@@ -20,7 +20,7 @@ Description-Content-Type: text/markdown
|
|
|
20
20
|
<!-- prettier-ignore-start -->
|
|
21
21
|
> [!Note]
|
|
22
22
|
> This project was automatically generated by [OpenAPI Python Client](https://github.com/openapi-generators/openapi-python-client)
|
|
23
|
-
> from [this OpenAPI spec](https://
|
|
23
|
+
> from [this OpenAPI spec](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/blob/main/openapi.json).
|
|
24
24
|
<!-- prettier-ignore-end -->
|
|
25
25
|
|
|
26
26
|
## Installation
|
|
@@ -48,19 +48,19 @@ Now build a request and use it when calling an endpoint
|
|
|
48
48
|
```python
|
|
49
49
|
import os
|
|
50
50
|
|
|
51
|
-
from circuit_breaker_labs.api.evaluations import
|
|
52
|
-
from circuit_breaker_labs.models import
|
|
51
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
52
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
53
53
|
|
|
54
54
|
with client as client:
|
|
55
|
-
request =
|
|
56
|
-
0.5,
|
|
57
|
-
3,
|
|
58
|
-
2,
|
|
59
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
60
|
-
"anthropic/claude-3.7-sonnet",
|
|
55
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
56
|
+
threshold=0.5,
|
|
57
|
+
variations=3,
|
|
58
|
+
maximum_iteration_layers=2,
|
|
59
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
60
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
61
61
|
)
|
|
62
62
|
|
|
63
|
-
run_tests_response =
|
|
63
|
+
run_tests_response = singleturn_evaluate_system_prompt_post.sync(
|
|
64
64
|
client=client,
|
|
65
65
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
66
66
|
body=request,
|
|
@@ -72,19 +72,19 @@ Or do the same thing with an async version:
|
|
|
72
72
|
```python
|
|
73
73
|
import os
|
|
74
74
|
|
|
75
|
-
from circuit_breaker_labs.api.evaluations import
|
|
76
|
-
from circuit_breaker_labs.models import
|
|
75
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
76
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
77
77
|
|
|
78
78
|
async with client as client:
|
|
79
|
-
request =
|
|
80
|
-
0.5,
|
|
81
|
-
3,
|
|
82
|
-
2,
|
|
83
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
84
|
-
"anthropic/claude-3.7-sonnet",
|
|
79
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
80
|
+
threshold=0.5,
|
|
81
|
+
variations=3,
|
|
82
|
+
maximum_iteration_layers=2,
|
|
83
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
84
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
85
85
|
)
|
|
86
86
|
|
|
87
|
-
run_tests_response =
|
|
87
|
+
run_tests_response = await singleturn_evaluate_system_prompt_post.asyncio(
|
|
88
88
|
client=client,
|
|
89
89
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
90
90
|
body=request,
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
<!-- prettier-ignore-start -->
|
|
10
10
|
> [!Note]
|
|
11
11
|
> This project was automatically generated by [OpenAPI Python Client](https://github.com/openapi-generators/openapi-python-client)
|
|
12
|
-
> from [this OpenAPI spec](https://
|
|
12
|
+
> from [this OpenAPI spec](https://github.com/circuitbreakerlabs/circuitbreakerlabs-python/blob/main/openapi.json).
|
|
13
13
|
<!-- prettier-ignore-end -->
|
|
14
14
|
|
|
15
15
|
## Installation
|
|
@@ -37,19 +37,19 @@ Now build a request and use it when calling an endpoint
|
|
|
37
37
|
```python
|
|
38
38
|
import os
|
|
39
39
|
|
|
40
|
-
from circuit_breaker_labs.api.evaluations import
|
|
41
|
-
from circuit_breaker_labs.models import
|
|
40
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
41
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
42
42
|
|
|
43
43
|
with client as client:
|
|
44
|
-
request =
|
|
45
|
-
0.5,
|
|
46
|
-
3,
|
|
47
|
-
2,
|
|
48
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
49
|
-
"anthropic/claude-3.7-sonnet",
|
|
44
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
45
|
+
threshold=0.5,
|
|
46
|
+
variations=3,
|
|
47
|
+
maximum_iteration_layers=2,
|
|
48
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
49
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
50
50
|
)
|
|
51
51
|
|
|
52
|
-
run_tests_response =
|
|
52
|
+
run_tests_response = singleturn_evaluate_system_prompt_post.sync(
|
|
53
53
|
client=client,
|
|
54
54
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
55
55
|
body=request,
|
|
@@ -61,19 +61,19 @@ Or do the same thing with an async version:
|
|
|
61
61
|
```python
|
|
62
62
|
import os
|
|
63
63
|
|
|
64
|
-
from circuit_breaker_labs.api.evaluations import
|
|
65
|
-
from circuit_breaker_labs.models import
|
|
64
|
+
from circuit_breaker_labs.api.evaluations import singleturn_evaluate_system_prompt_post
|
|
65
|
+
from circuit_breaker_labs.models import SingleTurnEvaluateSystemPromptRequest
|
|
66
66
|
|
|
67
67
|
async with client as client:
|
|
68
|
-
request =
|
|
69
|
-
0.5,
|
|
70
|
-
3,
|
|
71
|
-
2,
|
|
72
|
-
os.getenv("SYSTEM_PROMPT"),
|
|
73
|
-
"anthropic/claude-3.7-sonnet",
|
|
68
|
+
request = SingleTurnEvaluateSystemPromptRequest(
|
|
69
|
+
threshold=0.5,
|
|
70
|
+
variations=3,
|
|
71
|
+
maximum_iteration_layers=2,
|
|
72
|
+
system_prompt=os.getenv("SYSTEM_PROMPT"),
|
|
73
|
+
openrouter_model_name="anthropic/claude-3.7-sonnet",
|
|
74
74
|
)
|
|
75
75
|
|
|
76
|
-
run_tests_response =
|
|
76
|
+
run_tests_response = await singleturn_evaluate_system_prompt_post.asyncio(
|
|
77
77
|
client=client,
|
|
78
78
|
cbl_api_key=os.getenv("CBL_API_KEY"),
|
|
79
79
|
body=request,
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
from http import HTTPStatus
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
|
|
6
|
+
from ... import errors
|
|
7
|
+
from ...client import AuthenticatedClient, Client
|
|
8
|
+
from ...models.http_validation_error import HTTPValidationError
|
|
9
|
+
from ...models.internal_server_error_response import InternalServerErrorResponse
|
|
10
|
+
from ...models.multi_turn_evaluate_system_prompt_request import MultiTurnEvaluateSystemPromptRequest
|
|
11
|
+
from ...models.multi_turn_run_tests_response import MultiTurnRunTestsResponse
|
|
12
|
+
from ...models.not_found_response import NotFoundResponse
|
|
13
|
+
from ...models.quota_exceeded_response import QuotaExceededResponse
|
|
14
|
+
from ...models.unauthorized_response import UnauthorizedResponse
|
|
15
|
+
from ...types import Response
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _get_kwargs(
|
|
19
|
+
*,
|
|
20
|
+
body: MultiTurnEvaluateSystemPromptRequest,
|
|
21
|
+
cbl_api_key: str,
|
|
22
|
+
) -> dict[str, Any]:
|
|
23
|
+
headers: dict[str, Any] = {}
|
|
24
|
+
headers["cbl-api-key"] = cbl_api_key
|
|
25
|
+
|
|
26
|
+
_kwargs: dict[str, Any] = {
|
|
27
|
+
"method": "post",
|
|
28
|
+
"url": "/multiturn_evaluate_system_prompt",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
_kwargs["json"] = body.to_dict()
|
|
32
|
+
|
|
33
|
+
headers["Content-Type"] = "application/json"
|
|
34
|
+
|
|
35
|
+
_kwargs["headers"] = headers
|
|
36
|
+
return _kwargs
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parse_response(
|
|
40
|
+
*, client: AuthenticatedClient | Client, response: httpx.Response
|
|
41
|
+
) -> (
|
|
42
|
+
HTTPValidationError
|
|
43
|
+
| InternalServerErrorResponse
|
|
44
|
+
| MultiTurnRunTestsResponse
|
|
45
|
+
| NotFoundResponse
|
|
46
|
+
| QuotaExceededResponse
|
|
47
|
+
| UnauthorizedResponse
|
|
48
|
+
| None
|
|
49
|
+
):
|
|
50
|
+
if response.status_code == 200:
|
|
51
|
+
response_200 = MultiTurnRunTestsResponse.from_dict(response.json())
|
|
52
|
+
|
|
53
|
+
return response_200
|
|
54
|
+
|
|
55
|
+
if response.status_code == 401:
|
|
56
|
+
response_401 = UnauthorizedResponse.from_dict(response.json())
|
|
57
|
+
|
|
58
|
+
return response_401
|
|
59
|
+
|
|
60
|
+
if response.status_code == 403:
|
|
61
|
+
response_403 = QuotaExceededResponse.from_dict(response.json())
|
|
62
|
+
|
|
63
|
+
return response_403
|
|
64
|
+
|
|
65
|
+
if response.status_code == 404:
|
|
66
|
+
response_404 = NotFoundResponse.from_dict(response.json())
|
|
67
|
+
|
|
68
|
+
return response_404
|
|
69
|
+
|
|
70
|
+
if response.status_code == 422:
|
|
71
|
+
response_422 = HTTPValidationError.from_dict(response.json())
|
|
72
|
+
|
|
73
|
+
return response_422
|
|
74
|
+
|
|
75
|
+
if response.status_code == 500:
|
|
76
|
+
response_500 = InternalServerErrorResponse.from_dict(response.json())
|
|
77
|
+
|
|
78
|
+
return response_500
|
|
79
|
+
|
|
80
|
+
if client.raise_on_unexpected_status:
|
|
81
|
+
raise errors.UnexpectedStatus(response.status_code, response.content)
|
|
82
|
+
else:
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _build_response(
|
|
87
|
+
*, client: AuthenticatedClient | Client, response: httpx.Response
|
|
88
|
+
) -> Response[
|
|
89
|
+
HTTPValidationError
|
|
90
|
+
| InternalServerErrorResponse
|
|
91
|
+
| MultiTurnRunTestsResponse
|
|
92
|
+
| NotFoundResponse
|
|
93
|
+
| QuotaExceededResponse
|
|
94
|
+
| UnauthorizedResponse
|
|
95
|
+
]:
|
|
96
|
+
return Response(
|
|
97
|
+
status_code=HTTPStatus(response.status_code),
|
|
98
|
+
content=response.content,
|
|
99
|
+
headers=response.headers,
|
|
100
|
+
parsed=_parse_response(client=client, response=response),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def sync_detailed(
|
|
105
|
+
*,
|
|
106
|
+
client: AuthenticatedClient | Client,
|
|
107
|
+
body: MultiTurnEvaluateSystemPromptRequest,
|
|
108
|
+
cbl_api_key: str,
|
|
109
|
+
) -> Response[
|
|
110
|
+
HTTPValidationError
|
|
111
|
+
| InternalServerErrorResponse
|
|
112
|
+
| MultiTurnRunTestsResponse
|
|
113
|
+
| NotFoundResponse
|
|
114
|
+
| QuotaExceededResponse
|
|
115
|
+
| UnauthorizedResponse
|
|
116
|
+
]:
|
|
117
|
+
"""Multi-turn Evaluate System Prompt
|
|
118
|
+
|
|
119
|
+
Run multi-turn safety tests against a system prompt.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
123
|
+
body (MultiTurnEvaluateSystemPromptRequest):
|
|
124
|
+
|
|
125
|
+
Raises:
|
|
126
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
127
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
Response[HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse]
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
kwargs = _get_kwargs(
|
|
134
|
+
body=body,
|
|
135
|
+
cbl_api_key=cbl_api_key,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
response = client.get_httpx_client().request(
|
|
139
|
+
**kwargs,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return _build_response(client=client, response=response)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def sync(
|
|
146
|
+
*,
|
|
147
|
+
client: AuthenticatedClient | Client,
|
|
148
|
+
body: MultiTurnEvaluateSystemPromptRequest,
|
|
149
|
+
cbl_api_key: str,
|
|
150
|
+
) -> (
|
|
151
|
+
HTTPValidationError
|
|
152
|
+
| InternalServerErrorResponse
|
|
153
|
+
| MultiTurnRunTestsResponse
|
|
154
|
+
| NotFoundResponse
|
|
155
|
+
| QuotaExceededResponse
|
|
156
|
+
| UnauthorizedResponse
|
|
157
|
+
| None
|
|
158
|
+
):
|
|
159
|
+
"""Multi-turn Evaluate System Prompt
|
|
160
|
+
|
|
161
|
+
Run multi-turn safety tests against a system prompt.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
165
|
+
body (MultiTurnEvaluateSystemPromptRequest):
|
|
166
|
+
|
|
167
|
+
Raises:
|
|
168
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
169
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
return sync_detailed(
|
|
176
|
+
client=client,
|
|
177
|
+
body=body,
|
|
178
|
+
cbl_api_key=cbl_api_key,
|
|
179
|
+
).parsed
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
async def asyncio_detailed(
|
|
183
|
+
*,
|
|
184
|
+
client: AuthenticatedClient | Client,
|
|
185
|
+
body: MultiTurnEvaluateSystemPromptRequest,
|
|
186
|
+
cbl_api_key: str,
|
|
187
|
+
) -> Response[
|
|
188
|
+
HTTPValidationError
|
|
189
|
+
| InternalServerErrorResponse
|
|
190
|
+
| MultiTurnRunTestsResponse
|
|
191
|
+
| NotFoundResponse
|
|
192
|
+
| QuotaExceededResponse
|
|
193
|
+
| UnauthorizedResponse
|
|
194
|
+
]:
|
|
195
|
+
"""Multi-turn Evaluate System Prompt
|
|
196
|
+
|
|
197
|
+
Run multi-turn safety tests against a system prompt.
|
|
198
|
+
|
|
199
|
+
Args:
|
|
200
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
201
|
+
body (MultiTurnEvaluateSystemPromptRequest):
|
|
202
|
+
|
|
203
|
+
Raises:
|
|
204
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
205
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
206
|
+
|
|
207
|
+
Returns:
|
|
208
|
+
Response[HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse]
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
kwargs = _get_kwargs(
|
|
212
|
+
body=body,
|
|
213
|
+
cbl_api_key=cbl_api_key,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
response = await client.get_async_httpx_client().request(**kwargs)
|
|
217
|
+
|
|
218
|
+
return _build_response(client=client, response=response)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
async def asyncio(
|
|
222
|
+
*,
|
|
223
|
+
client: AuthenticatedClient | Client,
|
|
224
|
+
body: MultiTurnEvaluateSystemPromptRequest,
|
|
225
|
+
cbl_api_key: str,
|
|
226
|
+
) -> (
|
|
227
|
+
HTTPValidationError
|
|
228
|
+
| InternalServerErrorResponse
|
|
229
|
+
| MultiTurnRunTestsResponse
|
|
230
|
+
| NotFoundResponse
|
|
231
|
+
| QuotaExceededResponse
|
|
232
|
+
| UnauthorizedResponse
|
|
233
|
+
| None
|
|
234
|
+
):
|
|
235
|
+
"""Multi-turn Evaluate System Prompt
|
|
236
|
+
|
|
237
|
+
Run multi-turn safety tests against a system prompt.
|
|
238
|
+
|
|
239
|
+
Args:
|
|
240
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
241
|
+
body (MultiTurnEvaluateSystemPromptRequest):
|
|
242
|
+
|
|
243
|
+
Raises:
|
|
244
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
245
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
return (
|
|
252
|
+
await asyncio_detailed(
|
|
253
|
+
client=client,
|
|
254
|
+
body=body,
|
|
255
|
+
cbl_api_key=cbl_api_key,
|
|
256
|
+
)
|
|
257
|
+
).parsed
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
from http import HTTPStatus
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
|
|
6
|
+
from ... import errors
|
|
7
|
+
from ...client import AuthenticatedClient, Client
|
|
8
|
+
from ...models.http_validation_error import HTTPValidationError
|
|
9
|
+
from ...models.internal_server_error_response import InternalServerErrorResponse
|
|
10
|
+
from ...models.multi_turn_evaluate_open_ai_finetune_request import MultiTurnEvaluateOpenAiFinetuneRequest
|
|
11
|
+
from ...models.multi_turn_run_tests_response import MultiTurnRunTestsResponse
|
|
12
|
+
from ...models.not_found_response import NotFoundResponse
|
|
13
|
+
from ...models.quota_exceeded_response import QuotaExceededResponse
|
|
14
|
+
from ...models.unauthorized_response import UnauthorizedResponse
|
|
15
|
+
from ...types import Response
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _get_kwargs(
|
|
19
|
+
*,
|
|
20
|
+
body: MultiTurnEvaluateOpenAiFinetuneRequest,
|
|
21
|
+
cbl_api_key: str,
|
|
22
|
+
openai_api_key: str,
|
|
23
|
+
) -> dict[str, Any]:
|
|
24
|
+
headers: dict[str, Any] = {}
|
|
25
|
+
headers["cbl-api-key"] = cbl_api_key
|
|
26
|
+
|
|
27
|
+
headers["openai-api-key"] = openai_api_key
|
|
28
|
+
|
|
29
|
+
_kwargs: dict[str, Any] = {
|
|
30
|
+
"method": "post",
|
|
31
|
+
"url": "/multiturn_evaluate_openai_finetune",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
_kwargs["json"] = body.to_dict()
|
|
35
|
+
|
|
36
|
+
headers["Content-Type"] = "application/json"
|
|
37
|
+
|
|
38
|
+
_kwargs["headers"] = headers
|
|
39
|
+
return _kwargs
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _parse_response(
|
|
43
|
+
*, client: AuthenticatedClient | Client, response: httpx.Response
|
|
44
|
+
) -> (
|
|
45
|
+
HTTPValidationError
|
|
46
|
+
| InternalServerErrorResponse
|
|
47
|
+
| MultiTurnRunTestsResponse
|
|
48
|
+
| NotFoundResponse
|
|
49
|
+
| QuotaExceededResponse
|
|
50
|
+
| UnauthorizedResponse
|
|
51
|
+
| None
|
|
52
|
+
):
|
|
53
|
+
if response.status_code == 200:
|
|
54
|
+
response_200 = MultiTurnRunTestsResponse.from_dict(response.json())
|
|
55
|
+
|
|
56
|
+
return response_200
|
|
57
|
+
|
|
58
|
+
if response.status_code == 401:
|
|
59
|
+
response_401 = UnauthorizedResponse.from_dict(response.json())
|
|
60
|
+
|
|
61
|
+
return response_401
|
|
62
|
+
|
|
63
|
+
if response.status_code == 403:
|
|
64
|
+
response_403 = QuotaExceededResponse.from_dict(response.json())
|
|
65
|
+
|
|
66
|
+
return response_403
|
|
67
|
+
|
|
68
|
+
if response.status_code == 404:
|
|
69
|
+
response_404 = NotFoundResponse.from_dict(response.json())
|
|
70
|
+
|
|
71
|
+
return response_404
|
|
72
|
+
|
|
73
|
+
if response.status_code == 422:
|
|
74
|
+
response_422 = HTTPValidationError.from_dict(response.json())
|
|
75
|
+
|
|
76
|
+
return response_422
|
|
77
|
+
|
|
78
|
+
if response.status_code == 500:
|
|
79
|
+
response_500 = InternalServerErrorResponse.from_dict(response.json())
|
|
80
|
+
|
|
81
|
+
return response_500
|
|
82
|
+
|
|
83
|
+
if client.raise_on_unexpected_status:
|
|
84
|
+
raise errors.UnexpectedStatus(response.status_code, response.content)
|
|
85
|
+
else:
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _build_response(
|
|
90
|
+
*, client: AuthenticatedClient | Client, response: httpx.Response
|
|
91
|
+
) -> Response[
|
|
92
|
+
HTTPValidationError
|
|
93
|
+
| InternalServerErrorResponse
|
|
94
|
+
| MultiTurnRunTestsResponse
|
|
95
|
+
| NotFoundResponse
|
|
96
|
+
| QuotaExceededResponse
|
|
97
|
+
| UnauthorizedResponse
|
|
98
|
+
]:
|
|
99
|
+
return Response(
|
|
100
|
+
status_code=HTTPStatus(response.status_code),
|
|
101
|
+
content=response.content,
|
|
102
|
+
headers=response.headers,
|
|
103
|
+
parsed=_parse_response(client=client, response=response),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def sync_detailed(
|
|
108
|
+
*,
|
|
109
|
+
client: AuthenticatedClient | Client,
|
|
110
|
+
body: MultiTurnEvaluateOpenAiFinetuneRequest,
|
|
111
|
+
cbl_api_key: str,
|
|
112
|
+
openai_api_key: str,
|
|
113
|
+
) -> Response[
|
|
114
|
+
HTTPValidationError
|
|
115
|
+
| InternalServerErrorResponse
|
|
116
|
+
| MultiTurnRunTestsResponse
|
|
117
|
+
| NotFoundResponse
|
|
118
|
+
| QuotaExceededResponse
|
|
119
|
+
| UnauthorizedResponse
|
|
120
|
+
]:
|
|
121
|
+
"""Multi-turn Evaluate OpenAI Fine Tune
|
|
122
|
+
|
|
123
|
+
Run multi-turn safety tests against an OpenAI fine-tuned model.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
127
|
+
openai_api_key (str):
|
|
128
|
+
The OpenAI API Key owned by a [service account](https://platform.openai.com/docs/api-
|
|
129
|
+
reference/project-service-accounts) within the same project as the finetuned model. The
|
|
130
|
+
API key should minimally have 'Request' permissions for 'Model Capabilities'.
|
|
131
|
+
|
|
132
|
+
You can create a new API key associated with a service account and project
|
|
133
|
+
[here](https://platform.openai.com/api-keys).
|
|
134
|
+
|
|
135
|
+
body (MultiTurnEvaluateOpenAiFinetuneRequest):
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
139
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Response[HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse]
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
kwargs = _get_kwargs(
|
|
146
|
+
body=body,
|
|
147
|
+
cbl_api_key=cbl_api_key,
|
|
148
|
+
openai_api_key=openai_api_key,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
response = client.get_httpx_client().request(
|
|
152
|
+
**kwargs,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
return _build_response(client=client, response=response)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def sync(
|
|
159
|
+
*,
|
|
160
|
+
client: AuthenticatedClient | Client,
|
|
161
|
+
body: MultiTurnEvaluateOpenAiFinetuneRequest,
|
|
162
|
+
cbl_api_key: str,
|
|
163
|
+
openai_api_key: str,
|
|
164
|
+
) -> (
|
|
165
|
+
HTTPValidationError
|
|
166
|
+
| InternalServerErrorResponse
|
|
167
|
+
| MultiTurnRunTestsResponse
|
|
168
|
+
| NotFoundResponse
|
|
169
|
+
| QuotaExceededResponse
|
|
170
|
+
| UnauthorizedResponse
|
|
171
|
+
| None
|
|
172
|
+
):
|
|
173
|
+
"""Multi-turn Evaluate OpenAI Fine Tune
|
|
174
|
+
|
|
175
|
+
Run multi-turn safety tests against an OpenAI fine-tuned model.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
179
|
+
openai_api_key (str):
|
|
180
|
+
The OpenAI API Key owned by a [service account](https://platform.openai.com/docs/api-
|
|
181
|
+
reference/project-service-accounts) within the same project as the finetuned model. The
|
|
182
|
+
API key should minimally have 'Request' permissions for 'Model Capabilities'.
|
|
183
|
+
|
|
184
|
+
You can create a new API key associated with a service account and project
|
|
185
|
+
[here](https://platform.openai.com/api-keys).
|
|
186
|
+
|
|
187
|
+
body (MultiTurnEvaluateOpenAiFinetuneRequest):
|
|
188
|
+
|
|
189
|
+
Raises:
|
|
190
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
191
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
return sync_detailed(
|
|
198
|
+
client=client,
|
|
199
|
+
body=body,
|
|
200
|
+
cbl_api_key=cbl_api_key,
|
|
201
|
+
openai_api_key=openai_api_key,
|
|
202
|
+
).parsed
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
async def asyncio_detailed(
|
|
206
|
+
*,
|
|
207
|
+
client: AuthenticatedClient | Client,
|
|
208
|
+
body: MultiTurnEvaluateOpenAiFinetuneRequest,
|
|
209
|
+
cbl_api_key: str,
|
|
210
|
+
openai_api_key: str,
|
|
211
|
+
) -> Response[
|
|
212
|
+
HTTPValidationError
|
|
213
|
+
| InternalServerErrorResponse
|
|
214
|
+
| MultiTurnRunTestsResponse
|
|
215
|
+
| NotFoundResponse
|
|
216
|
+
| QuotaExceededResponse
|
|
217
|
+
| UnauthorizedResponse
|
|
218
|
+
]:
|
|
219
|
+
"""Multi-turn Evaluate OpenAI Fine Tune
|
|
220
|
+
|
|
221
|
+
Run multi-turn safety tests against an OpenAI fine-tuned model.
|
|
222
|
+
|
|
223
|
+
Args:
|
|
224
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
225
|
+
openai_api_key (str):
|
|
226
|
+
The OpenAI API Key owned by a [service account](https://platform.openai.com/docs/api-
|
|
227
|
+
reference/project-service-accounts) within the same project as the finetuned model. The
|
|
228
|
+
API key should minimally have 'Request' permissions for 'Model Capabilities'.
|
|
229
|
+
|
|
230
|
+
You can create a new API key associated with a service account and project
|
|
231
|
+
[here](https://platform.openai.com/api-keys).
|
|
232
|
+
|
|
233
|
+
body (MultiTurnEvaluateOpenAiFinetuneRequest):
|
|
234
|
+
|
|
235
|
+
Raises:
|
|
236
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
237
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
Response[HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse]
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
kwargs = _get_kwargs(
|
|
244
|
+
body=body,
|
|
245
|
+
cbl_api_key=cbl_api_key,
|
|
246
|
+
openai_api_key=openai_api_key,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
response = await client.get_async_httpx_client().request(**kwargs)
|
|
250
|
+
|
|
251
|
+
return _build_response(client=client, response=response)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
async def asyncio(
|
|
255
|
+
*,
|
|
256
|
+
client: AuthenticatedClient | Client,
|
|
257
|
+
body: MultiTurnEvaluateOpenAiFinetuneRequest,
|
|
258
|
+
cbl_api_key: str,
|
|
259
|
+
openai_api_key: str,
|
|
260
|
+
) -> (
|
|
261
|
+
HTTPValidationError
|
|
262
|
+
| InternalServerErrorResponse
|
|
263
|
+
| MultiTurnRunTestsResponse
|
|
264
|
+
| NotFoundResponse
|
|
265
|
+
| QuotaExceededResponse
|
|
266
|
+
| UnauthorizedResponse
|
|
267
|
+
| None
|
|
268
|
+
):
|
|
269
|
+
"""Multi-turn Evaluate OpenAI Fine Tune
|
|
270
|
+
|
|
271
|
+
Run multi-turn safety tests against an OpenAI fine-tuned model.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
cbl_api_key (str): Circuit Breaker Labs API Key
|
|
275
|
+
openai_api_key (str):
|
|
276
|
+
The OpenAI API Key owned by a [service account](https://platform.openai.com/docs/api-
|
|
277
|
+
reference/project-service-accounts) within the same project as the finetuned model. The
|
|
278
|
+
API key should minimally have 'Request' permissions for 'Model Capabilities'.
|
|
279
|
+
|
|
280
|
+
You can create a new API key associated with a service account and project
|
|
281
|
+
[here](https://platform.openai.com/api-keys).
|
|
282
|
+
|
|
283
|
+
body (MultiTurnEvaluateOpenAiFinetuneRequest):
|
|
284
|
+
|
|
285
|
+
Raises:
|
|
286
|
+
errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
|
|
287
|
+
httpx.TimeoutException: If the request takes longer than Client.timeout.
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
HTTPValidationError | InternalServerErrorResponse | MultiTurnRunTestsResponse | NotFoundResponse | QuotaExceededResponse | UnauthorizedResponse
|
|
291
|
+
"""
|
|
292
|
+
|
|
293
|
+
return (
|
|
294
|
+
await asyncio_detailed(
|
|
295
|
+
client=client,
|
|
296
|
+
body=body,
|
|
297
|
+
cbl_api_key=cbl_api_key,
|
|
298
|
+
openai_api_key=openai_api_key,
|
|
299
|
+
)
|
|
300
|
+
).parsed
|