deeprails 1.7.0__tar.gz → 1.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deeprails might be problematic. Click here for more details.
- deeprails-1.8.0/.release-please-manifest.json +3 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/CHANGELOG.md +8 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/PKG-INFO +33 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/README.md +32 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/pyproject.toml +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_version.py +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/resources/defend.py +4 -4
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/resources/evaluate.py +2 -2
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/resources/monitor.py +2 -2
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/defend_submit_event_params.py +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/evaluate_create_params.py +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/evaluation.py +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_submit_event_params.py +1 -1
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/api_resources/test_defend.py +10 -6
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_client.py +10 -10
- deeprails-1.7.0/.release-please-manifest.json +0 -3
- {deeprails-1.7.0 → deeprails-1.8.0}/.gitignore +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/CONTRIBUTING.md +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/LICENSE +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/api.md +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/bin/check-release-environment +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/bin/publish-pypi +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/examples/.keep +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/noxfile.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/release-please-config.json +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/requirements-dev.lock +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/requirements.lock +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_base_client.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_client.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_compat.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_constants.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_exceptions.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_files.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_models.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_qs.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_resource.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_streaming.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_types.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_compat.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_datetime_parse.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_logs.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_proxy.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_reflection.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_resources_proxy.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_streams.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_sync.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_transform.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_typing.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/_utils/_utils.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/lib/.keep +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/py.typed +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/resources/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/api_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/defend_create_workflow_params.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/defend_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/defend_update_workflow_params.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_create_params.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_retrieve_params.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_retrieve_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_submit_event_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/monitor_update_params.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/src/deeprails/types/workflow_event_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/api_resources/__init__.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/api_resources/test_evaluate.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/api_resources/test_monitor.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/conftest.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/sample_file.txt +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_deepcopy.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_extract_files.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_files.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_models.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_qs.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_required_args.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_response.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_streaming.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_transform.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_utils/test_datetime_parse.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_utils/test_proxy.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/test_utils/test_typing.py +0 -0
- {deeprails-1.7.0 → deeprails-1.8.0}/tests/utils.py +0 -0
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.8.0 (2025-10-22)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v1.7.0...v1.8.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.7.0...v1.8.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** manual updates ([4b46121](https://github.com/deeprails/deeprails-sdk-python/commit/4b461213615578ca0382f044201c8343d4e9f167))
|
|
10
|
+
|
|
3
11
|
## 1.7.0 (2025-10-22)
|
|
4
12
|
|
|
5
13
|
Full Changelog: [v1.6.1...v1.7.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.6.1...v1.7.0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: deeprails
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0
|
|
4
4
|
Summary: The official Python library for the deeprails API
|
|
5
5
|
Project-URL: Homepage, https://docs.deeprails.com/
|
|
6
6
|
Project-URL: Repository, https://github.com/deeprails/deeprails-sdk-python
|
|
@@ -69,6 +69,10 @@ defend_response = client.defend.create_workflow(
|
|
|
69
69
|
improvement_action="fixit",
|
|
70
70
|
name="Push Alert Workflow",
|
|
71
71
|
type="custom",
|
|
72
|
+
custom_hallucination_threshold_values={
|
|
73
|
+
"completeness": 0.7,
|
|
74
|
+
"instruction_adherence": 0.75,
|
|
75
|
+
},
|
|
72
76
|
)
|
|
73
77
|
print(defend_response.workflow_id)
|
|
74
78
|
```
|
|
@@ -97,6 +101,10 @@ async def main() -> None:
|
|
|
97
101
|
improvement_action="fixit",
|
|
98
102
|
name="Push Alert Workflow",
|
|
99
103
|
type="custom",
|
|
104
|
+
custom_hallucination_threshold_values={
|
|
105
|
+
"completeness": 0.7,
|
|
106
|
+
"instruction_adherence": 0.75,
|
|
107
|
+
},
|
|
100
108
|
)
|
|
101
109
|
print(defend_response.workflow_id)
|
|
102
110
|
|
|
@@ -134,6 +142,10 @@ async def main() -> None:
|
|
|
134
142
|
improvement_action="fixit",
|
|
135
143
|
name="Push Alert Workflow",
|
|
136
144
|
type="custom",
|
|
145
|
+
custom_hallucination_threshold_values={
|
|
146
|
+
"completeness": 0.7,
|
|
147
|
+
"instruction_adherence": 0.75,
|
|
148
|
+
},
|
|
137
149
|
)
|
|
138
150
|
print(defend_response.workflow_id)
|
|
139
151
|
|
|
@@ -189,6 +201,10 @@ try:
|
|
|
189
201
|
improvement_action="fixit",
|
|
190
202
|
name="Push Alert Workflow",
|
|
191
203
|
type="custom",
|
|
204
|
+
custom_hallucination_threshold_values={
|
|
205
|
+
"completeness": 0.7,
|
|
206
|
+
"instruction_adherence": 0.75,
|
|
207
|
+
},
|
|
192
208
|
)
|
|
193
209
|
except deeprails.APIConnectionError as e:
|
|
194
210
|
print("The server could not be reached")
|
|
@@ -236,6 +252,10 @@ client.with_options(max_retries=5).defend.create_workflow(
|
|
|
236
252
|
improvement_action="fixit",
|
|
237
253
|
name="Push Alert Workflow",
|
|
238
254
|
type="custom",
|
|
255
|
+
custom_hallucination_threshold_values={
|
|
256
|
+
"completeness": 0.7,
|
|
257
|
+
"instruction_adherence": 0.75,
|
|
258
|
+
},
|
|
239
259
|
)
|
|
240
260
|
```
|
|
241
261
|
|
|
@@ -263,6 +283,10 @@ client.with_options(timeout=5.0).defend.create_workflow(
|
|
|
263
283
|
improvement_action="fixit",
|
|
264
284
|
name="Push Alert Workflow",
|
|
265
285
|
type="custom",
|
|
286
|
+
custom_hallucination_threshold_values={
|
|
287
|
+
"completeness": 0.7,
|
|
288
|
+
"instruction_adherence": 0.75,
|
|
289
|
+
},
|
|
266
290
|
)
|
|
267
291
|
```
|
|
268
292
|
|
|
@@ -308,6 +332,10 @@ response = client.defend.with_raw_response.create_workflow(
|
|
|
308
332
|
improvement_action="fixit",
|
|
309
333
|
name="Push Alert Workflow",
|
|
310
334
|
type="custom",
|
|
335
|
+
custom_hallucination_threshold_values={
|
|
336
|
+
"completeness": 0.7,
|
|
337
|
+
"instruction_adherence": 0.75,
|
|
338
|
+
},
|
|
311
339
|
)
|
|
312
340
|
print(response.headers.get('X-My-Header'))
|
|
313
341
|
|
|
@@ -330,6 +358,10 @@ with client.defend.with_streaming_response.create_workflow(
|
|
|
330
358
|
improvement_action="fixit",
|
|
331
359
|
name="Push Alert Workflow",
|
|
332
360
|
type="custom",
|
|
361
|
+
custom_hallucination_threshold_values={
|
|
362
|
+
"completeness": 0.7,
|
|
363
|
+
"instruction_adherence": 0.75,
|
|
364
|
+
},
|
|
333
365
|
) as response:
|
|
334
366
|
print(response.headers.get("X-My-Header"))
|
|
335
367
|
|
|
@@ -34,6 +34,10 @@ defend_response = client.defend.create_workflow(
|
|
|
34
34
|
improvement_action="fixit",
|
|
35
35
|
name="Push Alert Workflow",
|
|
36
36
|
type="custom",
|
|
37
|
+
custom_hallucination_threshold_values={
|
|
38
|
+
"completeness": 0.7,
|
|
39
|
+
"instruction_adherence": 0.75,
|
|
40
|
+
},
|
|
37
41
|
)
|
|
38
42
|
print(defend_response.workflow_id)
|
|
39
43
|
```
|
|
@@ -62,6 +66,10 @@ async def main() -> None:
|
|
|
62
66
|
improvement_action="fixit",
|
|
63
67
|
name="Push Alert Workflow",
|
|
64
68
|
type="custom",
|
|
69
|
+
custom_hallucination_threshold_values={
|
|
70
|
+
"completeness": 0.7,
|
|
71
|
+
"instruction_adherence": 0.75,
|
|
72
|
+
},
|
|
65
73
|
)
|
|
66
74
|
print(defend_response.workflow_id)
|
|
67
75
|
|
|
@@ -99,6 +107,10 @@ async def main() -> None:
|
|
|
99
107
|
improvement_action="fixit",
|
|
100
108
|
name="Push Alert Workflow",
|
|
101
109
|
type="custom",
|
|
110
|
+
custom_hallucination_threshold_values={
|
|
111
|
+
"completeness": 0.7,
|
|
112
|
+
"instruction_adherence": 0.75,
|
|
113
|
+
},
|
|
102
114
|
)
|
|
103
115
|
print(defend_response.workflow_id)
|
|
104
116
|
|
|
@@ -154,6 +166,10 @@ try:
|
|
|
154
166
|
improvement_action="fixit",
|
|
155
167
|
name="Push Alert Workflow",
|
|
156
168
|
type="custom",
|
|
169
|
+
custom_hallucination_threshold_values={
|
|
170
|
+
"completeness": 0.7,
|
|
171
|
+
"instruction_adherence": 0.75,
|
|
172
|
+
},
|
|
157
173
|
)
|
|
158
174
|
except deeprails.APIConnectionError as e:
|
|
159
175
|
print("The server could not be reached")
|
|
@@ -201,6 +217,10 @@ client.with_options(max_retries=5).defend.create_workflow(
|
|
|
201
217
|
improvement_action="fixit",
|
|
202
218
|
name="Push Alert Workflow",
|
|
203
219
|
type="custom",
|
|
220
|
+
custom_hallucination_threshold_values={
|
|
221
|
+
"completeness": 0.7,
|
|
222
|
+
"instruction_adherence": 0.75,
|
|
223
|
+
},
|
|
204
224
|
)
|
|
205
225
|
```
|
|
206
226
|
|
|
@@ -228,6 +248,10 @@ client.with_options(timeout=5.0).defend.create_workflow(
|
|
|
228
248
|
improvement_action="fixit",
|
|
229
249
|
name="Push Alert Workflow",
|
|
230
250
|
type="custom",
|
|
251
|
+
custom_hallucination_threshold_values={
|
|
252
|
+
"completeness": 0.7,
|
|
253
|
+
"instruction_adherence": 0.75,
|
|
254
|
+
},
|
|
231
255
|
)
|
|
232
256
|
```
|
|
233
257
|
|
|
@@ -273,6 +297,10 @@ response = client.defend.with_raw_response.create_workflow(
|
|
|
273
297
|
improvement_action="fixit",
|
|
274
298
|
name="Push Alert Workflow",
|
|
275
299
|
type="custom",
|
|
300
|
+
custom_hallucination_threshold_values={
|
|
301
|
+
"completeness": 0.7,
|
|
302
|
+
"instruction_adherence": 0.75,
|
|
303
|
+
},
|
|
276
304
|
)
|
|
277
305
|
print(response.headers.get('X-My-Header'))
|
|
278
306
|
|
|
@@ -295,6 +323,10 @@ with client.defend.with_streaming_response.create_workflow(
|
|
|
295
323
|
improvement_action="fixit",
|
|
296
324
|
name="Push Alert Workflow",
|
|
297
325
|
type="custom",
|
|
326
|
+
custom_hallucination_threshold_values={
|
|
327
|
+
"completeness": 0.7,
|
|
328
|
+
"instruction_adherence": 0.75,
|
|
329
|
+
},
|
|
298
330
|
) as response:
|
|
299
331
|
print(response.headers.get("X-My-Header"))
|
|
300
332
|
|
|
@@ -214,8 +214,8 @@ class DefendResource(SyncAPIResource):
|
|
|
214
214
|
|
|
215
215
|
Args:
|
|
216
216
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
217
|
-
contain at least `user_prompt` or `system_prompt` field. For
|
|
218
|
-
|
|
217
|
+
contain at least `user_prompt` or `system_prompt` field. For the
|
|
218
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
219
219
|
|
|
220
220
|
model_output: Output generated by the LLM to be evaluated.
|
|
221
221
|
|
|
@@ -492,8 +492,8 @@ class AsyncDefendResource(AsyncAPIResource):
|
|
|
492
492
|
|
|
493
493
|
Args:
|
|
494
494
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
495
|
-
contain at least `user_prompt` or `system_prompt` field. For
|
|
496
|
-
|
|
495
|
+
contain at least `user_prompt` or `system_prompt` field. For the
|
|
496
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
497
497
|
|
|
498
498
|
model_output: Output generated by the LLM to be evaluated.
|
|
499
499
|
|
|
@@ -77,7 +77,7 @@ class EvaluateResource(SyncAPIResource):
|
|
|
77
77
|
Args:
|
|
78
78
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
79
79
|
contain at least `user_prompt` or `system_prompt` field. For
|
|
80
|
-
|
|
80
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
81
81
|
|
|
82
82
|
model_output: Output generated by the LLM to be evaluated.
|
|
83
83
|
|
|
@@ -208,7 +208,7 @@ class AsyncEvaluateResource(AsyncAPIResource):
|
|
|
208
208
|
Args:
|
|
209
209
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
210
210
|
contain at least `user_prompt` or `system_prompt` field. For
|
|
211
|
-
|
|
211
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
212
212
|
|
|
213
213
|
model_output: Output generated by the LLM to be evaluated.
|
|
214
214
|
|
|
@@ -221,7 +221,7 @@ class MonitorResource(SyncAPIResource):
|
|
|
221
221
|
|
|
222
222
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
223
223
|
contain at least a `user_prompt` or `system_prompt` field. For
|
|
224
|
-
|
|
224
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
225
225
|
|
|
226
226
|
model_output: Output generated by the LLM to be evaluated.
|
|
227
227
|
|
|
@@ -459,7 +459,7 @@ class AsyncMonitorResource(AsyncAPIResource):
|
|
|
459
459
|
|
|
460
460
|
model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
461
461
|
contain at least a `user_prompt` or `system_prompt` field. For
|
|
462
|
-
|
|
462
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
463
463
|
|
|
464
464
|
model_output: Output generated by the LLM to be evaluated.
|
|
465
465
|
|
|
@@ -12,7 +12,7 @@ class DefendSubmitEventParams(TypedDict, total=False):
|
|
|
12
12
|
"""A dictionary of inputs sent to the LLM to generate output.
|
|
13
13
|
|
|
14
14
|
The dictionary must contain at least `user_prompt` or `system_prompt` field. For
|
|
15
|
-
|
|
15
|
+
the ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
16
16
|
"""
|
|
17
17
|
|
|
18
18
|
model_output: Required[str]
|
|
@@ -13,7 +13,7 @@ class EvaluateCreateParams(TypedDict, total=False):
|
|
|
13
13
|
"""A dictionary of inputs sent to the LLM to generate output.
|
|
14
14
|
|
|
15
15
|
The dictionary must contain at least `user_prompt` or `system_prompt` field. For
|
|
16
|
-
|
|
16
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
19
|
model_output: Required[str]
|
|
@@ -33,7 +33,7 @@ class Evaluation(BaseModel):
|
|
|
33
33
|
"""A dictionary of inputs sent to the LLM to generate output.
|
|
34
34
|
|
|
35
35
|
The dictionary must contain at least `user_prompt` or `system_prompt` field. For
|
|
36
|
-
|
|
36
|
+
ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
37
37
|
"""
|
|
38
38
|
|
|
39
39
|
api_model_output: str = FieldInfo(alias="model_output")
|
|
@@ -32,7 +32,7 @@ class MonitorSubmitEventParams(TypedDict, total=False):
|
|
|
32
32
|
"""A dictionary of inputs sent to the LLM to generate output.
|
|
33
33
|
|
|
34
34
|
The dictionary must contain at least a `user_prompt` or `system_prompt` field.
|
|
35
|
-
For
|
|
35
|
+
For ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
36
36
|
"""
|
|
37
37
|
|
|
38
38
|
model_output: Required[str]
|
|
@@ -27,6 +27,7 @@ class TestDefend:
|
|
|
27
27
|
improvement_action="regen",
|
|
28
28
|
name="name",
|
|
29
29
|
type="automatic",
|
|
30
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
30
31
|
)
|
|
31
32
|
assert_matches_type(DefendResponse, defend, path=["response"])
|
|
32
33
|
|
|
@@ -37,10 +38,9 @@ class TestDefend:
|
|
|
37
38
|
improvement_action="regen",
|
|
38
39
|
name="name",
|
|
39
40
|
type="automatic",
|
|
40
|
-
automatic_hallucination_tolerance_levels={"
|
|
41
|
-
custom_hallucination_threshold_values={"foo": 0},
|
|
41
|
+
automatic_hallucination_tolerance_levels={"correctness": "low"},
|
|
42
42
|
description="description",
|
|
43
|
-
max_improvement_attempt=
|
|
43
|
+
max_improvement_attempt=2,
|
|
44
44
|
)
|
|
45
45
|
assert_matches_type(DefendResponse, defend, path=["response"])
|
|
46
46
|
|
|
@@ -51,6 +51,7 @@ class TestDefend:
|
|
|
51
51
|
improvement_action="regen",
|
|
52
52
|
name="name",
|
|
53
53
|
type="automatic",
|
|
54
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
54
55
|
)
|
|
55
56
|
|
|
56
57
|
assert response.is_closed is True
|
|
@@ -65,6 +66,7 @@ class TestDefend:
|
|
|
65
66
|
improvement_action="regen",
|
|
66
67
|
name="name",
|
|
67
68
|
type="automatic",
|
|
69
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
68
70
|
) as response:
|
|
69
71
|
assert not response.is_closed
|
|
70
72
|
assert response.http_request.headers.get("X-Stainless-Lang") == "python"
|
|
@@ -308,6 +310,7 @@ class TestAsyncDefend:
|
|
|
308
310
|
improvement_action="regen",
|
|
309
311
|
name="name",
|
|
310
312
|
type="automatic",
|
|
313
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
311
314
|
)
|
|
312
315
|
assert_matches_type(DefendResponse, defend, path=["response"])
|
|
313
316
|
|
|
@@ -318,10 +321,9 @@ class TestAsyncDefend:
|
|
|
318
321
|
improvement_action="regen",
|
|
319
322
|
name="name",
|
|
320
323
|
type="automatic",
|
|
321
|
-
automatic_hallucination_tolerance_levels={"
|
|
322
|
-
custom_hallucination_threshold_values={"foo": 0},
|
|
324
|
+
automatic_hallucination_tolerance_levels={"correctness": "low"},
|
|
323
325
|
description="description",
|
|
324
|
-
max_improvement_attempt=
|
|
326
|
+
max_improvement_attempt=2,
|
|
325
327
|
)
|
|
326
328
|
assert_matches_type(DefendResponse, defend, path=["response"])
|
|
327
329
|
|
|
@@ -332,6 +334,7 @@ class TestAsyncDefend:
|
|
|
332
334
|
improvement_action="regen",
|
|
333
335
|
name="name",
|
|
334
336
|
type="automatic",
|
|
337
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
335
338
|
)
|
|
336
339
|
|
|
337
340
|
assert response.is_closed is True
|
|
@@ -346,6 +349,7 @@ class TestAsyncDefend:
|
|
|
346
349
|
improvement_action="regen",
|
|
347
350
|
name="name",
|
|
348
351
|
type="automatic",
|
|
352
|
+
automatic_hallucination_tolerance_levels={"completeness": "medium"},
|
|
349
353
|
) as response:
|
|
350
354
|
assert not response.is_closed
|
|
351
355
|
assert response.http_request.headers.get("X-Stainless-Lang") == "python"
|
|
@@ -718,7 +718,7 @@ class TestDeeprails:
|
|
|
718
718
|
|
|
719
719
|
with pytest.raises(APITimeoutError):
|
|
720
720
|
client.defend.with_streaming_response.create_workflow(
|
|
721
|
-
improvement_action="regen", name="name", type="automatic"
|
|
721
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
722
722
|
).__enter__()
|
|
723
723
|
|
|
724
724
|
assert _get_open_connections(self.client) == 0
|
|
@@ -730,7 +730,7 @@ class TestDeeprails:
|
|
|
730
730
|
|
|
731
731
|
with pytest.raises(APIStatusError):
|
|
732
732
|
client.defend.with_streaming_response.create_workflow(
|
|
733
|
-
improvement_action="regen", name="name", type="automatic"
|
|
733
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
734
734
|
).__enter__()
|
|
735
735
|
assert _get_open_connections(self.client) == 0
|
|
736
736
|
|
|
@@ -761,7 +761,7 @@ class TestDeeprails:
|
|
|
761
761
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
762
762
|
|
|
763
763
|
response = client.defend.with_raw_response.create_workflow(
|
|
764
|
-
improvement_action="regen", name="name", type="automatic"
|
|
764
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
765
765
|
)
|
|
766
766
|
|
|
767
767
|
assert response.retries_taken == failures_before_success
|
|
@@ -787,7 +787,7 @@ class TestDeeprails:
|
|
|
787
787
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
788
788
|
|
|
789
789
|
response = client.defend.with_raw_response.create_workflow(
|
|
790
|
-
improvement_action="regen", name="name", type="automatic", extra_headers={"x-stainless-retry-count": Omit()}
|
|
790
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}, extra_headers={"x-stainless-retry-count": Omit()}
|
|
791
791
|
)
|
|
792
792
|
|
|
793
793
|
assert len(response.http_request.headers.get_list("x-stainless-retry-count")) == 0
|
|
@@ -812,7 +812,7 @@ class TestDeeprails:
|
|
|
812
812
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
813
813
|
|
|
814
814
|
response = client.defend.with_raw_response.create_workflow(
|
|
815
|
-
improvement_action="regen", name="name", type="automatic", extra_headers={"x-stainless-retry-count": "42"}
|
|
815
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}, extra_headers={"x-stainless-retry-count": "42"}
|
|
816
816
|
)
|
|
817
817
|
|
|
818
818
|
assert response.http_request.headers.get("x-stainless-retry-count") == "42"
|
|
@@ -1545,7 +1545,7 @@ class TestAsyncDeeprails:
|
|
|
1545
1545
|
|
|
1546
1546
|
with pytest.raises(APITimeoutError):
|
|
1547
1547
|
await async_client.defend.with_streaming_response.create_workflow(
|
|
1548
|
-
improvement_action="regen", name="name", type="automatic"
|
|
1548
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
1549
1549
|
).__aenter__()
|
|
1550
1550
|
|
|
1551
1551
|
assert _get_open_connections(self.client) == 0
|
|
@@ -1559,7 +1559,7 @@ class TestAsyncDeeprails:
|
|
|
1559
1559
|
|
|
1560
1560
|
with pytest.raises(APIStatusError):
|
|
1561
1561
|
await async_client.defend.with_streaming_response.create_workflow(
|
|
1562
|
-
improvement_action="regen", name="name", type="automatic"
|
|
1562
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
1563
1563
|
).__aenter__()
|
|
1564
1564
|
assert _get_open_connections(self.client) == 0
|
|
1565
1565
|
|
|
@@ -1591,7 +1591,7 @@ class TestAsyncDeeprails:
|
|
|
1591
1591
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
1592
1592
|
|
|
1593
1593
|
response = await client.defend.with_raw_response.create_workflow(
|
|
1594
|
-
improvement_action="regen", name="name", type="automatic"
|
|
1594
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}
|
|
1595
1595
|
)
|
|
1596
1596
|
|
|
1597
1597
|
assert response.retries_taken == failures_before_success
|
|
@@ -1618,7 +1618,7 @@ class TestAsyncDeeprails:
|
|
|
1618
1618
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
1619
1619
|
|
|
1620
1620
|
response = await client.defend.with_raw_response.create_workflow(
|
|
1621
|
-
improvement_action="regen", name="name", type="automatic", extra_headers={"x-stainless-retry-count": Omit()}
|
|
1621
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}, extra_headers={"x-stainless-retry-count": Omit()}
|
|
1622
1622
|
)
|
|
1623
1623
|
|
|
1624
1624
|
assert len(response.http_request.headers.get_list("x-stainless-retry-count")) == 0
|
|
@@ -1644,7 +1644,7 @@ class TestAsyncDeeprails:
|
|
|
1644
1644
|
respx_mock.post("/defend").mock(side_effect=retry_handler)
|
|
1645
1645
|
|
|
1646
1646
|
response = await client.defend.with_raw_response.create_workflow(
|
|
1647
|
-
improvement_action="regen", name="name", type="automatic", extra_headers={"x-stainless-retry-count": "42"}
|
|
1647
|
+
improvement_action="regen", name="name", type="automatic", automatic_hallucination_tolerance_levels={"correctness": "medium"}, extra_headers={"x-stainless-retry-count": "42"}
|
|
1648
1648
|
)
|
|
1649
1649
|
|
|
1650
1650
|
assert response.http_request.headers.get("x-stainless-retry-count") == "42"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|