deeprails 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeprails/__init__.py +0 -2
- deeprails/_client.py +13 -72
- deeprails/_version.py +1 -1
- deeprails/resources/{defend/defend.py → defend.py} +235 -44
- deeprails/resources/evaluate.py +4 -4
- deeprails/resources/monitor.py +4 -4
- deeprails/types/__init__.py +2 -0
- deeprails/types/{defend/event_submit_event_params.py → defend_submit_event_params.py} +2 -2
- deeprails/types/{defend/workflow_event_response.py → workflow_event_response.py} +1 -1
- {deeprails-1.0.0.dist-info → deeprails-1.2.0.dist-info}/METADATA +50 -223
- {deeprails-1.0.0.dist-info → deeprails-1.2.0.dist-info}/RECORD +13 -16
- deeprails/resources/defend/__init__.py +0 -33
- deeprails/resources/defend/events.py +0 -311
- deeprails/types/defend/__init__.py +0 -6
- {deeprails-1.0.0.dist-info → deeprails-1.2.0.dist-info}/WHEEL +0 -0
- {deeprails-1.0.0.dist-info → deeprails-1.2.0.dist-info}/licenses/LICENSE +0 -0
deeprails/__init__.py
CHANGED
|
@@ -6,7 +6,6 @@ from . import types
|
|
|
6
6
|
from ._types import NOT_GIVEN, Omit, NoneType, NotGiven, Transport, ProxiesTypes, omit, not_given
|
|
7
7
|
from ._utils import file_from_path
|
|
8
8
|
from ._client import (
|
|
9
|
-
ENVIRONMENTS,
|
|
10
9
|
Client,
|
|
11
10
|
Stream,
|
|
12
11
|
Timeout,
|
|
@@ -74,7 +73,6 @@ __all__ = [
|
|
|
74
73
|
"AsyncStream",
|
|
75
74
|
"Deeprails",
|
|
76
75
|
"AsyncDeeprails",
|
|
77
|
-
"ENVIRONMENTS",
|
|
78
76
|
"file_from_path",
|
|
79
77
|
"BaseModel",
|
|
80
78
|
"DEFAULT_TIMEOUT",
|
deeprails/_client.py
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
|
-
from typing import Any,
|
|
7
|
-
from typing_extensions import Self,
|
|
6
|
+
from typing import Any, Mapping
|
|
7
|
+
from typing_extensions import Self, override
|
|
8
8
|
|
|
9
9
|
import httpx
|
|
10
10
|
|
|
@@ -21,7 +21,7 @@ from ._types import (
|
|
|
21
21
|
)
|
|
22
22
|
from ._utils import is_given, get_async_library
|
|
23
23
|
from ._version import __version__
|
|
24
|
-
from .resources import monitor, evaluate
|
|
24
|
+
from .resources import defend, monitor, evaluate
|
|
25
25
|
from ._streaming import Stream as Stream, AsyncStream as AsyncStream
|
|
26
26
|
from ._exceptions import APIStatusError, DeeprailsError
|
|
27
27
|
from ._base_client import (
|
|
@@ -29,10 +29,8 @@ from ._base_client import (
|
|
|
29
29
|
SyncAPIClient,
|
|
30
30
|
AsyncAPIClient,
|
|
31
31
|
)
|
|
32
|
-
from .resources.defend import defend
|
|
33
32
|
|
|
34
33
|
__all__ = [
|
|
35
|
-
"ENVIRONMENTS",
|
|
36
34
|
"Timeout",
|
|
37
35
|
"Transport",
|
|
38
36
|
"ProxiesTypes",
|
|
@@ -43,11 +41,6 @@ __all__ = [
|
|
|
43
41
|
"AsyncClient",
|
|
44
42
|
]
|
|
45
43
|
|
|
46
|
-
ENVIRONMENTS: Dict[str, str] = {
|
|
47
|
-
"production": "https://api.deeprails.com",
|
|
48
|
-
"environment_1": "https://dev-api.deeprails",
|
|
49
|
-
}
|
|
50
|
-
|
|
51
44
|
|
|
52
45
|
class Deeprails(SyncAPIClient):
|
|
53
46
|
defend: defend.DefendResource
|
|
@@ -59,14 +52,11 @@ class Deeprails(SyncAPIClient):
|
|
|
59
52
|
# client options
|
|
60
53
|
api_key: str
|
|
61
54
|
|
|
62
|
-
_environment: Literal["production", "environment_1"] | NotGiven
|
|
63
|
-
|
|
64
55
|
def __init__(
|
|
65
56
|
self,
|
|
66
57
|
*,
|
|
67
58
|
api_key: str | None = None,
|
|
68
|
-
|
|
69
|
-
base_url: str | httpx.URL | None | NotGiven = not_given,
|
|
59
|
+
base_url: str | httpx.URL | None = None,
|
|
70
60
|
timeout: float | Timeout | None | NotGiven = not_given,
|
|
71
61
|
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
72
62
|
default_headers: Mapping[str, str] | None = None,
|
|
@@ -97,31 +87,10 @@ class Deeprails(SyncAPIClient):
|
|
|
97
87
|
)
|
|
98
88
|
self.api_key = api_key
|
|
99
89
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# cast required because mypy doesn't understand the type narrowing
|
|
105
|
-
base_url = cast("str | httpx.URL", base_url) # pyright: ignore[reportUnnecessaryCast]
|
|
106
|
-
elif is_given(environment):
|
|
107
|
-
if base_url_env and base_url is not None:
|
|
108
|
-
raise ValueError(
|
|
109
|
-
"Ambiguous URL; The `DEEPRAILS_BASE_URL` env var and the `environment` argument are given. If you want to use the environment, you must pass base_url=None",
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
try:
|
|
113
|
-
base_url = ENVIRONMENTS[environment]
|
|
114
|
-
except KeyError as exc:
|
|
115
|
-
raise ValueError(f"Unknown environment: {environment}") from exc
|
|
116
|
-
elif base_url_env is not None:
|
|
117
|
-
base_url = base_url_env
|
|
118
|
-
else:
|
|
119
|
-
self._environment = environment = "production"
|
|
120
|
-
|
|
121
|
-
try:
|
|
122
|
-
base_url = ENVIRONMENTS[environment]
|
|
123
|
-
except KeyError as exc:
|
|
124
|
-
raise ValueError(f"Unknown environment: {environment}") from exc
|
|
90
|
+
if base_url is None:
|
|
91
|
+
base_url = os.environ.get("DEEPRAILS_BASE_URL")
|
|
92
|
+
if base_url is None:
|
|
93
|
+
base_url = f"https://api.deeprails.com"
|
|
125
94
|
|
|
126
95
|
super().__init__(
|
|
127
96
|
version=__version__,
|
|
@@ -164,7 +133,6 @@ class Deeprails(SyncAPIClient):
|
|
|
164
133
|
self,
|
|
165
134
|
*,
|
|
166
135
|
api_key: str | None = None,
|
|
167
|
-
environment: Literal["production", "environment_1"] | None = None,
|
|
168
136
|
base_url: str | httpx.URL | None = None,
|
|
169
137
|
timeout: float | Timeout | None | NotGiven = not_given,
|
|
170
138
|
http_client: httpx.Client | None = None,
|
|
@@ -200,7 +168,6 @@ class Deeprails(SyncAPIClient):
|
|
|
200
168
|
return self.__class__(
|
|
201
169
|
api_key=api_key or self.api_key,
|
|
202
170
|
base_url=base_url or self.base_url,
|
|
203
|
-
environment=environment or self._environment,
|
|
204
171
|
timeout=self.timeout if isinstance(timeout, NotGiven) else timeout,
|
|
205
172
|
http_client=http_client,
|
|
206
173
|
max_retries=max_retries if is_given(max_retries) else self.max_retries,
|
|
@@ -257,14 +224,11 @@ class AsyncDeeprails(AsyncAPIClient):
|
|
|
257
224
|
# client options
|
|
258
225
|
api_key: str
|
|
259
226
|
|
|
260
|
-
_environment: Literal["production", "environment_1"] | NotGiven
|
|
261
|
-
|
|
262
227
|
def __init__(
|
|
263
228
|
self,
|
|
264
229
|
*,
|
|
265
230
|
api_key: str | None = None,
|
|
266
|
-
|
|
267
|
-
base_url: str | httpx.URL | None | NotGiven = not_given,
|
|
231
|
+
base_url: str | httpx.URL | None = None,
|
|
268
232
|
timeout: float | Timeout | None | NotGiven = not_given,
|
|
269
233
|
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
270
234
|
default_headers: Mapping[str, str] | None = None,
|
|
@@ -295,31 +259,10 @@ class AsyncDeeprails(AsyncAPIClient):
|
|
|
295
259
|
)
|
|
296
260
|
self.api_key = api_key
|
|
297
261
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
# cast required because mypy doesn't understand the type narrowing
|
|
303
|
-
base_url = cast("str | httpx.URL", base_url) # pyright: ignore[reportUnnecessaryCast]
|
|
304
|
-
elif is_given(environment):
|
|
305
|
-
if base_url_env and base_url is not None:
|
|
306
|
-
raise ValueError(
|
|
307
|
-
"Ambiguous URL; The `DEEPRAILS_BASE_URL` env var and the `environment` argument are given. If you want to use the environment, you must pass base_url=None",
|
|
308
|
-
)
|
|
309
|
-
|
|
310
|
-
try:
|
|
311
|
-
base_url = ENVIRONMENTS[environment]
|
|
312
|
-
except KeyError as exc:
|
|
313
|
-
raise ValueError(f"Unknown environment: {environment}") from exc
|
|
314
|
-
elif base_url_env is not None:
|
|
315
|
-
base_url = base_url_env
|
|
316
|
-
else:
|
|
317
|
-
self._environment = environment = "production"
|
|
318
|
-
|
|
319
|
-
try:
|
|
320
|
-
base_url = ENVIRONMENTS[environment]
|
|
321
|
-
except KeyError as exc:
|
|
322
|
-
raise ValueError(f"Unknown environment: {environment}") from exc
|
|
262
|
+
if base_url is None:
|
|
263
|
+
base_url = os.environ.get("DEEPRAILS_BASE_URL")
|
|
264
|
+
if base_url is None:
|
|
265
|
+
base_url = f"https://api.deeprails.com"
|
|
323
266
|
|
|
324
267
|
super().__init__(
|
|
325
268
|
version=__version__,
|
|
@@ -362,7 +305,6 @@ class AsyncDeeprails(AsyncAPIClient):
|
|
|
362
305
|
self,
|
|
363
306
|
*,
|
|
364
307
|
api_key: str | None = None,
|
|
365
|
-
environment: Literal["production", "environment_1"] | None = None,
|
|
366
308
|
base_url: str | httpx.URL | None = None,
|
|
367
309
|
timeout: float | Timeout | None | NotGiven = not_given,
|
|
368
310
|
http_client: httpx.AsyncClient | None = None,
|
|
@@ -398,7 +340,6 @@ class AsyncDeeprails(AsyncAPIClient):
|
|
|
398
340
|
return self.__class__(
|
|
399
341
|
api_key=api_key or self.api_key,
|
|
400
342
|
base_url=base_url or self.base_url,
|
|
401
|
-
environment=environment or self._environment,
|
|
402
343
|
timeout=self.timeout if isinstance(timeout, NotGiven) else timeout,
|
|
403
344
|
http_client=http_client,
|
|
404
345
|
max_retries=max_retries if is_given(max_retries) else self.max_retries,
|
deeprails/_version.py
CHANGED
|
@@ -7,43 +7,32 @@ from typing_extensions import Literal
|
|
|
7
7
|
|
|
8
8
|
import httpx
|
|
9
9
|
|
|
10
|
-
from
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
AsyncEventsResourceWithStreamingResponse,
|
|
17
|
-
)
|
|
18
|
-
from ...types import defend_create_workflow_params, defend_update_workflow_params
|
|
19
|
-
from ..._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
|
20
|
-
from ..._utils import maybe_transform, async_maybe_transform
|
|
21
|
-
from ..._compat import cached_property
|
|
22
|
-
from ..._resource import SyncAPIResource, AsyncAPIResource
|
|
23
|
-
from ..._response import (
|
|
10
|
+
from ..types import defend_submit_event_params, defend_create_workflow_params, defend_update_workflow_params
|
|
11
|
+
from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
|
12
|
+
from .._utils import maybe_transform, async_maybe_transform
|
|
13
|
+
from .._compat import cached_property
|
|
14
|
+
from .._resource import SyncAPIResource, AsyncAPIResource
|
|
15
|
+
from .._response import (
|
|
24
16
|
to_raw_response_wrapper,
|
|
25
17
|
to_streamed_response_wrapper,
|
|
26
18
|
async_to_raw_response_wrapper,
|
|
27
19
|
async_to_streamed_response_wrapper,
|
|
28
20
|
)
|
|
29
|
-
from
|
|
30
|
-
from
|
|
21
|
+
from .._base_client import make_request_options
|
|
22
|
+
from ..types.defend_response import DefendResponse
|
|
23
|
+
from ..types.workflow_event_response import WorkflowEventResponse
|
|
31
24
|
|
|
32
25
|
__all__ = ["DefendResource", "AsyncDefendResource"]
|
|
33
26
|
|
|
34
27
|
|
|
35
28
|
class DefendResource(SyncAPIResource):
|
|
36
|
-
@cached_property
|
|
37
|
-
def events(self) -> EventsResource:
|
|
38
|
-
return EventsResource(self._client)
|
|
39
|
-
|
|
40
29
|
@cached_property
|
|
41
30
|
def with_raw_response(self) -> DefendResourceWithRawResponse:
|
|
42
31
|
"""
|
|
43
32
|
This property can be used as a prefix for any HTTP method call to return
|
|
44
33
|
the raw response object instead of the parsed content.
|
|
45
34
|
|
|
46
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
35
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
47
36
|
"""
|
|
48
37
|
return DefendResourceWithRawResponse(self)
|
|
49
38
|
|
|
@@ -52,7 +41,7 @@ class DefendResource(SyncAPIResource):
|
|
|
52
41
|
"""
|
|
53
42
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
54
43
|
|
|
55
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
44
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
56
45
|
"""
|
|
57
46
|
return DefendResourceWithStreamingResponse(self)
|
|
58
47
|
|
|
@@ -134,6 +123,42 @@ class DefendResource(SyncAPIResource):
|
|
|
134
123
|
cast_to=DefendResponse,
|
|
135
124
|
)
|
|
136
125
|
|
|
126
|
+
def retrieve_event(
|
|
127
|
+
self,
|
|
128
|
+
event_id: str,
|
|
129
|
+
*,
|
|
130
|
+
workflow_id: str,
|
|
131
|
+
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
132
|
+
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
133
|
+
extra_headers: Headers | None = None,
|
|
134
|
+
extra_query: Query | None = None,
|
|
135
|
+
extra_body: Body | None = None,
|
|
136
|
+
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
|
137
|
+
) -> WorkflowEventResponse:
|
|
138
|
+
"""
|
|
139
|
+
Retrieve a specific event of a guardrail workflow.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
extra_headers: Send extra headers
|
|
143
|
+
|
|
144
|
+
extra_query: Add additional query parameters to the request
|
|
145
|
+
|
|
146
|
+
extra_body: Add additional JSON properties to the request
|
|
147
|
+
|
|
148
|
+
timeout: Override the client-level default timeout for this request, in seconds
|
|
149
|
+
"""
|
|
150
|
+
if not workflow_id:
|
|
151
|
+
raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
|
|
152
|
+
if not event_id:
|
|
153
|
+
raise ValueError(f"Expected a non-empty value for `event_id` but received {event_id!r}")
|
|
154
|
+
return self._get(
|
|
155
|
+
f"/defend/{workflow_id}/events/{event_id}",
|
|
156
|
+
options=make_request_options(
|
|
157
|
+
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
|
158
|
+
),
|
|
159
|
+
cast_to=WorkflowEventResponse,
|
|
160
|
+
)
|
|
161
|
+
|
|
137
162
|
def retrieve_workflow(
|
|
138
163
|
self,
|
|
139
164
|
workflow_id: str,
|
|
@@ -167,6 +192,69 @@ class DefendResource(SyncAPIResource):
|
|
|
167
192
|
cast_to=DefendResponse,
|
|
168
193
|
)
|
|
169
194
|
|
|
195
|
+
def submit_event(
|
|
196
|
+
self,
|
|
197
|
+
workflow_id: str,
|
|
198
|
+
*,
|
|
199
|
+
model_input: defend_submit_event_params.ModelInput,
|
|
200
|
+
model_output: str,
|
|
201
|
+
model_used: str,
|
|
202
|
+
nametag: str,
|
|
203
|
+
run_mode: Literal["precision_plus", "precision", "smart", "economy"],
|
|
204
|
+
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
205
|
+
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
206
|
+
extra_headers: Headers | None = None,
|
|
207
|
+
extra_query: Query | None = None,
|
|
208
|
+
extra_body: Body | None = None,
|
|
209
|
+
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
|
210
|
+
) -> WorkflowEventResponse:
|
|
211
|
+
"""
|
|
212
|
+
Submit a model input and output pair to a workflow for evaluation.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
model_input: A dictionary of inputs sent to the LLM to generate output. This must contain a
|
|
216
|
+
`user_prompt` field and an optional `context` field. Additional properties are
|
|
217
|
+
allowed.
|
|
218
|
+
|
|
219
|
+
model_output: Output generated by the LLM to be evaluated.
|
|
220
|
+
|
|
221
|
+
model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
222
|
+
|
|
223
|
+
nametag: An optional, user-defined tag for the event.
|
|
224
|
+
|
|
225
|
+
run_mode: Run mode for the workflow event. The run mode allows the user to optimize for
|
|
226
|
+
speed, accuracy, and cost by determining which models are used to evaluate the
|
|
227
|
+
event. Available run modes include `precision_plus`, `precision`, `smart`, and
|
|
228
|
+
`economy`. Defaults to `smart`.
|
|
229
|
+
|
|
230
|
+
extra_headers: Send extra headers
|
|
231
|
+
|
|
232
|
+
extra_query: Add additional query parameters to the request
|
|
233
|
+
|
|
234
|
+
extra_body: Add additional JSON properties to the request
|
|
235
|
+
|
|
236
|
+
timeout: Override the client-level default timeout for this request, in seconds
|
|
237
|
+
"""
|
|
238
|
+
if not workflow_id:
|
|
239
|
+
raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
|
|
240
|
+
return self._post(
|
|
241
|
+
f"/defend/{workflow_id}/events",
|
|
242
|
+
body=maybe_transform(
|
|
243
|
+
{
|
|
244
|
+
"model_input": model_input,
|
|
245
|
+
"model_output": model_output,
|
|
246
|
+
"model_used": model_used,
|
|
247
|
+
"nametag": nametag,
|
|
248
|
+
"run_mode": run_mode,
|
|
249
|
+
},
|
|
250
|
+
defend_submit_event_params.DefendSubmitEventParams,
|
|
251
|
+
),
|
|
252
|
+
options=make_request_options(
|
|
253
|
+
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
|
254
|
+
),
|
|
255
|
+
cast_to=WorkflowEventResponse,
|
|
256
|
+
)
|
|
257
|
+
|
|
170
258
|
def update_workflow(
|
|
171
259
|
self,
|
|
172
260
|
workflow_id: str,
|
|
@@ -219,17 +307,13 @@ class DefendResource(SyncAPIResource):
|
|
|
219
307
|
|
|
220
308
|
|
|
221
309
|
class AsyncDefendResource(AsyncAPIResource):
|
|
222
|
-
@cached_property
|
|
223
|
-
def events(self) -> AsyncEventsResource:
|
|
224
|
-
return AsyncEventsResource(self._client)
|
|
225
|
-
|
|
226
310
|
@cached_property
|
|
227
311
|
def with_raw_response(self) -> AsyncDefendResourceWithRawResponse:
|
|
228
312
|
"""
|
|
229
313
|
This property can be used as a prefix for any HTTP method call to return
|
|
230
314
|
the raw response object instead of the parsed content.
|
|
231
315
|
|
|
232
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
316
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
233
317
|
"""
|
|
234
318
|
return AsyncDefendResourceWithRawResponse(self)
|
|
235
319
|
|
|
@@ -238,7 +322,7 @@ class AsyncDefendResource(AsyncAPIResource):
|
|
|
238
322
|
"""
|
|
239
323
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
240
324
|
|
|
241
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
325
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
242
326
|
"""
|
|
243
327
|
return AsyncDefendResourceWithStreamingResponse(self)
|
|
244
328
|
|
|
@@ -320,6 +404,42 @@ class AsyncDefendResource(AsyncAPIResource):
|
|
|
320
404
|
cast_to=DefendResponse,
|
|
321
405
|
)
|
|
322
406
|
|
|
407
|
+
async def retrieve_event(
|
|
408
|
+
self,
|
|
409
|
+
event_id: str,
|
|
410
|
+
*,
|
|
411
|
+
workflow_id: str,
|
|
412
|
+
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
413
|
+
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
414
|
+
extra_headers: Headers | None = None,
|
|
415
|
+
extra_query: Query | None = None,
|
|
416
|
+
extra_body: Body | None = None,
|
|
417
|
+
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
|
418
|
+
) -> WorkflowEventResponse:
|
|
419
|
+
"""
|
|
420
|
+
Retrieve a specific event of a guardrail workflow.
|
|
421
|
+
|
|
422
|
+
Args:
|
|
423
|
+
extra_headers: Send extra headers
|
|
424
|
+
|
|
425
|
+
extra_query: Add additional query parameters to the request
|
|
426
|
+
|
|
427
|
+
extra_body: Add additional JSON properties to the request
|
|
428
|
+
|
|
429
|
+
timeout: Override the client-level default timeout for this request, in seconds
|
|
430
|
+
"""
|
|
431
|
+
if not workflow_id:
|
|
432
|
+
raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
|
|
433
|
+
if not event_id:
|
|
434
|
+
raise ValueError(f"Expected a non-empty value for `event_id` but received {event_id!r}")
|
|
435
|
+
return await self._get(
|
|
436
|
+
f"/defend/{workflow_id}/events/{event_id}",
|
|
437
|
+
options=make_request_options(
|
|
438
|
+
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
|
439
|
+
),
|
|
440
|
+
cast_to=WorkflowEventResponse,
|
|
441
|
+
)
|
|
442
|
+
|
|
323
443
|
async def retrieve_workflow(
|
|
324
444
|
self,
|
|
325
445
|
workflow_id: str,
|
|
@@ -353,6 +473,69 @@ class AsyncDefendResource(AsyncAPIResource):
|
|
|
353
473
|
cast_to=DefendResponse,
|
|
354
474
|
)
|
|
355
475
|
|
|
476
|
+
async def submit_event(
|
|
477
|
+
self,
|
|
478
|
+
workflow_id: str,
|
|
479
|
+
*,
|
|
480
|
+
model_input: defend_submit_event_params.ModelInput,
|
|
481
|
+
model_output: str,
|
|
482
|
+
model_used: str,
|
|
483
|
+
nametag: str,
|
|
484
|
+
run_mode: Literal["precision_plus", "precision", "smart", "economy"],
|
|
485
|
+
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
486
|
+
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
487
|
+
extra_headers: Headers | None = None,
|
|
488
|
+
extra_query: Query | None = None,
|
|
489
|
+
extra_body: Body | None = None,
|
|
490
|
+
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
|
491
|
+
) -> WorkflowEventResponse:
|
|
492
|
+
"""
|
|
493
|
+
Submit a model input and output pair to a workflow for evaluation.
|
|
494
|
+
|
|
495
|
+
Args:
|
|
496
|
+
model_input: A dictionary of inputs sent to the LLM to generate output. This must contain a
|
|
497
|
+
`user_prompt` field and an optional `context` field. Additional properties are
|
|
498
|
+
allowed.
|
|
499
|
+
|
|
500
|
+
model_output: Output generated by the LLM to be evaluated.
|
|
501
|
+
|
|
502
|
+
model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
503
|
+
|
|
504
|
+
nametag: An optional, user-defined tag for the event.
|
|
505
|
+
|
|
506
|
+
run_mode: Run mode for the workflow event. The run mode allows the user to optimize for
|
|
507
|
+
speed, accuracy, and cost by determining which models are used to evaluate the
|
|
508
|
+
event. Available run modes include `precision_plus`, `precision`, `smart`, and
|
|
509
|
+
`economy`. Defaults to `smart`.
|
|
510
|
+
|
|
511
|
+
extra_headers: Send extra headers
|
|
512
|
+
|
|
513
|
+
extra_query: Add additional query parameters to the request
|
|
514
|
+
|
|
515
|
+
extra_body: Add additional JSON properties to the request
|
|
516
|
+
|
|
517
|
+
timeout: Override the client-level default timeout for this request, in seconds
|
|
518
|
+
"""
|
|
519
|
+
if not workflow_id:
|
|
520
|
+
raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
|
|
521
|
+
return await self._post(
|
|
522
|
+
f"/defend/{workflow_id}/events",
|
|
523
|
+
body=await async_maybe_transform(
|
|
524
|
+
{
|
|
525
|
+
"model_input": model_input,
|
|
526
|
+
"model_output": model_output,
|
|
527
|
+
"model_used": model_used,
|
|
528
|
+
"nametag": nametag,
|
|
529
|
+
"run_mode": run_mode,
|
|
530
|
+
},
|
|
531
|
+
defend_submit_event_params.DefendSubmitEventParams,
|
|
532
|
+
),
|
|
533
|
+
options=make_request_options(
|
|
534
|
+
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
|
535
|
+
),
|
|
536
|
+
cast_to=WorkflowEventResponse,
|
|
537
|
+
)
|
|
538
|
+
|
|
356
539
|
async def update_workflow(
|
|
357
540
|
self,
|
|
358
541
|
workflow_id: str,
|
|
@@ -411,17 +594,19 @@ class DefendResourceWithRawResponse:
|
|
|
411
594
|
self.create_workflow = to_raw_response_wrapper(
|
|
412
595
|
defend.create_workflow,
|
|
413
596
|
)
|
|
597
|
+
self.retrieve_event = to_raw_response_wrapper(
|
|
598
|
+
defend.retrieve_event,
|
|
599
|
+
)
|
|
414
600
|
self.retrieve_workflow = to_raw_response_wrapper(
|
|
415
601
|
defend.retrieve_workflow,
|
|
416
602
|
)
|
|
603
|
+
self.submit_event = to_raw_response_wrapper(
|
|
604
|
+
defend.submit_event,
|
|
605
|
+
)
|
|
417
606
|
self.update_workflow = to_raw_response_wrapper(
|
|
418
607
|
defend.update_workflow,
|
|
419
608
|
)
|
|
420
609
|
|
|
421
|
-
@cached_property
|
|
422
|
-
def events(self) -> EventsResourceWithRawResponse:
|
|
423
|
-
return EventsResourceWithRawResponse(self._defend.events)
|
|
424
|
-
|
|
425
610
|
|
|
426
611
|
class AsyncDefendResourceWithRawResponse:
|
|
427
612
|
def __init__(self, defend: AsyncDefendResource) -> None:
|
|
@@ -430,17 +615,19 @@ class AsyncDefendResourceWithRawResponse:
|
|
|
430
615
|
self.create_workflow = async_to_raw_response_wrapper(
|
|
431
616
|
defend.create_workflow,
|
|
432
617
|
)
|
|
618
|
+
self.retrieve_event = async_to_raw_response_wrapper(
|
|
619
|
+
defend.retrieve_event,
|
|
620
|
+
)
|
|
433
621
|
self.retrieve_workflow = async_to_raw_response_wrapper(
|
|
434
622
|
defend.retrieve_workflow,
|
|
435
623
|
)
|
|
624
|
+
self.submit_event = async_to_raw_response_wrapper(
|
|
625
|
+
defend.submit_event,
|
|
626
|
+
)
|
|
436
627
|
self.update_workflow = async_to_raw_response_wrapper(
|
|
437
628
|
defend.update_workflow,
|
|
438
629
|
)
|
|
439
630
|
|
|
440
|
-
@cached_property
|
|
441
|
-
def events(self) -> AsyncEventsResourceWithRawResponse:
|
|
442
|
-
return AsyncEventsResourceWithRawResponse(self._defend.events)
|
|
443
|
-
|
|
444
631
|
|
|
445
632
|
class DefendResourceWithStreamingResponse:
|
|
446
633
|
def __init__(self, defend: DefendResource) -> None:
|
|
@@ -449,17 +636,19 @@ class DefendResourceWithStreamingResponse:
|
|
|
449
636
|
self.create_workflow = to_streamed_response_wrapper(
|
|
450
637
|
defend.create_workflow,
|
|
451
638
|
)
|
|
639
|
+
self.retrieve_event = to_streamed_response_wrapper(
|
|
640
|
+
defend.retrieve_event,
|
|
641
|
+
)
|
|
452
642
|
self.retrieve_workflow = to_streamed_response_wrapper(
|
|
453
643
|
defend.retrieve_workflow,
|
|
454
644
|
)
|
|
645
|
+
self.submit_event = to_streamed_response_wrapper(
|
|
646
|
+
defend.submit_event,
|
|
647
|
+
)
|
|
455
648
|
self.update_workflow = to_streamed_response_wrapper(
|
|
456
649
|
defend.update_workflow,
|
|
457
650
|
)
|
|
458
651
|
|
|
459
|
-
@cached_property
|
|
460
|
-
def events(self) -> EventsResourceWithStreamingResponse:
|
|
461
|
-
return EventsResourceWithStreamingResponse(self._defend.events)
|
|
462
|
-
|
|
463
652
|
|
|
464
653
|
class AsyncDefendResourceWithStreamingResponse:
|
|
465
654
|
def __init__(self, defend: AsyncDefendResource) -> None:
|
|
@@ -468,13 +657,15 @@ class AsyncDefendResourceWithStreamingResponse:
|
|
|
468
657
|
self.create_workflow = async_to_streamed_response_wrapper(
|
|
469
658
|
defend.create_workflow,
|
|
470
659
|
)
|
|
660
|
+
self.retrieve_event = async_to_streamed_response_wrapper(
|
|
661
|
+
defend.retrieve_event,
|
|
662
|
+
)
|
|
471
663
|
self.retrieve_workflow = async_to_streamed_response_wrapper(
|
|
472
664
|
defend.retrieve_workflow,
|
|
473
665
|
)
|
|
666
|
+
self.submit_event = async_to_streamed_response_wrapper(
|
|
667
|
+
defend.submit_event,
|
|
668
|
+
)
|
|
474
669
|
self.update_workflow = async_to_streamed_response_wrapper(
|
|
475
670
|
defend.update_workflow,
|
|
476
671
|
)
|
|
477
|
-
|
|
478
|
-
@cached_property
|
|
479
|
-
def events(self) -> AsyncEventsResourceWithStreamingResponse:
|
|
480
|
-
return AsyncEventsResourceWithStreamingResponse(self._defend.events)
|
deeprails/resources/evaluate.py
CHANGED
|
@@ -31,7 +31,7 @@ class EvaluateResource(SyncAPIResource):
|
|
|
31
31
|
This property can be used as a prefix for any HTTP method call to return
|
|
32
32
|
the raw response object instead of the parsed content.
|
|
33
33
|
|
|
34
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
34
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
35
35
|
"""
|
|
36
36
|
return EvaluateResourceWithRawResponse(self)
|
|
37
37
|
|
|
@@ -40,7 +40,7 @@ class EvaluateResource(SyncAPIResource):
|
|
|
40
40
|
"""
|
|
41
41
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
42
42
|
|
|
43
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
43
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
44
44
|
"""
|
|
45
45
|
return EvaluateResourceWithStreamingResponse(self)
|
|
46
46
|
|
|
@@ -162,7 +162,7 @@ class AsyncEvaluateResource(AsyncAPIResource):
|
|
|
162
162
|
This property can be used as a prefix for any HTTP method call to return
|
|
163
163
|
the raw response object instead of the parsed content.
|
|
164
164
|
|
|
165
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
165
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
166
166
|
"""
|
|
167
167
|
return AsyncEvaluateResourceWithRawResponse(self)
|
|
168
168
|
|
|
@@ -171,7 +171,7 @@ class AsyncEvaluateResource(AsyncAPIResource):
|
|
|
171
171
|
"""
|
|
172
172
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
173
173
|
|
|
174
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
174
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
175
175
|
"""
|
|
176
176
|
return AsyncEvaluateResourceWithStreamingResponse(self)
|
|
177
177
|
|
deeprails/resources/monitor.py
CHANGED
|
@@ -33,7 +33,7 @@ class MonitorResource(SyncAPIResource):
|
|
|
33
33
|
This property can be used as a prefix for any HTTP method call to return
|
|
34
34
|
the raw response object instead of the parsed content.
|
|
35
35
|
|
|
36
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
36
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
37
37
|
"""
|
|
38
38
|
return MonitorResourceWithRawResponse(self)
|
|
39
39
|
|
|
@@ -42,7 +42,7 @@ class MonitorResource(SyncAPIResource):
|
|
|
42
42
|
"""
|
|
43
43
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
44
44
|
|
|
45
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
45
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
46
46
|
"""
|
|
47
47
|
return MonitorResourceWithStreamingResponse(self)
|
|
48
48
|
|
|
@@ -267,7 +267,7 @@ class AsyncMonitorResource(AsyncAPIResource):
|
|
|
267
267
|
This property can be used as a prefix for any HTTP method call to return
|
|
268
268
|
the raw response object instead of the parsed content.
|
|
269
269
|
|
|
270
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
270
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
|
|
271
271
|
"""
|
|
272
272
|
return AsyncMonitorResourceWithRawResponse(self)
|
|
273
273
|
|
|
@@ -276,7 +276,7 @@ class AsyncMonitorResource(AsyncAPIResource):
|
|
|
276
276
|
"""
|
|
277
277
|
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
|
278
278
|
|
|
279
|
-
For more information, see https://www.github.com/deeprails/deeprails-python
|
|
279
|
+
For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
|
|
280
280
|
"""
|
|
281
281
|
return AsyncMonitorResourceWithStreamingResponse(self)
|
|
282
282
|
|