agenta 0.24.1__py3-none-any.whl → 0.24.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenta might be problematic. Click here for more details.
- agenta/cli/variant_commands.py +15 -10
- agenta/client/Readme.md +72 -64
- agenta/client/api.py +1 -1
- agenta/client/backend/__init__.py +14 -9
- agenta/client/backend/apps/client.py +1669 -0
- agenta/client/backend/bases/client.py +190 -0
- agenta/client/backend/client.py +2102 -868
- agenta/client/backend/configs/client.py +598 -0
- agenta/client/backend/containers/client.py +638 -0
- agenta/client/backend/{resources/containers → containers}/types/container_templates_response.py +1 -2
- agenta/client/backend/core/__init__.py +29 -0
- agenta/client/backend/core/client_wrapper.py +42 -9
- agenta/client/backend/core/datetime_utils.py +1 -1
- agenta/client/backend/core/file.py +43 -0
- agenta/client/backend/core/http_client.py +553 -0
- agenta/client/backend/core/jsonable_encoder.py +33 -39
- agenta/client/backend/core/pydantic_utilities.py +212 -0
- agenta/client/backend/core/query_encoder.py +60 -0
- agenta/client/backend/core/remove_none_from_dict.py +2 -2
- agenta/client/backend/core/request_options.py +32 -0
- agenta/client/backend/core/serialization.py +179 -0
- agenta/client/backend/environments/client.py +190 -0
- agenta/client/backend/evaluations/client.py +1462 -0
- agenta/client/backend/evaluators/client.py +911 -0
- agenta/client/backend/observability/client.py +1271 -0
- agenta/client/backend/testsets/client.py +1132 -0
- agenta/client/backend/types/__init__.py +8 -6
- agenta/client/backend/types/aggregated_result.py +14 -29
- agenta/client/backend/types/aggregated_result_evaluator_config.py +1 -2
- agenta/client/backend/types/app.py +13 -28
- agenta/client/backend/types/app_variant_response.py +21 -37
- agenta/client/backend/types/app_variant_revision.py +17 -32
- agenta/client/backend/types/base_output.py +13 -28
- agenta/client/backend/types/body_import_testset.py +16 -31
- agenta/client/backend/types/config_db.py +16 -31
- agenta/client/backend/types/correct_answer.py +22 -0
- agenta/client/backend/types/create_app_output.py +13 -28
- agenta/client/backend/types/create_span.py +33 -50
- agenta/client/backend/types/create_trace_response.py +16 -31
- agenta/client/backend/types/docker_env_vars.py +13 -28
- agenta/client/backend/types/environment_output.py +21 -36
- agenta/client/backend/types/environment_output_extended.py +21 -36
- agenta/client/backend/types/environment_revision.py +18 -33
- agenta/client/backend/types/error.py +16 -31
- agenta/client/backend/types/evaluation.py +20 -34
- agenta/client/backend/types/evaluation_scenario.py +18 -33
- agenta/client/backend/types/evaluation_scenario_input.py +16 -31
- agenta/client/backend/types/evaluation_scenario_output.py +18 -33
- agenta/client/backend/types/evaluation_scenario_result.py +14 -29
- agenta/client/backend/types/evaluation_scenario_score_update.py +13 -28
- agenta/client/backend/types/evaluation_status_enum.py +11 -33
- agenta/client/backend/types/evaluation_type.py +3 -21
- agenta/client/backend/types/evaluator.py +18 -32
- agenta/client/backend/types/evaluator_config.py +20 -33
- agenta/client/backend/types/get_config_response.py +16 -31
- agenta/client/backend/types/http_validation_error.py +14 -29
- agenta/client/backend/types/human_evaluation.py +17 -32
- agenta/client/backend/types/human_evaluation_scenario.py +21 -37
- agenta/client/backend/types/human_evaluation_scenario_input.py +13 -28
- agenta/client/backend/types/human_evaluation_scenario_output.py +13 -28
- agenta/client/backend/types/human_evaluation_scenario_update.py +26 -41
- agenta/client/backend/types/human_evaluation_update.py +14 -29
- agenta/client/backend/types/image.py +18 -33
- agenta/client/backend/types/invite_request.py +13 -28
- agenta/client/backend/types/list_api_keys_response.py +18 -33
- agenta/client/backend/types/llm_run_rate_limit.py +13 -28
- agenta/client/backend/types/llm_tokens.py +16 -31
- agenta/client/backend/types/lm_providers_enum.py +21 -0
- agenta/client/backend/types/new_human_evaluation.py +13 -28
- agenta/client/backend/types/new_testset.py +16 -31
- agenta/client/backend/types/organization.py +22 -36
- agenta/client/backend/types/organization_output.py +13 -28
- agenta/client/backend/types/outputs.py +5 -0
- agenta/client/backend/types/permission.py +36 -137
- agenta/client/backend/types/result.py +17 -32
- agenta/client/backend/types/simple_evaluation_output.py +13 -28
- agenta/client/backend/types/span.py +23 -38
- agenta/client/backend/types/span_detail.py +26 -40
- agenta/client/backend/types/span_status_code.py +1 -25
- agenta/client/backend/types/span_variant.py +16 -31
- agenta/client/backend/types/template.py +14 -29
- agenta/client/backend/types/template_image_info.py +21 -35
- agenta/client/backend/types/test_set_output_response.py +16 -32
- agenta/client/backend/types/test_set_simple_response.py +13 -28
- agenta/client/backend/types/trace_detail.py +26 -40
- agenta/client/backend/types/update_app_output.py +22 -0
- agenta/client/backend/types/uri.py +13 -28
- agenta/client/backend/types/validation_error.py +13 -28
- agenta/client/backend/types/variant_action.py +14 -29
- agenta/client/backend/types/variant_action_enum.py +1 -19
- agenta/client/backend/types/with_pagination.py +14 -30
- agenta/client/backend/types/workspace_member_response.py +14 -29
- agenta/client/backend/types/workspace_permission.py +18 -33
- agenta/client/backend/types/workspace_response.py +20 -35
- agenta/client/backend/types/workspace_role.py +11 -37
- agenta/client/backend/types/workspace_role_response.py +17 -32
- agenta/client/backend/variants/client.py +1447 -0
- agenta/client/backend/variants/types/add_variant_from_base_and_config_response.py +8 -0
- agenta/sdk/tracing/llm_tracing.py +10 -12
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/METADATA +1 -1
- agenta-0.24.2.dist-info/RECORD +175 -0
- agenta/client/backend/resources/__init__.py +0 -31
- agenta/client/backend/resources/apps/client.py +0 -977
- agenta/client/backend/resources/bases/client.py +0 -127
- agenta/client/backend/resources/configs/client.py +0 -377
- agenta/client/backend/resources/containers/client.py +0 -383
- agenta/client/backend/resources/environments/client.py +0 -131
- agenta/client/backend/resources/evaluations/client.py +0 -1008
- agenta/client/backend/resources/evaluators/client.py +0 -594
- agenta/client/backend/resources/observability/client.py +0 -1187
- agenta/client/backend/resources/testsets/client.py +0 -689
- agenta/client/backend/resources/variants/client.py +0 -796
- agenta/client/backend/resources/variants/types/add_variant_from_base_and_config_response.py +0 -7
- agenta/client/backend/types/evaluation_webhook.py +0 -36
- agenta/client/backend/types/feedback.py +0 -40
- agenta/client/backend/types/span_kind.py +0 -49
- agenta-0.24.1.dist-info/RECORD +0 -169
- /agenta/client/backend/{resources/apps → apps}/__init__.py +0 -0
- /agenta/client/backend/{resources/bases → bases}/__init__.py +0 -0
- /agenta/client/backend/{resources/configs → configs}/__init__.py +0 -0
- /agenta/client/backend/{resources/containers → containers}/__init__.py +0 -0
- /agenta/client/backend/{resources/containers → containers}/types/__init__.py +0 -0
- /agenta/client/backend/{resources/environments → environments}/__init__.py +0 -0
- /agenta/client/backend/{resources/evaluations → evaluations}/__init__.py +0 -0
- /agenta/client/backend/{resources/evaluators → evaluators}/__init__.py +0 -0
- /agenta/client/backend/{resources/observability → observability}/__init__.py +0 -0
- /agenta/client/backend/{resources/testsets → testsets}/__init__.py +0 -0
- /agenta/client/backend/{resources/variants → variants}/__init__.py +0 -0
- /agenta/client/backend/{resources/variants → variants}/types/__init__.py +0 -0
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/WHEEL +0 -0
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/entry_points.txt +0 -0
|
@@ -1,594 +0,0 @@
|
|
|
1
|
-
# This file was auto-generated by Fern from our API Definition.
|
|
2
|
-
|
|
3
|
-
import typing
|
|
4
|
-
import urllib.parse
|
|
5
|
-
from json.decoder import JSONDecodeError
|
|
6
|
-
|
|
7
|
-
from ...core.api_error import ApiError
|
|
8
|
-
from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
|
|
9
|
-
from ...core.jsonable_encoder import jsonable_encoder
|
|
10
|
-
from ...core.remove_none_from_dict import remove_none_from_dict
|
|
11
|
-
from ...errors.unprocessable_entity_error import UnprocessableEntityError
|
|
12
|
-
from ...types.evaluator import Evaluator
|
|
13
|
-
from ...types.evaluator_config import EvaluatorConfig
|
|
14
|
-
from ...types.http_validation_error import HttpValidationError
|
|
15
|
-
|
|
16
|
-
try:
|
|
17
|
-
import pydantic.v1 as pydantic # type: ignore
|
|
18
|
-
except ImportError:
|
|
19
|
-
import pydantic # type: ignore
|
|
20
|
-
|
|
21
|
-
# this is used as the default value for optional parameters
|
|
22
|
-
OMIT = typing.cast(typing.Any, ...)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class EvaluatorsClient:
|
|
26
|
-
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
27
|
-
self._client_wrapper = client_wrapper
|
|
28
|
-
|
|
29
|
-
def get_evaluators_endpoint(self) -> typing.List[Evaluator]:
|
|
30
|
-
"""
|
|
31
|
-
Endpoint to fetch a list of evaluators.
|
|
32
|
-
|
|
33
|
-
Returns:
|
|
34
|
-
List[Evaluator]: A list of evaluator objects.
|
|
35
|
-
|
|
36
|
-
---
|
|
37
|
-
from agenta.client import AgentaApi
|
|
38
|
-
|
|
39
|
-
client = AgentaApi(
|
|
40
|
-
api_key="YOUR_API_KEY",
|
|
41
|
-
base_url="https://yourhost.com/path/to/api",
|
|
42
|
-
)
|
|
43
|
-
client.evaluators.get_evaluators_endpoint()
|
|
44
|
-
"""
|
|
45
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
46
|
-
"GET",
|
|
47
|
-
urllib.parse.urljoin(
|
|
48
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators"
|
|
49
|
-
),
|
|
50
|
-
headers=self._client_wrapper.get_headers(),
|
|
51
|
-
timeout=60,
|
|
52
|
-
)
|
|
53
|
-
if 200 <= _response.status_code < 300:
|
|
54
|
-
return pydantic.parse_obj_as(typing.List[Evaluator], _response.json()) # type: ignore
|
|
55
|
-
try:
|
|
56
|
-
_response_json = _response.json()
|
|
57
|
-
except JSONDecodeError:
|
|
58
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
59
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
60
|
-
|
|
61
|
-
def get_evaluator_configs(self, *, app_id: str) -> typing.List[EvaluatorConfig]:
|
|
62
|
-
"""
|
|
63
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
64
|
-
|
|
65
|
-
Args:
|
|
66
|
-
app_id (str): The ID of the app.
|
|
67
|
-
|
|
68
|
-
Returns:
|
|
69
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
70
|
-
|
|
71
|
-
Parameters:
|
|
72
|
-
- app_id: str.
|
|
73
|
-
---
|
|
74
|
-
from agenta.client import AgentaApi
|
|
75
|
-
|
|
76
|
-
client = AgentaApi(
|
|
77
|
-
api_key="YOUR_API_KEY",
|
|
78
|
-
base_url="https://yourhost.com/path/to/api",
|
|
79
|
-
)
|
|
80
|
-
client.evaluators.get_evaluator_configs(
|
|
81
|
-
app_id="app_id",
|
|
82
|
-
)
|
|
83
|
-
"""
|
|
84
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
85
|
-
"GET",
|
|
86
|
-
urllib.parse.urljoin(
|
|
87
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators/configs"
|
|
88
|
-
),
|
|
89
|
-
params=remove_none_from_dict({"app_id": app_id}),
|
|
90
|
-
headers=self._client_wrapper.get_headers(),
|
|
91
|
-
timeout=60,
|
|
92
|
-
)
|
|
93
|
-
if 200 <= _response.status_code < 300:
|
|
94
|
-
return pydantic.parse_obj_as(typing.List[EvaluatorConfig], _response.json()) # type: ignore
|
|
95
|
-
if _response.status_code == 422:
|
|
96
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
97
|
-
try:
|
|
98
|
-
_response_json = _response.json()
|
|
99
|
-
except JSONDecodeError:
|
|
100
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
101
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
102
|
-
|
|
103
|
-
def create_new_evaluator_config(
|
|
104
|
-
self,
|
|
105
|
-
*,
|
|
106
|
-
app_id: str,
|
|
107
|
-
name: str,
|
|
108
|
-
evaluator_key: str,
|
|
109
|
-
settings_values: typing.Dict[str, typing.Any],
|
|
110
|
-
) -> EvaluatorConfig:
|
|
111
|
-
"""
|
|
112
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
113
|
-
|
|
114
|
-
Args:
|
|
115
|
-
app_id (str): The ID of the app.
|
|
116
|
-
|
|
117
|
-
Returns:
|
|
118
|
-
EvaluatorConfigDB: Evaluator configuration api model.
|
|
119
|
-
|
|
120
|
-
Parameters:
|
|
121
|
-
- app_id: str.
|
|
122
|
-
|
|
123
|
-
- name: str.
|
|
124
|
-
|
|
125
|
-
- evaluator_key: str.
|
|
126
|
-
|
|
127
|
-
- settings_values: typing.Dict[str, typing.Any].
|
|
128
|
-
---
|
|
129
|
-
from agenta.client import AgentaApi
|
|
130
|
-
|
|
131
|
-
client = AgentaApi(
|
|
132
|
-
api_key="YOUR_API_KEY",
|
|
133
|
-
base_url="https://yourhost.com/path/to/api",
|
|
134
|
-
)
|
|
135
|
-
client.evaluators.create_new_evaluator_config(
|
|
136
|
-
app_id="app_id",
|
|
137
|
-
name="name",
|
|
138
|
-
evaluator_key="evaluator_key",
|
|
139
|
-
settings_values={},
|
|
140
|
-
)
|
|
141
|
-
"""
|
|
142
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
143
|
-
"POST",
|
|
144
|
-
urllib.parse.urljoin(
|
|
145
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators/configs"
|
|
146
|
-
),
|
|
147
|
-
json=jsonable_encoder(
|
|
148
|
-
{
|
|
149
|
-
"app_id": app_id,
|
|
150
|
-
"name": name,
|
|
151
|
-
"evaluator_key": evaluator_key,
|
|
152
|
-
"settings_values": settings_values,
|
|
153
|
-
}
|
|
154
|
-
),
|
|
155
|
-
headers=self._client_wrapper.get_headers(),
|
|
156
|
-
timeout=60,
|
|
157
|
-
)
|
|
158
|
-
if 200 <= _response.status_code < 300:
|
|
159
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
160
|
-
if _response.status_code == 422:
|
|
161
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
162
|
-
try:
|
|
163
|
-
_response_json = _response.json()
|
|
164
|
-
except JSONDecodeError:
|
|
165
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
166
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
167
|
-
|
|
168
|
-
def get_evaluator_config(self, evaluator_config_id: str) -> EvaluatorConfig:
|
|
169
|
-
"""
|
|
170
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
171
|
-
|
|
172
|
-
Returns:
|
|
173
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
174
|
-
|
|
175
|
-
Parameters:
|
|
176
|
-
- evaluator_config_id: str.
|
|
177
|
-
---
|
|
178
|
-
from agenta.client import AgentaApi
|
|
179
|
-
|
|
180
|
-
client = AgentaApi(
|
|
181
|
-
api_key="YOUR_API_KEY",
|
|
182
|
-
base_url="https://yourhost.com/path/to/api",
|
|
183
|
-
)
|
|
184
|
-
client.evaluators.get_evaluator_config(
|
|
185
|
-
evaluator_config_id="evaluator_config_id",
|
|
186
|
-
)
|
|
187
|
-
"""
|
|
188
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
189
|
-
"GET",
|
|
190
|
-
urllib.parse.urljoin(
|
|
191
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
192
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
193
|
-
),
|
|
194
|
-
headers=self._client_wrapper.get_headers(),
|
|
195
|
-
timeout=60,
|
|
196
|
-
)
|
|
197
|
-
if 200 <= _response.status_code < 300:
|
|
198
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
199
|
-
if _response.status_code == 422:
|
|
200
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
201
|
-
try:
|
|
202
|
-
_response_json = _response.json()
|
|
203
|
-
except JSONDecodeError:
|
|
204
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
205
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
206
|
-
|
|
207
|
-
def update_evaluator_config(
|
|
208
|
-
self,
|
|
209
|
-
evaluator_config_id: str,
|
|
210
|
-
*,
|
|
211
|
-
name: typing.Optional[str] = OMIT,
|
|
212
|
-
evaluator_key: typing.Optional[str] = OMIT,
|
|
213
|
-
settings_values: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
|
|
214
|
-
) -> EvaluatorConfig:
|
|
215
|
-
"""
|
|
216
|
-
Endpoint to update evaluator configurations for a specific app.
|
|
217
|
-
|
|
218
|
-
Returns:
|
|
219
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
220
|
-
|
|
221
|
-
Parameters:
|
|
222
|
-
- evaluator_config_id: str.
|
|
223
|
-
|
|
224
|
-
- name: typing.Optional[str].
|
|
225
|
-
|
|
226
|
-
- evaluator_key: typing.Optional[str].
|
|
227
|
-
|
|
228
|
-
- settings_values: typing.Optional[typing.Dict[str, typing.Any]].
|
|
229
|
-
---
|
|
230
|
-
from agenta.client import AgentaApi
|
|
231
|
-
|
|
232
|
-
client = AgentaApi(
|
|
233
|
-
api_key="YOUR_API_KEY",
|
|
234
|
-
base_url="https://yourhost.com/path/to/api",
|
|
235
|
-
)
|
|
236
|
-
client.evaluators.update_evaluator_config(
|
|
237
|
-
evaluator_config_id="evaluator_config_id",
|
|
238
|
-
)
|
|
239
|
-
"""
|
|
240
|
-
_request: typing.Dict[str, typing.Any] = {}
|
|
241
|
-
if name is not OMIT:
|
|
242
|
-
_request["name"] = name
|
|
243
|
-
if evaluator_key is not OMIT:
|
|
244
|
-
_request["evaluator_key"] = evaluator_key
|
|
245
|
-
if settings_values is not OMIT:
|
|
246
|
-
_request["settings_values"] = settings_values
|
|
247
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
248
|
-
"PUT",
|
|
249
|
-
urllib.parse.urljoin(
|
|
250
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
251
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
252
|
-
),
|
|
253
|
-
json=jsonable_encoder(_request),
|
|
254
|
-
headers=self._client_wrapper.get_headers(),
|
|
255
|
-
timeout=60,
|
|
256
|
-
)
|
|
257
|
-
if 200 <= _response.status_code < 300:
|
|
258
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
259
|
-
if _response.status_code == 422:
|
|
260
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
261
|
-
try:
|
|
262
|
-
_response_json = _response.json()
|
|
263
|
-
except JSONDecodeError:
|
|
264
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
265
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
266
|
-
|
|
267
|
-
def delete_evaluator_config(self, evaluator_config_id: str) -> bool:
|
|
268
|
-
"""
|
|
269
|
-
Endpoint to delete a specific evaluator configuration.
|
|
270
|
-
|
|
271
|
-
Args:
|
|
272
|
-
evaluator_config_id (str): The unique identifier of the evaluator configuration.
|
|
273
|
-
|
|
274
|
-
Returns:
|
|
275
|
-
bool: True if deletion was successful, False otherwise.
|
|
276
|
-
|
|
277
|
-
Parameters:
|
|
278
|
-
- evaluator_config_id: str.
|
|
279
|
-
---
|
|
280
|
-
from agenta.client import AgentaApi
|
|
281
|
-
|
|
282
|
-
client = AgentaApi(
|
|
283
|
-
api_key="YOUR_API_KEY",
|
|
284
|
-
base_url="https://yourhost.com/path/to/api",
|
|
285
|
-
)
|
|
286
|
-
client.evaluators.delete_evaluator_config(
|
|
287
|
-
evaluator_config_id="evaluator_config_id",
|
|
288
|
-
)
|
|
289
|
-
"""
|
|
290
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
291
|
-
"DELETE",
|
|
292
|
-
urllib.parse.urljoin(
|
|
293
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
294
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
295
|
-
),
|
|
296
|
-
headers=self._client_wrapper.get_headers(),
|
|
297
|
-
timeout=60,
|
|
298
|
-
)
|
|
299
|
-
if 200 <= _response.status_code < 300:
|
|
300
|
-
return pydantic.parse_obj_as(bool, _response.json()) # type: ignore
|
|
301
|
-
if _response.status_code == 422:
|
|
302
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
303
|
-
try:
|
|
304
|
-
_response_json = _response.json()
|
|
305
|
-
except JSONDecodeError:
|
|
306
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
307
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
class AsyncEvaluatorsClient:
|
|
311
|
-
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
312
|
-
self._client_wrapper = client_wrapper
|
|
313
|
-
|
|
314
|
-
async def get_evaluators_endpoint(self) -> typing.List[Evaluator]:
|
|
315
|
-
"""
|
|
316
|
-
Endpoint to fetch a list of evaluators.
|
|
317
|
-
|
|
318
|
-
Returns:
|
|
319
|
-
List[Evaluator]: A list of evaluator objects.
|
|
320
|
-
|
|
321
|
-
---
|
|
322
|
-
from agenta.client import AsyncAgentaApi
|
|
323
|
-
|
|
324
|
-
client = AsyncAgentaApi(
|
|
325
|
-
api_key="YOUR_API_KEY",
|
|
326
|
-
base_url="https://yourhost.com/path/to/api",
|
|
327
|
-
)
|
|
328
|
-
await client.evaluators.get_evaluators_endpoint()
|
|
329
|
-
"""
|
|
330
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
331
|
-
"GET",
|
|
332
|
-
urllib.parse.urljoin(
|
|
333
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators"
|
|
334
|
-
),
|
|
335
|
-
headers=self._client_wrapper.get_headers(),
|
|
336
|
-
timeout=60,
|
|
337
|
-
)
|
|
338
|
-
if 200 <= _response.status_code < 300:
|
|
339
|
-
return pydantic.parse_obj_as(typing.List[Evaluator], _response.json()) # type: ignore
|
|
340
|
-
try:
|
|
341
|
-
_response_json = _response.json()
|
|
342
|
-
except JSONDecodeError:
|
|
343
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
344
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
345
|
-
|
|
346
|
-
async def get_evaluator_configs(
|
|
347
|
-
self, *, app_id: str
|
|
348
|
-
) -> typing.List[EvaluatorConfig]:
|
|
349
|
-
"""
|
|
350
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
351
|
-
|
|
352
|
-
Args:
|
|
353
|
-
app_id (str): The ID of the app.
|
|
354
|
-
|
|
355
|
-
Returns:
|
|
356
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
357
|
-
|
|
358
|
-
Parameters:
|
|
359
|
-
- app_id: str.
|
|
360
|
-
---
|
|
361
|
-
from agenta.client import AsyncAgentaApi
|
|
362
|
-
|
|
363
|
-
client = AsyncAgentaApi(
|
|
364
|
-
api_key="YOUR_API_KEY",
|
|
365
|
-
base_url="https://yourhost.com/path/to/api",
|
|
366
|
-
)
|
|
367
|
-
await client.evaluators.get_evaluator_configs(
|
|
368
|
-
app_id="app_id",
|
|
369
|
-
)
|
|
370
|
-
"""
|
|
371
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
372
|
-
"GET",
|
|
373
|
-
urllib.parse.urljoin(
|
|
374
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators/configs"
|
|
375
|
-
),
|
|
376
|
-
params=remove_none_from_dict({"app_id": app_id}),
|
|
377
|
-
headers=self._client_wrapper.get_headers(),
|
|
378
|
-
timeout=60,
|
|
379
|
-
)
|
|
380
|
-
if 200 <= _response.status_code < 300:
|
|
381
|
-
return pydantic.parse_obj_as(typing.List[EvaluatorConfig], _response.json()) # type: ignore
|
|
382
|
-
if _response.status_code == 422:
|
|
383
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
384
|
-
try:
|
|
385
|
-
_response_json = _response.json()
|
|
386
|
-
except JSONDecodeError:
|
|
387
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
388
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
389
|
-
|
|
390
|
-
async def create_new_evaluator_config(
|
|
391
|
-
self,
|
|
392
|
-
*,
|
|
393
|
-
app_id: str,
|
|
394
|
-
name: str,
|
|
395
|
-
evaluator_key: str,
|
|
396
|
-
settings_values: typing.Dict[str, typing.Any],
|
|
397
|
-
) -> EvaluatorConfig:
|
|
398
|
-
"""
|
|
399
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
400
|
-
|
|
401
|
-
Args:
|
|
402
|
-
app_id (str): The ID of the app.
|
|
403
|
-
|
|
404
|
-
Returns:
|
|
405
|
-
EvaluatorConfigDB: Evaluator configuration api model.
|
|
406
|
-
|
|
407
|
-
Parameters:
|
|
408
|
-
- app_id: str.
|
|
409
|
-
|
|
410
|
-
- name: str.
|
|
411
|
-
|
|
412
|
-
- evaluator_key: str.
|
|
413
|
-
|
|
414
|
-
- settings_values: typing.Dict[str, typing.Any].
|
|
415
|
-
---
|
|
416
|
-
from agenta.client import AsyncAgentaApi
|
|
417
|
-
|
|
418
|
-
client = AsyncAgentaApi(
|
|
419
|
-
api_key="YOUR_API_KEY",
|
|
420
|
-
base_url="https://yourhost.com/path/to/api",
|
|
421
|
-
)
|
|
422
|
-
await client.evaluators.create_new_evaluator_config(
|
|
423
|
-
app_id="app_id",
|
|
424
|
-
name="name",
|
|
425
|
-
evaluator_key="evaluator_key",
|
|
426
|
-
settings_values={},
|
|
427
|
-
)
|
|
428
|
-
"""
|
|
429
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
430
|
-
"POST",
|
|
431
|
-
urllib.parse.urljoin(
|
|
432
|
-
f"{self._client_wrapper.get_base_url()}/", "evaluators/configs"
|
|
433
|
-
),
|
|
434
|
-
json=jsonable_encoder(
|
|
435
|
-
{
|
|
436
|
-
"app_id": app_id,
|
|
437
|
-
"name": name,
|
|
438
|
-
"evaluator_key": evaluator_key,
|
|
439
|
-
"settings_values": settings_values,
|
|
440
|
-
}
|
|
441
|
-
),
|
|
442
|
-
headers=self._client_wrapper.get_headers(),
|
|
443
|
-
timeout=60,
|
|
444
|
-
)
|
|
445
|
-
if 200 <= _response.status_code < 300:
|
|
446
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
447
|
-
if _response.status_code == 422:
|
|
448
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
449
|
-
try:
|
|
450
|
-
_response_json = _response.json()
|
|
451
|
-
except JSONDecodeError:
|
|
452
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
453
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
454
|
-
|
|
455
|
-
async def get_evaluator_config(self, evaluator_config_id: str) -> EvaluatorConfig:
|
|
456
|
-
"""
|
|
457
|
-
Endpoint to fetch evaluator configurations for a specific app.
|
|
458
|
-
|
|
459
|
-
Returns:
|
|
460
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
461
|
-
|
|
462
|
-
Parameters:
|
|
463
|
-
- evaluator_config_id: str.
|
|
464
|
-
---
|
|
465
|
-
from agenta.client import AsyncAgentaApi
|
|
466
|
-
|
|
467
|
-
client = AsyncAgentaApi(
|
|
468
|
-
api_key="YOUR_API_KEY",
|
|
469
|
-
base_url="https://yourhost.com/path/to/api",
|
|
470
|
-
)
|
|
471
|
-
await client.evaluators.get_evaluator_config(
|
|
472
|
-
evaluator_config_id="evaluator_config_id",
|
|
473
|
-
)
|
|
474
|
-
"""
|
|
475
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
476
|
-
"GET",
|
|
477
|
-
urllib.parse.urljoin(
|
|
478
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
479
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
480
|
-
),
|
|
481
|
-
headers=self._client_wrapper.get_headers(),
|
|
482
|
-
timeout=60,
|
|
483
|
-
)
|
|
484
|
-
if 200 <= _response.status_code < 300:
|
|
485
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
486
|
-
if _response.status_code == 422:
|
|
487
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
488
|
-
try:
|
|
489
|
-
_response_json = _response.json()
|
|
490
|
-
except JSONDecodeError:
|
|
491
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
492
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
493
|
-
|
|
494
|
-
async def update_evaluator_config(
|
|
495
|
-
self,
|
|
496
|
-
evaluator_config_id: str,
|
|
497
|
-
*,
|
|
498
|
-
name: typing.Optional[str] = OMIT,
|
|
499
|
-
evaluator_key: typing.Optional[str] = OMIT,
|
|
500
|
-
settings_values: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
|
|
501
|
-
) -> EvaluatorConfig:
|
|
502
|
-
"""
|
|
503
|
-
Endpoint to update evaluator configurations for a specific app.
|
|
504
|
-
|
|
505
|
-
Returns:
|
|
506
|
-
List[EvaluatorConfigDB]: A list of evaluator configuration objects.
|
|
507
|
-
|
|
508
|
-
Parameters:
|
|
509
|
-
- evaluator_config_id: str.
|
|
510
|
-
|
|
511
|
-
- name: typing.Optional[str].
|
|
512
|
-
|
|
513
|
-
- evaluator_key: typing.Optional[str].
|
|
514
|
-
|
|
515
|
-
- settings_values: typing.Optional[typing.Dict[str, typing.Any]].
|
|
516
|
-
---
|
|
517
|
-
from agenta.client import AsyncAgentaApi
|
|
518
|
-
|
|
519
|
-
client = AsyncAgentaApi(
|
|
520
|
-
api_key="YOUR_API_KEY",
|
|
521
|
-
base_url="https://yourhost.com/path/to/api",
|
|
522
|
-
)
|
|
523
|
-
await client.evaluators.update_evaluator_config(
|
|
524
|
-
evaluator_config_id="evaluator_config_id",
|
|
525
|
-
)
|
|
526
|
-
"""
|
|
527
|
-
_request: typing.Dict[str, typing.Any] = {}
|
|
528
|
-
if name is not OMIT:
|
|
529
|
-
_request["name"] = name
|
|
530
|
-
if evaluator_key is not OMIT:
|
|
531
|
-
_request["evaluator_key"] = evaluator_key
|
|
532
|
-
if settings_values is not OMIT:
|
|
533
|
-
_request["settings_values"] = settings_values
|
|
534
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
535
|
-
"PUT",
|
|
536
|
-
urllib.parse.urljoin(
|
|
537
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
538
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
539
|
-
),
|
|
540
|
-
json=jsonable_encoder(_request),
|
|
541
|
-
headers=self._client_wrapper.get_headers(),
|
|
542
|
-
timeout=60,
|
|
543
|
-
)
|
|
544
|
-
if 200 <= _response.status_code < 300:
|
|
545
|
-
return pydantic.parse_obj_as(EvaluatorConfig, _response.json()) # type: ignore
|
|
546
|
-
if _response.status_code == 422:
|
|
547
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
548
|
-
try:
|
|
549
|
-
_response_json = _response.json()
|
|
550
|
-
except JSONDecodeError:
|
|
551
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
552
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
553
|
-
|
|
554
|
-
async def delete_evaluator_config(self, evaluator_config_id: str) -> bool:
|
|
555
|
-
"""
|
|
556
|
-
Endpoint to delete a specific evaluator configuration.
|
|
557
|
-
|
|
558
|
-
Args:
|
|
559
|
-
evaluator_config_id (str): The unique identifier of the evaluator configuration.
|
|
560
|
-
|
|
561
|
-
Returns:
|
|
562
|
-
bool: True if deletion was successful, False otherwise.
|
|
563
|
-
|
|
564
|
-
Parameters:
|
|
565
|
-
- evaluator_config_id: str.
|
|
566
|
-
---
|
|
567
|
-
from agenta.client import AsyncAgentaApi
|
|
568
|
-
|
|
569
|
-
client = AsyncAgentaApi(
|
|
570
|
-
api_key="YOUR_API_KEY",
|
|
571
|
-
base_url="https://yourhost.com/path/to/api",
|
|
572
|
-
)
|
|
573
|
-
await client.evaluators.delete_evaluator_config(
|
|
574
|
-
evaluator_config_id="evaluator_config_id",
|
|
575
|
-
)
|
|
576
|
-
"""
|
|
577
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
578
|
-
"DELETE",
|
|
579
|
-
urllib.parse.urljoin(
|
|
580
|
-
f"{self._client_wrapper.get_base_url()}/",
|
|
581
|
-
f"evaluators/configs/{evaluator_config_id}",
|
|
582
|
-
),
|
|
583
|
-
headers=self._client_wrapper.get_headers(),
|
|
584
|
-
timeout=60,
|
|
585
|
-
)
|
|
586
|
-
if 200 <= _response.status_code < 300:
|
|
587
|
-
return pydantic.parse_obj_as(bool, _response.json()) # type: ignore
|
|
588
|
-
if _response.status_code == 422:
|
|
589
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
590
|
-
try:
|
|
591
|
-
_response_json = _response.json()
|
|
592
|
-
except JSONDecodeError:
|
|
593
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
594
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|