agenta 0.24.1__py3-none-any.whl → 0.24.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenta might be problematic. Click here for more details.
- agenta/cli/variant_commands.py +15 -10
- agenta/client/Readme.md +72 -64
- agenta/client/api.py +1 -1
- agenta/client/backend/__init__.py +14 -9
- agenta/client/backend/apps/client.py +1669 -0
- agenta/client/backend/bases/client.py +190 -0
- agenta/client/backend/client.py +2102 -868
- agenta/client/backend/configs/client.py +598 -0
- agenta/client/backend/containers/client.py +638 -0
- agenta/client/backend/{resources/containers → containers}/types/container_templates_response.py +1 -2
- agenta/client/backend/core/__init__.py +29 -0
- agenta/client/backend/core/client_wrapper.py +42 -9
- agenta/client/backend/core/datetime_utils.py +1 -1
- agenta/client/backend/core/file.py +43 -0
- agenta/client/backend/core/http_client.py +553 -0
- agenta/client/backend/core/jsonable_encoder.py +33 -39
- agenta/client/backend/core/pydantic_utilities.py +212 -0
- agenta/client/backend/core/query_encoder.py +60 -0
- agenta/client/backend/core/remove_none_from_dict.py +2 -2
- agenta/client/backend/core/request_options.py +32 -0
- agenta/client/backend/core/serialization.py +179 -0
- agenta/client/backend/environments/client.py +190 -0
- agenta/client/backend/evaluations/client.py +1462 -0
- agenta/client/backend/evaluators/client.py +911 -0
- agenta/client/backend/observability/client.py +1271 -0
- agenta/client/backend/testsets/client.py +1132 -0
- agenta/client/backend/types/__init__.py +8 -6
- agenta/client/backend/types/aggregated_result.py +14 -29
- agenta/client/backend/types/aggregated_result_evaluator_config.py +1 -2
- agenta/client/backend/types/app.py +13 -28
- agenta/client/backend/types/app_variant_response.py +21 -37
- agenta/client/backend/types/app_variant_revision.py +17 -32
- agenta/client/backend/types/base_output.py +13 -28
- agenta/client/backend/types/body_import_testset.py +16 -31
- agenta/client/backend/types/config_db.py +16 -31
- agenta/client/backend/types/correct_answer.py +22 -0
- agenta/client/backend/types/create_app_output.py +13 -28
- agenta/client/backend/types/create_span.py +33 -50
- agenta/client/backend/types/create_trace_response.py +16 -31
- agenta/client/backend/types/docker_env_vars.py +13 -28
- agenta/client/backend/types/environment_output.py +21 -36
- agenta/client/backend/types/environment_output_extended.py +21 -36
- agenta/client/backend/types/environment_revision.py +18 -33
- agenta/client/backend/types/error.py +16 -31
- agenta/client/backend/types/evaluation.py +20 -34
- agenta/client/backend/types/evaluation_scenario.py +18 -33
- agenta/client/backend/types/evaluation_scenario_input.py +16 -31
- agenta/client/backend/types/evaluation_scenario_output.py +18 -33
- agenta/client/backend/types/evaluation_scenario_result.py +14 -29
- agenta/client/backend/types/evaluation_scenario_score_update.py +13 -28
- agenta/client/backend/types/evaluation_status_enum.py +11 -33
- agenta/client/backend/types/evaluation_type.py +3 -21
- agenta/client/backend/types/evaluator.py +18 -32
- agenta/client/backend/types/evaluator_config.py +20 -33
- agenta/client/backend/types/get_config_response.py +16 -31
- agenta/client/backend/types/http_validation_error.py +14 -29
- agenta/client/backend/types/human_evaluation.py +17 -32
- agenta/client/backend/types/human_evaluation_scenario.py +21 -37
- agenta/client/backend/types/human_evaluation_scenario_input.py +13 -28
- agenta/client/backend/types/human_evaluation_scenario_output.py +13 -28
- agenta/client/backend/types/human_evaluation_scenario_update.py +26 -41
- agenta/client/backend/types/human_evaluation_update.py +14 -29
- agenta/client/backend/types/image.py +18 -33
- agenta/client/backend/types/invite_request.py +13 -28
- agenta/client/backend/types/list_api_keys_response.py +18 -33
- agenta/client/backend/types/llm_run_rate_limit.py +13 -28
- agenta/client/backend/types/llm_tokens.py +16 -31
- agenta/client/backend/types/lm_providers_enum.py +21 -0
- agenta/client/backend/types/new_human_evaluation.py +13 -28
- agenta/client/backend/types/new_testset.py +16 -31
- agenta/client/backend/types/organization.py +22 -36
- agenta/client/backend/types/organization_output.py +13 -28
- agenta/client/backend/types/outputs.py +5 -0
- agenta/client/backend/types/permission.py +36 -137
- agenta/client/backend/types/result.py +17 -32
- agenta/client/backend/types/simple_evaluation_output.py +13 -28
- agenta/client/backend/types/span.py +23 -38
- agenta/client/backend/types/span_detail.py +26 -40
- agenta/client/backend/types/span_status_code.py +1 -25
- agenta/client/backend/types/span_variant.py +16 -31
- agenta/client/backend/types/template.py +14 -29
- agenta/client/backend/types/template_image_info.py +21 -35
- agenta/client/backend/types/test_set_output_response.py +16 -32
- agenta/client/backend/types/test_set_simple_response.py +13 -28
- agenta/client/backend/types/trace_detail.py +26 -40
- agenta/client/backend/types/update_app_output.py +22 -0
- agenta/client/backend/types/uri.py +13 -28
- agenta/client/backend/types/validation_error.py +13 -28
- agenta/client/backend/types/variant_action.py +14 -29
- agenta/client/backend/types/variant_action_enum.py +1 -19
- agenta/client/backend/types/with_pagination.py +14 -30
- agenta/client/backend/types/workspace_member_response.py +14 -29
- agenta/client/backend/types/workspace_permission.py +18 -33
- agenta/client/backend/types/workspace_response.py +20 -35
- agenta/client/backend/types/workspace_role.py +11 -37
- agenta/client/backend/types/workspace_role_response.py +17 -32
- agenta/client/backend/variants/client.py +1447 -0
- agenta/client/backend/variants/types/add_variant_from_base_and_config_response.py +8 -0
- agenta/sdk/tracing/llm_tracing.py +10 -12
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/METADATA +1 -1
- agenta-0.24.2.dist-info/RECORD +175 -0
- agenta/client/backend/resources/__init__.py +0 -31
- agenta/client/backend/resources/apps/client.py +0 -977
- agenta/client/backend/resources/bases/client.py +0 -127
- agenta/client/backend/resources/configs/client.py +0 -377
- agenta/client/backend/resources/containers/client.py +0 -383
- agenta/client/backend/resources/environments/client.py +0 -131
- agenta/client/backend/resources/evaluations/client.py +0 -1008
- agenta/client/backend/resources/evaluators/client.py +0 -594
- agenta/client/backend/resources/observability/client.py +0 -1187
- agenta/client/backend/resources/testsets/client.py +0 -689
- agenta/client/backend/resources/variants/client.py +0 -796
- agenta/client/backend/resources/variants/types/add_variant_from_base_and_config_response.py +0 -7
- agenta/client/backend/types/evaluation_webhook.py +0 -36
- agenta/client/backend/types/feedback.py +0 -40
- agenta/client/backend/types/span_kind.py +0 -49
- agenta-0.24.1.dist-info/RECORD +0 -169
- /agenta/client/backend/{resources/apps → apps}/__init__.py +0 -0
- /agenta/client/backend/{resources/bases → bases}/__init__.py +0 -0
- /agenta/client/backend/{resources/configs → configs}/__init__.py +0 -0
- /agenta/client/backend/{resources/containers → containers}/__init__.py +0 -0
- /agenta/client/backend/{resources/containers → containers}/types/__init__.py +0 -0
- /agenta/client/backend/{resources/environments → environments}/__init__.py +0 -0
- /agenta/client/backend/{resources/evaluations → evaluations}/__init__.py +0 -0
- /agenta/client/backend/{resources/evaluators → evaluators}/__init__.py +0 -0
- /agenta/client/backend/{resources/observability → observability}/__init__.py +0 -0
- /agenta/client/backend/{resources/testsets → testsets}/__init__.py +0 -0
- /agenta/client/backend/{resources/variants → variants}/__init__.py +0 -0
- /agenta/client/backend/{resources/variants → variants}/types/__init__.py +0 -0
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/WHEEL +0 -0
- {agenta-0.24.1.dist-info → agenta-0.24.2.dist-info}/entry_points.txt +0 -0
|
@@ -1,689 +0,0 @@
|
|
|
1
|
-
# This file was auto-generated by Fern from our API Definition.
|
|
2
|
-
|
|
3
|
-
import typing
|
|
4
|
-
import urllib.parse
|
|
5
|
-
from json.decoder import JSONDecodeError
|
|
6
|
-
|
|
7
|
-
from ...core.api_error import ApiError
|
|
8
|
-
from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
|
|
9
|
-
from ...core.jsonable_encoder import jsonable_encoder
|
|
10
|
-
from ...core.remove_none_from_dict import remove_none_from_dict
|
|
11
|
-
from ...errors.unprocessable_entity_error import UnprocessableEntityError
|
|
12
|
-
from ...types.http_validation_error import HttpValidationError
|
|
13
|
-
from ...types.new_testset import NewTestset
|
|
14
|
-
from ...types.test_set_output_response import TestSetOutputResponse
|
|
15
|
-
from ...types.test_set_simple_response import TestSetSimpleResponse
|
|
16
|
-
|
|
17
|
-
try:
|
|
18
|
-
import pydantic.v1 as pydantic # type: ignore
|
|
19
|
-
except ImportError:
|
|
20
|
-
import pydantic # type: ignore
|
|
21
|
-
|
|
22
|
-
# this is used as the default value for optional parameters
|
|
23
|
-
OMIT = typing.cast(typing.Any, ...)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class TestsetsClient:
|
|
27
|
-
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
28
|
-
self._client_wrapper = client_wrapper
|
|
29
|
-
|
|
30
|
-
def upload_file(
|
|
31
|
-
self,
|
|
32
|
-
*,
|
|
33
|
-
upload_type: typing.Optional[str] = None,
|
|
34
|
-
file: typing.IO,
|
|
35
|
-
testset_name: typing.Optional[str] = None,
|
|
36
|
-
app_id: typing.Optional[str] = None,
|
|
37
|
-
) -> TestSetSimpleResponse:
|
|
38
|
-
"""
|
|
39
|
-
Uploads a CSV or JSON file and saves its data to MongoDB.
|
|
40
|
-
|
|
41
|
-
Args:
|
|
42
|
-
upload_type : Either a json or csv file.
|
|
43
|
-
file (UploadFile): The CSV or JSON file to upload.
|
|
44
|
-
testset_name (Optional): the name of the testset if provided.
|
|
45
|
-
|
|
46
|
-
Returns:
|
|
47
|
-
dict: The result of the upload process.
|
|
48
|
-
|
|
49
|
-
Parameters:
|
|
50
|
-
- upload_type: typing.Optional[str].
|
|
51
|
-
|
|
52
|
-
- file: typing.IO.
|
|
53
|
-
|
|
54
|
-
- testset_name: typing.Optional[str].
|
|
55
|
-
|
|
56
|
-
- app_id: typing.Optional[str].
|
|
57
|
-
---
|
|
58
|
-
from agenta.client import AgentaApi
|
|
59
|
-
|
|
60
|
-
client = AgentaApi(
|
|
61
|
-
api_key="YOUR_API_KEY",
|
|
62
|
-
base_url="https://yourhost.com/path/to/api",
|
|
63
|
-
)
|
|
64
|
-
client.testsets.upload_file()
|
|
65
|
-
"""
|
|
66
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
67
|
-
"POST",
|
|
68
|
-
urllib.parse.urljoin(
|
|
69
|
-
f"{self._client_wrapper.get_base_url()}/", "testsets/upload"
|
|
70
|
-
),
|
|
71
|
-
data=jsonable_encoder(
|
|
72
|
-
{
|
|
73
|
-
"upload_type": upload_type,
|
|
74
|
-
"testset_name": testset_name,
|
|
75
|
-
"app_id": app_id,
|
|
76
|
-
}
|
|
77
|
-
),
|
|
78
|
-
files={"file": file},
|
|
79
|
-
headers=self._client_wrapper.get_headers(),
|
|
80
|
-
timeout=60,
|
|
81
|
-
)
|
|
82
|
-
if 200 <= _response.status_code < 300:
|
|
83
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
84
|
-
if _response.status_code == 422:
|
|
85
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
86
|
-
try:
|
|
87
|
-
_response_json = _response.json()
|
|
88
|
-
except JSONDecodeError:
|
|
89
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
90
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
91
|
-
|
|
92
|
-
def import_testset(self) -> TestSetSimpleResponse:
|
|
93
|
-
"""
|
|
94
|
-
Import JSON testset data from an endpoint and save it to MongoDB.
|
|
95
|
-
|
|
96
|
-
Args:
|
|
97
|
-
endpoint (str): An endpoint URL to import data from.
|
|
98
|
-
testset_name (str): the name of the testset if provided.
|
|
99
|
-
|
|
100
|
-
Returns:
|
|
101
|
-
dict: The result of the import process.
|
|
102
|
-
|
|
103
|
-
---
|
|
104
|
-
from agenta.client import AgentaApi
|
|
105
|
-
|
|
106
|
-
client = AgentaApi(
|
|
107
|
-
api_key="YOUR_API_KEY",
|
|
108
|
-
base_url="https://yourhost.com/path/to/api",
|
|
109
|
-
)
|
|
110
|
-
client.testsets.import_testset()
|
|
111
|
-
"""
|
|
112
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
113
|
-
"POST",
|
|
114
|
-
urllib.parse.urljoin(
|
|
115
|
-
f"{self._client_wrapper.get_base_url()}/", "testsets/endpoint"
|
|
116
|
-
),
|
|
117
|
-
headers=self._client_wrapper.get_headers(),
|
|
118
|
-
timeout=60,
|
|
119
|
-
)
|
|
120
|
-
if 200 <= _response.status_code < 300:
|
|
121
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
122
|
-
if _response.status_code == 422:
|
|
123
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
124
|
-
try:
|
|
125
|
-
_response_json = _response.json()
|
|
126
|
-
except JSONDecodeError:
|
|
127
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
128
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
129
|
-
|
|
130
|
-
def create_testset(
|
|
131
|
-
self, app_id: str, *, request: NewTestset
|
|
132
|
-
) -> TestSetSimpleResponse:
|
|
133
|
-
"""
|
|
134
|
-
Create a testset with given name and app_name, save the testset to MongoDB.
|
|
135
|
-
|
|
136
|
-
Args:
|
|
137
|
-
name (str): name of the test set.
|
|
138
|
-
app_name (str): name of the application.
|
|
139
|
-
testset (Dict[str, str]): test set data.
|
|
140
|
-
|
|
141
|
-
Returns:
|
|
142
|
-
str: The id of the test set created.
|
|
143
|
-
|
|
144
|
-
Parameters:
|
|
145
|
-
- app_id: str.
|
|
146
|
-
|
|
147
|
-
- request: NewTestset.
|
|
148
|
-
---
|
|
149
|
-
from agenta import NewTestset
|
|
150
|
-
from agenta.client import AgentaApi
|
|
151
|
-
|
|
152
|
-
client = AgentaApi(
|
|
153
|
-
api_key="YOUR_API_KEY",
|
|
154
|
-
base_url="https://yourhost.com/path/to/api",
|
|
155
|
-
)
|
|
156
|
-
client.testsets.create_testset(
|
|
157
|
-
app_id="app_id",
|
|
158
|
-
request=NewTestset(
|
|
159
|
-
name="name",
|
|
160
|
-
csvdata=[{"csvdata": "csvdata"}],
|
|
161
|
-
),
|
|
162
|
-
)
|
|
163
|
-
"""
|
|
164
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
165
|
-
"POST",
|
|
166
|
-
urllib.parse.urljoin(
|
|
167
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{app_id}"
|
|
168
|
-
),
|
|
169
|
-
json=jsonable_encoder(request),
|
|
170
|
-
headers=self._client_wrapper.get_headers(),
|
|
171
|
-
timeout=60,
|
|
172
|
-
)
|
|
173
|
-
if 200 <= _response.status_code < 300:
|
|
174
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
175
|
-
if _response.status_code == 422:
|
|
176
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
177
|
-
try:
|
|
178
|
-
_response_json = _response.json()
|
|
179
|
-
except JSONDecodeError:
|
|
180
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
181
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
182
|
-
|
|
183
|
-
def get_single_testset(self, testset_id: str) -> typing.Any:
|
|
184
|
-
"""
|
|
185
|
-
Fetch a specific testset in a MongoDB collection using its \_id.
|
|
186
|
-
|
|
187
|
-
Args:
|
|
188
|
-
testset_id (str): The \_id of the testset to fetch.
|
|
189
|
-
|
|
190
|
-
Returns:
|
|
191
|
-
The requested testset if found, else an HTTPException.
|
|
192
|
-
|
|
193
|
-
Parameters:
|
|
194
|
-
- testset_id: str.
|
|
195
|
-
---
|
|
196
|
-
from agenta.client import AgentaApi
|
|
197
|
-
|
|
198
|
-
client = AgentaApi(
|
|
199
|
-
api_key="YOUR_API_KEY",
|
|
200
|
-
base_url="https://yourhost.com/path/to/api",
|
|
201
|
-
)
|
|
202
|
-
client.testsets.get_single_testset(
|
|
203
|
-
testset_id="testset_id",
|
|
204
|
-
)
|
|
205
|
-
"""
|
|
206
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
207
|
-
"GET",
|
|
208
|
-
urllib.parse.urljoin(
|
|
209
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{testset_id}"
|
|
210
|
-
),
|
|
211
|
-
headers=self._client_wrapper.get_headers(),
|
|
212
|
-
timeout=60,
|
|
213
|
-
)
|
|
214
|
-
if 200 <= _response.status_code < 300:
|
|
215
|
-
return pydantic.parse_obj_as(typing.Any, _response.json()) # type: ignore
|
|
216
|
-
if _response.status_code == 422:
|
|
217
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
218
|
-
try:
|
|
219
|
-
_response_json = _response.json()
|
|
220
|
-
except JSONDecodeError:
|
|
221
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
222
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
223
|
-
|
|
224
|
-
def update_testset(self, testset_id: str, *, request: NewTestset) -> typing.Any:
|
|
225
|
-
"""
|
|
226
|
-
Update a testset with given id, update the testset in MongoDB.
|
|
227
|
-
|
|
228
|
-
Args:
|
|
229
|
-
testset_id (str): id of the test set to be updated.
|
|
230
|
-
csvdata (NewTestset): New data to replace the old testset.
|
|
231
|
-
|
|
232
|
-
Returns:
|
|
233
|
-
str: The id of the test set updated.
|
|
234
|
-
|
|
235
|
-
Parameters:
|
|
236
|
-
- testset_id: str.
|
|
237
|
-
|
|
238
|
-
- request: NewTestset.
|
|
239
|
-
---
|
|
240
|
-
from agenta import NewTestset
|
|
241
|
-
from agenta.client import AgentaApi
|
|
242
|
-
|
|
243
|
-
client = AgentaApi(
|
|
244
|
-
api_key="YOUR_API_KEY",
|
|
245
|
-
base_url="https://yourhost.com/path/to/api",
|
|
246
|
-
)
|
|
247
|
-
client.testsets.update_testset(
|
|
248
|
-
testset_id="testset_id",
|
|
249
|
-
request=NewTestset(
|
|
250
|
-
name="name",
|
|
251
|
-
csvdata=[{"csvdata": "csvdata"}],
|
|
252
|
-
),
|
|
253
|
-
)
|
|
254
|
-
"""
|
|
255
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
256
|
-
"PUT",
|
|
257
|
-
urllib.parse.urljoin(
|
|
258
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{testset_id}"
|
|
259
|
-
),
|
|
260
|
-
json=jsonable_encoder(request),
|
|
261
|
-
headers=self._client_wrapper.get_headers(),
|
|
262
|
-
timeout=60,
|
|
263
|
-
)
|
|
264
|
-
if 200 <= _response.status_code < 300:
|
|
265
|
-
return pydantic.parse_obj_as(typing.Any, _response.json()) # type: ignore
|
|
266
|
-
if _response.status_code == 422:
|
|
267
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
268
|
-
try:
|
|
269
|
-
_response_json = _response.json()
|
|
270
|
-
except JSONDecodeError:
|
|
271
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
272
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
273
|
-
|
|
274
|
-
def get_testsets(self, *, app_id: str) -> typing.List[TestSetOutputResponse]:
|
|
275
|
-
"""
|
|
276
|
-
Get all testsets.
|
|
277
|
-
|
|
278
|
-
Returns:
|
|
279
|
-
|
|
280
|
-
- A list of testset objects.
|
|
281
|
-
|
|
282
|
-
Raises:
|
|
283
|
-
|
|
284
|
-
- `HTTPException` with status code 404 if no testsets are found.
|
|
285
|
-
|
|
286
|
-
Parameters:
|
|
287
|
-
- app_id: str.
|
|
288
|
-
---
|
|
289
|
-
from agenta.client import AgentaApi
|
|
290
|
-
|
|
291
|
-
client = AgentaApi(
|
|
292
|
-
api_key="YOUR_API_KEY",
|
|
293
|
-
base_url="https://yourhost.com/path/to/api",
|
|
294
|
-
)
|
|
295
|
-
client.testsets.get_testsets(
|
|
296
|
-
app_id="app_id",
|
|
297
|
-
)
|
|
298
|
-
"""
|
|
299
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
300
|
-
"GET",
|
|
301
|
-
urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "testsets"),
|
|
302
|
-
params=remove_none_from_dict({"app_id": app_id}),
|
|
303
|
-
headers=self._client_wrapper.get_headers(),
|
|
304
|
-
timeout=60,
|
|
305
|
-
)
|
|
306
|
-
if 200 <= _response.status_code < 300:
|
|
307
|
-
return pydantic.parse_obj_as(typing.List[TestSetOutputResponse], _response.json()) # type: ignore
|
|
308
|
-
if _response.status_code == 422:
|
|
309
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
310
|
-
try:
|
|
311
|
-
_response_json = _response.json()
|
|
312
|
-
except JSONDecodeError:
|
|
313
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
314
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
315
|
-
|
|
316
|
-
def delete_testsets(self, *, testset_ids: typing.List[str]) -> typing.List[str]:
|
|
317
|
-
"""
|
|
318
|
-
Delete specific testsets based on their unique IDs.
|
|
319
|
-
|
|
320
|
-
Args:
|
|
321
|
-
testset_ids (List[str]): The unique identifiers of the testsets to delete.
|
|
322
|
-
|
|
323
|
-
Returns:
|
|
324
|
-
A list of the deleted testsets' IDs.
|
|
325
|
-
|
|
326
|
-
Parameters:
|
|
327
|
-
- testset_ids: typing.List[str].
|
|
328
|
-
---
|
|
329
|
-
from agenta.client import AgentaApi
|
|
330
|
-
|
|
331
|
-
client = AgentaApi(
|
|
332
|
-
api_key="YOUR_API_KEY",
|
|
333
|
-
base_url="https://yourhost.com/path/to/api",
|
|
334
|
-
)
|
|
335
|
-
client.testsets.delete_testsets(
|
|
336
|
-
testset_ids=["testset_ids"],
|
|
337
|
-
)
|
|
338
|
-
"""
|
|
339
|
-
_response = self._client_wrapper.httpx_client.request(
|
|
340
|
-
"DELETE",
|
|
341
|
-
urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "testsets"),
|
|
342
|
-
json=jsonable_encoder({"testset_ids": testset_ids}),
|
|
343
|
-
headers=self._client_wrapper.get_headers(),
|
|
344
|
-
timeout=60,
|
|
345
|
-
)
|
|
346
|
-
if 200 <= _response.status_code < 300:
|
|
347
|
-
return pydantic.parse_obj_as(typing.List[str], _response.json()) # type: ignore
|
|
348
|
-
if _response.status_code == 422:
|
|
349
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
350
|
-
try:
|
|
351
|
-
_response_json = _response.json()
|
|
352
|
-
except JSONDecodeError:
|
|
353
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
354
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
class AsyncTestsetsClient:
|
|
358
|
-
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
359
|
-
self._client_wrapper = client_wrapper
|
|
360
|
-
|
|
361
|
-
async def upload_file(
|
|
362
|
-
self,
|
|
363
|
-
*,
|
|
364
|
-
upload_type: typing.Optional[str] = None,
|
|
365
|
-
file: typing.IO,
|
|
366
|
-
testset_name: typing.Optional[str] = None,
|
|
367
|
-
app_id: typing.Optional[str] = None,
|
|
368
|
-
) -> TestSetSimpleResponse:
|
|
369
|
-
"""
|
|
370
|
-
Uploads a CSV or JSON file and saves its data to MongoDB.
|
|
371
|
-
|
|
372
|
-
Args:
|
|
373
|
-
upload_type : Either a json or csv file.
|
|
374
|
-
file (UploadFile): The CSV or JSON file to upload.
|
|
375
|
-
testset_name (Optional): the name of the testset if provided.
|
|
376
|
-
|
|
377
|
-
Returns:
|
|
378
|
-
dict: The result of the upload process.
|
|
379
|
-
|
|
380
|
-
Parameters:
|
|
381
|
-
- upload_type: typing.Optional[str].
|
|
382
|
-
|
|
383
|
-
- file: typing.IO.
|
|
384
|
-
|
|
385
|
-
- testset_name: typing.Optional[str].
|
|
386
|
-
|
|
387
|
-
- app_id: typing.Optional[str].
|
|
388
|
-
---
|
|
389
|
-
from agenta.client import AsyncAgentaApi
|
|
390
|
-
|
|
391
|
-
client = AsyncAgentaApi(
|
|
392
|
-
api_key="YOUR_API_KEY",
|
|
393
|
-
base_url="https://yourhost.com/path/to/api",
|
|
394
|
-
)
|
|
395
|
-
await client.testsets.upload_file()
|
|
396
|
-
"""
|
|
397
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
398
|
-
"POST",
|
|
399
|
-
urllib.parse.urljoin(
|
|
400
|
-
f"{self._client_wrapper.get_base_url()}/", "testsets/upload"
|
|
401
|
-
),
|
|
402
|
-
data=jsonable_encoder(
|
|
403
|
-
{
|
|
404
|
-
"upload_type": upload_type,
|
|
405
|
-
"testset_name": testset_name,
|
|
406
|
-
"app_id": app_id,
|
|
407
|
-
}
|
|
408
|
-
),
|
|
409
|
-
files={"file": file},
|
|
410
|
-
headers=self._client_wrapper.get_headers(),
|
|
411
|
-
timeout=60,
|
|
412
|
-
)
|
|
413
|
-
if 200 <= _response.status_code < 300:
|
|
414
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
415
|
-
if _response.status_code == 422:
|
|
416
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
417
|
-
try:
|
|
418
|
-
_response_json = _response.json()
|
|
419
|
-
except JSONDecodeError:
|
|
420
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
421
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
422
|
-
|
|
423
|
-
async def import_testset(self) -> TestSetSimpleResponse:
|
|
424
|
-
"""
|
|
425
|
-
Import JSON testset data from an endpoint and save it to MongoDB.
|
|
426
|
-
|
|
427
|
-
Args:
|
|
428
|
-
endpoint (str): An endpoint URL to import data from.
|
|
429
|
-
testset_name (str): the name of the testset if provided.
|
|
430
|
-
|
|
431
|
-
Returns:
|
|
432
|
-
dict: The result of the import process.
|
|
433
|
-
|
|
434
|
-
---
|
|
435
|
-
from agenta.client import AsyncAgentaApi
|
|
436
|
-
|
|
437
|
-
client = AsyncAgentaApi(
|
|
438
|
-
api_key="YOUR_API_KEY",
|
|
439
|
-
base_url="https://yourhost.com/path/to/api",
|
|
440
|
-
)
|
|
441
|
-
await client.testsets.import_testset()
|
|
442
|
-
"""
|
|
443
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
444
|
-
"POST",
|
|
445
|
-
urllib.parse.urljoin(
|
|
446
|
-
f"{self._client_wrapper.get_base_url()}/", "testsets/endpoint"
|
|
447
|
-
),
|
|
448
|
-
headers=self._client_wrapper.get_headers(),
|
|
449
|
-
timeout=60,
|
|
450
|
-
)
|
|
451
|
-
if 200 <= _response.status_code < 300:
|
|
452
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
453
|
-
if _response.status_code == 422:
|
|
454
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
455
|
-
try:
|
|
456
|
-
_response_json = _response.json()
|
|
457
|
-
except JSONDecodeError:
|
|
458
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
459
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
460
|
-
|
|
461
|
-
async def create_testset(
|
|
462
|
-
self, app_id: str, *, request: NewTestset
|
|
463
|
-
) -> TestSetSimpleResponse:
|
|
464
|
-
"""
|
|
465
|
-
Create a testset with given name and app_name, save the testset to MongoDB.
|
|
466
|
-
|
|
467
|
-
Args:
|
|
468
|
-
name (str): name of the test set.
|
|
469
|
-
app_name (str): name of the application.
|
|
470
|
-
testset (Dict[str, str]): test set data.
|
|
471
|
-
|
|
472
|
-
Returns:
|
|
473
|
-
str: The id of the test set created.
|
|
474
|
-
|
|
475
|
-
Parameters:
|
|
476
|
-
- app_id: str.
|
|
477
|
-
|
|
478
|
-
- request: NewTestset.
|
|
479
|
-
---
|
|
480
|
-
from agenta import NewTestset
|
|
481
|
-
from agenta.client import AsyncAgentaApi
|
|
482
|
-
|
|
483
|
-
client = AsyncAgentaApi(
|
|
484
|
-
api_key="YOUR_API_KEY",
|
|
485
|
-
base_url="https://yourhost.com/path/to/api",
|
|
486
|
-
)
|
|
487
|
-
await client.testsets.create_testset(
|
|
488
|
-
app_id="app_id",
|
|
489
|
-
request=NewTestset(
|
|
490
|
-
name="name",
|
|
491
|
-
csvdata=[{"csvdata": "csvdata"}],
|
|
492
|
-
),
|
|
493
|
-
)
|
|
494
|
-
"""
|
|
495
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
496
|
-
"POST",
|
|
497
|
-
urllib.parse.urljoin(
|
|
498
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{app_id}"
|
|
499
|
-
),
|
|
500
|
-
json=jsonable_encoder(request),
|
|
501
|
-
headers=self._client_wrapper.get_headers(),
|
|
502
|
-
timeout=60,
|
|
503
|
-
)
|
|
504
|
-
if 200 <= _response.status_code < 300:
|
|
505
|
-
return pydantic.parse_obj_as(TestSetSimpleResponse, _response.json()) # type: ignore
|
|
506
|
-
if _response.status_code == 422:
|
|
507
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
508
|
-
try:
|
|
509
|
-
_response_json = _response.json()
|
|
510
|
-
except JSONDecodeError:
|
|
511
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
512
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
513
|
-
|
|
514
|
-
async def get_single_testset(self, testset_id: str) -> typing.Any:
|
|
515
|
-
"""
|
|
516
|
-
Fetch a specific testset in a MongoDB collection using its \_id.
|
|
517
|
-
|
|
518
|
-
Args:
|
|
519
|
-
testset_id (str): The \_id of the testset to fetch.
|
|
520
|
-
|
|
521
|
-
Returns:
|
|
522
|
-
The requested testset if found, else an HTTPException.
|
|
523
|
-
|
|
524
|
-
Parameters:
|
|
525
|
-
- testset_id: str.
|
|
526
|
-
---
|
|
527
|
-
from agenta.client import AsyncAgentaApi
|
|
528
|
-
|
|
529
|
-
client = AsyncAgentaApi(
|
|
530
|
-
api_key="YOUR_API_KEY",
|
|
531
|
-
base_url="https://yourhost.com/path/to/api",
|
|
532
|
-
)
|
|
533
|
-
await client.testsets.get_single_testset(
|
|
534
|
-
testset_id="testset_id",
|
|
535
|
-
)
|
|
536
|
-
"""
|
|
537
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
538
|
-
"GET",
|
|
539
|
-
urllib.parse.urljoin(
|
|
540
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{testset_id}"
|
|
541
|
-
),
|
|
542
|
-
headers=self._client_wrapper.get_headers(),
|
|
543
|
-
timeout=60,
|
|
544
|
-
)
|
|
545
|
-
if 200 <= _response.status_code < 300:
|
|
546
|
-
return pydantic.parse_obj_as(typing.Any, _response.json()) # type: ignore
|
|
547
|
-
if _response.status_code == 422:
|
|
548
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
549
|
-
try:
|
|
550
|
-
_response_json = _response.json()
|
|
551
|
-
except JSONDecodeError:
|
|
552
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
553
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
554
|
-
|
|
555
|
-
async def update_testset(
|
|
556
|
-
self, testset_id: str, *, request: NewTestset
|
|
557
|
-
) -> typing.Any:
|
|
558
|
-
"""
|
|
559
|
-
Update a testset with given id, update the testset in MongoDB.
|
|
560
|
-
|
|
561
|
-
Args:
|
|
562
|
-
testset_id (str): id of the test set to be updated.
|
|
563
|
-
csvdata (NewTestset): New data to replace the old testset.
|
|
564
|
-
|
|
565
|
-
Returns:
|
|
566
|
-
str: The id of the test set updated.
|
|
567
|
-
|
|
568
|
-
Parameters:
|
|
569
|
-
- testset_id: str.
|
|
570
|
-
|
|
571
|
-
- request: NewTestset.
|
|
572
|
-
---
|
|
573
|
-
from agenta import NewTestset
|
|
574
|
-
from agenta.client import AsyncAgentaApi
|
|
575
|
-
|
|
576
|
-
client = AsyncAgentaApi(
|
|
577
|
-
api_key="YOUR_API_KEY",
|
|
578
|
-
base_url="https://yourhost.com/path/to/api",
|
|
579
|
-
)
|
|
580
|
-
await client.testsets.update_testset(
|
|
581
|
-
testset_id="testset_id",
|
|
582
|
-
request=NewTestset(
|
|
583
|
-
name="name",
|
|
584
|
-
csvdata=[{"csvdata": "csvdata"}],
|
|
585
|
-
),
|
|
586
|
-
)
|
|
587
|
-
"""
|
|
588
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
589
|
-
"PUT",
|
|
590
|
-
urllib.parse.urljoin(
|
|
591
|
-
f"{self._client_wrapper.get_base_url()}/", f"testsets/{testset_id}"
|
|
592
|
-
),
|
|
593
|
-
json=jsonable_encoder(request),
|
|
594
|
-
headers=self._client_wrapper.get_headers(),
|
|
595
|
-
timeout=60,
|
|
596
|
-
)
|
|
597
|
-
if 200 <= _response.status_code < 300:
|
|
598
|
-
return pydantic.parse_obj_as(typing.Any, _response.json()) # type: ignore
|
|
599
|
-
if _response.status_code == 422:
|
|
600
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
601
|
-
try:
|
|
602
|
-
_response_json = _response.json()
|
|
603
|
-
except JSONDecodeError:
|
|
604
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
605
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
606
|
-
|
|
607
|
-
async def get_testsets(self, *, app_id: str) -> typing.List[TestSetOutputResponse]:
|
|
608
|
-
"""
|
|
609
|
-
Get all testsets.
|
|
610
|
-
|
|
611
|
-
Returns:
|
|
612
|
-
|
|
613
|
-
- A list of testset objects.
|
|
614
|
-
|
|
615
|
-
Raises:
|
|
616
|
-
|
|
617
|
-
- `HTTPException` with status code 404 if no testsets are found.
|
|
618
|
-
|
|
619
|
-
Parameters:
|
|
620
|
-
- app_id: str.
|
|
621
|
-
---
|
|
622
|
-
from agenta.client import AsyncAgentaApi
|
|
623
|
-
|
|
624
|
-
client = AsyncAgentaApi(
|
|
625
|
-
api_key="YOUR_API_KEY",
|
|
626
|
-
base_url="https://yourhost.com/path/to/api",
|
|
627
|
-
)
|
|
628
|
-
await client.testsets.get_testsets(
|
|
629
|
-
app_id="app_id",
|
|
630
|
-
)
|
|
631
|
-
"""
|
|
632
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
633
|
-
"GET",
|
|
634
|
-
urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "testsets"),
|
|
635
|
-
params=remove_none_from_dict({"app_id": app_id}),
|
|
636
|
-
headers=self._client_wrapper.get_headers(),
|
|
637
|
-
timeout=60,
|
|
638
|
-
)
|
|
639
|
-
if 200 <= _response.status_code < 300:
|
|
640
|
-
return pydantic.parse_obj_as(typing.List[TestSetOutputResponse], _response.json()) # type: ignore
|
|
641
|
-
if _response.status_code == 422:
|
|
642
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
643
|
-
try:
|
|
644
|
-
_response_json = _response.json()
|
|
645
|
-
except JSONDecodeError:
|
|
646
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
647
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
648
|
-
|
|
649
|
-
async def delete_testsets(
|
|
650
|
-
self, *, testset_ids: typing.List[str]
|
|
651
|
-
) -> typing.List[str]:
|
|
652
|
-
"""
|
|
653
|
-
Delete specific testsets based on their unique IDs.
|
|
654
|
-
|
|
655
|
-
Args:
|
|
656
|
-
testset_ids (List[str]): The unique identifiers of the testsets to delete.
|
|
657
|
-
|
|
658
|
-
Returns:
|
|
659
|
-
A list of the deleted testsets' IDs.
|
|
660
|
-
|
|
661
|
-
Parameters:
|
|
662
|
-
- testset_ids: typing.List[str].
|
|
663
|
-
---
|
|
664
|
-
from agenta.client import AsyncAgentaApi
|
|
665
|
-
|
|
666
|
-
client = AsyncAgentaApi(
|
|
667
|
-
api_key="YOUR_API_KEY",
|
|
668
|
-
base_url="https://yourhost.com/path/to/api",
|
|
669
|
-
)
|
|
670
|
-
await client.testsets.delete_testsets(
|
|
671
|
-
testset_ids=["testset_ids"],
|
|
672
|
-
)
|
|
673
|
-
"""
|
|
674
|
-
_response = await self._client_wrapper.httpx_client.request(
|
|
675
|
-
"DELETE",
|
|
676
|
-
urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "testsets"),
|
|
677
|
-
json=jsonable_encoder({"testset_ids": testset_ids}),
|
|
678
|
-
headers=self._client_wrapper.get_headers(),
|
|
679
|
-
timeout=60,
|
|
680
|
-
)
|
|
681
|
-
if 200 <= _response.status_code < 300:
|
|
682
|
-
return pydantic.parse_obj_as(typing.List[str], _response.json()) # type: ignore
|
|
683
|
-
if _response.status_code == 422:
|
|
684
|
-
raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json())) # type: ignore
|
|
685
|
-
try:
|
|
686
|
-
_response_json = _response.json()
|
|
687
|
-
except JSONDecodeError:
|
|
688
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
689
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|