agenta 0.24.1a0__py3-none-any.whl → 0.24.2a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of agenta might be problematic. Click here for more details.

Files changed (132) hide show
  1. agenta/cli/variant_commands.py +15 -10
  2. agenta/client/Readme.md +72 -64
  3. agenta/client/api.py +1 -1
  4. agenta/client/backend/__init__.py +14 -9
  5. agenta/client/backend/apps/client.py +1669 -0
  6. agenta/client/backend/bases/client.py +190 -0
  7. agenta/client/backend/client.py +2102 -868
  8. agenta/client/backend/configs/client.py +598 -0
  9. agenta/client/backend/containers/client.py +638 -0
  10. agenta/client/backend/{resources/containers → containers}/types/container_templates_response.py +1 -2
  11. agenta/client/backend/core/__init__.py +29 -0
  12. agenta/client/backend/core/client_wrapper.py +42 -9
  13. agenta/client/backend/core/datetime_utils.py +1 -1
  14. agenta/client/backend/core/file.py +43 -0
  15. agenta/client/backend/core/http_client.py +553 -0
  16. agenta/client/backend/core/jsonable_encoder.py +33 -39
  17. agenta/client/backend/core/pydantic_utilities.py +212 -0
  18. agenta/client/backend/core/query_encoder.py +60 -0
  19. agenta/client/backend/core/remove_none_from_dict.py +2 -2
  20. agenta/client/backend/core/request_options.py +32 -0
  21. agenta/client/backend/core/serialization.py +179 -0
  22. agenta/client/backend/environments/client.py +190 -0
  23. agenta/client/backend/evaluations/client.py +1462 -0
  24. agenta/client/backend/evaluators/client.py +911 -0
  25. agenta/client/backend/observability/client.py +1271 -0
  26. agenta/client/backend/testsets/client.py +1132 -0
  27. agenta/client/backend/types/__init__.py +8 -6
  28. agenta/client/backend/types/aggregated_result.py +14 -29
  29. agenta/client/backend/types/aggregated_result_evaluator_config.py +1 -2
  30. agenta/client/backend/types/app.py +13 -28
  31. agenta/client/backend/types/app_variant_response.py +21 -37
  32. agenta/client/backend/types/app_variant_revision.py +17 -32
  33. agenta/client/backend/types/base_output.py +13 -28
  34. agenta/client/backend/types/body_import_testset.py +16 -31
  35. agenta/client/backend/types/config_db.py +16 -31
  36. agenta/client/backend/types/correct_answer.py +22 -0
  37. agenta/client/backend/types/create_app_output.py +13 -28
  38. agenta/client/backend/types/create_span.py +33 -50
  39. agenta/client/backend/types/create_trace_response.py +16 -31
  40. agenta/client/backend/types/docker_env_vars.py +13 -28
  41. agenta/client/backend/types/environment_output.py +21 -36
  42. agenta/client/backend/types/environment_output_extended.py +21 -36
  43. agenta/client/backend/types/environment_revision.py +18 -33
  44. agenta/client/backend/types/error.py +16 -31
  45. agenta/client/backend/types/evaluation.py +20 -34
  46. agenta/client/backend/types/evaluation_scenario.py +18 -33
  47. agenta/client/backend/types/evaluation_scenario_input.py +16 -31
  48. agenta/client/backend/types/evaluation_scenario_output.py +18 -33
  49. agenta/client/backend/types/evaluation_scenario_result.py +14 -29
  50. agenta/client/backend/types/evaluation_scenario_score_update.py +13 -28
  51. agenta/client/backend/types/evaluation_status_enum.py +11 -33
  52. agenta/client/backend/types/evaluation_type.py +3 -21
  53. agenta/client/backend/types/evaluator.py +18 -32
  54. agenta/client/backend/types/evaluator_config.py +20 -33
  55. agenta/client/backend/types/get_config_response.py +16 -31
  56. agenta/client/backend/types/http_validation_error.py +14 -29
  57. agenta/client/backend/types/human_evaluation.py +17 -32
  58. agenta/client/backend/types/human_evaluation_scenario.py +21 -37
  59. agenta/client/backend/types/human_evaluation_scenario_input.py +13 -28
  60. agenta/client/backend/types/human_evaluation_scenario_output.py +13 -28
  61. agenta/client/backend/types/human_evaluation_scenario_update.py +26 -41
  62. agenta/client/backend/types/human_evaluation_update.py +14 -29
  63. agenta/client/backend/types/image.py +18 -33
  64. agenta/client/backend/types/invite_request.py +13 -28
  65. agenta/client/backend/types/list_api_keys_response.py +18 -33
  66. agenta/client/backend/types/llm_run_rate_limit.py +13 -28
  67. agenta/client/backend/types/llm_tokens.py +16 -31
  68. agenta/client/backend/types/lm_providers_enum.py +21 -0
  69. agenta/client/backend/types/new_human_evaluation.py +13 -28
  70. agenta/client/backend/types/new_testset.py +16 -31
  71. agenta/client/backend/types/organization.py +22 -36
  72. agenta/client/backend/types/organization_output.py +13 -28
  73. agenta/client/backend/types/outputs.py +5 -0
  74. agenta/client/backend/types/permission.py +36 -137
  75. agenta/client/backend/types/result.py +17 -32
  76. agenta/client/backend/types/simple_evaluation_output.py +13 -28
  77. agenta/client/backend/types/span.py +23 -38
  78. agenta/client/backend/types/span_detail.py +26 -40
  79. agenta/client/backend/types/span_status_code.py +1 -25
  80. agenta/client/backend/types/span_variant.py +16 -31
  81. agenta/client/backend/types/template.py +14 -29
  82. agenta/client/backend/types/template_image_info.py +21 -35
  83. agenta/client/backend/types/test_set_output_response.py +16 -32
  84. agenta/client/backend/types/test_set_simple_response.py +13 -28
  85. agenta/client/backend/types/trace_detail.py +26 -40
  86. agenta/client/backend/types/update_app_output.py +22 -0
  87. agenta/client/backend/types/uri.py +13 -28
  88. agenta/client/backend/types/validation_error.py +13 -28
  89. agenta/client/backend/types/variant_action.py +14 -29
  90. agenta/client/backend/types/variant_action_enum.py +1 -19
  91. agenta/client/backend/types/with_pagination.py +14 -30
  92. agenta/client/backend/types/workspace_member_response.py +14 -29
  93. agenta/client/backend/types/workspace_permission.py +18 -33
  94. agenta/client/backend/types/workspace_response.py +20 -35
  95. agenta/client/backend/types/workspace_role.py +11 -37
  96. agenta/client/backend/types/workspace_role_response.py +17 -32
  97. agenta/client/backend/variants/client.py +1447 -0
  98. agenta/client/backend/variants/types/add_variant_from_base_and_config_response.py +8 -0
  99. agenta/sdk/decorators/llm_entrypoint.py +8 -13
  100. agenta/sdk/tracing/llm_tracing.py +1 -1
  101. {agenta-0.24.1a0.dist-info → agenta-0.24.2a1.dist-info}/METADATA +1 -1
  102. agenta-0.24.2a1.dist-info/RECORD +175 -0
  103. agenta/client/backend/resources/__init__.py +0 -31
  104. agenta/client/backend/resources/apps/client.py +0 -977
  105. agenta/client/backend/resources/bases/client.py +0 -127
  106. agenta/client/backend/resources/configs/client.py +0 -377
  107. agenta/client/backend/resources/containers/client.py +0 -383
  108. agenta/client/backend/resources/environments/client.py +0 -131
  109. agenta/client/backend/resources/evaluations/client.py +0 -1008
  110. agenta/client/backend/resources/evaluators/client.py +0 -594
  111. agenta/client/backend/resources/observability/client.py +0 -1187
  112. agenta/client/backend/resources/testsets/client.py +0 -689
  113. agenta/client/backend/resources/variants/client.py +0 -796
  114. agenta/client/backend/resources/variants/types/add_variant_from_base_and_config_response.py +0 -7
  115. agenta/client/backend/types/evaluation_webhook.py +0 -36
  116. agenta/client/backend/types/feedback.py +0 -40
  117. agenta/client/backend/types/span_kind.py +0 -49
  118. agenta-0.24.1a0.dist-info/RECORD +0 -169
  119. /agenta/client/backend/{resources/apps → apps}/__init__.py +0 -0
  120. /agenta/client/backend/{resources/bases → bases}/__init__.py +0 -0
  121. /agenta/client/backend/{resources/configs → configs}/__init__.py +0 -0
  122. /agenta/client/backend/{resources/containers → containers}/__init__.py +0 -0
  123. /agenta/client/backend/{resources/containers → containers}/types/__init__.py +0 -0
  124. /agenta/client/backend/{resources/environments → environments}/__init__.py +0 -0
  125. /agenta/client/backend/{resources/evaluations → evaluations}/__init__.py +0 -0
  126. /agenta/client/backend/{resources/evaluators → evaluators}/__init__.py +0 -0
  127. /agenta/client/backend/{resources/observability → observability}/__init__.py +0 -0
  128. /agenta/client/backend/{resources/testsets → testsets}/__init__.py +0 -0
  129. /agenta/client/backend/{resources/variants → variants}/__init__.py +0 -0
  130. /agenta/client/backend/{resources/variants → variants}/types/__init__.py +0 -0
  131. {agenta-0.24.1a0.dist-info → agenta-0.24.2a1.dist-info}/WHEEL +0 -0
  132. {agenta-0.24.1a0.dist-info → agenta-0.24.2a1.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,1132 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+ from ..core.client_wrapper import SyncClientWrapper
5
+ from .. import core
6
+ from ..core.request_options import RequestOptions
7
+ from ..types.test_set_simple_response import TestSetSimpleResponse
8
+ from ..core.pydantic_utilities import parse_obj_as
9
+ from ..errors.unprocessable_entity_error import UnprocessableEntityError
10
+ from ..types.http_validation_error import HttpValidationError
11
+ from json.decoder import JSONDecodeError
12
+ from ..core.api_error import ApiError
13
+ from ..core.jsonable_encoder import jsonable_encoder
14
+ from ..types.test_set_output_response import TestSetOutputResponse
15
+ from ..core.client_wrapper import AsyncClientWrapper
16
+
17
+ # this is used as the default value for optional parameters
18
+ OMIT = typing.cast(typing.Any, ...)
19
+
20
+
21
+ class TestsetsClient:
22
+ def __init__(self, *, client_wrapper: SyncClientWrapper):
23
+ self._client_wrapper = client_wrapper
24
+
25
+ def upload_file(
26
+ self,
27
+ *,
28
+ file: core.File,
29
+ upload_type: typing.Optional[str] = None,
30
+ testset_name: typing.Optional[str] = None,
31
+ app_id: typing.Optional[str] = None,
32
+ request_options: typing.Optional[RequestOptions] = None,
33
+ ) -> TestSetSimpleResponse:
34
+ """
35
+ Uploads a CSV or JSON file and saves its data to MongoDB.
36
+
37
+ Args:
38
+ upload_type : Either a json or csv file.
39
+ file (UploadFile): The CSV or JSON file to upload.
40
+ testset_name (Optional): the name of the testset if provided.
41
+
42
+ Returns:
43
+ dict: The result of the upload process.
44
+
45
+ Parameters
46
+ ----------
47
+ file : core.File
48
+ See core.File for more documentation
49
+
50
+ upload_type : typing.Optional[str]
51
+
52
+ testset_name : typing.Optional[str]
53
+
54
+ app_id : typing.Optional[str]
55
+
56
+ request_options : typing.Optional[RequestOptions]
57
+ Request-specific configuration.
58
+
59
+ Returns
60
+ -------
61
+ TestSetSimpleResponse
62
+ Successful Response
63
+
64
+ Examples
65
+ --------
66
+ from agenta import AgentaApi
67
+
68
+ client = AgentaApi(
69
+ api_key="YOUR_API_KEY",
70
+ base_url="https://yourhost.com/path/to/api",
71
+ )
72
+ client.testsets.upload_file()
73
+ """
74
+ _response = self._client_wrapper.httpx_client.request(
75
+ "testsets/upload",
76
+ method="POST",
77
+ data={
78
+ "upload_type": upload_type,
79
+ "testset_name": testset_name,
80
+ "app_id": app_id,
81
+ },
82
+ files={
83
+ "file": file,
84
+ },
85
+ request_options=request_options,
86
+ omit=OMIT,
87
+ )
88
+ try:
89
+ if 200 <= _response.status_code < 300:
90
+ return typing.cast(
91
+ TestSetSimpleResponse,
92
+ parse_obj_as(
93
+ type_=TestSetSimpleResponse, # type: ignore
94
+ object_=_response.json(),
95
+ ),
96
+ )
97
+ if _response.status_code == 422:
98
+ raise UnprocessableEntityError(
99
+ typing.cast(
100
+ HttpValidationError,
101
+ parse_obj_as(
102
+ type_=HttpValidationError, # type: ignore
103
+ object_=_response.json(),
104
+ ),
105
+ )
106
+ )
107
+ _response_json = _response.json()
108
+ except JSONDecodeError:
109
+ raise ApiError(status_code=_response.status_code, body=_response.text)
110
+ raise ApiError(status_code=_response.status_code, body=_response_json)
111
+
112
+ def import_testset(
113
+ self, *, request_options: typing.Optional[RequestOptions] = None
114
+ ) -> TestSetSimpleResponse:
115
+ """
116
+ Import JSON testset data from an endpoint and save it to MongoDB.
117
+
118
+ Args:
119
+ endpoint (str): An endpoint URL to import data from.
120
+ testset_name (str): the name of the testset if provided.
121
+
122
+ Returns:
123
+ dict: The result of the import process.
124
+
125
+ Parameters
126
+ ----------
127
+ request_options : typing.Optional[RequestOptions]
128
+ Request-specific configuration.
129
+
130
+ Returns
131
+ -------
132
+ TestSetSimpleResponse
133
+ Successful Response
134
+
135
+ Examples
136
+ --------
137
+ from agenta import AgentaApi
138
+
139
+ client = AgentaApi(
140
+ api_key="YOUR_API_KEY",
141
+ base_url="https://yourhost.com/path/to/api",
142
+ )
143
+ client.testsets.import_testset()
144
+ """
145
+ _response = self._client_wrapper.httpx_client.request(
146
+ "testsets/endpoint",
147
+ method="POST",
148
+ request_options=request_options,
149
+ )
150
+ try:
151
+ if 200 <= _response.status_code < 300:
152
+ return typing.cast(
153
+ TestSetSimpleResponse,
154
+ parse_obj_as(
155
+ type_=TestSetSimpleResponse, # type: ignore
156
+ object_=_response.json(),
157
+ ),
158
+ )
159
+ if _response.status_code == 422:
160
+ raise UnprocessableEntityError(
161
+ typing.cast(
162
+ HttpValidationError,
163
+ parse_obj_as(
164
+ type_=HttpValidationError, # type: ignore
165
+ object_=_response.json(),
166
+ ),
167
+ )
168
+ )
169
+ _response_json = _response.json()
170
+ except JSONDecodeError:
171
+ raise ApiError(status_code=_response.status_code, body=_response.text)
172
+ raise ApiError(status_code=_response.status_code, body=_response_json)
173
+
174
+ def create_testset(
175
+ self,
176
+ app_id: str,
177
+ *,
178
+ name: str,
179
+ csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]],
180
+ request_options: typing.Optional[RequestOptions] = None,
181
+ ) -> TestSetSimpleResponse:
182
+ """
183
+ Create a testset with given name and app_name, save the testset to MongoDB.
184
+
185
+ Args:
186
+ name (str): name of the test set.
187
+ app_name (str): name of the application.
188
+ testset (Dict[str, str]): test set data.
189
+
190
+ Returns:
191
+ str: The id of the test set created.
192
+
193
+ Parameters
194
+ ----------
195
+ app_id : str
196
+
197
+ name : str
198
+
199
+ csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]]
200
+
201
+ request_options : typing.Optional[RequestOptions]
202
+ Request-specific configuration.
203
+
204
+ Returns
205
+ -------
206
+ TestSetSimpleResponse
207
+ Successful Response
208
+
209
+ Examples
210
+ --------
211
+ from agenta import AgentaApi
212
+
213
+ client = AgentaApi(
214
+ api_key="YOUR_API_KEY",
215
+ base_url="https://yourhost.com/path/to/api",
216
+ )
217
+ client.testsets.create_testset(
218
+ app_id="app_id",
219
+ name="name",
220
+ csvdata=[{"key": "value"}],
221
+ )
222
+ """
223
+ _response = self._client_wrapper.httpx_client.request(
224
+ f"testsets/{jsonable_encoder(app_id)}",
225
+ method="POST",
226
+ json={
227
+ "name": name,
228
+ "csvdata": csvdata,
229
+ },
230
+ request_options=request_options,
231
+ omit=OMIT,
232
+ )
233
+ try:
234
+ if 200 <= _response.status_code < 300:
235
+ return typing.cast(
236
+ TestSetSimpleResponse,
237
+ parse_obj_as(
238
+ type_=TestSetSimpleResponse, # type: ignore
239
+ object_=_response.json(),
240
+ ),
241
+ )
242
+ if _response.status_code == 422:
243
+ raise UnprocessableEntityError(
244
+ typing.cast(
245
+ HttpValidationError,
246
+ parse_obj_as(
247
+ type_=HttpValidationError, # type: ignore
248
+ object_=_response.json(),
249
+ ),
250
+ )
251
+ )
252
+ _response_json = _response.json()
253
+ except JSONDecodeError:
254
+ raise ApiError(status_code=_response.status_code, body=_response.text)
255
+ raise ApiError(status_code=_response.status_code, body=_response_json)
256
+
257
+ def get_single_testset(
258
+ self,
259
+ testset_id: str,
260
+ *,
261
+ request_options: typing.Optional[RequestOptions] = None,
262
+ ) -> typing.Optional[typing.Any]:
263
+ """
264
+ Fetch a specific testset in a MongoDB collection using its id.
265
+
266
+ Args:
267
+ testset_id (str): The id of the testset to fetch.
268
+
269
+ Returns:
270
+ The requested testset if found, else an HTTPException.
271
+
272
+ Parameters
273
+ ----------
274
+ testset_id : str
275
+
276
+ request_options : typing.Optional[RequestOptions]
277
+ Request-specific configuration.
278
+
279
+ Returns
280
+ -------
281
+ typing.Optional[typing.Any]
282
+ Successful Response
283
+
284
+ Examples
285
+ --------
286
+ from agenta import AgentaApi
287
+
288
+ client = AgentaApi(
289
+ api_key="YOUR_API_KEY",
290
+ base_url="https://yourhost.com/path/to/api",
291
+ )
292
+ client.testsets.get_single_testset(
293
+ testset_id="testset_id",
294
+ )
295
+ """
296
+ _response = self._client_wrapper.httpx_client.request(
297
+ f"testsets/{jsonable_encoder(testset_id)}",
298
+ method="GET",
299
+ request_options=request_options,
300
+ )
301
+ try:
302
+ if 200 <= _response.status_code < 300:
303
+ return typing.cast(
304
+ typing.Optional[typing.Any],
305
+ parse_obj_as(
306
+ type_=typing.Optional[typing.Any], # type: ignore
307
+ object_=_response.json(),
308
+ ),
309
+ )
310
+ if _response.status_code == 422:
311
+ raise UnprocessableEntityError(
312
+ typing.cast(
313
+ HttpValidationError,
314
+ parse_obj_as(
315
+ type_=HttpValidationError, # type: ignore
316
+ object_=_response.json(),
317
+ ),
318
+ )
319
+ )
320
+ _response_json = _response.json()
321
+ except JSONDecodeError:
322
+ raise ApiError(status_code=_response.status_code, body=_response.text)
323
+ raise ApiError(status_code=_response.status_code, body=_response_json)
324
+
325
+ def update_testset(
326
+ self,
327
+ testset_id: str,
328
+ *,
329
+ name: str,
330
+ csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]],
331
+ request_options: typing.Optional[RequestOptions] = None,
332
+ ) -> typing.Optional[typing.Any]:
333
+ """
334
+ Update a testset with given id, update the testset in MongoDB.
335
+
336
+ Args:
337
+ testset_id (str): id of the test set to be updated.
338
+ csvdata (NewTestset): New data to replace the old testset.
339
+
340
+ Returns:
341
+ str: The id of the test set updated.
342
+
343
+ Parameters
344
+ ----------
345
+ testset_id : str
346
+
347
+ name : str
348
+
349
+ csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]]
350
+
351
+ request_options : typing.Optional[RequestOptions]
352
+ Request-specific configuration.
353
+
354
+ Returns
355
+ -------
356
+ typing.Optional[typing.Any]
357
+ Successful Response
358
+
359
+ Examples
360
+ --------
361
+ from agenta import AgentaApi
362
+
363
+ client = AgentaApi(
364
+ api_key="YOUR_API_KEY",
365
+ base_url="https://yourhost.com/path/to/api",
366
+ )
367
+ client.testsets.update_testset(
368
+ testset_id="testset_id",
369
+ name="name",
370
+ csvdata=[{"key": "value"}],
371
+ )
372
+ """
373
+ _response = self._client_wrapper.httpx_client.request(
374
+ f"testsets/{jsonable_encoder(testset_id)}",
375
+ method="PUT",
376
+ json={
377
+ "name": name,
378
+ "csvdata": csvdata,
379
+ },
380
+ request_options=request_options,
381
+ omit=OMIT,
382
+ )
383
+ try:
384
+ if 200 <= _response.status_code < 300:
385
+ return typing.cast(
386
+ typing.Optional[typing.Any],
387
+ parse_obj_as(
388
+ type_=typing.Optional[typing.Any], # type: ignore
389
+ object_=_response.json(),
390
+ ),
391
+ )
392
+ if _response.status_code == 422:
393
+ raise UnprocessableEntityError(
394
+ typing.cast(
395
+ HttpValidationError,
396
+ parse_obj_as(
397
+ type_=HttpValidationError, # type: ignore
398
+ object_=_response.json(),
399
+ ),
400
+ )
401
+ )
402
+ _response_json = _response.json()
403
+ except JSONDecodeError:
404
+ raise ApiError(status_code=_response.status_code, body=_response.text)
405
+ raise ApiError(status_code=_response.status_code, body=_response_json)
406
+
407
+ def get_testsets(
408
+ self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
409
+ ) -> typing.List[TestSetOutputResponse]:
410
+ """
411
+ Get all testsets.
412
+
413
+ Returns:
414
+
415
+ - A list of testset objects.
416
+
417
+ Raises:
418
+
419
+ - `HTTPException` with status code 404 if no testsets are found.
420
+
421
+ Parameters
422
+ ----------
423
+ app_id : str
424
+
425
+ request_options : typing.Optional[RequestOptions]
426
+ Request-specific configuration.
427
+
428
+ Returns
429
+ -------
430
+ typing.List[TestSetOutputResponse]
431
+ Successful Response
432
+
433
+ Examples
434
+ --------
435
+ from agenta import AgentaApi
436
+
437
+ client = AgentaApi(
438
+ api_key="YOUR_API_KEY",
439
+ base_url="https://yourhost.com/path/to/api",
440
+ )
441
+ client.testsets.get_testsets(
442
+ app_id="app_id",
443
+ )
444
+ """
445
+ _response = self._client_wrapper.httpx_client.request(
446
+ "testsets",
447
+ method="GET",
448
+ params={
449
+ "app_id": app_id,
450
+ },
451
+ request_options=request_options,
452
+ )
453
+ try:
454
+ if 200 <= _response.status_code < 300:
455
+ return typing.cast(
456
+ typing.List[TestSetOutputResponse],
457
+ parse_obj_as(
458
+ type_=typing.List[TestSetOutputResponse], # type: ignore
459
+ object_=_response.json(),
460
+ ),
461
+ )
462
+ if _response.status_code == 422:
463
+ raise UnprocessableEntityError(
464
+ typing.cast(
465
+ HttpValidationError,
466
+ parse_obj_as(
467
+ type_=HttpValidationError, # type: ignore
468
+ object_=_response.json(),
469
+ ),
470
+ )
471
+ )
472
+ _response_json = _response.json()
473
+ except JSONDecodeError:
474
+ raise ApiError(status_code=_response.status_code, body=_response.text)
475
+ raise ApiError(status_code=_response.status_code, body=_response_json)
476
+
477
+ def delete_testsets(
478
+ self,
479
+ *,
480
+ testset_ids: typing.Sequence[str],
481
+ request_options: typing.Optional[RequestOptions] = None,
482
+ ) -> typing.List[str]:
483
+ """
484
+ Delete specific testsets based on their unique IDs.
485
+
486
+ Args:
487
+ testset_ids (List[str]): The unique identifiers of the testsets to delete.
488
+
489
+ Returns:
490
+ A list of the deleted testsets' IDs.
491
+
492
+ Parameters
493
+ ----------
494
+ testset_ids : typing.Sequence[str]
495
+
496
+ request_options : typing.Optional[RequestOptions]
497
+ Request-specific configuration.
498
+
499
+ Returns
500
+ -------
501
+ typing.List[str]
502
+ Successful Response
503
+
504
+ Examples
505
+ --------
506
+ from agenta import AgentaApi
507
+
508
+ client = AgentaApi(
509
+ api_key="YOUR_API_KEY",
510
+ base_url="https://yourhost.com/path/to/api",
511
+ )
512
+ client.testsets.delete_testsets(
513
+ testset_ids=["testset_ids"],
514
+ )
515
+ """
516
+ _response = self._client_wrapper.httpx_client.request(
517
+ "testsets",
518
+ method="DELETE",
519
+ json={
520
+ "testset_ids": testset_ids,
521
+ },
522
+ request_options=request_options,
523
+ omit=OMIT,
524
+ )
525
+ try:
526
+ if 200 <= _response.status_code < 300:
527
+ return typing.cast(
528
+ typing.List[str],
529
+ parse_obj_as(
530
+ type_=typing.List[str], # type: ignore
531
+ object_=_response.json(),
532
+ ),
533
+ )
534
+ if _response.status_code == 422:
535
+ raise UnprocessableEntityError(
536
+ typing.cast(
537
+ HttpValidationError,
538
+ parse_obj_as(
539
+ type_=HttpValidationError, # type: ignore
540
+ object_=_response.json(),
541
+ ),
542
+ )
543
+ )
544
+ _response_json = _response.json()
545
+ except JSONDecodeError:
546
+ raise ApiError(status_code=_response.status_code, body=_response.text)
547
+ raise ApiError(status_code=_response.status_code, body=_response_json)
548
+
549
+
550
+ class AsyncTestsetsClient:
551
+ def __init__(self, *, client_wrapper: AsyncClientWrapper):
552
+ self._client_wrapper = client_wrapper
553
+
554
+ async def upload_file(
555
+ self,
556
+ *,
557
+ file: core.File,
558
+ upload_type: typing.Optional[str] = None,
559
+ testset_name: typing.Optional[str] = None,
560
+ app_id: typing.Optional[str] = None,
561
+ request_options: typing.Optional[RequestOptions] = None,
562
+ ) -> TestSetSimpleResponse:
563
+ """
564
+ Uploads a CSV or JSON file and saves its data to MongoDB.
565
+
566
+ Args:
567
+ upload_type : Either a json or csv file.
568
+ file (UploadFile): The CSV or JSON file to upload.
569
+ testset_name (Optional): the name of the testset if provided.
570
+
571
+ Returns:
572
+ dict: The result of the upload process.
573
+
574
+ Parameters
575
+ ----------
576
+ file : core.File
577
+ See core.File for more documentation
578
+
579
+ upload_type : typing.Optional[str]
580
+
581
+ testset_name : typing.Optional[str]
582
+
583
+ app_id : typing.Optional[str]
584
+
585
+ request_options : typing.Optional[RequestOptions]
586
+ Request-specific configuration.
587
+
588
+ Returns
589
+ -------
590
+ TestSetSimpleResponse
591
+ Successful Response
592
+
593
+ Examples
594
+ --------
595
+ import asyncio
596
+
597
+ from agenta import AsyncAgentaApi
598
+
599
+ client = AsyncAgentaApi(
600
+ api_key="YOUR_API_KEY",
601
+ base_url="https://yourhost.com/path/to/api",
602
+ )
603
+
604
+
605
+ async def main() -> None:
606
+ await client.testsets.upload_file()
607
+
608
+
609
+ asyncio.run(main())
610
+ """
611
+ _response = await self._client_wrapper.httpx_client.request(
612
+ "testsets/upload",
613
+ method="POST",
614
+ data={
615
+ "upload_type": upload_type,
616
+ "testset_name": testset_name,
617
+ "app_id": app_id,
618
+ },
619
+ files={
620
+ "file": file,
621
+ },
622
+ request_options=request_options,
623
+ omit=OMIT,
624
+ )
625
+ try:
626
+ if 200 <= _response.status_code < 300:
627
+ return typing.cast(
628
+ TestSetSimpleResponse,
629
+ parse_obj_as(
630
+ type_=TestSetSimpleResponse, # type: ignore
631
+ object_=_response.json(),
632
+ ),
633
+ )
634
+ if _response.status_code == 422:
635
+ raise UnprocessableEntityError(
636
+ typing.cast(
637
+ HttpValidationError,
638
+ parse_obj_as(
639
+ type_=HttpValidationError, # type: ignore
640
+ object_=_response.json(),
641
+ ),
642
+ )
643
+ )
644
+ _response_json = _response.json()
645
+ except JSONDecodeError:
646
+ raise ApiError(status_code=_response.status_code, body=_response.text)
647
+ raise ApiError(status_code=_response.status_code, body=_response_json)
648
+
649
+ async def import_testset(
650
+ self, *, request_options: typing.Optional[RequestOptions] = None
651
+ ) -> TestSetSimpleResponse:
652
+ """
653
+ Import JSON testset data from an endpoint and save it to MongoDB.
654
+
655
+ Args:
656
+ endpoint (str): An endpoint URL to import data from.
657
+ testset_name (str): the name of the testset if provided.
658
+
659
+ Returns:
660
+ dict: The result of the import process.
661
+
662
+ Parameters
663
+ ----------
664
+ request_options : typing.Optional[RequestOptions]
665
+ Request-specific configuration.
666
+
667
+ Returns
668
+ -------
669
+ TestSetSimpleResponse
670
+ Successful Response
671
+
672
+ Examples
673
+ --------
674
+ import asyncio
675
+
676
+ from agenta import AsyncAgentaApi
677
+
678
+ client = AsyncAgentaApi(
679
+ api_key="YOUR_API_KEY",
680
+ base_url="https://yourhost.com/path/to/api",
681
+ )
682
+
683
+
684
+ async def main() -> None:
685
+ await client.testsets.import_testset()
686
+
687
+
688
+ asyncio.run(main())
689
+ """
690
+ _response = await self._client_wrapper.httpx_client.request(
691
+ "testsets/endpoint",
692
+ method="POST",
693
+ request_options=request_options,
694
+ )
695
+ try:
696
+ if 200 <= _response.status_code < 300:
697
+ return typing.cast(
698
+ TestSetSimpleResponse,
699
+ parse_obj_as(
700
+ type_=TestSetSimpleResponse, # type: ignore
701
+ object_=_response.json(),
702
+ ),
703
+ )
704
+ if _response.status_code == 422:
705
+ raise UnprocessableEntityError(
706
+ typing.cast(
707
+ HttpValidationError,
708
+ parse_obj_as(
709
+ type_=HttpValidationError, # type: ignore
710
+ object_=_response.json(),
711
+ ),
712
+ )
713
+ )
714
+ _response_json = _response.json()
715
+ except JSONDecodeError:
716
+ raise ApiError(status_code=_response.status_code, body=_response.text)
717
+ raise ApiError(status_code=_response.status_code, body=_response_json)
718
+
719
+ async def create_testset(
720
+ self,
721
+ app_id: str,
722
+ *,
723
+ name: str,
724
+ csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]],
725
+ request_options: typing.Optional[RequestOptions] = None,
726
+ ) -> TestSetSimpleResponse:
727
+ """
728
+ Create a testset with given name and app_name, save the testset to MongoDB.
729
+
730
+ Args:
731
+ name (str): name of the test set.
732
+ app_name (str): name of the application.
733
+ testset (Dict[str, str]): test set data.
734
+
735
+ Returns:
736
+ str: The id of the test set created.
737
+
738
+ Parameters
739
+ ----------
740
+ app_id : str
741
+
742
+ name : str
743
+
744
+ csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]]
745
+
746
+ request_options : typing.Optional[RequestOptions]
747
+ Request-specific configuration.
748
+
749
+ Returns
750
+ -------
751
+ TestSetSimpleResponse
752
+ Successful Response
753
+
754
+ Examples
755
+ --------
756
+ import asyncio
757
+
758
+ from agenta import AsyncAgentaApi
759
+
760
+ client = AsyncAgentaApi(
761
+ api_key="YOUR_API_KEY",
762
+ base_url="https://yourhost.com/path/to/api",
763
+ )
764
+
765
+
766
+ async def main() -> None:
767
+ await client.testsets.create_testset(
768
+ app_id="app_id",
769
+ name="name",
770
+ csvdata=[{"key": "value"}],
771
+ )
772
+
773
+
774
+ asyncio.run(main())
775
+ """
776
+ _response = await self._client_wrapper.httpx_client.request(
777
+ f"testsets/{jsonable_encoder(app_id)}",
778
+ method="POST",
779
+ json={
780
+ "name": name,
781
+ "csvdata": csvdata,
782
+ },
783
+ request_options=request_options,
784
+ omit=OMIT,
785
+ )
786
+ try:
787
+ if 200 <= _response.status_code < 300:
788
+ return typing.cast(
789
+ TestSetSimpleResponse,
790
+ parse_obj_as(
791
+ type_=TestSetSimpleResponse, # type: ignore
792
+ object_=_response.json(),
793
+ ),
794
+ )
795
+ if _response.status_code == 422:
796
+ raise UnprocessableEntityError(
797
+ typing.cast(
798
+ HttpValidationError,
799
+ parse_obj_as(
800
+ type_=HttpValidationError, # type: ignore
801
+ object_=_response.json(),
802
+ ),
803
+ )
804
+ )
805
+ _response_json = _response.json()
806
+ except JSONDecodeError:
807
+ raise ApiError(status_code=_response.status_code, body=_response.text)
808
+ raise ApiError(status_code=_response.status_code, body=_response_json)
809
+
810
+ async def get_single_testset(
811
+ self,
812
+ testset_id: str,
813
+ *,
814
+ request_options: typing.Optional[RequestOptions] = None,
815
+ ) -> typing.Optional[typing.Any]:
816
+ """
817
+ Fetch a specific testset in a MongoDB collection using its id.
818
+
819
+ Args:
820
+ testset_id (str): The id of the testset to fetch.
821
+
822
+ Returns:
823
+ The requested testset if found, else an HTTPException.
824
+
825
+ Parameters
826
+ ----------
827
+ testset_id : str
828
+
829
+ request_options : typing.Optional[RequestOptions]
830
+ Request-specific configuration.
831
+
832
+ Returns
833
+ -------
834
+ typing.Optional[typing.Any]
835
+ Successful Response
836
+
837
+ Examples
838
+ --------
839
+ import asyncio
840
+
841
+ from agenta import AsyncAgentaApi
842
+
843
+ client = AsyncAgentaApi(
844
+ api_key="YOUR_API_KEY",
845
+ base_url="https://yourhost.com/path/to/api",
846
+ )
847
+
848
+
849
+ async def main() -> None:
850
+ await client.testsets.get_single_testset(
851
+ testset_id="testset_id",
852
+ )
853
+
854
+
855
+ asyncio.run(main())
856
+ """
857
+ _response = await self._client_wrapper.httpx_client.request(
858
+ f"testsets/{jsonable_encoder(testset_id)}",
859
+ method="GET",
860
+ request_options=request_options,
861
+ )
862
+ try:
863
+ if 200 <= _response.status_code < 300:
864
+ return typing.cast(
865
+ typing.Optional[typing.Any],
866
+ parse_obj_as(
867
+ type_=typing.Optional[typing.Any], # type: ignore
868
+ object_=_response.json(),
869
+ ),
870
+ )
871
+ if _response.status_code == 422:
872
+ raise UnprocessableEntityError(
873
+ typing.cast(
874
+ HttpValidationError,
875
+ parse_obj_as(
876
+ type_=HttpValidationError, # type: ignore
877
+ object_=_response.json(),
878
+ ),
879
+ )
880
+ )
881
+ _response_json = _response.json()
882
+ except JSONDecodeError:
883
+ raise ApiError(status_code=_response.status_code, body=_response.text)
884
+ raise ApiError(status_code=_response.status_code, body=_response_json)
885
+
886
+ async def update_testset(
887
+ self,
888
+ testset_id: str,
889
+ *,
890
+ name: str,
891
+ csvdata: typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]],
892
+ request_options: typing.Optional[RequestOptions] = None,
893
+ ) -> typing.Optional[typing.Any]:
894
+ """
895
+ Update a testset with given id, update the testset in MongoDB.
896
+
897
+ Args:
898
+ testset_id (str): id of the test set to be updated.
899
+ csvdata (NewTestset): New data to replace the old testset.
900
+
901
+ Returns:
902
+ str: The id of the test set updated.
903
+
904
+ Parameters
905
+ ----------
906
+ testset_id : str
907
+
908
+ name : str
909
+
910
+ csvdata : typing.Sequence[typing.Dict[str, typing.Optional[typing.Any]]]
911
+
912
+ request_options : typing.Optional[RequestOptions]
913
+ Request-specific configuration.
914
+
915
+ Returns
916
+ -------
917
+ typing.Optional[typing.Any]
918
+ Successful Response
919
+
920
+ Examples
921
+ --------
922
+ import asyncio
923
+
924
+ from agenta import AsyncAgentaApi
925
+
926
+ client = AsyncAgentaApi(
927
+ api_key="YOUR_API_KEY",
928
+ base_url="https://yourhost.com/path/to/api",
929
+ )
930
+
931
+
932
+ async def main() -> None:
933
+ await client.testsets.update_testset(
934
+ testset_id="testset_id",
935
+ name="name",
936
+ csvdata=[{"key": "value"}],
937
+ )
938
+
939
+
940
+ asyncio.run(main())
941
+ """
942
+ _response = await self._client_wrapper.httpx_client.request(
943
+ f"testsets/{jsonable_encoder(testset_id)}",
944
+ method="PUT",
945
+ json={
946
+ "name": name,
947
+ "csvdata": csvdata,
948
+ },
949
+ request_options=request_options,
950
+ omit=OMIT,
951
+ )
952
+ try:
953
+ if 200 <= _response.status_code < 300:
954
+ return typing.cast(
955
+ typing.Optional[typing.Any],
956
+ parse_obj_as(
957
+ type_=typing.Optional[typing.Any], # type: ignore
958
+ object_=_response.json(),
959
+ ),
960
+ )
961
+ if _response.status_code == 422:
962
+ raise UnprocessableEntityError(
963
+ typing.cast(
964
+ HttpValidationError,
965
+ parse_obj_as(
966
+ type_=HttpValidationError, # type: ignore
967
+ object_=_response.json(),
968
+ ),
969
+ )
970
+ )
971
+ _response_json = _response.json()
972
+ except JSONDecodeError:
973
+ raise ApiError(status_code=_response.status_code, body=_response.text)
974
+ raise ApiError(status_code=_response.status_code, body=_response_json)
975
+
976
+ async def get_testsets(
977
+ self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
978
+ ) -> typing.List[TestSetOutputResponse]:
979
+ """
980
+ Get all testsets.
981
+
982
+ Returns:
983
+
984
+ - A list of testset objects.
985
+
986
+ Raises:
987
+
988
+ - `HTTPException` with status code 404 if no testsets are found.
989
+
990
+ Parameters
991
+ ----------
992
+ app_id : str
993
+
994
+ request_options : typing.Optional[RequestOptions]
995
+ Request-specific configuration.
996
+
997
+ Returns
998
+ -------
999
+ typing.List[TestSetOutputResponse]
1000
+ Successful Response
1001
+
1002
+ Examples
1003
+ --------
1004
+ import asyncio
1005
+
1006
+ from agenta import AsyncAgentaApi
1007
+
1008
+ client = AsyncAgentaApi(
1009
+ api_key="YOUR_API_KEY",
1010
+ base_url="https://yourhost.com/path/to/api",
1011
+ )
1012
+
1013
+
1014
+ async def main() -> None:
1015
+ await client.testsets.get_testsets(
1016
+ app_id="app_id",
1017
+ )
1018
+
1019
+
1020
+ asyncio.run(main())
1021
+ """
1022
+ _response = await self._client_wrapper.httpx_client.request(
1023
+ "testsets",
1024
+ method="GET",
1025
+ params={
1026
+ "app_id": app_id,
1027
+ },
1028
+ request_options=request_options,
1029
+ )
1030
+ try:
1031
+ if 200 <= _response.status_code < 300:
1032
+ return typing.cast(
1033
+ typing.List[TestSetOutputResponse],
1034
+ parse_obj_as(
1035
+ type_=typing.List[TestSetOutputResponse], # type: ignore
1036
+ object_=_response.json(),
1037
+ ),
1038
+ )
1039
+ if _response.status_code == 422:
1040
+ raise UnprocessableEntityError(
1041
+ typing.cast(
1042
+ HttpValidationError,
1043
+ parse_obj_as(
1044
+ type_=HttpValidationError, # type: ignore
1045
+ object_=_response.json(),
1046
+ ),
1047
+ )
1048
+ )
1049
+ _response_json = _response.json()
1050
+ except JSONDecodeError:
1051
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1052
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1053
+
1054
+ async def delete_testsets(
1055
+ self,
1056
+ *,
1057
+ testset_ids: typing.Sequence[str],
1058
+ request_options: typing.Optional[RequestOptions] = None,
1059
+ ) -> typing.List[str]:
1060
+ """
1061
+ Delete specific testsets based on their unique IDs.
1062
+
1063
+ Args:
1064
+ testset_ids (List[str]): The unique identifiers of the testsets to delete.
1065
+
1066
+ Returns:
1067
+ A list of the deleted testsets' IDs.
1068
+
1069
+ Parameters
1070
+ ----------
1071
+ testset_ids : typing.Sequence[str]
1072
+
1073
+ request_options : typing.Optional[RequestOptions]
1074
+ Request-specific configuration.
1075
+
1076
+ Returns
1077
+ -------
1078
+ typing.List[str]
1079
+ Successful Response
1080
+
1081
+ Examples
1082
+ --------
1083
+ import asyncio
1084
+
1085
+ from agenta import AsyncAgentaApi
1086
+
1087
+ client = AsyncAgentaApi(
1088
+ api_key="YOUR_API_KEY",
1089
+ base_url="https://yourhost.com/path/to/api",
1090
+ )
1091
+
1092
+
1093
+ async def main() -> None:
1094
+ await client.testsets.delete_testsets(
1095
+ testset_ids=["testset_ids"],
1096
+ )
1097
+
1098
+
1099
+ asyncio.run(main())
1100
+ """
1101
+ _response = await self._client_wrapper.httpx_client.request(
1102
+ "testsets",
1103
+ method="DELETE",
1104
+ json={
1105
+ "testset_ids": testset_ids,
1106
+ },
1107
+ request_options=request_options,
1108
+ omit=OMIT,
1109
+ )
1110
+ try:
1111
+ if 200 <= _response.status_code < 300:
1112
+ return typing.cast(
1113
+ typing.List[str],
1114
+ parse_obj_as(
1115
+ type_=typing.List[str], # type: ignore
1116
+ object_=_response.json(),
1117
+ ),
1118
+ )
1119
+ if _response.status_code == 422:
1120
+ raise UnprocessableEntityError(
1121
+ typing.cast(
1122
+ HttpValidationError,
1123
+ parse_obj_as(
1124
+ type_=HttpValidationError, # type: ignore
1125
+ object_=_response.json(),
1126
+ ),
1127
+ )
1128
+ )
1129
+ _response_json = _response.json()
1130
+ except JSONDecodeError:
1131
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1132
+ raise ApiError(status_code=_response.status_code, body=_response_json)