agenta 0.48.9__py3-none-any.whl → 0.48.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenta might be problematic. Click here for more details.
- agenta/__init__.py +3 -2
- agenta/client/Readme.md +258 -80
- agenta/client/__init__.py +205 -29
- agenta/client/backend/__init__.py +461 -0
- agenta/client/backend/access_control/__init__.py +2 -0
- agenta/client/backend/access_control/client.py +53 -73
- agenta/client/backend/access_control/raw_client.py +180 -0
- agenta/client/backend/admin/__init__.py +2 -0
- agenta/client/backend/admin/client.py +473 -268
- agenta/client/backend/admin/raw_client.py +1017 -0
- agenta/client/backend/api_keys/__init__.py +2 -0
- agenta/client/backend/api_keys/client.py +43 -276
- agenta/client/backend/api_keys/raw_client.py +364 -0
- agenta/client/backend/apps/__init__.py +2 -0
- agenta/client/backend/apps/client.py +132 -895
- agenta/client/backend/apps/raw_client.py +1516 -0
- agenta/client/backend/bases/__init__.py +2 -0
- agenta/client/backend/bases/client.py +33 -73
- agenta/client/backend/bases/raw_client.py +179 -0
- agenta/client/backend/billing/__init__.py +3 -0
- agenta/client/backend/billing/client.py +564 -0
- agenta/client/backend/billing/raw_client.py +805 -0
- agenta/client/backend/client.py +1268 -0
- agenta/client/backend/configs/__init__.py +2 -0
- agenta/client/backend/configs/client.py +49 -361
- agenta/client/backend/configs/raw_client.py +402 -0
- agenta/client/backend/containers/__init__.py +1 -3
- agenta/client/backend/containers/client.py +25 -548
- agenta/client/backend/containers/raw_client.py +112 -0
- agenta/client/backend/core/__init__.py +5 -0
- agenta/client/backend/core/api_error.py +12 -6
- agenta/client/backend/core/client_wrapper.py +4 -4
- agenta/client/backend/core/file.py +1 -3
- agenta/client/backend/core/force_multipart.py +16 -0
- agenta/client/backend/core/http_client.py +78 -34
- agenta/client/backend/core/http_response.py +55 -0
- agenta/client/backend/core/jsonable_encoder.py +0 -1
- agenta/client/backend/core/pydantic_utilities.py +88 -113
- agenta/client/backend/core/serialization.py +9 -3
- agenta/client/backend/environment.py +7 -0
- agenta/client/backend/environments/__init__.py +2 -0
- agenta/client/backend/environments/client.py +43 -79
- agenta/client/backend/environments/raw_client.py +193 -0
- agenta/client/backend/errors/__init__.py +2 -0
- agenta/client/backend/errors/unprocessable_entity_error.py +8 -2
- agenta/client/backend/evals/__init__.py +3 -0
- agenta/client/backend/evals/client.py +1042 -0
- agenta/client/backend/evals/raw_client.py +1549 -0
- agenta/client/backend/evaluations/__init__.py +2 -0
- agenta/client/backend/evaluations/client.py +106 -590
- agenta/client/backend/evaluations/raw_client.py +1344 -0
- agenta/client/backend/evaluators/__init__.py +2 -0
- agenta/client/backend/evaluators/client.py +99 -516
- agenta/client/backend/evaluators/raw_client.py +1182 -0
- agenta/client/backend/human_evaluations/__init__.py +2 -0
- agenta/client/backend/human_evaluations/client.py +120 -680
- agenta/client/backend/human_evaluations/raw_client.py +1577 -0
- agenta/client/backend/observability/__init__.py +14 -2
- agenta/client/backend/observability/client.py +185 -341
- agenta/client/backend/observability/raw_client.py +943 -0
- agenta/client/backend/observability/types/__init__.py +10 -2
- agenta/client/backend/observability/types/{format.py → fetch_trace_by_id_request_trace_id.py} +1 -1
- agenta/client/backend/observability/types/fetch_trace_by_id_response.py +15 -0
- agenta/client/backend/observability/types/query_analytics_response.py +2 -1
- agenta/client/backend/observability/types/query_traces_response.py +7 -3
- agenta/client/backend/organization/__init__.py +2 -0
- agenta/client/backend/organization/client.py +105 -361
- agenta/client/backend/organization/raw_client.py +774 -0
- agenta/client/backend/raw_client.py +1432 -0
- agenta/client/backend/scopes/__init__.py +2 -0
- agenta/client/backend/scopes/client.py +31 -45
- agenta/client/backend/scopes/raw_client.py +105 -0
- agenta/client/backend/testsets/__init__.py +14 -0
- agenta/client/backend/testsets/client.py +1098 -653
- agenta/client/backend/testsets/raw_client.py +2348 -0
- agenta/client/backend/testsets/types/__init__.py +17 -0
- agenta/client/backend/testsets/types/create_testset_from_file_request_file_type.py +7 -0
- agenta/client/backend/testsets/types/fetch_testset_to_file_request_file_type.py +7 -0
- agenta/client/backend/testsets/types/update_testset_from_file_request_file_type.py +7 -0
- agenta/client/backend/tracing/__init__.py +7 -0
- agenta/client/backend/tracing/client.py +782 -0
- agenta/client/backend/tracing/raw_client.py +1223 -0
- agenta/client/backend/tracing/types/__init__.py +8 -0
- agenta/client/backend/{types/variant_action_enum.py → tracing/types/fetch_trace_request_trace_id.py} +1 -1
- agenta/client/backend/tracing/types/remove_trace_request_trace_id.py +5 -0
- agenta/client/backend/types/__init__.py +153 -26
- agenta/client/backend/types/account_request.py +24 -0
- agenta/client/backend/types/account_response.py +5 -7
- agenta/client/backend/types/agenta_node_dto.py +13 -13
- agenta/client/backend/types/agenta_node_dto_nodes_value.py +1 -0
- agenta/client/backend/types/agenta_nodes_response.py +14 -8
- agenta/client/backend/types/agenta_root_dto.py +16 -8
- agenta/client/backend/types/agenta_roots_response.py +16 -8
- agenta/client/backend/types/agenta_tree_dto.py +16 -8
- agenta/client/backend/types/agenta_trees_response.py +16 -8
- agenta/client/backend/types/aggregated_result.py +5 -7
- agenta/client/backend/types/aggregated_result_evaluator_config.py +1 -0
- agenta/client/backend/types/analytics_response.py +4 -6
- agenta/client/backend/types/annotation.py +50 -0
- agenta/client/backend/types/annotation_create.py +39 -0
- agenta/client/backend/types/annotation_edit.py +31 -0
- agenta/client/backend/types/annotation_kind.py +5 -0
- agenta/client/backend/types/{uri.py → annotation_link.py} +6 -7
- agenta/client/backend/types/{provider_key_dto.py → annotation_link_response.py} +6 -7
- agenta/client/backend/types/annotation_query.py +40 -0
- agenta/client/backend/types/annotation_query_request.py +20 -0
- agenta/client/backend/types/annotation_reference.py +21 -0
- agenta/client/backend/types/annotation_references.py +22 -0
- agenta/client/backend/types/{docker_env_vars.py → annotation_response.py} +6 -7
- agenta/client/backend/types/annotation_source.py +5 -0
- agenta/client/backend/types/annotations_response.py +24 -0
- agenta/client/backend/types/app.py +3 -5
- agenta/client/backend/types/app_variant_response.py +3 -6
- agenta/client/backend/types/app_variant_revision.py +5 -6
- agenta/client/backend/types/artifact.py +44 -0
- agenta/client/backend/types/base_output.py +3 -5
- agenta/client/backend/types/body_fetch_workflow_revision.py +21 -0
- agenta/client/backend/types/body_import_testset.py +3 -5
- agenta/client/backend/types/bucket_dto.py +4 -6
- agenta/client/backend/types/collect_status_response.py +3 -5
- agenta/client/backend/types/config_db.py +3 -5
- agenta/client/backend/types/config_dto.py +5 -7
- agenta/client/backend/types/config_response_model.py +5 -7
- agenta/client/backend/types/correct_answer.py +3 -5
- agenta/client/backend/types/create_app_output.py +3 -5
- agenta/client/backend/types/custom_model_settings_dto.py +3 -5
- agenta/client/backend/types/custom_provider_dto.py +6 -9
- agenta/client/backend/types/custom_provider_kind.py +5 -5
- agenta/client/backend/types/custom_provider_settings_dto.py +3 -5
- agenta/client/backend/types/data.py +2 -1
- agenta/client/backend/types/delete_evaluation.py +3 -5
- agenta/client/backend/types/environment_output.py +3 -5
- agenta/client/backend/types/environment_output_extended.py +4 -6
- agenta/client/backend/types/environment_revision.py +5 -5
- agenta/client/backend/types/error.py +3 -5
- agenta/client/backend/types/evaluation.py +6 -8
- agenta/client/backend/types/evaluation_scenario.py +5 -7
- agenta/client/backend/types/evaluation_scenario_input.py +3 -5
- agenta/client/backend/types/evaluation_scenario_output.py +4 -6
- agenta/client/backend/types/evaluation_scenario_result.py +4 -6
- agenta/client/backend/types/evaluator.py +31 -12
- agenta/client/backend/types/evaluator_config.py +3 -5
- agenta/client/backend/types/evaluator_flags.py +21 -0
- agenta/client/backend/types/evaluator_mapping_output_interface.py +3 -5
- agenta/client/backend/types/evaluator_output_interface.py +3 -5
- agenta/client/backend/types/evaluator_query.py +32 -0
- agenta/client/backend/types/evaluator_query_request.py +30 -0
- agenta/client/backend/types/evaluator_request.py +20 -0
- agenta/client/backend/types/evaluator_response.py +21 -0
- agenta/client/backend/types/evaluators_response.py +21 -0
- agenta/client/backend/types/exception_dto.py +3 -5
- agenta/client/backend/types/{o_tel_spans_response.py → extended_o_tel_tracing_response.py} +5 -7
- agenta/client/backend/types/focus.py +5 -0
- agenta/client/backend/types/format.py +5 -0
- agenta/client/backend/types/full_json_input.py +34 -0
- agenta/client/backend/types/full_json_output.py +29 -0
- agenta/client/backend/types/get_config_response.py +3 -5
- agenta/client/backend/types/{header_dto.py → header.py} +4 -6
- agenta/client/backend/types/http_validation_error.py +4 -6
- agenta/client/backend/types/human_evaluation.py +3 -5
- agenta/client/backend/types/human_evaluation_scenario.py +4 -6
- agenta/client/backend/types/human_evaluation_scenario_input.py +3 -5
- agenta/client/backend/types/human_evaluation_scenario_output.py +3 -5
- agenta/client/backend/types/invite_request.py +4 -6
- agenta/client/backend/types/legacy_analytics_response.py +4 -6
- agenta/client/backend/types/legacy_data_point.py +3 -5
- agenta/client/backend/types/legacy_evaluator.py +26 -0
- agenta/client/backend/types/legacy_scope_request.py +4 -6
- agenta/client/backend/types/legacy_scopes_response.py +3 -5
- agenta/client/backend/types/legacy_subscription_request.py +19 -0
- agenta/client/backend/types/legacy_user_request.py +5 -7
- agenta/client/backend/types/legacy_user_response.py +3 -5
- agenta/client/backend/types/lifecycle_dto.py +3 -5
- agenta/client/backend/types/link_dto.py +4 -6
- agenta/client/backend/types/list_api_keys_response.py +3 -5
- agenta/client/backend/types/llm_run_rate_limit.py +3 -5
- agenta/client/backend/types/meta_request.py +30 -0
- agenta/client/backend/types/metrics_dto.py +3 -5
- agenta/client/backend/types/new_testset.py +3 -5
- agenta/client/backend/types/node_dto.py +4 -6
- agenta/client/backend/types/o_tel_context_dto.py +3 -5
- agenta/client/backend/types/o_tel_event.py +35 -0
- agenta/client/backend/types/o_tel_event_dto.py +3 -5
- agenta/client/backend/types/o_tel_extra_dto.py +4 -6
- agenta/client/backend/types/o_tel_flat_span.py +56 -0
- agenta/client/backend/types/o_tel_flat_span_input_end_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_input_start_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_output_end_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_output_start_time.py +6 -0
- agenta/client/backend/types/o_tel_link.py +34 -0
- agenta/client/backend/types/o_tel_link_dto.py +4 -6
- agenta/client/backend/types/o_tel_links_response.py +22 -0
- agenta/client/backend/types/o_tel_span.py +58 -0
- agenta/client/backend/types/o_tel_span_dto.py +8 -10
- agenta/client/backend/types/o_tel_span_input_end_time.py +6 -0
- agenta/client/backend/types/o_tel_span_input_spans_value.py +7 -0
- agenta/client/backend/types/o_tel_span_input_start_time.py +6 -0
- agenta/client/backend/types/o_tel_span_output_end_time.py +6 -0
- agenta/client/backend/types/o_tel_span_output_spans_value.py +30 -0
- agenta/client/backend/types/o_tel_span_output_start_time.py +6 -0
- agenta/client/backend/types/o_tel_spans_tree.py +22 -0
- agenta/client/backend/types/o_tel_spans_tree_input_spans_value.py +7 -0
- agenta/client/backend/types/o_tel_spans_tree_output_spans_value.py +5 -0
- agenta/client/backend/types/o_tel_status_code.py +1 -1
- agenta/client/backend/types/o_tel_tracing_data_response.py +22 -0
- agenta/client/backend/types/o_tel_tracing_request.py +22 -0
- agenta/client/backend/types/o_tel_tracing_response.py +27 -0
- agenta/client/backend/types/organization.py +3 -5
- agenta/client/backend/types/organization_details.py +3 -5
- agenta/client/backend/types/organization_membership_request.py +5 -7
- agenta/client/backend/types/organization_output.py +3 -5
- agenta/client/backend/types/organization_request.py +3 -5
- agenta/client/backend/types/parent_dto.py +3 -5
- agenta/client/backend/types/permission.py +11 -0
- agenta/client/backend/types/plan.py +14 -0
- agenta/client/backend/types/project_membership_request.py +5 -7
- agenta/client/backend/types/project_request.py +4 -6
- agenta/client/backend/types/project_scope.py +5 -7
- agenta/client/backend/types/projects_response.py +3 -5
- agenta/client/backend/types/recursive_types.py +23 -0
- agenta/client/backend/types/reference.py +18 -5
- agenta/client/backend/types/reference_dto.py +4 -5
- agenta/client/backend/types/reference_request_model.py +4 -5
- agenta/client/backend/types/result.py +4 -6
- agenta/client/backend/types/root_dto.py +3 -5
- agenta/client/backend/types/scopes_response_model.py +4 -6
- agenta/client/backend/types/secret_dto.py +5 -7
- agenta/client/backend/types/secret_response_dto.py +11 -11
- agenta/client/backend/types/simple_evaluation_output.py +4 -6
- agenta/client/backend/types/span_dto.py +18 -14
- agenta/client/backend/types/span_dto_nodes_value.py +1 -1
- agenta/client/backend/types/standard_provider_dto.py +5 -7
- agenta/client/backend/types/standard_provider_settings_dto.py +3 -5
- agenta/client/backend/types/status_dto.py +4 -6
- agenta/client/backend/types/tags_request.py +30 -0
- agenta/client/backend/types/test_set_output_response.py +5 -7
- agenta/client/backend/types/test_set_simple_response.py +3 -5
- agenta/client/backend/types/testcase_response.py +33 -0
- agenta/client/backend/types/testset.py +46 -0
- agenta/client/backend/types/testset_request.py +20 -0
- agenta/client/backend/types/testset_response.py +21 -0
- agenta/client/backend/types/testsets_response.py +21 -0
- agenta/client/backend/types/time_dto.py +3 -5
- agenta/client/backend/types/timestamp.py +6 -0
- agenta/client/backend/types/tree_dto.py +4 -6
- agenta/client/backend/types/update_app_output.py +3 -5
- agenta/client/backend/types/user_request.py +3 -5
- agenta/client/backend/types/validation_error.py +4 -6
- agenta/client/backend/types/workflow_artifact.py +45 -0
- agenta/client/backend/types/workflow_data.py +20 -0
- agenta/client/backend/types/workflow_flags.py +21 -0
- agenta/client/backend/types/workflow_request.py +20 -0
- agenta/client/backend/types/workflow_response.py +21 -0
- agenta/client/backend/types/workflow_revision.py +57 -0
- agenta/client/backend/types/workflow_revision_request.py +20 -0
- agenta/client/backend/types/workflow_revision_response.py +21 -0
- agenta/client/backend/types/workflow_revisions_response.py +21 -0
- agenta/client/backend/types/workflow_variant.py +48 -0
- agenta/client/backend/types/workflow_variant_request.py +20 -0
- agenta/client/backend/types/workflow_variant_response.py +21 -0
- agenta/client/backend/types/workflow_variants_response.py +21 -0
- agenta/client/backend/types/workflows_response.py +21 -0
- agenta/client/backend/types/workspace.py +3 -5
- agenta/client/backend/types/workspace_member_response.py +4 -6
- agenta/client/backend/types/workspace_membership_request.py +5 -7
- agenta/client/backend/types/workspace_permission.py +5 -7
- agenta/client/backend/types/workspace_request.py +4 -6
- agenta/client/backend/types/workspace_response.py +4 -6
- agenta/client/backend/variants/__init__.py +2 -0
- agenta/client/backend/variants/client.py +306 -1651
- agenta/client/backend/variants/raw_client.py +2482 -0
- agenta/client/backend/variants/types/__init__.py +2 -0
- agenta/client/backend/variants/types/add_variant_from_base_and_config_response.py +1 -0
- agenta/client/backend/vault/__init__.py +2 -0
- agenta/client/backend/vault/client.py +69 -323
- agenta/client/backend/vault/raw_client.py +616 -0
- agenta/client/backend/workflows/__init__.py +3 -0
- agenta/client/backend/workflows/client.py +2398 -0
- agenta/client/backend/workflows/raw_client.py +3639 -0
- agenta/client/backend/workspace/__init__.py +2 -0
- agenta/client/backend/workspace/client.py +46 -147
- agenta/client/backend/workspace/raw_client.py +376 -0
- agenta/client/types.py +4 -0
- agenta/sdk/types.py +28 -1
- {agenta-0.48.9.dist-info → agenta-0.48.10.dist-info}/METADATA +1 -1
- agenta-0.48.10.dist-info/RECORD +362 -0
- agenta/client/backend/containers/types/__init__.py +0 -5
- agenta/client/backend/containers/types/container_templates_response.py +0 -6
- agenta/client/backend/types/image.py +0 -25
- agenta/client/backend/types/provider_kind.py +0 -21
- agenta/client/backend/types/template.py +0 -23
- agenta/client/backend/types/template_image_info.py +0 -29
- agenta/client/backend/types/variant_action.py +0 -22
- agenta-0.48.9.dist-info/RECORD +0 -255
- {agenta-0.48.9.dist-info → agenta-0.48.10.dist-info}/WHEEL +0 -0
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
# This file was auto-generated by Fern from our API Definition.
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
|
|
5
6
|
from ..core.request_options import RequestOptions
|
|
6
|
-
from ..core.pydantic_utilities import parse_obj_as
|
|
7
|
-
from ..errors.unprocessable_entity_error import UnprocessableEntityError
|
|
8
|
-
from ..types.http_validation_error import HttpValidationError
|
|
9
|
-
from json.decoder import JSONDecodeError
|
|
10
|
-
from ..core.api_error import ApiError
|
|
11
7
|
from ..types.evaluation import Evaluation
|
|
12
|
-
from ..types.llm_run_rate_limit import LlmRunRateLimit
|
|
13
|
-
from ..core.serialization import convert_and_respect_annotation_metadata
|
|
14
|
-
from ..core.jsonable_encoder import jsonable_encoder
|
|
15
8
|
from ..types.evaluation_scenario import EvaluationScenario
|
|
16
|
-
from ..
|
|
9
|
+
from ..types.llm_run_rate_limit import LlmRunRateLimit
|
|
10
|
+
from .raw_client import AsyncRawEvaluationsClient, RawEvaluationsClient
|
|
17
11
|
|
|
18
12
|
# this is used as the default value for optional parameters
|
|
19
13
|
OMIT = typing.cast(typing.Any, ...)
|
|
@@ -21,7 +15,18 @@ OMIT = typing.cast(typing.Any, ...)
|
|
|
21
15
|
|
|
22
16
|
class EvaluationsClient:
|
|
23
17
|
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
24
|
-
self.
|
|
18
|
+
self._raw_client = RawEvaluationsClient(client_wrapper=client_wrapper)
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def with_raw_response(self) -> RawEvaluationsClient:
|
|
22
|
+
"""
|
|
23
|
+
Retrieves a raw implementation of this client that returns raw responses.
|
|
24
|
+
|
|
25
|
+
Returns
|
|
26
|
+
-------
|
|
27
|
+
RawEvaluationsClient
|
|
28
|
+
"""
|
|
29
|
+
return self._raw_client
|
|
25
30
|
|
|
26
31
|
def fetch_evaluation_ids(
|
|
27
32
|
self,
|
|
@@ -64,44 +69,17 @@ class EvaluationsClient:
|
|
|
64
69
|
|
|
65
70
|
client = AgentaApi(
|
|
66
71
|
api_key="YOUR_API_KEY",
|
|
67
|
-
base_url="https://yourhost.com/path/to/api",
|
|
68
72
|
)
|
|
69
73
|
client.evaluations.fetch_evaluation_ids(
|
|
70
74
|
resource_type="resource_type",
|
|
71
75
|
)
|
|
72
76
|
"""
|
|
73
|
-
_response = self.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
params={
|
|
77
|
-
"resource_type": resource_type,
|
|
78
|
-
"resource_ids": resource_ids,
|
|
79
|
-
},
|
|
77
|
+
_response = self._raw_client.fetch_evaluation_ids(
|
|
78
|
+
resource_type=resource_type,
|
|
79
|
+
resource_ids=resource_ids,
|
|
80
80
|
request_options=request_options,
|
|
81
81
|
)
|
|
82
|
-
|
|
83
|
-
if 200 <= _response.status_code < 300:
|
|
84
|
-
return typing.cast(
|
|
85
|
-
typing.List[str],
|
|
86
|
-
parse_obj_as(
|
|
87
|
-
type_=typing.List[str], # type: ignore
|
|
88
|
-
object_=_response.json(),
|
|
89
|
-
),
|
|
90
|
-
)
|
|
91
|
-
if _response.status_code == 422:
|
|
92
|
-
raise UnprocessableEntityError(
|
|
93
|
-
typing.cast(
|
|
94
|
-
HttpValidationError,
|
|
95
|
-
parse_obj_as(
|
|
96
|
-
type_=HttpValidationError, # type: ignore
|
|
97
|
-
object_=_response.json(),
|
|
98
|
-
),
|
|
99
|
-
)
|
|
100
|
-
)
|
|
101
|
-
_response_json = _response.json()
|
|
102
|
-
except JSONDecodeError:
|
|
103
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
104
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
82
|
+
return _response.data
|
|
105
83
|
|
|
106
84
|
def fetch_list_evaluations(
|
|
107
85
|
self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
|
|
@@ -133,55 +111,24 @@ class EvaluationsClient:
|
|
|
133
111
|
|
|
134
112
|
client = AgentaApi(
|
|
135
113
|
api_key="YOUR_API_KEY",
|
|
136
|
-
base_url="https://yourhost.com/path/to/api",
|
|
137
114
|
)
|
|
138
115
|
client.evaluations.fetch_list_evaluations(
|
|
139
116
|
app_id="app_id",
|
|
140
117
|
)
|
|
141
118
|
"""
|
|
142
|
-
_response = self.
|
|
143
|
-
|
|
144
|
-
method="GET",
|
|
145
|
-
params={
|
|
146
|
-
"app_id": app_id,
|
|
147
|
-
},
|
|
148
|
-
request_options=request_options,
|
|
119
|
+
_response = self._raw_client.fetch_list_evaluations(
|
|
120
|
+
app_id=app_id, request_options=request_options
|
|
149
121
|
)
|
|
150
|
-
|
|
151
|
-
if 200 <= _response.status_code < 300:
|
|
152
|
-
return typing.cast(
|
|
153
|
-
typing.List[Evaluation],
|
|
154
|
-
parse_obj_as(
|
|
155
|
-
type_=typing.List[Evaluation], # type: ignore
|
|
156
|
-
object_=_response.json(),
|
|
157
|
-
),
|
|
158
|
-
)
|
|
159
|
-
if _response.status_code == 422:
|
|
160
|
-
raise UnprocessableEntityError(
|
|
161
|
-
typing.cast(
|
|
162
|
-
HttpValidationError,
|
|
163
|
-
parse_obj_as(
|
|
164
|
-
type_=HttpValidationError, # type: ignore
|
|
165
|
-
object_=_response.json(),
|
|
166
|
-
),
|
|
167
|
-
)
|
|
168
|
-
)
|
|
169
|
-
_response_json = _response.json()
|
|
170
|
-
except JSONDecodeError:
|
|
171
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
172
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
122
|
+
return _response.data
|
|
173
123
|
|
|
174
124
|
def create_evaluation(
|
|
175
125
|
self,
|
|
176
126
|
*,
|
|
177
127
|
app_id: str,
|
|
178
|
-
|
|
128
|
+
revisions_ids: typing.Sequence[str],
|
|
179
129
|
evaluators_configs: typing.Sequence[str],
|
|
180
130
|
testset_id: str,
|
|
181
131
|
rate_limit: LlmRunRateLimit,
|
|
182
|
-
lm_providers_keys: typing.Optional[
|
|
183
|
-
typing.Dict[str, typing.Optional[str]]
|
|
184
|
-
] = OMIT,
|
|
185
132
|
correct_answer_column: typing.Optional[str] = OMIT,
|
|
186
133
|
request_options: typing.Optional[RequestOptions] = None,
|
|
187
134
|
) -> typing.List[Evaluation]:
|
|
@@ -196,7 +143,7 @@ class EvaluationsClient:
|
|
|
196
143
|
----------
|
|
197
144
|
app_id : str
|
|
198
145
|
|
|
199
|
-
|
|
146
|
+
revisions_ids : typing.Sequence[str]
|
|
200
147
|
|
|
201
148
|
evaluators_configs : typing.Sequence[str]
|
|
202
149
|
|
|
@@ -204,8 +151,6 @@ class EvaluationsClient:
|
|
|
204
151
|
|
|
205
152
|
rate_limit : LlmRunRateLimit
|
|
206
153
|
|
|
207
|
-
lm_providers_keys : typing.Optional[typing.Dict[str, typing.Optional[str]]]
|
|
208
|
-
|
|
209
154
|
correct_answer_column : typing.Optional[str]
|
|
210
155
|
|
|
211
156
|
request_options : typing.Optional[RequestOptions]
|
|
@@ -222,11 +167,10 @@ class EvaluationsClient:
|
|
|
222
167
|
|
|
223
168
|
client = AgentaApi(
|
|
224
169
|
api_key="YOUR_API_KEY",
|
|
225
|
-
base_url="https://yourhost.com/path/to/api",
|
|
226
170
|
)
|
|
227
171
|
client.evaluations.create_evaluation(
|
|
228
172
|
app_id="app_id",
|
|
229
|
-
|
|
173
|
+
revisions_ids=["revisions_ids"],
|
|
230
174
|
evaluators_configs=["evaluators_configs"],
|
|
231
175
|
testset_id="testset_id",
|
|
232
176
|
rate_limit=LlmRunRateLimit(
|
|
@@ -237,49 +181,16 @@ class EvaluationsClient:
|
|
|
237
181
|
),
|
|
238
182
|
)
|
|
239
183
|
"""
|
|
240
|
-
_response = self.
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
"testset_id": testset_id,
|
|
248
|
-
"rate_limit": convert_and_respect_annotation_metadata(
|
|
249
|
-
object_=rate_limit, annotation=LlmRunRateLimit, direction="write"
|
|
250
|
-
),
|
|
251
|
-
"lm_providers_keys": lm_providers_keys,
|
|
252
|
-
"correct_answer_column": correct_answer_column,
|
|
253
|
-
},
|
|
254
|
-
headers={
|
|
255
|
-
"content-type": "application/json",
|
|
256
|
-
},
|
|
184
|
+
_response = self._raw_client.create_evaluation(
|
|
185
|
+
app_id=app_id,
|
|
186
|
+
revisions_ids=revisions_ids,
|
|
187
|
+
evaluators_configs=evaluators_configs,
|
|
188
|
+
testset_id=testset_id,
|
|
189
|
+
rate_limit=rate_limit,
|
|
190
|
+
correct_answer_column=correct_answer_column,
|
|
257
191
|
request_options=request_options,
|
|
258
|
-
omit=OMIT,
|
|
259
192
|
)
|
|
260
|
-
|
|
261
|
-
if 200 <= _response.status_code < 300:
|
|
262
|
-
return typing.cast(
|
|
263
|
-
typing.List[Evaluation],
|
|
264
|
-
parse_obj_as(
|
|
265
|
-
type_=typing.List[Evaluation], # type: ignore
|
|
266
|
-
object_=_response.json(),
|
|
267
|
-
),
|
|
268
|
-
)
|
|
269
|
-
if _response.status_code == 422:
|
|
270
|
-
raise UnprocessableEntityError(
|
|
271
|
-
typing.cast(
|
|
272
|
-
HttpValidationError,
|
|
273
|
-
parse_obj_as(
|
|
274
|
-
type_=HttpValidationError, # type: ignore
|
|
275
|
-
object_=_response.json(),
|
|
276
|
-
),
|
|
277
|
-
)
|
|
278
|
-
)
|
|
279
|
-
_response_json = _response.json()
|
|
280
|
-
except JSONDecodeError:
|
|
281
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
282
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
193
|
+
return _response.data
|
|
283
194
|
|
|
284
195
|
def delete_evaluations(
|
|
285
196
|
self,
|
|
@@ -314,44 +225,15 @@ class EvaluationsClient:
|
|
|
314
225
|
|
|
315
226
|
client = AgentaApi(
|
|
316
227
|
api_key="YOUR_API_KEY",
|
|
317
|
-
base_url="https://yourhost.com/path/to/api",
|
|
318
228
|
)
|
|
319
229
|
client.evaluations.delete_evaluations(
|
|
320
230
|
evaluations_ids=["evaluations_ids"],
|
|
321
231
|
)
|
|
322
232
|
"""
|
|
323
|
-
_response = self.
|
|
324
|
-
|
|
325
|
-
method="DELETE",
|
|
326
|
-
json={
|
|
327
|
-
"evaluations_ids": evaluations_ids,
|
|
328
|
-
},
|
|
329
|
-
request_options=request_options,
|
|
330
|
-
omit=OMIT,
|
|
233
|
+
_response = self._raw_client.delete_evaluations(
|
|
234
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
331
235
|
)
|
|
332
|
-
|
|
333
|
-
if 200 <= _response.status_code < 300:
|
|
334
|
-
return typing.cast(
|
|
335
|
-
typing.List[str],
|
|
336
|
-
parse_obj_as(
|
|
337
|
-
type_=typing.List[str], # type: ignore
|
|
338
|
-
object_=_response.json(),
|
|
339
|
-
),
|
|
340
|
-
)
|
|
341
|
-
if _response.status_code == 422:
|
|
342
|
-
raise UnprocessableEntityError(
|
|
343
|
-
typing.cast(
|
|
344
|
-
HttpValidationError,
|
|
345
|
-
parse_obj_as(
|
|
346
|
-
type_=HttpValidationError, # type: ignore
|
|
347
|
-
object_=_response.json(),
|
|
348
|
-
),
|
|
349
|
-
)
|
|
350
|
-
)
|
|
351
|
-
_response_json = _response.json()
|
|
352
|
-
except JSONDecodeError:
|
|
353
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
354
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
236
|
+
return _response.data
|
|
355
237
|
|
|
356
238
|
def fetch_evaluation_status(
|
|
357
239
|
self,
|
|
@@ -387,40 +269,15 @@ class EvaluationsClient:
|
|
|
387
269
|
|
|
388
270
|
client = AgentaApi(
|
|
389
271
|
api_key="YOUR_API_KEY",
|
|
390
|
-
base_url="https://yourhost.com/path/to/api",
|
|
391
272
|
)
|
|
392
273
|
client.evaluations.fetch_evaluation_status(
|
|
393
274
|
evaluation_id="evaluation_id",
|
|
394
275
|
)
|
|
395
276
|
"""
|
|
396
|
-
_response = self.
|
|
397
|
-
|
|
398
|
-
method="GET",
|
|
399
|
-
request_options=request_options,
|
|
277
|
+
_response = self._raw_client.fetch_evaluation_status(
|
|
278
|
+
evaluation_id, request_options=request_options
|
|
400
279
|
)
|
|
401
|
-
|
|
402
|
-
if 200 <= _response.status_code < 300:
|
|
403
|
-
return typing.cast(
|
|
404
|
-
typing.Optional[typing.Any],
|
|
405
|
-
parse_obj_as(
|
|
406
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
407
|
-
object_=_response.json(),
|
|
408
|
-
),
|
|
409
|
-
)
|
|
410
|
-
if _response.status_code == 422:
|
|
411
|
-
raise UnprocessableEntityError(
|
|
412
|
-
typing.cast(
|
|
413
|
-
HttpValidationError,
|
|
414
|
-
parse_obj_as(
|
|
415
|
-
type_=HttpValidationError, # type: ignore
|
|
416
|
-
object_=_response.json(),
|
|
417
|
-
),
|
|
418
|
-
)
|
|
419
|
-
)
|
|
420
|
-
_response_json = _response.json()
|
|
421
|
-
except JSONDecodeError:
|
|
422
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
423
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
280
|
+
return _response.data
|
|
424
281
|
|
|
425
282
|
def fetch_evaluation_results(
|
|
426
283
|
self,
|
|
@@ -456,40 +313,15 @@ class EvaluationsClient:
|
|
|
456
313
|
|
|
457
314
|
client = AgentaApi(
|
|
458
315
|
api_key="YOUR_API_KEY",
|
|
459
|
-
base_url="https://yourhost.com/path/to/api",
|
|
460
316
|
)
|
|
461
317
|
client.evaluations.fetch_evaluation_results(
|
|
462
318
|
evaluation_id="evaluation_id",
|
|
463
319
|
)
|
|
464
320
|
"""
|
|
465
|
-
_response = self.
|
|
466
|
-
|
|
467
|
-
method="GET",
|
|
468
|
-
request_options=request_options,
|
|
321
|
+
_response = self._raw_client.fetch_evaluation_results(
|
|
322
|
+
evaluation_id, request_options=request_options
|
|
469
323
|
)
|
|
470
|
-
|
|
471
|
-
if 200 <= _response.status_code < 300:
|
|
472
|
-
return typing.cast(
|
|
473
|
-
typing.Optional[typing.Any],
|
|
474
|
-
parse_obj_as(
|
|
475
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
476
|
-
object_=_response.json(),
|
|
477
|
-
),
|
|
478
|
-
)
|
|
479
|
-
if _response.status_code == 422:
|
|
480
|
-
raise UnprocessableEntityError(
|
|
481
|
-
typing.cast(
|
|
482
|
-
HttpValidationError,
|
|
483
|
-
parse_obj_as(
|
|
484
|
-
type_=HttpValidationError, # type: ignore
|
|
485
|
-
object_=_response.json(),
|
|
486
|
-
),
|
|
487
|
-
)
|
|
488
|
-
)
|
|
489
|
-
_response_json = _response.json()
|
|
490
|
-
except JSONDecodeError:
|
|
491
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
492
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
324
|
+
return _response.data
|
|
493
325
|
|
|
494
326
|
def fetch_evaluation_scenarios(
|
|
495
327
|
self,
|
|
@@ -527,40 +359,15 @@ class EvaluationsClient:
|
|
|
527
359
|
|
|
528
360
|
client = AgentaApi(
|
|
529
361
|
api_key="YOUR_API_KEY",
|
|
530
|
-
base_url="https://yourhost.com/path/to/api",
|
|
531
362
|
)
|
|
532
363
|
client.evaluations.fetch_evaluation_scenarios(
|
|
533
364
|
evaluation_id="evaluation_id",
|
|
534
365
|
)
|
|
535
366
|
"""
|
|
536
|
-
_response = self.
|
|
537
|
-
|
|
538
|
-
method="GET",
|
|
539
|
-
request_options=request_options,
|
|
367
|
+
_response = self._raw_client.fetch_evaluation_scenarios(
|
|
368
|
+
evaluation_id, request_options=request_options
|
|
540
369
|
)
|
|
541
|
-
|
|
542
|
-
if 200 <= _response.status_code < 300:
|
|
543
|
-
return typing.cast(
|
|
544
|
-
typing.List[EvaluationScenario],
|
|
545
|
-
parse_obj_as(
|
|
546
|
-
type_=typing.List[EvaluationScenario], # type: ignore
|
|
547
|
-
object_=_response.json(),
|
|
548
|
-
),
|
|
549
|
-
)
|
|
550
|
-
if _response.status_code == 422:
|
|
551
|
-
raise UnprocessableEntityError(
|
|
552
|
-
typing.cast(
|
|
553
|
-
HttpValidationError,
|
|
554
|
-
parse_obj_as(
|
|
555
|
-
type_=HttpValidationError, # type: ignore
|
|
556
|
-
object_=_response.json(),
|
|
557
|
-
),
|
|
558
|
-
)
|
|
559
|
-
)
|
|
560
|
-
_response_json = _response.json()
|
|
561
|
-
except JSONDecodeError:
|
|
562
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
563
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
370
|
+
return _response.data
|
|
564
371
|
|
|
565
372
|
def fetch_evaluation(
|
|
566
373
|
self,
|
|
@@ -595,42 +402,17 @@ class EvaluationsClient:
|
|
|
595
402
|
|
|
596
403
|
client = AgentaApi(
|
|
597
404
|
api_key="YOUR_API_KEY",
|
|
598
|
-
base_url="https://yourhost.com/path/to/api",
|
|
599
405
|
)
|
|
600
406
|
client.evaluations.fetch_evaluation(
|
|
601
407
|
evaluation_id="evaluation_id",
|
|
602
408
|
)
|
|
603
409
|
"""
|
|
604
|
-
_response = self.
|
|
605
|
-
|
|
606
|
-
method="GET",
|
|
607
|
-
request_options=request_options,
|
|
410
|
+
_response = self._raw_client.fetch_evaluation(
|
|
411
|
+
evaluation_id, request_options=request_options
|
|
608
412
|
)
|
|
609
|
-
|
|
610
|
-
if 200 <= _response.status_code < 300:
|
|
611
|
-
return typing.cast(
|
|
612
|
-
Evaluation,
|
|
613
|
-
parse_obj_as(
|
|
614
|
-
type_=Evaluation, # type: ignore
|
|
615
|
-
object_=_response.json(),
|
|
616
|
-
),
|
|
617
|
-
)
|
|
618
|
-
if _response.status_code == 422:
|
|
619
|
-
raise UnprocessableEntityError(
|
|
620
|
-
typing.cast(
|
|
621
|
-
HttpValidationError,
|
|
622
|
-
parse_obj_as(
|
|
623
|
-
type_=HttpValidationError, # type: ignore
|
|
624
|
-
object_=_response.json(),
|
|
625
|
-
),
|
|
626
|
-
)
|
|
627
|
-
)
|
|
628
|
-
_response_json = _response.json()
|
|
629
|
-
except JSONDecodeError:
|
|
630
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
631
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
413
|
+
return _response.data
|
|
632
414
|
|
|
633
|
-
def
|
|
415
|
+
def fetch_evaluation_scenarios_comparison_results(
|
|
634
416
|
self,
|
|
635
417
|
*,
|
|
636
418
|
evaluations_ids: str,
|
|
@@ -666,48 +448,31 @@ class EvaluationsClient:
|
|
|
666
448
|
|
|
667
449
|
client = AgentaApi(
|
|
668
450
|
api_key="YOUR_API_KEY",
|
|
669
|
-
base_url="https://yourhost.com/path/to/api",
|
|
670
451
|
)
|
|
671
|
-
client.evaluations.
|
|
452
|
+
client.evaluations.fetch_evaluation_scenarios_comparison_results(
|
|
672
453
|
evaluations_ids="evaluations_ids",
|
|
673
454
|
)
|
|
674
455
|
"""
|
|
675
|
-
_response = self.
|
|
676
|
-
|
|
677
|
-
method="GET",
|
|
678
|
-
params={
|
|
679
|
-
"evaluations_ids": evaluations_ids,
|
|
680
|
-
},
|
|
681
|
-
request_options=request_options,
|
|
456
|
+
_response = self._raw_client.fetch_evaluation_scenarios_comparison_results(
|
|
457
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
682
458
|
)
|
|
683
|
-
|
|
684
|
-
if 200 <= _response.status_code < 300:
|
|
685
|
-
return typing.cast(
|
|
686
|
-
typing.Optional[typing.Any],
|
|
687
|
-
parse_obj_as(
|
|
688
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
689
|
-
object_=_response.json(),
|
|
690
|
-
),
|
|
691
|
-
)
|
|
692
|
-
if _response.status_code == 422:
|
|
693
|
-
raise UnprocessableEntityError(
|
|
694
|
-
typing.cast(
|
|
695
|
-
HttpValidationError,
|
|
696
|
-
parse_obj_as(
|
|
697
|
-
type_=HttpValidationError, # type: ignore
|
|
698
|
-
object_=_response.json(),
|
|
699
|
-
),
|
|
700
|
-
)
|
|
701
|
-
)
|
|
702
|
-
_response_json = _response.json()
|
|
703
|
-
except JSONDecodeError:
|
|
704
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
705
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
459
|
+
return _response.data
|
|
706
460
|
|
|
707
461
|
|
|
708
462
|
class AsyncEvaluationsClient:
|
|
709
463
|
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
710
|
-
self.
|
|
464
|
+
self._raw_client = AsyncRawEvaluationsClient(client_wrapper=client_wrapper)
|
|
465
|
+
|
|
466
|
+
@property
|
|
467
|
+
def with_raw_response(self) -> AsyncRawEvaluationsClient:
|
|
468
|
+
"""
|
|
469
|
+
Retrieves a raw implementation of this client that returns raw responses.
|
|
470
|
+
|
|
471
|
+
Returns
|
|
472
|
+
-------
|
|
473
|
+
AsyncRawEvaluationsClient
|
|
474
|
+
"""
|
|
475
|
+
return self._raw_client
|
|
711
476
|
|
|
712
477
|
async def fetch_evaluation_ids(
|
|
713
478
|
self,
|
|
@@ -752,7 +517,6 @@ class AsyncEvaluationsClient:
|
|
|
752
517
|
|
|
753
518
|
client = AsyncAgentaApi(
|
|
754
519
|
api_key="YOUR_API_KEY",
|
|
755
|
-
base_url="https://yourhost.com/path/to/api",
|
|
756
520
|
)
|
|
757
521
|
|
|
758
522
|
|
|
@@ -764,38 +528,12 @@ class AsyncEvaluationsClient:
|
|
|
764
528
|
|
|
765
529
|
asyncio.run(main())
|
|
766
530
|
"""
|
|
767
|
-
_response = await self.
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
params={
|
|
771
|
-
"resource_type": resource_type,
|
|
772
|
-
"resource_ids": resource_ids,
|
|
773
|
-
},
|
|
531
|
+
_response = await self._raw_client.fetch_evaluation_ids(
|
|
532
|
+
resource_type=resource_type,
|
|
533
|
+
resource_ids=resource_ids,
|
|
774
534
|
request_options=request_options,
|
|
775
535
|
)
|
|
776
|
-
|
|
777
|
-
if 200 <= _response.status_code < 300:
|
|
778
|
-
return typing.cast(
|
|
779
|
-
typing.List[str],
|
|
780
|
-
parse_obj_as(
|
|
781
|
-
type_=typing.List[str], # type: ignore
|
|
782
|
-
object_=_response.json(),
|
|
783
|
-
),
|
|
784
|
-
)
|
|
785
|
-
if _response.status_code == 422:
|
|
786
|
-
raise UnprocessableEntityError(
|
|
787
|
-
typing.cast(
|
|
788
|
-
HttpValidationError,
|
|
789
|
-
parse_obj_as(
|
|
790
|
-
type_=HttpValidationError, # type: ignore
|
|
791
|
-
object_=_response.json(),
|
|
792
|
-
),
|
|
793
|
-
)
|
|
794
|
-
)
|
|
795
|
-
_response_json = _response.json()
|
|
796
|
-
except JSONDecodeError:
|
|
797
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
798
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
536
|
+
return _response.data
|
|
799
537
|
|
|
800
538
|
async def fetch_list_evaluations(
|
|
801
539
|
self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
|
|
@@ -829,7 +567,6 @@ class AsyncEvaluationsClient:
|
|
|
829
567
|
|
|
830
568
|
client = AsyncAgentaApi(
|
|
831
569
|
api_key="YOUR_API_KEY",
|
|
832
|
-
base_url="https://yourhost.com/path/to/api",
|
|
833
570
|
)
|
|
834
571
|
|
|
835
572
|
|
|
@@ -841,49 +578,19 @@ class AsyncEvaluationsClient:
|
|
|
841
578
|
|
|
842
579
|
asyncio.run(main())
|
|
843
580
|
"""
|
|
844
|
-
_response = await self.
|
|
845
|
-
|
|
846
|
-
method="GET",
|
|
847
|
-
params={
|
|
848
|
-
"app_id": app_id,
|
|
849
|
-
},
|
|
850
|
-
request_options=request_options,
|
|
581
|
+
_response = await self._raw_client.fetch_list_evaluations(
|
|
582
|
+
app_id=app_id, request_options=request_options
|
|
851
583
|
)
|
|
852
|
-
|
|
853
|
-
if 200 <= _response.status_code < 300:
|
|
854
|
-
return typing.cast(
|
|
855
|
-
typing.List[Evaluation],
|
|
856
|
-
parse_obj_as(
|
|
857
|
-
type_=typing.List[Evaluation], # type: ignore
|
|
858
|
-
object_=_response.json(),
|
|
859
|
-
),
|
|
860
|
-
)
|
|
861
|
-
if _response.status_code == 422:
|
|
862
|
-
raise UnprocessableEntityError(
|
|
863
|
-
typing.cast(
|
|
864
|
-
HttpValidationError,
|
|
865
|
-
parse_obj_as(
|
|
866
|
-
type_=HttpValidationError, # type: ignore
|
|
867
|
-
object_=_response.json(),
|
|
868
|
-
),
|
|
869
|
-
)
|
|
870
|
-
)
|
|
871
|
-
_response_json = _response.json()
|
|
872
|
-
except JSONDecodeError:
|
|
873
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
874
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
584
|
+
return _response.data
|
|
875
585
|
|
|
876
586
|
async def create_evaluation(
|
|
877
587
|
self,
|
|
878
588
|
*,
|
|
879
589
|
app_id: str,
|
|
880
|
-
|
|
590
|
+
revisions_ids: typing.Sequence[str],
|
|
881
591
|
evaluators_configs: typing.Sequence[str],
|
|
882
592
|
testset_id: str,
|
|
883
593
|
rate_limit: LlmRunRateLimit,
|
|
884
|
-
lm_providers_keys: typing.Optional[
|
|
885
|
-
typing.Dict[str, typing.Optional[str]]
|
|
886
|
-
] = OMIT,
|
|
887
594
|
correct_answer_column: typing.Optional[str] = OMIT,
|
|
888
595
|
request_options: typing.Optional[RequestOptions] = None,
|
|
889
596
|
) -> typing.List[Evaluation]:
|
|
@@ -898,7 +605,7 @@ class AsyncEvaluationsClient:
|
|
|
898
605
|
----------
|
|
899
606
|
app_id : str
|
|
900
607
|
|
|
901
|
-
|
|
608
|
+
revisions_ids : typing.Sequence[str]
|
|
902
609
|
|
|
903
610
|
evaluators_configs : typing.Sequence[str]
|
|
904
611
|
|
|
@@ -906,8 +613,6 @@ class AsyncEvaluationsClient:
|
|
|
906
613
|
|
|
907
614
|
rate_limit : LlmRunRateLimit
|
|
908
615
|
|
|
909
|
-
lm_providers_keys : typing.Optional[typing.Dict[str, typing.Optional[str]]]
|
|
910
|
-
|
|
911
616
|
correct_answer_column : typing.Optional[str]
|
|
912
617
|
|
|
913
618
|
request_options : typing.Optional[RequestOptions]
|
|
@@ -926,14 +631,13 @@ class AsyncEvaluationsClient:
|
|
|
926
631
|
|
|
927
632
|
client = AsyncAgentaApi(
|
|
928
633
|
api_key="YOUR_API_KEY",
|
|
929
|
-
base_url="https://yourhost.com/path/to/api",
|
|
930
634
|
)
|
|
931
635
|
|
|
932
636
|
|
|
933
637
|
async def main() -> None:
|
|
934
638
|
await client.evaluations.create_evaluation(
|
|
935
639
|
app_id="app_id",
|
|
936
|
-
|
|
640
|
+
revisions_ids=["revisions_ids"],
|
|
937
641
|
evaluators_configs=["evaluators_configs"],
|
|
938
642
|
testset_id="testset_id",
|
|
939
643
|
rate_limit=LlmRunRateLimit(
|
|
@@ -947,49 +651,16 @@ class AsyncEvaluationsClient:
|
|
|
947
651
|
|
|
948
652
|
asyncio.run(main())
|
|
949
653
|
"""
|
|
950
|
-
_response = await self.
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
"testset_id": testset_id,
|
|
958
|
-
"rate_limit": convert_and_respect_annotation_metadata(
|
|
959
|
-
object_=rate_limit, annotation=LlmRunRateLimit, direction="write"
|
|
960
|
-
),
|
|
961
|
-
"lm_providers_keys": lm_providers_keys,
|
|
962
|
-
"correct_answer_column": correct_answer_column,
|
|
963
|
-
},
|
|
964
|
-
headers={
|
|
965
|
-
"content-type": "application/json",
|
|
966
|
-
},
|
|
654
|
+
_response = await self._raw_client.create_evaluation(
|
|
655
|
+
app_id=app_id,
|
|
656
|
+
revisions_ids=revisions_ids,
|
|
657
|
+
evaluators_configs=evaluators_configs,
|
|
658
|
+
testset_id=testset_id,
|
|
659
|
+
rate_limit=rate_limit,
|
|
660
|
+
correct_answer_column=correct_answer_column,
|
|
967
661
|
request_options=request_options,
|
|
968
|
-
omit=OMIT,
|
|
969
662
|
)
|
|
970
|
-
|
|
971
|
-
if 200 <= _response.status_code < 300:
|
|
972
|
-
return typing.cast(
|
|
973
|
-
typing.List[Evaluation],
|
|
974
|
-
parse_obj_as(
|
|
975
|
-
type_=typing.List[Evaluation], # type: ignore
|
|
976
|
-
object_=_response.json(),
|
|
977
|
-
),
|
|
978
|
-
)
|
|
979
|
-
if _response.status_code == 422:
|
|
980
|
-
raise UnprocessableEntityError(
|
|
981
|
-
typing.cast(
|
|
982
|
-
HttpValidationError,
|
|
983
|
-
parse_obj_as(
|
|
984
|
-
type_=HttpValidationError, # type: ignore
|
|
985
|
-
object_=_response.json(),
|
|
986
|
-
),
|
|
987
|
-
)
|
|
988
|
-
)
|
|
989
|
-
_response_json = _response.json()
|
|
990
|
-
except JSONDecodeError:
|
|
991
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
992
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
663
|
+
return _response.data
|
|
993
664
|
|
|
994
665
|
async def delete_evaluations(
|
|
995
666
|
self,
|
|
@@ -1026,7 +697,6 @@ class AsyncEvaluationsClient:
|
|
|
1026
697
|
|
|
1027
698
|
client = AsyncAgentaApi(
|
|
1028
699
|
api_key="YOUR_API_KEY",
|
|
1029
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1030
700
|
)
|
|
1031
701
|
|
|
1032
702
|
|
|
@@ -1038,38 +708,10 @@ class AsyncEvaluationsClient:
|
|
|
1038
708
|
|
|
1039
709
|
asyncio.run(main())
|
|
1040
710
|
"""
|
|
1041
|
-
_response = await self.
|
|
1042
|
-
|
|
1043
|
-
method="DELETE",
|
|
1044
|
-
json={
|
|
1045
|
-
"evaluations_ids": evaluations_ids,
|
|
1046
|
-
},
|
|
1047
|
-
request_options=request_options,
|
|
1048
|
-
omit=OMIT,
|
|
711
|
+
_response = await self._raw_client.delete_evaluations(
|
|
712
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
1049
713
|
)
|
|
1050
|
-
|
|
1051
|
-
if 200 <= _response.status_code < 300:
|
|
1052
|
-
return typing.cast(
|
|
1053
|
-
typing.List[str],
|
|
1054
|
-
parse_obj_as(
|
|
1055
|
-
type_=typing.List[str], # type: ignore
|
|
1056
|
-
object_=_response.json(),
|
|
1057
|
-
),
|
|
1058
|
-
)
|
|
1059
|
-
if _response.status_code == 422:
|
|
1060
|
-
raise UnprocessableEntityError(
|
|
1061
|
-
typing.cast(
|
|
1062
|
-
HttpValidationError,
|
|
1063
|
-
parse_obj_as(
|
|
1064
|
-
type_=HttpValidationError, # type: ignore
|
|
1065
|
-
object_=_response.json(),
|
|
1066
|
-
),
|
|
1067
|
-
)
|
|
1068
|
-
)
|
|
1069
|
-
_response_json = _response.json()
|
|
1070
|
-
except JSONDecodeError:
|
|
1071
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1072
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
714
|
+
return _response.data
|
|
1073
715
|
|
|
1074
716
|
async def fetch_evaluation_status(
|
|
1075
717
|
self,
|
|
@@ -1107,7 +749,6 @@ class AsyncEvaluationsClient:
|
|
|
1107
749
|
|
|
1108
750
|
client = AsyncAgentaApi(
|
|
1109
751
|
api_key="YOUR_API_KEY",
|
|
1110
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1111
752
|
)
|
|
1112
753
|
|
|
1113
754
|
|
|
@@ -1119,34 +760,10 @@ class AsyncEvaluationsClient:
|
|
|
1119
760
|
|
|
1120
761
|
asyncio.run(main())
|
|
1121
762
|
"""
|
|
1122
|
-
_response = await self.
|
|
1123
|
-
|
|
1124
|
-
method="GET",
|
|
1125
|
-
request_options=request_options,
|
|
763
|
+
_response = await self._raw_client.fetch_evaluation_status(
|
|
764
|
+
evaluation_id, request_options=request_options
|
|
1126
765
|
)
|
|
1127
|
-
|
|
1128
|
-
if 200 <= _response.status_code < 300:
|
|
1129
|
-
return typing.cast(
|
|
1130
|
-
typing.Optional[typing.Any],
|
|
1131
|
-
parse_obj_as(
|
|
1132
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1133
|
-
object_=_response.json(),
|
|
1134
|
-
),
|
|
1135
|
-
)
|
|
1136
|
-
if _response.status_code == 422:
|
|
1137
|
-
raise UnprocessableEntityError(
|
|
1138
|
-
typing.cast(
|
|
1139
|
-
HttpValidationError,
|
|
1140
|
-
parse_obj_as(
|
|
1141
|
-
type_=HttpValidationError, # type: ignore
|
|
1142
|
-
object_=_response.json(),
|
|
1143
|
-
),
|
|
1144
|
-
)
|
|
1145
|
-
)
|
|
1146
|
-
_response_json = _response.json()
|
|
1147
|
-
except JSONDecodeError:
|
|
1148
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1149
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
766
|
+
return _response.data
|
|
1150
767
|
|
|
1151
768
|
async def fetch_evaluation_results(
|
|
1152
769
|
self,
|
|
@@ -1184,7 +801,6 @@ class AsyncEvaluationsClient:
|
|
|
1184
801
|
|
|
1185
802
|
client = AsyncAgentaApi(
|
|
1186
803
|
api_key="YOUR_API_KEY",
|
|
1187
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1188
804
|
)
|
|
1189
805
|
|
|
1190
806
|
|
|
@@ -1196,34 +812,10 @@ class AsyncEvaluationsClient:
|
|
|
1196
812
|
|
|
1197
813
|
asyncio.run(main())
|
|
1198
814
|
"""
|
|
1199
|
-
_response = await self.
|
|
1200
|
-
|
|
1201
|
-
method="GET",
|
|
1202
|
-
request_options=request_options,
|
|
815
|
+
_response = await self._raw_client.fetch_evaluation_results(
|
|
816
|
+
evaluation_id, request_options=request_options
|
|
1203
817
|
)
|
|
1204
|
-
|
|
1205
|
-
if 200 <= _response.status_code < 300:
|
|
1206
|
-
return typing.cast(
|
|
1207
|
-
typing.Optional[typing.Any],
|
|
1208
|
-
parse_obj_as(
|
|
1209
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1210
|
-
object_=_response.json(),
|
|
1211
|
-
),
|
|
1212
|
-
)
|
|
1213
|
-
if _response.status_code == 422:
|
|
1214
|
-
raise UnprocessableEntityError(
|
|
1215
|
-
typing.cast(
|
|
1216
|
-
HttpValidationError,
|
|
1217
|
-
parse_obj_as(
|
|
1218
|
-
type_=HttpValidationError, # type: ignore
|
|
1219
|
-
object_=_response.json(),
|
|
1220
|
-
),
|
|
1221
|
-
)
|
|
1222
|
-
)
|
|
1223
|
-
_response_json = _response.json()
|
|
1224
|
-
except JSONDecodeError:
|
|
1225
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1226
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
818
|
+
return _response.data
|
|
1227
819
|
|
|
1228
820
|
async def fetch_evaluation_scenarios(
|
|
1229
821
|
self,
|
|
@@ -1263,7 +855,6 @@ class AsyncEvaluationsClient:
|
|
|
1263
855
|
|
|
1264
856
|
client = AsyncAgentaApi(
|
|
1265
857
|
api_key="YOUR_API_KEY",
|
|
1266
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1267
858
|
)
|
|
1268
859
|
|
|
1269
860
|
|
|
@@ -1275,34 +866,10 @@ class AsyncEvaluationsClient:
|
|
|
1275
866
|
|
|
1276
867
|
asyncio.run(main())
|
|
1277
868
|
"""
|
|
1278
|
-
_response = await self.
|
|
1279
|
-
|
|
1280
|
-
method="GET",
|
|
1281
|
-
request_options=request_options,
|
|
869
|
+
_response = await self._raw_client.fetch_evaluation_scenarios(
|
|
870
|
+
evaluation_id, request_options=request_options
|
|
1282
871
|
)
|
|
1283
|
-
|
|
1284
|
-
if 200 <= _response.status_code < 300:
|
|
1285
|
-
return typing.cast(
|
|
1286
|
-
typing.List[EvaluationScenario],
|
|
1287
|
-
parse_obj_as(
|
|
1288
|
-
type_=typing.List[EvaluationScenario], # type: ignore
|
|
1289
|
-
object_=_response.json(),
|
|
1290
|
-
),
|
|
1291
|
-
)
|
|
1292
|
-
if _response.status_code == 422:
|
|
1293
|
-
raise UnprocessableEntityError(
|
|
1294
|
-
typing.cast(
|
|
1295
|
-
HttpValidationError,
|
|
1296
|
-
parse_obj_as(
|
|
1297
|
-
type_=HttpValidationError, # type: ignore
|
|
1298
|
-
object_=_response.json(),
|
|
1299
|
-
),
|
|
1300
|
-
)
|
|
1301
|
-
)
|
|
1302
|
-
_response_json = _response.json()
|
|
1303
|
-
except JSONDecodeError:
|
|
1304
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1305
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
872
|
+
return _response.data
|
|
1306
873
|
|
|
1307
874
|
async def fetch_evaluation(
|
|
1308
875
|
self,
|
|
@@ -1339,7 +906,6 @@ class AsyncEvaluationsClient:
|
|
|
1339
906
|
|
|
1340
907
|
client = AsyncAgentaApi(
|
|
1341
908
|
api_key="YOUR_API_KEY",
|
|
1342
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1343
909
|
)
|
|
1344
910
|
|
|
1345
911
|
|
|
@@ -1351,36 +917,12 @@ class AsyncEvaluationsClient:
|
|
|
1351
917
|
|
|
1352
918
|
asyncio.run(main())
|
|
1353
919
|
"""
|
|
1354
|
-
_response = await self.
|
|
1355
|
-
|
|
1356
|
-
method="GET",
|
|
1357
|
-
request_options=request_options,
|
|
920
|
+
_response = await self._raw_client.fetch_evaluation(
|
|
921
|
+
evaluation_id, request_options=request_options
|
|
1358
922
|
)
|
|
1359
|
-
|
|
1360
|
-
if 200 <= _response.status_code < 300:
|
|
1361
|
-
return typing.cast(
|
|
1362
|
-
Evaluation,
|
|
1363
|
-
parse_obj_as(
|
|
1364
|
-
type_=Evaluation, # type: ignore
|
|
1365
|
-
object_=_response.json(),
|
|
1366
|
-
),
|
|
1367
|
-
)
|
|
1368
|
-
if _response.status_code == 422:
|
|
1369
|
-
raise UnprocessableEntityError(
|
|
1370
|
-
typing.cast(
|
|
1371
|
-
HttpValidationError,
|
|
1372
|
-
parse_obj_as(
|
|
1373
|
-
type_=HttpValidationError, # type: ignore
|
|
1374
|
-
object_=_response.json(),
|
|
1375
|
-
),
|
|
1376
|
-
)
|
|
1377
|
-
)
|
|
1378
|
-
_response_json = _response.json()
|
|
1379
|
-
except JSONDecodeError:
|
|
1380
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1381
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
923
|
+
return _response.data
|
|
1382
924
|
|
|
1383
|
-
async def
|
|
925
|
+
async def fetch_evaluation_scenarios_comparison_results(
|
|
1384
926
|
self,
|
|
1385
927
|
*,
|
|
1386
928
|
evaluations_ids: str,
|
|
@@ -1418,46 +960,20 @@ class AsyncEvaluationsClient:
|
|
|
1418
960
|
|
|
1419
961
|
client = AsyncAgentaApi(
|
|
1420
962
|
api_key="YOUR_API_KEY",
|
|
1421
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1422
963
|
)
|
|
1423
964
|
|
|
1424
965
|
|
|
1425
966
|
async def main() -> None:
|
|
1426
|
-
await client.evaluations.
|
|
967
|
+
await client.evaluations.fetch_evaluation_scenarios_comparison_results(
|
|
1427
968
|
evaluations_ids="evaluations_ids",
|
|
1428
969
|
)
|
|
1429
970
|
|
|
1430
971
|
|
|
1431
972
|
asyncio.run(main())
|
|
1432
973
|
"""
|
|
1433
|
-
_response =
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
"evaluations_ids": evaluations_ids,
|
|
1438
|
-
},
|
|
1439
|
-
request_options=request_options,
|
|
974
|
+
_response = (
|
|
975
|
+
await self._raw_client.fetch_evaluation_scenarios_comparison_results(
|
|
976
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
977
|
+
)
|
|
1440
978
|
)
|
|
1441
|
-
|
|
1442
|
-
if 200 <= _response.status_code < 300:
|
|
1443
|
-
return typing.cast(
|
|
1444
|
-
typing.Optional[typing.Any],
|
|
1445
|
-
parse_obj_as(
|
|
1446
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1447
|
-
object_=_response.json(),
|
|
1448
|
-
),
|
|
1449
|
-
)
|
|
1450
|
-
if _response.status_code == 422:
|
|
1451
|
-
raise UnprocessableEntityError(
|
|
1452
|
-
typing.cast(
|
|
1453
|
-
HttpValidationError,
|
|
1454
|
-
parse_obj_as(
|
|
1455
|
-
type_=HttpValidationError, # type: ignore
|
|
1456
|
-
object_=_response.json(),
|
|
1457
|
-
),
|
|
1458
|
-
)
|
|
1459
|
-
)
|
|
1460
|
-
_response_json = _response.json()
|
|
1461
|
-
except JSONDecodeError:
|
|
1462
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1463
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
979
|
+
return _response.data
|