agenta 0.48.9__py3-none-any.whl → 0.49.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenta might be problematic. Click here for more details.
- agenta/__init__.py +3 -2
- agenta/client/Readme.md +258 -80
- agenta/client/__init__.py +205 -29
- agenta/client/backend/__init__.py +461 -0
- agenta/client/backend/access_control/__init__.py +2 -0
- agenta/client/backend/access_control/client.py +53 -73
- agenta/client/backend/access_control/raw_client.py +180 -0
- agenta/client/backend/admin/__init__.py +2 -0
- agenta/client/backend/admin/client.py +473 -268
- agenta/client/backend/admin/raw_client.py +1017 -0
- agenta/client/backend/api_keys/__init__.py +2 -0
- agenta/client/backend/api_keys/client.py +43 -276
- agenta/client/backend/api_keys/raw_client.py +364 -0
- agenta/client/backend/apps/__init__.py +2 -0
- agenta/client/backend/apps/client.py +132 -895
- agenta/client/backend/apps/raw_client.py +1516 -0
- agenta/client/backend/bases/__init__.py +2 -0
- agenta/client/backend/bases/client.py +33 -73
- agenta/client/backend/bases/raw_client.py +179 -0
- agenta/client/backend/billing/__init__.py +3 -0
- agenta/client/backend/billing/client.py +564 -0
- agenta/client/backend/billing/raw_client.py +805 -0
- agenta/client/backend/client.py +1268 -0
- agenta/client/backend/configs/__init__.py +2 -0
- agenta/client/backend/configs/client.py +49 -361
- agenta/client/backend/configs/raw_client.py +402 -0
- agenta/client/backend/containers/__init__.py +1 -3
- agenta/client/backend/containers/client.py +25 -548
- agenta/client/backend/containers/raw_client.py +112 -0
- agenta/client/backend/core/__init__.py +5 -0
- agenta/client/backend/core/api_error.py +12 -6
- agenta/client/backend/core/client_wrapper.py +4 -4
- agenta/client/backend/core/file.py +1 -3
- agenta/client/backend/core/force_multipart.py +16 -0
- agenta/client/backend/core/http_client.py +78 -34
- agenta/client/backend/core/http_response.py +55 -0
- agenta/client/backend/core/jsonable_encoder.py +0 -1
- agenta/client/backend/core/pydantic_utilities.py +88 -113
- agenta/client/backend/core/serialization.py +9 -3
- agenta/client/backend/environment.py +7 -0
- agenta/client/backend/environments/__init__.py +2 -0
- agenta/client/backend/environments/client.py +43 -79
- agenta/client/backend/environments/raw_client.py +193 -0
- agenta/client/backend/errors/__init__.py +2 -0
- agenta/client/backend/errors/unprocessable_entity_error.py +8 -2
- agenta/client/backend/evals/__init__.py +3 -0
- agenta/client/backend/evals/client.py +1042 -0
- agenta/client/backend/evals/raw_client.py +1549 -0
- agenta/client/backend/evaluations/__init__.py +2 -0
- agenta/client/backend/evaluations/client.py +106 -590
- agenta/client/backend/evaluations/raw_client.py +1344 -0
- agenta/client/backend/evaluators/__init__.py +2 -0
- agenta/client/backend/evaluators/client.py +99 -516
- agenta/client/backend/evaluators/raw_client.py +1182 -0
- agenta/client/backend/human_evaluations/__init__.py +2 -0
- agenta/client/backend/human_evaluations/client.py +120 -680
- agenta/client/backend/human_evaluations/raw_client.py +1577 -0
- agenta/client/backend/observability/__init__.py +14 -2
- agenta/client/backend/observability/client.py +185 -341
- agenta/client/backend/observability/raw_client.py +943 -0
- agenta/client/backend/observability/types/__init__.py +10 -2
- agenta/client/backend/observability/types/{format.py → fetch_trace_by_id_request_trace_id.py} +1 -1
- agenta/client/backend/observability/types/fetch_trace_by_id_response.py +15 -0
- agenta/client/backend/observability/types/query_analytics_response.py +2 -1
- agenta/client/backend/observability/types/query_traces_response.py +7 -3
- agenta/client/backend/organization/__init__.py +2 -0
- agenta/client/backend/organization/client.py +105 -361
- agenta/client/backend/organization/raw_client.py +774 -0
- agenta/client/backend/raw_client.py +1432 -0
- agenta/client/backend/scopes/__init__.py +2 -0
- agenta/client/backend/scopes/client.py +31 -45
- agenta/client/backend/scopes/raw_client.py +105 -0
- agenta/client/backend/testsets/__init__.py +14 -0
- agenta/client/backend/testsets/client.py +1098 -653
- agenta/client/backend/testsets/raw_client.py +2348 -0
- agenta/client/backend/testsets/types/__init__.py +17 -0
- agenta/client/backend/testsets/types/create_testset_from_file_request_file_type.py +7 -0
- agenta/client/backend/testsets/types/fetch_testset_to_file_request_file_type.py +7 -0
- agenta/client/backend/testsets/types/update_testset_from_file_request_file_type.py +7 -0
- agenta/client/backend/tracing/__init__.py +7 -0
- agenta/client/backend/tracing/client.py +782 -0
- agenta/client/backend/tracing/raw_client.py +1223 -0
- agenta/client/backend/tracing/types/__init__.py +8 -0
- agenta/client/backend/{types/variant_action_enum.py → tracing/types/fetch_trace_request_trace_id.py} +1 -1
- agenta/client/backend/tracing/types/remove_trace_request_trace_id.py +5 -0
- agenta/client/backend/types/__init__.py +153 -26
- agenta/client/backend/types/account_request.py +24 -0
- agenta/client/backend/types/account_response.py +5 -7
- agenta/client/backend/types/agenta_node_dto.py +13 -13
- agenta/client/backend/types/agenta_node_dto_nodes_value.py +1 -0
- agenta/client/backend/types/agenta_nodes_response.py +14 -8
- agenta/client/backend/types/agenta_root_dto.py +16 -8
- agenta/client/backend/types/agenta_roots_response.py +16 -8
- agenta/client/backend/types/agenta_tree_dto.py +16 -8
- agenta/client/backend/types/agenta_trees_response.py +16 -8
- agenta/client/backend/types/aggregated_result.py +5 -7
- agenta/client/backend/types/aggregated_result_evaluator_config.py +1 -0
- agenta/client/backend/types/analytics_response.py +4 -6
- agenta/client/backend/types/annotation.py +50 -0
- agenta/client/backend/types/annotation_create.py +39 -0
- agenta/client/backend/types/annotation_edit.py +31 -0
- agenta/client/backend/types/annotation_kind.py +5 -0
- agenta/client/backend/types/{uri.py → annotation_link.py} +6 -7
- agenta/client/backend/types/{provider_key_dto.py → annotation_link_response.py} +6 -7
- agenta/client/backend/types/annotation_query.py +40 -0
- agenta/client/backend/types/annotation_query_request.py +20 -0
- agenta/client/backend/types/annotation_reference.py +21 -0
- agenta/client/backend/types/annotation_references.py +22 -0
- agenta/client/backend/types/{docker_env_vars.py → annotation_response.py} +6 -7
- agenta/client/backend/types/annotation_source.py +5 -0
- agenta/client/backend/types/annotations_response.py +24 -0
- agenta/client/backend/types/app.py +3 -5
- agenta/client/backend/types/app_variant_response.py +3 -6
- agenta/client/backend/types/app_variant_revision.py +5 -6
- agenta/client/backend/types/artifact.py +44 -0
- agenta/client/backend/types/base_output.py +3 -5
- agenta/client/backend/types/body_fetch_workflow_revision.py +21 -0
- agenta/client/backend/types/body_import_testset.py +3 -5
- agenta/client/backend/types/bucket_dto.py +4 -6
- agenta/client/backend/types/collect_status_response.py +3 -5
- agenta/client/backend/types/config_db.py +3 -5
- agenta/client/backend/types/config_dto.py +5 -7
- agenta/client/backend/types/config_response_model.py +5 -7
- agenta/client/backend/types/correct_answer.py +3 -5
- agenta/client/backend/types/create_app_output.py +3 -5
- agenta/client/backend/types/custom_model_settings_dto.py +3 -5
- agenta/client/backend/types/custom_provider_dto.py +6 -9
- agenta/client/backend/types/custom_provider_kind.py +5 -5
- agenta/client/backend/types/custom_provider_settings_dto.py +3 -5
- agenta/client/backend/types/data.py +2 -1
- agenta/client/backend/types/delete_evaluation.py +3 -5
- agenta/client/backend/types/environment_output.py +3 -5
- agenta/client/backend/types/environment_output_extended.py +4 -6
- agenta/client/backend/types/environment_revision.py +5 -5
- agenta/client/backend/types/error.py +3 -5
- agenta/client/backend/types/evaluation.py +6 -8
- agenta/client/backend/types/evaluation_scenario.py +5 -7
- agenta/client/backend/types/evaluation_scenario_input.py +3 -5
- agenta/client/backend/types/evaluation_scenario_output.py +4 -6
- agenta/client/backend/types/evaluation_scenario_result.py +4 -6
- agenta/client/backend/types/evaluator.py +31 -12
- agenta/client/backend/types/evaluator_config.py +3 -5
- agenta/client/backend/types/evaluator_flags.py +21 -0
- agenta/client/backend/types/evaluator_mapping_output_interface.py +3 -5
- agenta/client/backend/types/evaluator_output_interface.py +3 -5
- agenta/client/backend/types/evaluator_query.py +32 -0
- agenta/client/backend/types/evaluator_query_request.py +30 -0
- agenta/client/backend/types/evaluator_request.py +20 -0
- agenta/client/backend/types/evaluator_response.py +21 -0
- agenta/client/backend/types/evaluators_response.py +21 -0
- agenta/client/backend/types/exception_dto.py +3 -5
- agenta/client/backend/types/{o_tel_spans_response.py → extended_o_tel_tracing_response.py} +5 -7
- agenta/client/backend/types/focus.py +5 -0
- agenta/client/backend/types/format.py +5 -0
- agenta/client/backend/types/full_json_input.py +34 -0
- agenta/client/backend/types/full_json_output.py +29 -0
- agenta/client/backend/types/get_config_response.py +3 -5
- agenta/client/backend/types/{header_dto.py → header.py} +4 -6
- agenta/client/backend/types/http_validation_error.py +4 -6
- agenta/client/backend/types/human_evaluation.py +3 -5
- agenta/client/backend/types/human_evaluation_scenario.py +4 -6
- agenta/client/backend/types/human_evaluation_scenario_input.py +3 -5
- agenta/client/backend/types/human_evaluation_scenario_output.py +3 -5
- agenta/client/backend/types/invite_request.py +4 -6
- agenta/client/backend/types/legacy_analytics_response.py +4 -6
- agenta/client/backend/types/legacy_data_point.py +3 -5
- agenta/client/backend/types/legacy_evaluator.py +26 -0
- agenta/client/backend/types/legacy_scope_request.py +4 -6
- agenta/client/backend/types/legacy_scopes_response.py +3 -5
- agenta/client/backend/types/legacy_subscription_request.py +19 -0
- agenta/client/backend/types/legacy_user_request.py +5 -7
- agenta/client/backend/types/legacy_user_response.py +3 -5
- agenta/client/backend/types/lifecycle_dto.py +3 -5
- agenta/client/backend/types/link_dto.py +4 -6
- agenta/client/backend/types/list_api_keys_response.py +3 -5
- agenta/client/backend/types/llm_run_rate_limit.py +3 -5
- agenta/client/backend/types/meta_request.py +30 -0
- agenta/client/backend/types/metrics_dto.py +3 -5
- agenta/client/backend/types/new_testset.py +3 -5
- agenta/client/backend/types/node_dto.py +4 -6
- agenta/client/backend/types/o_tel_context_dto.py +3 -5
- agenta/client/backend/types/o_tel_event.py +35 -0
- agenta/client/backend/types/o_tel_event_dto.py +3 -5
- agenta/client/backend/types/o_tel_extra_dto.py +4 -6
- agenta/client/backend/types/o_tel_flat_span.py +56 -0
- agenta/client/backend/types/o_tel_flat_span_input_end_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_input_start_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_output_end_time.py +6 -0
- agenta/client/backend/types/o_tel_flat_span_output_start_time.py +6 -0
- agenta/client/backend/types/o_tel_link.py +34 -0
- agenta/client/backend/types/o_tel_link_dto.py +4 -6
- agenta/client/backend/types/o_tel_links_response.py +22 -0
- agenta/client/backend/types/o_tel_span.py +58 -0
- agenta/client/backend/types/o_tel_span_dto.py +8 -10
- agenta/client/backend/types/o_tel_span_input_end_time.py +6 -0
- agenta/client/backend/types/o_tel_span_input_spans_value.py +7 -0
- agenta/client/backend/types/o_tel_span_input_start_time.py +6 -0
- agenta/client/backend/types/o_tel_span_output_end_time.py +6 -0
- agenta/client/backend/types/o_tel_span_output_spans_value.py +30 -0
- agenta/client/backend/types/o_tel_span_output_start_time.py +6 -0
- agenta/client/backend/types/o_tel_spans_tree.py +22 -0
- agenta/client/backend/types/o_tel_spans_tree_input_spans_value.py +7 -0
- agenta/client/backend/types/o_tel_spans_tree_output_spans_value.py +5 -0
- agenta/client/backend/types/o_tel_status_code.py +1 -1
- agenta/client/backend/types/o_tel_tracing_data_response.py +22 -0
- agenta/client/backend/types/o_tel_tracing_request.py +22 -0
- agenta/client/backend/types/o_tel_tracing_response.py +27 -0
- agenta/client/backend/types/organization.py +3 -5
- agenta/client/backend/types/organization_details.py +3 -5
- agenta/client/backend/types/organization_membership_request.py +5 -7
- agenta/client/backend/types/organization_output.py +3 -5
- agenta/client/backend/types/organization_request.py +3 -5
- agenta/client/backend/types/parent_dto.py +3 -5
- agenta/client/backend/types/permission.py +11 -0
- agenta/client/backend/types/plan.py +14 -0
- agenta/client/backend/types/project_membership_request.py +5 -7
- agenta/client/backend/types/project_request.py +4 -6
- agenta/client/backend/types/project_scope.py +5 -7
- agenta/client/backend/types/projects_response.py +3 -5
- agenta/client/backend/types/recursive_types.py +23 -0
- agenta/client/backend/types/reference.py +18 -5
- agenta/client/backend/types/reference_dto.py +4 -5
- agenta/client/backend/types/reference_request_model.py +4 -5
- agenta/client/backend/types/result.py +4 -6
- agenta/client/backend/types/root_dto.py +3 -5
- agenta/client/backend/types/scopes_response_model.py +4 -6
- agenta/client/backend/types/secret_dto.py +5 -7
- agenta/client/backend/types/secret_response_dto.py +11 -11
- agenta/client/backend/types/simple_evaluation_output.py +4 -6
- agenta/client/backend/types/span_dto.py +18 -14
- agenta/client/backend/types/span_dto_nodes_value.py +1 -1
- agenta/client/backend/types/standard_provider_dto.py +5 -7
- agenta/client/backend/types/standard_provider_settings_dto.py +3 -5
- agenta/client/backend/types/status_dto.py +4 -6
- agenta/client/backend/types/tags_request.py +30 -0
- agenta/client/backend/types/test_set_output_response.py +5 -7
- agenta/client/backend/types/test_set_simple_response.py +3 -5
- agenta/client/backend/types/testcase_response.py +33 -0
- agenta/client/backend/types/testset.py +46 -0
- agenta/client/backend/types/testset_request.py +20 -0
- agenta/client/backend/types/testset_response.py +21 -0
- agenta/client/backend/types/testsets_response.py +21 -0
- agenta/client/backend/types/time_dto.py +3 -5
- agenta/client/backend/types/timestamp.py +6 -0
- agenta/client/backend/types/tree_dto.py +4 -6
- agenta/client/backend/types/update_app_output.py +3 -5
- agenta/client/backend/types/user_request.py +3 -5
- agenta/client/backend/types/validation_error.py +4 -6
- agenta/client/backend/types/workflow_artifact.py +45 -0
- agenta/client/backend/types/workflow_data.py +20 -0
- agenta/client/backend/types/workflow_flags.py +21 -0
- agenta/client/backend/types/workflow_request.py +20 -0
- agenta/client/backend/types/workflow_response.py +21 -0
- agenta/client/backend/types/workflow_revision.py +57 -0
- agenta/client/backend/types/workflow_revision_request.py +20 -0
- agenta/client/backend/types/workflow_revision_response.py +21 -0
- agenta/client/backend/types/workflow_revisions_response.py +21 -0
- agenta/client/backend/types/workflow_variant.py +48 -0
- agenta/client/backend/types/workflow_variant_request.py +20 -0
- agenta/client/backend/types/workflow_variant_response.py +21 -0
- agenta/client/backend/types/workflow_variants_response.py +21 -0
- agenta/client/backend/types/workflows_response.py +21 -0
- agenta/client/backend/types/workspace.py +3 -5
- agenta/client/backend/types/workspace_member_response.py +4 -6
- agenta/client/backend/types/workspace_membership_request.py +5 -7
- agenta/client/backend/types/workspace_permission.py +5 -7
- agenta/client/backend/types/workspace_request.py +4 -6
- agenta/client/backend/types/workspace_response.py +4 -6
- agenta/client/backend/variants/__init__.py +2 -0
- agenta/client/backend/variants/client.py +306 -1651
- agenta/client/backend/variants/raw_client.py +2482 -0
- agenta/client/backend/variants/types/__init__.py +2 -0
- agenta/client/backend/variants/types/add_variant_from_base_and_config_response.py +1 -0
- agenta/client/backend/vault/__init__.py +2 -0
- agenta/client/backend/vault/client.py +69 -323
- agenta/client/backend/vault/raw_client.py +616 -0
- agenta/client/backend/workflows/__init__.py +3 -0
- agenta/client/backend/workflows/client.py +2398 -0
- agenta/client/backend/workflows/raw_client.py +3639 -0
- agenta/client/backend/workspace/__init__.py +2 -0
- agenta/client/backend/workspace/client.py +46 -147
- agenta/client/backend/workspace/raw_client.py +376 -0
- agenta/client/types.py +4 -0
- agenta/sdk/litellm/mocks/__init__.py +144 -0
- agenta/sdk/middleware/auth.py +26 -25
- agenta/sdk/types.py +28 -1
- {agenta-0.48.9.dist-info → agenta-0.49.0.dist-info}/METADATA +1 -1
- agenta-0.49.0.dist-info/RECORD +362 -0
- agenta/client/backend/containers/types/__init__.py +0 -5
- agenta/client/backend/containers/types/container_templates_response.py +0 -6
- agenta/client/backend/types/image.py +0 -25
- agenta/client/backend/types/provider_kind.py +0 -21
- agenta/client/backend/types/template.py +0 -23
- agenta/client/backend/types/template_image_info.py +0 -29
- agenta/client/backend/types/variant_action.py +0 -22
- agenta-0.48.9.dist-info/RECORD +0 -255
- {agenta-0.48.9.dist-info → agenta-0.49.0.dist-info}/WHEEL +0 -0
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
# This file was auto-generated by Fern from our API Definition.
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
|
|
5
6
|
from ..core.request_options import RequestOptions
|
|
6
|
-
from ..types.human_evaluation import HumanEvaluation
|
|
7
|
-
from ..core.pydantic_utilities import parse_obj_as
|
|
8
|
-
from ..errors.unprocessable_entity_error import UnprocessableEntityError
|
|
9
|
-
from ..types.http_validation_error import HttpValidationError
|
|
10
|
-
from json.decoder import JSONDecodeError
|
|
11
|
-
from ..core.api_error import ApiError
|
|
12
|
-
from ..types.evaluation_type import EvaluationType
|
|
13
|
-
from ..types.simple_evaluation_output import SimpleEvaluationOutput
|
|
14
|
-
from ..core.jsonable_encoder import jsonable_encoder
|
|
15
7
|
from ..types.evaluation_status_enum import EvaluationStatusEnum
|
|
8
|
+
from ..types.evaluation_type import EvaluationType
|
|
9
|
+
from ..types.human_evaluation import HumanEvaluation
|
|
16
10
|
from ..types.human_evaluation_scenario import HumanEvaluationScenario
|
|
17
|
-
from ..types.score import Score
|
|
18
|
-
from ..types.human_evaluation_scenario_output import HumanEvaluationScenarioOutput
|
|
19
11
|
from ..types.human_evaluation_scenario_input import HumanEvaluationScenarioInput
|
|
20
|
-
from ..
|
|
21
|
-
from ..
|
|
12
|
+
from ..types.human_evaluation_scenario_output import HumanEvaluationScenarioOutput
|
|
13
|
+
from ..types.score import Score
|
|
14
|
+
from ..types.simple_evaluation_output import SimpleEvaluationOutput
|
|
15
|
+
from .raw_client import AsyncRawHumanEvaluationsClient, RawHumanEvaluationsClient
|
|
22
16
|
|
|
23
17
|
# this is used as the default value for optional parameters
|
|
24
18
|
OMIT = typing.cast(typing.Any, ...)
|
|
@@ -26,7 +20,18 @@ OMIT = typing.cast(typing.Any, ...)
|
|
|
26
20
|
|
|
27
21
|
class HumanEvaluationsClient:
|
|
28
22
|
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
29
|
-
self.
|
|
23
|
+
self._raw_client = RawHumanEvaluationsClient(client_wrapper=client_wrapper)
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def with_raw_response(self) -> RawHumanEvaluationsClient:
|
|
27
|
+
"""
|
|
28
|
+
Retrieves a raw implementation of this client that returns raw responses.
|
|
29
|
+
|
|
30
|
+
Returns
|
|
31
|
+
-------
|
|
32
|
+
RawHumanEvaluationsClient
|
|
33
|
+
"""
|
|
34
|
+
return self._raw_client
|
|
30
35
|
|
|
31
36
|
def fetch_list_human_evaluations(
|
|
32
37
|
self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
|
|
@@ -58,43 +63,15 @@ class HumanEvaluationsClient:
|
|
|
58
63
|
|
|
59
64
|
client = AgentaApi(
|
|
60
65
|
api_key="YOUR_API_KEY",
|
|
61
|
-
base_url="https://yourhost.com/path/to/api",
|
|
62
66
|
)
|
|
63
67
|
client.human_evaluations.fetch_list_human_evaluations(
|
|
64
68
|
app_id="app_id",
|
|
65
69
|
)
|
|
66
70
|
"""
|
|
67
|
-
_response = self.
|
|
68
|
-
|
|
69
|
-
method="GET",
|
|
70
|
-
params={
|
|
71
|
-
"app_id": app_id,
|
|
72
|
-
},
|
|
73
|
-
request_options=request_options,
|
|
71
|
+
_response = self._raw_client.fetch_list_human_evaluations(
|
|
72
|
+
app_id=app_id, request_options=request_options
|
|
74
73
|
)
|
|
75
|
-
|
|
76
|
-
if 200 <= _response.status_code < 300:
|
|
77
|
-
return typing.cast(
|
|
78
|
-
typing.List[HumanEvaluation],
|
|
79
|
-
parse_obj_as(
|
|
80
|
-
type_=typing.List[HumanEvaluation], # type: ignore
|
|
81
|
-
object_=_response.json(),
|
|
82
|
-
),
|
|
83
|
-
)
|
|
84
|
-
if _response.status_code == 422:
|
|
85
|
-
raise UnprocessableEntityError(
|
|
86
|
-
typing.cast(
|
|
87
|
-
HttpValidationError,
|
|
88
|
-
parse_obj_as(
|
|
89
|
-
type_=HttpValidationError, # type: ignore
|
|
90
|
-
object_=_response.json(),
|
|
91
|
-
),
|
|
92
|
-
)
|
|
93
|
-
)
|
|
94
|
-
_response_json = _response.json()
|
|
95
|
-
except JSONDecodeError:
|
|
96
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
97
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
74
|
+
return _response.data
|
|
98
75
|
|
|
99
76
|
def create_human_evaluation(
|
|
100
77
|
self,
|
|
@@ -142,7 +119,6 @@ class HumanEvaluationsClient:
|
|
|
142
119
|
|
|
143
120
|
client = AgentaApi(
|
|
144
121
|
api_key="YOUR_API_KEY",
|
|
145
|
-
base_url="https://yourhost.com/path/to/api",
|
|
146
122
|
)
|
|
147
123
|
client.human_evaluations.create_human_evaluation(
|
|
148
124
|
app_id="app_id",
|
|
@@ -153,46 +129,16 @@ class HumanEvaluationsClient:
|
|
|
153
129
|
status="status",
|
|
154
130
|
)
|
|
155
131
|
"""
|
|
156
|
-
_response = self.
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
"inputs": inputs,
|
|
164
|
-
"testset_id": testset_id,
|
|
165
|
-
"status": status,
|
|
166
|
-
},
|
|
167
|
-
headers={
|
|
168
|
-
"content-type": "application/json",
|
|
169
|
-
},
|
|
132
|
+
_response = self._raw_client.create_human_evaluation(
|
|
133
|
+
app_id=app_id,
|
|
134
|
+
variant_ids=variant_ids,
|
|
135
|
+
evaluation_type=evaluation_type,
|
|
136
|
+
inputs=inputs,
|
|
137
|
+
testset_id=testset_id,
|
|
138
|
+
status=status,
|
|
170
139
|
request_options=request_options,
|
|
171
|
-
omit=OMIT,
|
|
172
140
|
)
|
|
173
|
-
|
|
174
|
-
if 200 <= _response.status_code < 300:
|
|
175
|
-
return typing.cast(
|
|
176
|
-
SimpleEvaluationOutput,
|
|
177
|
-
parse_obj_as(
|
|
178
|
-
type_=SimpleEvaluationOutput, # type: ignore
|
|
179
|
-
object_=_response.json(),
|
|
180
|
-
),
|
|
181
|
-
)
|
|
182
|
-
if _response.status_code == 422:
|
|
183
|
-
raise UnprocessableEntityError(
|
|
184
|
-
typing.cast(
|
|
185
|
-
HttpValidationError,
|
|
186
|
-
parse_obj_as(
|
|
187
|
-
type_=HttpValidationError, # type: ignore
|
|
188
|
-
object_=_response.json(),
|
|
189
|
-
),
|
|
190
|
-
)
|
|
191
|
-
)
|
|
192
|
-
_response_json = _response.json()
|
|
193
|
-
except JSONDecodeError:
|
|
194
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
195
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
141
|
+
return _response.data
|
|
196
142
|
|
|
197
143
|
def delete_evaluations(
|
|
198
144
|
self,
|
|
@@ -227,44 +173,15 @@ class HumanEvaluationsClient:
|
|
|
227
173
|
|
|
228
174
|
client = AgentaApi(
|
|
229
175
|
api_key="YOUR_API_KEY",
|
|
230
|
-
base_url="https://yourhost.com/path/to/api",
|
|
231
176
|
)
|
|
232
177
|
client.human_evaluations.delete_evaluations(
|
|
233
178
|
evaluations_ids=["evaluations_ids"],
|
|
234
179
|
)
|
|
235
180
|
"""
|
|
236
|
-
_response = self.
|
|
237
|
-
|
|
238
|
-
method="DELETE",
|
|
239
|
-
json={
|
|
240
|
-
"evaluations_ids": evaluations_ids,
|
|
241
|
-
},
|
|
242
|
-
request_options=request_options,
|
|
243
|
-
omit=OMIT,
|
|
181
|
+
_response = self._raw_client.delete_evaluations(
|
|
182
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
244
183
|
)
|
|
245
|
-
|
|
246
|
-
if 200 <= _response.status_code < 300:
|
|
247
|
-
return typing.cast(
|
|
248
|
-
typing.List[str],
|
|
249
|
-
parse_obj_as(
|
|
250
|
-
type_=typing.List[str], # type: ignore
|
|
251
|
-
object_=_response.json(),
|
|
252
|
-
),
|
|
253
|
-
)
|
|
254
|
-
if _response.status_code == 422:
|
|
255
|
-
raise UnprocessableEntityError(
|
|
256
|
-
typing.cast(
|
|
257
|
-
HttpValidationError,
|
|
258
|
-
parse_obj_as(
|
|
259
|
-
type_=HttpValidationError, # type: ignore
|
|
260
|
-
object_=_response.json(),
|
|
261
|
-
),
|
|
262
|
-
)
|
|
263
|
-
)
|
|
264
|
-
_response_json = _response.json()
|
|
265
|
-
except JSONDecodeError:
|
|
266
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
267
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
184
|
+
return _response.data
|
|
268
185
|
|
|
269
186
|
def fetch_human_evaluation(
|
|
270
187
|
self,
|
|
@@ -299,40 +216,15 @@ class HumanEvaluationsClient:
|
|
|
299
216
|
|
|
300
217
|
client = AgentaApi(
|
|
301
218
|
api_key="YOUR_API_KEY",
|
|
302
|
-
base_url="https://yourhost.com/path/to/api",
|
|
303
219
|
)
|
|
304
220
|
client.human_evaluations.fetch_human_evaluation(
|
|
305
221
|
evaluation_id="evaluation_id",
|
|
306
222
|
)
|
|
307
223
|
"""
|
|
308
|
-
_response = self.
|
|
309
|
-
|
|
310
|
-
method="GET",
|
|
311
|
-
request_options=request_options,
|
|
224
|
+
_response = self._raw_client.fetch_human_evaluation(
|
|
225
|
+
evaluation_id, request_options=request_options
|
|
312
226
|
)
|
|
313
|
-
|
|
314
|
-
if 200 <= _response.status_code < 300:
|
|
315
|
-
return typing.cast(
|
|
316
|
-
HumanEvaluation,
|
|
317
|
-
parse_obj_as(
|
|
318
|
-
type_=HumanEvaluation, # type: ignore
|
|
319
|
-
object_=_response.json(),
|
|
320
|
-
),
|
|
321
|
-
)
|
|
322
|
-
if _response.status_code == 422:
|
|
323
|
-
raise UnprocessableEntityError(
|
|
324
|
-
typing.cast(
|
|
325
|
-
HttpValidationError,
|
|
326
|
-
parse_obj_as(
|
|
327
|
-
type_=HttpValidationError, # type: ignore
|
|
328
|
-
object_=_response.json(),
|
|
329
|
-
),
|
|
330
|
-
)
|
|
331
|
-
)
|
|
332
|
-
_response_json = _response.json()
|
|
333
|
-
except JSONDecodeError:
|
|
334
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
335
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
227
|
+
return _response.data
|
|
336
228
|
|
|
337
229
|
def update_human_evaluation(
|
|
338
230
|
self,
|
|
@@ -370,47 +262,15 @@ class HumanEvaluationsClient:
|
|
|
370
262
|
|
|
371
263
|
client = AgentaApi(
|
|
372
264
|
api_key="YOUR_API_KEY",
|
|
373
|
-
base_url="https://yourhost.com/path/to/api",
|
|
374
265
|
)
|
|
375
266
|
client.human_evaluations.update_human_evaluation(
|
|
376
267
|
evaluation_id="evaluation_id",
|
|
377
268
|
)
|
|
378
269
|
"""
|
|
379
|
-
_response = self.
|
|
380
|
-
|
|
381
|
-
method="PUT",
|
|
382
|
-
json={
|
|
383
|
-
"status": status,
|
|
384
|
-
},
|
|
385
|
-
headers={
|
|
386
|
-
"content-type": "application/json",
|
|
387
|
-
},
|
|
388
|
-
request_options=request_options,
|
|
389
|
-
omit=OMIT,
|
|
270
|
+
_response = self._raw_client.update_human_evaluation(
|
|
271
|
+
evaluation_id, status=status, request_options=request_options
|
|
390
272
|
)
|
|
391
|
-
|
|
392
|
-
if 200 <= _response.status_code < 300:
|
|
393
|
-
return typing.cast(
|
|
394
|
-
typing.Optional[typing.Any],
|
|
395
|
-
parse_obj_as(
|
|
396
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
397
|
-
object_=_response.json(),
|
|
398
|
-
),
|
|
399
|
-
)
|
|
400
|
-
if _response.status_code == 422:
|
|
401
|
-
raise UnprocessableEntityError(
|
|
402
|
-
typing.cast(
|
|
403
|
-
HttpValidationError,
|
|
404
|
-
parse_obj_as(
|
|
405
|
-
type_=HttpValidationError, # type: ignore
|
|
406
|
-
object_=_response.json(),
|
|
407
|
-
),
|
|
408
|
-
)
|
|
409
|
-
)
|
|
410
|
-
_response_json = _response.json()
|
|
411
|
-
except JSONDecodeError:
|
|
412
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
413
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
273
|
+
return _response.data
|
|
414
274
|
|
|
415
275
|
def fetch_human_evaluation_scenarios(
|
|
416
276
|
self,
|
|
@@ -448,40 +308,15 @@ class HumanEvaluationsClient:
|
|
|
448
308
|
|
|
449
309
|
client = AgentaApi(
|
|
450
310
|
api_key="YOUR_API_KEY",
|
|
451
|
-
base_url="https://yourhost.com/path/to/api",
|
|
452
311
|
)
|
|
453
312
|
client.human_evaluations.fetch_human_evaluation_scenarios(
|
|
454
313
|
evaluation_id="evaluation_id",
|
|
455
314
|
)
|
|
456
315
|
"""
|
|
457
|
-
_response = self.
|
|
458
|
-
|
|
459
|
-
method="GET",
|
|
460
|
-
request_options=request_options,
|
|
316
|
+
_response = self._raw_client.fetch_human_evaluation_scenarios(
|
|
317
|
+
evaluation_id, request_options=request_options
|
|
461
318
|
)
|
|
462
|
-
|
|
463
|
-
if 200 <= _response.status_code < 300:
|
|
464
|
-
return typing.cast(
|
|
465
|
-
typing.List[HumanEvaluationScenario],
|
|
466
|
-
parse_obj_as(
|
|
467
|
-
type_=typing.List[HumanEvaluationScenario], # type: ignore
|
|
468
|
-
object_=_response.json(),
|
|
469
|
-
),
|
|
470
|
-
)
|
|
471
|
-
if _response.status_code == 422:
|
|
472
|
-
raise UnprocessableEntityError(
|
|
473
|
-
typing.cast(
|
|
474
|
-
HttpValidationError,
|
|
475
|
-
parse_obj_as(
|
|
476
|
-
type_=HttpValidationError, # type: ignore
|
|
477
|
-
object_=_response.json(),
|
|
478
|
-
),
|
|
479
|
-
)
|
|
480
|
-
)
|
|
481
|
-
_response_json = _response.json()
|
|
482
|
-
except JSONDecodeError:
|
|
483
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
484
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
319
|
+
return _response.data
|
|
485
320
|
|
|
486
321
|
def update_evaluation_scenario_router(
|
|
487
322
|
self,
|
|
@@ -543,7 +378,6 @@ class HumanEvaluationsClient:
|
|
|
543
378
|
|
|
544
379
|
client = AgentaApi(
|
|
545
380
|
api_key="YOUR_API_KEY",
|
|
546
|
-
base_url="https://yourhost.com/path/to/api",
|
|
547
381
|
)
|
|
548
382
|
client.human_evaluations.update_evaluation_scenario_router(
|
|
549
383
|
evaluation_id="evaluation_id",
|
|
@@ -551,55 +385,20 @@ class HumanEvaluationsClient:
|
|
|
551
385
|
evaluation_type="human_a_b_testing",
|
|
552
386
|
)
|
|
553
387
|
"""
|
|
554
|
-
_response = self.
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
),
|
|
566
|
-
"inputs": convert_and_respect_annotation_metadata(
|
|
567
|
-
object_=inputs,
|
|
568
|
-
annotation=typing.Sequence[HumanEvaluationScenarioInput],
|
|
569
|
-
direction="write",
|
|
570
|
-
),
|
|
571
|
-
"is_pinned": is_pinned,
|
|
572
|
-
"note": note,
|
|
573
|
-
},
|
|
574
|
-
headers={
|
|
575
|
-
"content-type": "application/json",
|
|
576
|
-
},
|
|
388
|
+
_response = self._raw_client.update_evaluation_scenario_router(
|
|
389
|
+
evaluation_id,
|
|
390
|
+
evaluation_scenario_id,
|
|
391
|
+
evaluation_type,
|
|
392
|
+
vote=vote,
|
|
393
|
+
score=score,
|
|
394
|
+
correct_answer=correct_answer,
|
|
395
|
+
outputs=outputs,
|
|
396
|
+
inputs=inputs,
|
|
397
|
+
is_pinned=is_pinned,
|
|
398
|
+
note=note,
|
|
577
399
|
request_options=request_options,
|
|
578
|
-
omit=OMIT,
|
|
579
400
|
)
|
|
580
|
-
|
|
581
|
-
if 200 <= _response.status_code < 300:
|
|
582
|
-
return typing.cast(
|
|
583
|
-
typing.Optional[typing.Any],
|
|
584
|
-
parse_obj_as(
|
|
585
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
586
|
-
object_=_response.json(),
|
|
587
|
-
),
|
|
588
|
-
)
|
|
589
|
-
if _response.status_code == 422:
|
|
590
|
-
raise UnprocessableEntityError(
|
|
591
|
-
typing.cast(
|
|
592
|
-
HttpValidationError,
|
|
593
|
-
parse_obj_as(
|
|
594
|
-
type_=HttpValidationError, # type: ignore
|
|
595
|
-
object_=_response.json(),
|
|
596
|
-
),
|
|
597
|
-
)
|
|
598
|
-
)
|
|
599
|
-
_response_json = _response.json()
|
|
600
|
-
except JSONDecodeError:
|
|
601
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
602
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
401
|
+
return _response.data
|
|
603
402
|
|
|
604
403
|
def get_evaluation_scenario_score_router(
|
|
605
404
|
self,
|
|
@@ -634,40 +433,15 @@ class HumanEvaluationsClient:
|
|
|
634
433
|
|
|
635
434
|
client = AgentaApi(
|
|
636
435
|
api_key="YOUR_API_KEY",
|
|
637
|
-
base_url="https://yourhost.com/path/to/api",
|
|
638
436
|
)
|
|
639
437
|
client.human_evaluations.get_evaluation_scenario_score_router(
|
|
640
438
|
evaluation_scenario_id="evaluation_scenario_id",
|
|
641
439
|
)
|
|
642
440
|
"""
|
|
643
|
-
_response = self.
|
|
644
|
-
|
|
645
|
-
method="GET",
|
|
646
|
-
request_options=request_options,
|
|
441
|
+
_response = self._raw_client.get_evaluation_scenario_score_router(
|
|
442
|
+
evaluation_scenario_id, request_options=request_options
|
|
647
443
|
)
|
|
648
|
-
|
|
649
|
-
if 200 <= _response.status_code < 300:
|
|
650
|
-
return typing.cast(
|
|
651
|
-
typing.Dict[str, str],
|
|
652
|
-
parse_obj_as(
|
|
653
|
-
type_=typing.Dict[str, str], # type: ignore
|
|
654
|
-
object_=_response.json(),
|
|
655
|
-
),
|
|
656
|
-
)
|
|
657
|
-
if _response.status_code == 422:
|
|
658
|
-
raise UnprocessableEntityError(
|
|
659
|
-
typing.cast(
|
|
660
|
-
HttpValidationError,
|
|
661
|
-
parse_obj_as(
|
|
662
|
-
type_=HttpValidationError, # type: ignore
|
|
663
|
-
object_=_response.json(),
|
|
664
|
-
),
|
|
665
|
-
)
|
|
666
|
-
)
|
|
667
|
-
_response_json = _response.json()
|
|
668
|
-
except JSONDecodeError:
|
|
669
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
670
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
444
|
+
return _response.data
|
|
671
445
|
|
|
672
446
|
def update_evaluation_scenario_score_router(
|
|
673
447
|
self,
|
|
@@ -705,48 +479,16 @@ class HumanEvaluationsClient:
|
|
|
705
479
|
|
|
706
480
|
client = AgentaApi(
|
|
707
481
|
api_key="YOUR_API_KEY",
|
|
708
|
-
base_url="https://yourhost.com/path/to/api",
|
|
709
482
|
)
|
|
710
483
|
client.human_evaluations.update_evaluation_scenario_score_router(
|
|
711
484
|
evaluation_scenario_id="evaluation_scenario_id",
|
|
712
485
|
score=1.1,
|
|
713
486
|
)
|
|
714
487
|
"""
|
|
715
|
-
_response = self.
|
|
716
|
-
|
|
717
|
-
method="PUT",
|
|
718
|
-
json={
|
|
719
|
-
"score": score,
|
|
720
|
-
},
|
|
721
|
-
headers={
|
|
722
|
-
"content-type": "application/json",
|
|
723
|
-
},
|
|
724
|
-
request_options=request_options,
|
|
725
|
-
omit=OMIT,
|
|
488
|
+
_response = self._raw_client.update_evaluation_scenario_score_router(
|
|
489
|
+
evaluation_scenario_id, score=score, request_options=request_options
|
|
726
490
|
)
|
|
727
|
-
|
|
728
|
-
if 200 <= _response.status_code < 300:
|
|
729
|
-
return typing.cast(
|
|
730
|
-
typing.Optional[typing.Any],
|
|
731
|
-
parse_obj_as(
|
|
732
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
733
|
-
object_=_response.json(),
|
|
734
|
-
),
|
|
735
|
-
)
|
|
736
|
-
if _response.status_code == 422:
|
|
737
|
-
raise UnprocessableEntityError(
|
|
738
|
-
typing.cast(
|
|
739
|
-
HttpValidationError,
|
|
740
|
-
parse_obj_as(
|
|
741
|
-
type_=HttpValidationError, # type: ignore
|
|
742
|
-
object_=_response.json(),
|
|
743
|
-
),
|
|
744
|
-
)
|
|
745
|
-
)
|
|
746
|
-
_response_json = _response.json()
|
|
747
|
-
except JSONDecodeError:
|
|
748
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
749
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
491
|
+
return _response.data
|
|
750
492
|
|
|
751
493
|
def fetch_results(
|
|
752
494
|
self,
|
|
@@ -781,45 +523,31 @@ class HumanEvaluationsClient:
|
|
|
781
523
|
|
|
782
524
|
client = AgentaApi(
|
|
783
525
|
api_key="YOUR_API_KEY",
|
|
784
|
-
base_url="https://yourhost.com/path/to/api",
|
|
785
526
|
)
|
|
786
527
|
client.human_evaluations.fetch_results(
|
|
787
528
|
evaluation_id="evaluation_id",
|
|
788
529
|
)
|
|
789
530
|
"""
|
|
790
|
-
_response = self.
|
|
791
|
-
|
|
792
|
-
method="GET",
|
|
793
|
-
request_options=request_options,
|
|
531
|
+
_response = self._raw_client.fetch_results(
|
|
532
|
+
evaluation_id, request_options=request_options
|
|
794
533
|
)
|
|
795
|
-
|
|
796
|
-
if 200 <= _response.status_code < 300:
|
|
797
|
-
return typing.cast(
|
|
798
|
-
typing.Optional[typing.Any],
|
|
799
|
-
parse_obj_as(
|
|
800
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
801
|
-
object_=_response.json(),
|
|
802
|
-
),
|
|
803
|
-
)
|
|
804
|
-
if _response.status_code == 422:
|
|
805
|
-
raise UnprocessableEntityError(
|
|
806
|
-
typing.cast(
|
|
807
|
-
HttpValidationError,
|
|
808
|
-
parse_obj_as(
|
|
809
|
-
type_=HttpValidationError, # type: ignore
|
|
810
|
-
object_=_response.json(),
|
|
811
|
-
),
|
|
812
|
-
)
|
|
813
|
-
)
|
|
814
|
-
_response_json = _response.json()
|
|
815
|
-
except JSONDecodeError:
|
|
816
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
817
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
534
|
+
return _response.data
|
|
818
535
|
|
|
819
536
|
|
|
820
537
|
class AsyncHumanEvaluationsClient:
|
|
821
538
|
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
822
|
-
self.
|
|
539
|
+
self._raw_client = AsyncRawHumanEvaluationsClient(client_wrapper=client_wrapper)
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def with_raw_response(self) -> AsyncRawHumanEvaluationsClient:
|
|
543
|
+
"""
|
|
544
|
+
Retrieves a raw implementation of this client that returns raw responses.
|
|
545
|
+
|
|
546
|
+
Returns
|
|
547
|
+
-------
|
|
548
|
+
AsyncRawHumanEvaluationsClient
|
|
549
|
+
"""
|
|
550
|
+
return self._raw_client
|
|
823
551
|
|
|
824
552
|
async def fetch_list_human_evaluations(
|
|
825
553
|
self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
|
|
@@ -853,7 +581,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
853
581
|
|
|
854
582
|
client = AsyncAgentaApi(
|
|
855
583
|
api_key="YOUR_API_KEY",
|
|
856
|
-
base_url="https://yourhost.com/path/to/api",
|
|
857
584
|
)
|
|
858
585
|
|
|
859
586
|
|
|
@@ -865,37 +592,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
865
592
|
|
|
866
593
|
asyncio.run(main())
|
|
867
594
|
"""
|
|
868
|
-
_response = await self.
|
|
869
|
-
|
|
870
|
-
method="GET",
|
|
871
|
-
params={
|
|
872
|
-
"app_id": app_id,
|
|
873
|
-
},
|
|
874
|
-
request_options=request_options,
|
|
595
|
+
_response = await self._raw_client.fetch_list_human_evaluations(
|
|
596
|
+
app_id=app_id, request_options=request_options
|
|
875
597
|
)
|
|
876
|
-
|
|
877
|
-
if 200 <= _response.status_code < 300:
|
|
878
|
-
return typing.cast(
|
|
879
|
-
typing.List[HumanEvaluation],
|
|
880
|
-
parse_obj_as(
|
|
881
|
-
type_=typing.List[HumanEvaluation], # type: ignore
|
|
882
|
-
object_=_response.json(),
|
|
883
|
-
),
|
|
884
|
-
)
|
|
885
|
-
if _response.status_code == 422:
|
|
886
|
-
raise UnprocessableEntityError(
|
|
887
|
-
typing.cast(
|
|
888
|
-
HttpValidationError,
|
|
889
|
-
parse_obj_as(
|
|
890
|
-
type_=HttpValidationError, # type: ignore
|
|
891
|
-
object_=_response.json(),
|
|
892
|
-
),
|
|
893
|
-
)
|
|
894
|
-
)
|
|
895
|
-
_response_json = _response.json()
|
|
896
|
-
except JSONDecodeError:
|
|
897
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
898
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
598
|
+
return _response.data
|
|
899
599
|
|
|
900
600
|
async def create_human_evaluation(
|
|
901
601
|
self,
|
|
@@ -945,7 +645,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
945
645
|
|
|
946
646
|
client = AsyncAgentaApi(
|
|
947
647
|
api_key="YOUR_API_KEY",
|
|
948
|
-
base_url="https://yourhost.com/path/to/api",
|
|
949
648
|
)
|
|
950
649
|
|
|
951
650
|
|
|
@@ -962,46 +661,16 @@ class AsyncHumanEvaluationsClient:
|
|
|
962
661
|
|
|
963
662
|
asyncio.run(main())
|
|
964
663
|
"""
|
|
965
|
-
_response = await self.
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
"inputs": inputs,
|
|
973
|
-
"testset_id": testset_id,
|
|
974
|
-
"status": status,
|
|
975
|
-
},
|
|
976
|
-
headers={
|
|
977
|
-
"content-type": "application/json",
|
|
978
|
-
},
|
|
664
|
+
_response = await self._raw_client.create_human_evaluation(
|
|
665
|
+
app_id=app_id,
|
|
666
|
+
variant_ids=variant_ids,
|
|
667
|
+
evaluation_type=evaluation_type,
|
|
668
|
+
inputs=inputs,
|
|
669
|
+
testset_id=testset_id,
|
|
670
|
+
status=status,
|
|
979
671
|
request_options=request_options,
|
|
980
|
-
omit=OMIT,
|
|
981
672
|
)
|
|
982
|
-
|
|
983
|
-
if 200 <= _response.status_code < 300:
|
|
984
|
-
return typing.cast(
|
|
985
|
-
SimpleEvaluationOutput,
|
|
986
|
-
parse_obj_as(
|
|
987
|
-
type_=SimpleEvaluationOutput, # type: ignore
|
|
988
|
-
object_=_response.json(),
|
|
989
|
-
),
|
|
990
|
-
)
|
|
991
|
-
if _response.status_code == 422:
|
|
992
|
-
raise UnprocessableEntityError(
|
|
993
|
-
typing.cast(
|
|
994
|
-
HttpValidationError,
|
|
995
|
-
parse_obj_as(
|
|
996
|
-
type_=HttpValidationError, # type: ignore
|
|
997
|
-
object_=_response.json(),
|
|
998
|
-
),
|
|
999
|
-
)
|
|
1000
|
-
)
|
|
1001
|
-
_response_json = _response.json()
|
|
1002
|
-
except JSONDecodeError:
|
|
1003
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1004
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
673
|
+
return _response.data
|
|
1005
674
|
|
|
1006
675
|
async def delete_evaluations(
|
|
1007
676
|
self,
|
|
@@ -1038,7 +707,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1038
707
|
|
|
1039
708
|
client = AsyncAgentaApi(
|
|
1040
709
|
api_key="YOUR_API_KEY",
|
|
1041
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1042
710
|
)
|
|
1043
711
|
|
|
1044
712
|
|
|
@@ -1050,38 +718,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1050
718
|
|
|
1051
719
|
asyncio.run(main())
|
|
1052
720
|
"""
|
|
1053
|
-
_response = await self.
|
|
1054
|
-
|
|
1055
|
-
method="DELETE",
|
|
1056
|
-
json={
|
|
1057
|
-
"evaluations_ids": evaluations_ids,
|
|
1058
|
-
},
|
|
1059
|
-
request_options=request_options,
|
|
1060
|
-
omit=OMIT,
|
|
721
|
+
_response = await self._raw_client.delete_evaluations(
|
|
722
|
+
evaluations_ids=evaluations_ids, request_options=request_options
|
|
1061
723
|
)
|
|
1062
|
-
|
|
1063
|
-
if 200 <= _response.status_code < 300:
|
|
1064
|
-
return typing.cast(
|
|
1065
|
-
typing.List[str],
|
|
1066
|
-
parse_obj_as(
|
|
1067
|
-
type_=typing.List[str], # type: ignore
|
|
1068
|
-
object_=_response.json(),
|
|
1069
|
-
),
|
|
1070
|
-
)
|
|
1071
|
-
if _response.status_code == 422:
|
|
1072
|
-
raise UnprocessableEntityError(
|
|
1073
|
-
typing.cast(
|
|
1074
|
-
HttpValidationError,
|
|
1075
|
-
parse_obj_as(
|
|
1076
|
-
type_=HttpValidationError, # type: ignore
|
|
1077
|
-
object_=_response.json(),
|
|
1078
|
-
),
|
|
1079
|
-
)
|
|
1080
|
-
)
|
|
1081
|
-
_response_json = _response.json()
|
|
1082
|
-
except JSONDecodeError:
|
|
1083
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1084
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
724
|
+
return _response.data
|
|
1085
725
|
|
|
1086
726
|
async def fetch_human_evaluation(
|
|
1087
727
|
self,
|
|
@@ -1118,7 +758,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1118
758
|
|
|
1119
759
|
client = AsyncAgentaApi(
|
|
1120
760
|
api_key="YOUR_API_KEY",
|
|
1121
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1122
761
|
)
|
|
1123
762
|
|
|
1124
763
|
|
|
@@ -1130,34 +769,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1130
769
|
|
|
1131
770
|
asyncio.run(main())
|
|
1132
771
|
"""
|
|
1133
|
-
_response = await self.
|
|
1134
|
-
|
|
1135
|
-
method="GET",
|
|
1136
|
-
request_options=request_options,
|
|
772
|
+
_response = await self._raw_client.fetch_human_evaluation(
|
|
773
|
+
evaluation_id, request_options=request_options
|
|
1137
774
|
)
|
|
1138
|
-
|
|
1139
|
-
if 200 <= _response.status_code < 300:
|
|
1140
|
-
return typing.cast(
|
|
1141
|
-
HumanEvaluation,
|
|
1142
|
-
parse_obj_as(
|
|
1143
|
-
type_=HumanEvaluation, # type: ignore
|
|
1144
|
-
object_=_response.json(),
|
|
1145
|
-
),
|
|
1146
|
-
)
|
|
1147
|
-
if _response.status_code == 422:
|
|
1148
|
-
raise UnprocessableEntityError(
|
|
1149
|
-
typing.cast(
|
|
1150
|
-
HttpValidationError,
|
|
1151
|
-
parse_obj_as(
|
|
1152
|
-
type_=HttpValidationError, # type: ignore
|
|
1153
|
-
object_=_response.json(),
|
|
1154
|
-
),
|
|
1155
|
-
)
|
|
1156
|
-
)
|
|
1157
|
-
_response_json = _response.json()
|
|
1158
|
-
except JSONDecodeError:
|
|
1159
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1160
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
775
|
+
return _response.data
|
|
1161
776
|
|
|
1162
777
|
async def update_human_evaluation(
|
|
1163
778
|
self,
|
|
@@ -1197,7 +812,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1197
812
|
|
|
1198
813
|
client = AsyncAgentaApi(
|
|
1199
814
|
api_key="YOUR_API_KEY",
|
|
1200
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1201
815
|
)
|
|
1202
816
|
|
|
1203
817
|
|
|
@@ -1209,41 +823,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1209
823
|
|
|
1210
824
|
asyncio.run(main())
|
|
1211
825
|
"""
|
|
1212
|
-
_response = await self.
|
|
1213
|
-
|
|
1214
|
-
method="PUT",
|
|
1215
|
-
json={
|
|
1216
|
-
"status": status,
|
|
1217
|
-
},
|
|
1218
|
-
headers={
|
|
1219
|
-
"content-type": "application/json",
|
|
1220
|
-
},
|
|
1221
|
-
request_options=request_options,
|
|
1222
|
-
omit=OMIT,
|
|
826
|
+
_response = await self._raw_client.update_human_evaluation(
|
|
827
|
+
evaluation_id, status=status, request_options=request_options
|
|
1223
828
|
)
|
|
1224
|
-
|
|
1225
|
-
if 200 <= _response.status_code < 300:
|
|
1226
|
-
return typing.cast(
|
|
1227
|
-
typing.Optional[typing.Any],
|
|
1228
|
-
parse_obj_as(
|
|
1229
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1230
|
-
object_=_response.json(),
|
|
1231
|
-
),
|
|
1232
|
-
)
|
|
1233
|
-
if _response.status_code == 422:
|
|
1234
|
-
raise UnprocessableEntityError(
|
|
1235
|
-
typing.cast(
|
|
1236
|
-
HttpValidationError,
|
|
1237
|
-
parse_obj_as(
|
|
1238
|
-
type_=HttpValidationError, # type: ignore
|
|
1239
|
-
object_=_response.json(),
|
|
1240
|
-
),
|
|
1241
|
-
)
|
|
1242
|
-
)
|
|
1243
|
-
_response_json = _response.json()
|
|
1244
|
-
except JSONDecodeError:
|
|
1245
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1246
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
829
|
+
return _response.data
|
|
1247
830
|
|
|
1248
831
|
async def fetch_human_evaluation_scenarios(
|
|
1249
832
|
self,
|
|
@@ -1283,7 +866,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1283
866
|
|
|
1284
867
|
client = AsyncAgentaApi(
|
|
1285
868
|
api_key="YOUR_API_KEY",
|
|
1286
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1287
869
|
)
|
|
1288
870
|
|
|
1289
871
|
|
|
@@ -1295,34 +877,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1295
877
|
|
|
1296
878
|
asyncio.run(main())
|
|
1297
879
|
"""
|
|
1298
|
-
_response = await self.
|
|
1299
|
-
|
|
1300
|
-
method="GET",
|
|
1301
|
-
request_options=request_options,
|
|
880
|
+
_response = await self._raw_client.fetch_human_evaluation_scenarios(
|
|
881
|
+
evaluation_id, request_options=request_options
|
|
1302
882
|
)
|
|
1303
|
-
|
|
1304
|
-
if 200 <= _response.status_code < 300:
|
|
1305
|
-
return typing.cast(
|
|
1306
|
-
typing.List[HumanEvaluationScenario],
|
|
1307
|
-
parse_obj_as(
|
|
1308
|
-
type_=typing.List[HumanEvaluationScenario], # type: ignore
|
|
1309
|
-
object_=_response.json(),
|
|
1310
|
-
),
|
|
1311
|
-
)
|
|
1312
|
-
if _response.status_code == 422:
|
|
1313
|
-
raise UnprocessableEntityError(
|
|
1314
|
-
typing.cast(
|
|
1315
|
-
HttpValidationError,
|
|
1316
|
-
parse_obj_as(
|
|
1317
|
-
type_=HttpValidationError, # type: ignore
|
|
1318
|
-
object_=_response.json(),
|
|
1319
|
-
),
|
|
1320
|
-
)
|
|
1321
|
-
)
|
|
1322
|
-
_response_json = _response.json()
|
|
1323
|
-
except JSONDecodeError:
|
|
1324
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1325
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
883
|
+
return _response.data
|
|
1326
884
|
|
|
1327
885
|
async def update_evaluation_scenario_router(
|
|
1328
886
|
self,
|
|
@@ -1386,7 +944,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1386
944
|
|
|
1387
945
|
client = AsyncAgentaApi(
|
|
1388
946
|
api_key="YOUR_API_KEY",
|
|
1389
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1390
947
|
)
|
|
1391
948
|
|
|
1392
949
|
|
|
@@ -1400,55 +957,20 @@ class AsyncHumanEvaluationsClient:
|
|
|
1400
957
|
|
|
1401
958
|
asyncio.run(main())
|
|
1402
959
|
"""
|
|
1403
|
-
_response = await self.
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
),
|
|
1415
|
-
"inputs": convert_and_respect_annotation_metadata(
|
|
1416
|
-
object_=inputs,
|
|
1417
|
-
annotation=typing.Sequence[HumanEvaluationScenarioInput],
|
|
1418
|
-
direction="write",
|
|
1419
|
-
),
|
|
1420
|
-
"is_pinned": is_pinned,
|
|
1421
|
-
"note": note,
|
|
1422
|
-
},
|
|
1423
|
-
headers={
|
|
1424
|
-
"content-type": "application/json",
|
|
1425
|
-
},
|
|
960
|
+
_response = await self._raw_client.update_evaluation_scenario_router(
|
|
961
|
+
evaluation_id,
|
|
962
|
+
evaluation_scenario_id,
|
|
963
|
+
evaluation_type,
|
|
964
|
+
vote=vote,
|
|
965
|
+
score=score,
|
|
966
|
+
correct_answer=correct_answer,
|
|
967
|
+
outputs=outputs,
|
|
968
|
+
inputs=inputs,
|
|
969
|
+
is_pinned=is_pinned,
|
|
970
|
+
note=note,
|
|
1426
971
|
request_options=request_options,
|
|
1427
|
-
omit=OMIT,
|
|
1428
972
|
)
|
|
1429
|
-
|
|
1430
|
-
if 200 <= _response.status_code < 300:
|
|
1431
|
-
return typing.cast(
|
|
1432
|
-
typing.Optional[typing.Any],
|
|
1433
|
-
parse_obj_as(
|
|
1434
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1435
|
-
object_=_response.json(),
|
|
1436
|
-
),
|
|
1437
|
-
)
|
|
1438
|
-
if _response.status_code == 422:
|
|
1439
|
-
raise UnprocessableEntityError(
|
|
1440
|
-
typing.cast(
|
|
1441
|
-
HttpValidationError,
|
|
1442
|
-
parse_obj_as(
|
|
1443
|
-
type_=HttpValidationError, # type: ignore
|
|
1444
|
-
object_=_response.json(),
|
|
1445
|
-
),
|
|
1446
|
-
)
|
|
1447
|
-
)
|
|
1448
|
-
_response_json = _response.json()
|
|
1449
|
-
except JSONDecodeError:
|
|
1450
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1451
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
973
|
+
return _response.data
|
|
1452
974
|
|
|
1453
975
|
async def get_evaluation_scenario_score_router(
|
|
1454
976
|
self,
|
|
@@ -1485,7 +1007,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1485
1007
|
|
|
1486
1008
|
client = AsyncAgentaApi(
|
|
1487
1009
|
api_key="YOUR_API_KEY",
|
|
1488
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1489
1010
|
)
|
|
1490
1011
|
|
|
1491
1012
|
|
|
@@ -1497,34 +1018,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1497
1018
|
|
|
1498
1019
|
asyncio.run(main())
|
|
1499
1020
|
"""
|
|
1500
|
-
_response = await self.
|
|
1501
|
-
|
|
1502
|
-
method="GET",
|
|
1503
|
-
request_options=request_options,
|
|
1021
|
+
_response = await self._raw_client.get_evaluation_scenario_score_router(
|
|
1022
|
+
evaluation_scenario_id, request_options=request_options
|
|
1504
1023
|
)
|
|
1505
|
-
|
|
1506
|
-
if 200 <= _response.status_code < 300:
|
|
1507
|
-
return typing.cast(
|
|
1508
|
-
typing.Dict[str, str],
|
|
1509
|
-
parse_obj_as(
|
|
1510
|
-
type_=typing.Dict[str, str], # type: ignore
|
|
1511
|
-
object_=_response.json(),
|
|
1512
|
-
),
|
|
1513
|
-
)
|
|
1514
|
-
if _response.status_code == 422:
|
|
1515
|
-
raise UnprocessableEntityError(
|
|
1516
|
-
typing.cast(
|
|
1517
|
-
HttpValidationError,
|
|
1518
|
-
parse_obj_as(
|
|
1519
|
-
type_=HttpValidationError, # type: ignore
|
|
1520
|
-
object_=_response.json(),
|
|
1521
|
-
),
|
|
1522
|
-
)
|
|
1523
|
-
)
|
|
1524
|
-
_response_json = _response.json()
|
|
1525
|
-
except JSONDecodeError:
|
|
1526
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1527
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1024
|
+
return _response.data
|
|
1528
1025
|
|
|
1529
1026
|
async def update_evaluation_scenario_score_router(
|
|
1530
1027
|
self,
|
|
@@ -1564,7 +1061,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1564
1061
|
|
|
1565
1062
|
client = AsyncAgentaApi(
|
|
1566
1063
|
api_key="YOUR_API_KEY",
|
|
1567
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1568
1064
|
)
|
|
1569
1065
|
|
|
1570
1066
|
|
|
@@ -1577,41 +1073,10 @@ class AsyncHumanEvaluationsClient:
|
|
|
1577
1073
|
|
|
1578
1074
|
asyncio.run(main())
|
|
1579
1075
|
"""
|
|
1580
|
-
_response = await self.
|
|
1581
|
-
|
|
1582
|
-
method="PUT",
|
|
1583
|
-
json={
|
|
1584
|
-
"score": score,
|
|
1585
|
-
},
|
|
1586
|
-
headers={
|
|
1587
|
-
"content-type": "application/json",
|
|
1588
|
-
},
|
|
1589
|
-
request_options=request_options,
|
|
1590
|
-
omit=OMIT,
|
|
1076
|
+
_response = await self._raw_client.update_evaluation_scenario_score_router(
|
|
1077
|
+
evaluation_scenario_id, score=score, request_options=request_options
|
|
1591
1078
|
)
|
|
1592
|
-
|
|
1593
|
-
if 200 <= _response.status_code < 300:
|
|
1594
|
-
return typing.cast(
|
|
1595
|
-
typing.Optional[typing.Any],
|
|
1596
|
-
parse_obj_as(
|
|
1597
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1598
|
-
object_=_response.json(),
|
|
1599
|
-
),
|
|
1600
|
-
)
|
|
1601
|
-
if _response.status_code == 422:
|
|
1602
|
-
raise UnprocessableEntityError(
|
|
1603
|
-
typing.cast(
|
|
1604
|
-
HttpValidationError,
|
|
1605
|
-
parse_obj_as(
|
|
1606
|
-
type_=HttpValidationError, # type: ignore
|
|
1607
|
-
object_=_response.json(),
|
|
1608
|
-
),
|
|
1609
|
-
)
|
|
1610
|
-
)
|
|
1611
|
-
_response_json = _response.json()
|
|
1612
|
-
except JSONDecodeError:
|
|
1613
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1614
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1079
|
+
return _response.data
|
|
1615
1080
|
|
|
1616
1081
|
async def fetch_results(
|
|
1617
1082
|
self,
|
|
@@ -1648,7 +1113,6 @@ class AsyncHumanEvaluationsClient:
|
|
|
1648
1113
|
|
|
1649
1114
|
client = AsyncAgentaApi(
|
|
1650
1115
|
api_key="YOUR_API_KEY",
|
|
1651
|
-
base_url="https://yourhost.com/path/to/api",
|
|
1652
1116
|
)
|
|
1653
1117
|
|
|
1654
1118
|
|
|
@@ -1660,31 +1124,7 @@ class AsyncHumanEvaluationsClient:
|
|
|
1660
1124
|
|
|
1661
1125
|
asyncio.run(main())
|
|
1662
1126
|
"""
|
|
1663
|
-
_response = await self.
|
|
1664
|
-
|
|
1665
|
-
method="GET",
|
|
1666
|
-
request_options=request_options,
|
|
1127
|
+
_response = await self._raw_client.fetch_results(
|
|
1128
|
+
evaluation_id, request_options=request_options
|
|
1667
1129
|
)
|
|
1668
|
-
|
|
1669
|
-
if 200 <= _response.status_code < 300:
|
|
1670
|
-
return typing.cast(
|
|
1671
|
-
typing.Optional[typing.Any],
|
|
1672
|
-
parse_obj_as(
|
|
1673
|
-
type_=typing.Optional[typing.Any], # type: ignore
|
|
1674
|
-
object_=_response.json(),
|
|
1675
|
-
),
|
|
1676
|
-
)
|
|
1677
|
-
if _response.status_code == 422:
|
|
1678
|
-
raise UnprocessableEntityError(
|
|
1679
|
-
typing.cast(
|
|
1680
|
-
HttpValidationError,
|
|
1681
|
-
parse_obj_as(
|
|
1682
|
-
type_=HttpValidationError, # type: ignore
|
|
1683
|
-
object_=_response.json(),
|
|
1684
|
-
),
|
|
1685
|
-
)
|
|
1686
|
-
)
|
|
1687
|
-
_response_json = _response.json()
|
|
1688
|
-
except JSONDecodeError:
|
|
1689
|
-
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1690
|
-
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1130
|
+
return _response.data
|