agenta 0.32.0a1__py3-none-any.whl → 0.33.0__py3-none-any.whl

agenta/client/backend/human_evaluations/client.py

@@ -0,0 +1,1692 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+from ..core.client_wrapper import SyncClientWrapper
+from ..core.request_options import RequestOptions
+from ..types.human_evaluation import HumanEvaluation
+from ..core.pydantic_utilities import parse_obj_as
+from ..errors.unprocessable_entity_error import UnprocessableEntityError
+from ..types.http_validation_error import HttpValidationError
+from json.decoder import JSONDecodeError
+from ..core.api_error import ApiError
+from ..types.evaluation_type import EvaluationType
+from ..types.simple_evaluation_output import SimpleEvaluationOutput
+from ..core.jsonable_encoder import jsonable_encoder
+from ..types.evaluation_status_enum import EvaluationStatusEnum
+from ..types.human_evaluation_scenario import HumanEvaluationScenario
+from ..types.score import Score
+from ..types.human_evaluation_scenario_output import HumanEvaluationScenarioOutput
+from ..types.human_evaluation_scenario_input import HumanEvaluationScenarioInput
+from ..core.serialization import convert_and_respect_annotation_metadata
+from ..core.client_wrapper import AsyncClientWrapper
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class HumanEvaluationsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    def fetch_list_human_evaluations(
+        self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
+    ) -> typing.List[HumanEvaluation]:
+        """
+        Fetches a list of evaluations, optionally filtered by an app ID.
+
+        Args:
+            app_id (Optional[str]): An optional app ID to filter the evaluations.
+
+        Returns:
+            List[HumanEvaluation]: A list of evaluations.
+
+        Parameters
+        ----------
+        app_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[HumanEvaluation]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.fetch_list_human_evaluations(
+            app_id="app_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="GET",
+            params={
+                "app_id": app_id,
+            },
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[HumanEvaluation],
+                    parse_obj_as(
+                        type_=typing.List[HumanEvaluation],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def create_human_evaluation(
+        self,
+        *,
+        app_id: str,
+        variant_ids: typing.Sequence[str],
+        evaluation_type: EvaluationType,
+        inputs: typing.Sequence[str],
+        testset_id: str,
+        status: str,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> SimpleEvaluationOutput:
+        """
+        Creates a new comparison table document
+        Raises:
+            HTTPException: _description_
+        Returns:
+            _description_
+
+        Parameters
+        ----------
+        app_id : str
+
+        variant_ids : typing.Sequence[str]
+
+        evaluation_type : EvaluationType
+
+        inputs : typing.Sequence[str]
+
+        testset_id : str
+
+        status : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        SimpleEvaluationOutput
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.create_human_evaluation(
+            app_id="app_id",
+            variant_ids=["variant_ids"],
+            evaluation_type="human_a_b_testing",
+            inputs=["inputs"],
+            testset_id="testset_id",
+            status="status",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="POST",
+            json={
+                "app_id": app_id,
+                "variant_ids": variant_ids,
+                "evaluation_type": evaluation_type,
+                "inputs": inputs,
+                "testset_id": testset_id,
+                "status": status,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    SimpleEvaluationOutput,
+                    parse_obj_as(
+                        type_=SimpleEvaluationOutput,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def delete_evaluations(
+        self,
+        *,
+        evaluations_ids: typing.Sequence[str],
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[str]:
+        """
+        Delete specific comparison tables based on their unique IDs.
+
+        Args:
+            payload (List[str]): The unique identifiers of the comparison tables to delete.
+
+        Returns:
+        A list of the deleted comparison tables' IDs.
+
+        Parameters
+        ----------
+        evaluations_ids : typing.Sequence[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[str]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.delete_evaluations(
+            evaluations_ids=["evaluations_ids"],
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="DELETE",
+            json={
+                "evaluations_ids": evaluations_ids,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[str],
+                    parse_obj_as(
+                        type_=typing.List[str],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def fetch_human_evaluation(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> HumanEvaluation:
+        """
+        Fetches a single evaluation based on its ID.
+
+        Args:
+            evaluation_id (str): The ID of the evaluation to fetch.
+
+        Returns:
+            HumanEvaluation: The fetched evaluation.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        HumanEvaluation
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.fetch_human_evaluation(
+            evaluation_id="evaluation_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    HumanEvaluation,
+                    parse_obj_as(
+                        type_=HumanEvaluation,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def update_human_evaluation(
+        self,
+        evaluation_id: str,
+        *,
+        status: typing.Optional[EvaluationStatusEnum] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates an evaluation's status.
+
+        Raises:
+            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+
+        Returns:
+            None: A 204 No Content status code, indicating that the update was successful.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        status : typing.Optional[EvaluationStatusEnum]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.update_human_evaluation(
+            evaluation_id="evaluation_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}",
+            method="PUT",
+            json={
+                "status": status,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def fetch_human_evaluation_scenarios(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[HumanEvaluationScenario]:
+        """
+        Fetches evaluation scenarios for a given evaluation ID.
+
+        Arguments:
+            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.
+
+        Raises:
+            HTTPException: If the evaluation is not found or access is denied.
+
+        Returns:
+            List[EvaluationScenario]: A list of evaluation scenarios.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[HumanEvaluationScenario]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.fetch_human_evaluation_scenarios(
+            evaluation_id="evaluation_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/evaluation_scenarios",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[HumanEvaluationScenario],
+                    parse_obj_as(
+                        type_=typing.List[HumanEvaluationScenario],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def update_evaluation_scenario_router(
+        self,
+        evaluation_id: str,
+        evaluation_scenario_id: str,
+        evaluation_type: EvaluationType,
+        *,
+        vote: typing.Optional[str] = OMIT,
+        score: typing.Optional[Score] = OMIT,
+        correct_answer: typing.Optional[str] = OMIT,
+        outputs: typing.Optional[typing.Sequence[HumanEvaluationScenarioOutput]] = OMIT,
+        inputs: typing.Optional[typing.Sequence[HumanEvaluationScenarioInput]] = OMIT,
+        is_pinned: typing.Optional[bool] = OMIT,
+        note: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates an evaluation scenario's vote or score based on its type.
+
+        Raises:
+            HTTPException: If update fails or unauthorized.
+
+        Returns:
+            None: 204 No Content status code upon successful update.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        evaluation_scenario_id : str
+
+        evaluation_type : EvaluationType
+
+        vote : typing.Optional[str]
+
+        score : typing.Optional[Score]
+
+        correct_answer : typing.Optional[str]
+
+        outputs : typing.Optional[typing.Sequence[HumanEvaluationScenarioOutput]]
+
+        inputs : typing.Optional[typing.Sequence[HumanEvaluationScenarioInput]]
+
+        is_pinned : typing.Optional[bool]
+
+        note : typing.Optional[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.update_evaluation_scenario_router(
+            evaluation_id="evaluation_id",
+            evaluation_scenario_id="evaluation_scenario_id",
+            evaluation_type="human_a_b_testing",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/{jsonable_encoder(evaluation_type)}",
+            method="PUT",
+            json={
+                "vote": vote,
+                "score": score,
+                "correct_answer": correct_answer,
+                "outputs": convert_and_respect_annotation_metadata(
+                    object_=outputs,
+                    annotation=typing.Sequence[HumanEvaluationScenarioOutput],
+                    direction="write",
+                ),
+                "inputs": convert_and_respect_annotation_metadata(
+                    object_=inputs,
+                    annotation=typing.Sequence[HumanEvaluationScenarioInput],
+                    direction="write",
+                ),
+                "is_pinned": is_pinned,
+                "note": note,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def get_evaluation_scenario_score_router(
+        self,
+        evaluation_scenario_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Dict[str, str]:
+        """
+        Fetch the score of a specific evaluation scenario.
+
+        Args:
+            evaluation_scenario_id: The ID of the evaluation scenario to fetch.
+            stoken_session: Session data, verified by `verify_session`.
+
+        Returns:
+            Dictionary containing the scenario ID and its score.
+
+        Parameters
+        ----------
+        evaluation_scenario_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Dict[str, str]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.get_evaluation_scenario_score_router(
+            evaluation_scenario_id="evaluation_scenario_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/score",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Dict[str, str],
+                    parse_obj_as(
+                        type_=typing.Dict[str, str],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def update_evaluation_scenario_score_router(
+        self,
+        evaluation_scenario_id: str,
+        *,
+        score: float,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates the score of an evaluation scenario.
+
+        Raises:
+            HTTPException: Server error if the evaluation update fails.
+
+        Returns:
+            None: 204 No Content status code upon successful update.
+
+        Parameters
+        ----------
+        evaluation_scenario_id : str
+
+        score : float
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.update_evaluation_scenario_score_router(
+            evaluation_scenario_id="evaluation_scenario_id",
+            score=1.1,
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/score",
+            method="PUT",
+            json={
+                "score": score,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def fetch_results(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Fetch all the results for one the comparison table
+
+        Arguments:
+            evaluation_id -- _description_
+
+        Returns:
+            _description_
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.human_evaluations.fetch_results(
+            evaluation_id="evaluation_id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/results",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+
+class AsyncHumanEvaluationsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    async def fetch_list_human_evaluations(
+        self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
+    ) -> typing.List[HumanEvaluation]:
+        """
+        Fetches a list of evaluations, optionally filtered by an app ID.
+
+        Args:
+            app_id (Optional[str]): An optional app ID to filter the evaluations.
+
+        Returns:
+            List[HumanEvaluation]: A list of evaluations.
+
+        Parameters
+        ----------
+        app_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[HumanEvaluation]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.fetch_list_human_evaluations(
+                app_id="app_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="GET",
+            params={
+                "app_id": app_id,
+            },
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[HumanEvaluation],
+                    parse_obj_as(
+                        type_=typing.List[HumanEvaluation],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def create_human_evaluation(
+        self,
+        *,
+        app_id: str,
+        variant_ids: typing.Sequence[str],
+        evaluation_type: EvaluationType,
+        inputs: typing.Sequence[str],
+        testset_id: str,
+        status: str,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> SimpleEvaluationOutput:
+        """
+        Creates a new comparison table document
+        Raises:
+            HTTPException: _description_
+        Returns:
+            _description_
+
+        Parameters
+        ----------
+        app_id : str
+
+        variant_ids : typing.Sequence[str]
+
+        evaluation_type : EvaluationType
+
+        inputs : typing.Sequence[str]
+
+        testset_id : str
+
+        status : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        SimpleEvaluationOutput
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.create_human_evaluation(
+                app_id="app_id",
+                variant_ids=["variant_ids"],
+                evaluation_type="human_a_b_testing",
+                inputs=["inputs"],
+                testset_id="testset_id",
+                status="status",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="POST",
+            json={
+                "app_id": app_id,
+                "variant_ids": variant_ids,
+                "evaluation_type": evaluation_type,
+                "inputs": inputs,
+                "testset_id": testset_id,
+                "status": status,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    SimpleEvaluationOutput,
+                    parse_obj_as(
+                        type_=SimpleEvaluationOutput,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def delete_evaluations(
+        self,
+        *,
+        evaluations_ids: typing.Sequence[str],
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[str]:
+        """
+        Delete specific comparison tables based on their unique IDs.
+
+        Args:
+            payload (List[str]): The unique identifiers of the comparison tables to delete.
+
+        Returns:
+        A list of the deleted comparison tables' IDs.
+
+        Parameters
+        ----------
+        evaluations_ids : typing.Sequence[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[str]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.delete_evaluations(
+                evaluations_ids=["evaluations_ids"],
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "human-evaluations",
+            method="DELETE",
+            json={
+                "evaluations_ids": evaluations_ids,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[str],
+                    parse_obj_as(
+                        type_=typing.List[str],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def fetch_human_evaluation(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> HumanEvaluation:
+        """
+        Fetches a single evaluation based on its ID.
+
+        Args:
+            evaluation_id (str): The ID of the evaluation to fetch.
+
+        Returns:
+            HumanEvaluation: The fetched evaluation.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        HumanEvaluation
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.fetch_human_evaluation(
+                evaluation_id="evaluation_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    HumanEvaluation,
+                    parse_obj_as(
+                        type_=HumanEvaluation,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def update_human_evaluation(
+        self,
+        evaluation_id: str,
+        *,
+        status: typing.Optional[EvaluationStatusEnum] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates an evaluation's status.
+
+        Raises:
+            HTTPException: If the columns in the test set do not match with the inputs in the variant.
+
+        Returns:
+            None: A 204 No Content status code, indicating that the update was successful.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        status : typing.Optional[EvaluationStatusEnum]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.update_human_evaluation(
+                evaluation_id="evaluation_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}",
+            method="PUT",
+            json={
+                "status": status,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def fetch_human_evaluation_scenarios(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.List[HumanEvaluationScenario]:
+        """
+        Fetches evaluation scenarios for a given evaluation ID.
+
+        Arguments:
+            evaluation_id (str): The ID of the evaluation for which to fetch scenarios.
+
+        Raises:
+            HTTPException: If the evaluation is not found or access is denied.
+
+        Returns:
+            List[EvaluationScenario]: A list of evaluation scenarios.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[HumanEvaluationScenario]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.fetch_human_evaluation_scenarios(
+                evaluation_id="evaluation_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/evaluation_scenarios",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.List[HumanEvaluationScenario],
+                    parse_obj_as(
+                        type_=typing.List[HumanEvaluationScenario],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def update_evaluation_scenario_router(
+        self,
+        evaluation_id: str,
+        evaluation_scenario_id: str,
+        evaluation_type: EvaluationType,
+        *,
+        vote: typing.Optional[str] = OMIT,
+        score: typing.Optional[Score] = OMIT,
+        correct_answer: typing.Optional[str] = OMIT,
+        outputs: typing.Optional[typing.Sequence[HumanEvaluationScenarioOutput]] = OMIT,
+        inputs: typing.Optional[typing.Sequence[HumanEvaluationScenarioInput]] = OMIT,
+        is_pinned: typing.Optional[bool] = OMIT,
+        note: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates an evaluation scenario's vote or score based on its type.
+
+        Raises:
+            HTTPException: If update fails or unauthorized.
+
+        Returns:
+            None: 204 No Content status code upon successful update.
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        evaluation_scenario_id : str
+
+        evaluation_type : EvaluationType
+
+        vote : typing.Optional[str]
+
+        score : typing.Optional[Score]
+
+        correct_answer : typing.Optional[str]
+
+        outputs : typing.Optional[typing.Sequence[HumanEvaluationScenarioOutput]]
+
+        inputs : typing.Optional[typing.Sequence[HumanEvaluationScenarioInput]]
+
+        is_pinned : typing.Optional[bool]
+
+        note : typing.Optional[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.update_evaluation_scenario_router(
+                evaluation_id="evaluation_id",
+                evaluation_scenario_id="evaluation_scenario_id",
+                evaluation_type="human_a_b_testing",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/{jsonable_encoder(evaluation_type)}",
+            method="PUT",
+            json={
+                "vote": vote,
+                "score": score,
+                "correct_answer": correct_answer,
+                "outputs": convert_and_respect_annotation_metadata(
+                    object_=outputs,
+                    annotation=typing.Sequence[HumanEvaluationScenarioOutput],
+                    direction="write",
+                ),
+                "inputs": convert_and_respect_annotation_metadata(
+                    object_=inputs,
+                    annotation=typing.Sequence[HumanEvaluationScenarioInput],
+                    direction="write",
+                ),
+                "is_pinned": is_pinned,
+                "note": note,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def get_evaluation_scenario_score_router(
+        self,
+        evaluation_scenario_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Dict[str, str]:
+        """
+        Fetch the score of a specific evaluation scenario.
+
+        Args:
+            evaluation_scenario_id: The ID of the evaluation scenario to fetch.
+            stoken_session: Session data, verified by `verify_session`.
+
+        Returns:
+            Dictionary containing the scenario ID and its score.
+
+        Parameters
+        ----------
+        evaluation_scenario_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Dict[str, str]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.get_evaluation_scenario_score_router(
+                evaluation_scenario_id="evaluation_scenario_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/score",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Dict[str, str],
+                    parse_obj_as(
+                        type_=typing.Dict[str, str],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def update_evaluation_scenario_score_router(
+        self,
+        evaluation_scenario_id: str,
+        *,
+        score: float,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Updates the score of an evaluation scenario.
+
+        Raises:
+            HTTPException: Server error if the evaluation update fails.
+
+        Returns:
+            None: 204 No Content status code upon successful update.
+
+        Parameters
+        ----------
+        evaluation_scenario_id : str
+
+        score : float
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.update_evaluation_scenario_score_router(
+                evaluation_scenario_id="evaluation_scenario_id",
+                score=1.1,
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/evaluation_scenario/{jsonable_encoder(evaluation_scenario_id)}/score",
+            method="PUT",
+            json={
+                "score": score,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def fetch_results(
+        self,
+        evaluation_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Optional[typing.Any]:
+        """
+        Fetch all the results for one the comparison table
+
+        Arguments:
+            evaluation_id -- _description_
+
+        Returns:
+            _description_
+
+        Parameters
+        ----------
+        evaluation_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.Optional[typing.Any]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.human_evaluations.fetch_results(
+                evaluation_id="evaluation_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"human-evaluations/{jsonable_encoder(evaluation_id)}/results",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    typing.Optional[typing.Any],
+                    parse_obj_as(
+                        type_=typing.Optional[typing.Any],  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)