agenta 0.27.0a9__py3-none-any.whl → 0.27.0a13__py3-none-any.whl

agenta/client/backend/evaluators/client.py

@@ -7,10 +7,12 @@ from ..types.evaluator import Evaluator
 from ..core.pydantic_utilities import parse_obj_as
 from json.decoder import JSONDecodeError
 from ..core.api_error import ApiError
-from ..types.evaluator_config import EvaluatorConfig
+from ..types.evaluator_mapping_output_interface import EvaluatorMappingOutputInterface
 from ..errors.unprocessable_entity_error import UnprocessableEntityError
 from ..types.http_validation_error import HttpValidationError
+from ..types.evaluator_output_interface import EvaluatorOutputInterface
 from ..core.jsonable_encoder import jsonable_encoder
+from ..types.evaluator_config import EvaluatorConfig
 from ..core.client_wrapper import AsyncClientWrapper
 
 # this is used as the default value for optional parameters
@@ -69,6 +71,171 @@ class EvaluatorsClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    def evaluator_data_map(
+        self,
+        *,
+        inputs: typing.Dict[str, typing.Optional[typing.Any]],
+        mapping: typing.Dict[str, typing.Optional[typing.Any]],
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorMappingOutputInterface:
+        """
+        Endpoint to map the experiment data tree to evaluator interface.
+
+        Args:
+        request (Request): The request object.
+        payload (EvaluatorMappingInputInterface): The payload containing the request data.
+
+        Returns:
+        EvaluatorMappingOutputInterface: the evaluator mapping output object
+
+        Parameters
+        ----------
+        inputs : typing.Dict[str, typing.Optional[typing.Any]]
+
+        mapping : typing.Dict[str, typing.Optional[typing.Any]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorMappingOutputInterface
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.evaluators.evaluator_data_map(
+            inputs={"key": "value"},
+            mapping={"key": "value"},
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "evaluators/map",
+            method="POST",
+            json={
+                "inputs": inputs,
+                "mapping": mapping,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    EvaluatorMappingOutputInterface,
+                    parse_obj_as(
+                        type_=EvaluatorMappingOutputInterface,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def evaluator_run(
+        self,
+        evaluator_key: str,
+        *,
+        inputs: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = OMIT,
+        settings: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = OMIT,
+        credentials: typing.Optional[
+            typing.Dict[str, typing.Optional[typing.Any]]
+        ] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorOutputInterface:
+        """
+        Endpoint to evaluate LLM app run
+
+        Args:
+        request (Request): The request object.
+        evaluator_key (str): The key of the evaluator.
+        payload (EvaluatorInputInterface): The payload containing the request data.
+
+        Returns:
+        result: EvaluatorOutputInterface object containing the outputs.
+
+        Parameters
+        ----------
+        evaluator_key : str
+
+        inputs : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        settings : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        credentials : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorOutputInterface
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+        client.evaluators.evaluator_run(
+            evaluator_key="evaluator_key",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"evaluators/{jsonable_encoder(evaluator_key)}/run",
+            method="POST",
+            json={
+                "inputs": inputs,
+                "settings": settings,
+                "credentials": credentials,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    EvaluatorOutputInterface,
+                    parse_obj_as(
+                        type_=EvaluatorOutputInterface,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     def get_evaluator_configs(
         self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
     ) -> typing.List[EvaluatorConfig]:
@@ -500,6 +667,187 @@ class AsyncEvaluatorsClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    async def evaluator_data_map(
+        self,
+        *,
+        inputs: typing.Dict[str, typing.Optional[typing.Any]],
+        mapping: typing.Dict[str, typing.Optional[typing.Any]],
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorMappingOutputInterface:
+        """
+        Endpoint to map the experiment data tree to evaluator interface.
+
+        Args:
+        request (Request): The request object.
+        payload (EvaluatorMappingInputInterface): The payload containing the request data.
+
+        Returns:
+        EvaluatorMappingOutputInterface: the evaluator mapping output object
+
+        Parameters
+        ----------
+        inputs : typing.Dict[str, typing.Optional[typing.Any]]
+
+        mapping : typing.Dict[str, typing.Optional[typing.Any]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorMappingOutputInterface
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.evaluators.evaluator_data_map(
+                inputs={"key": "value"},
+                mapping={"key": "value"},
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "evaluators/map",
+            method="POST",
+            json={
+                "inputs": inputs,
+                "mapping": mapping,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    EvaluatorMappingOutputInterface,
+                    parse_obj_as(
+                        type_=EvaluatorMappingOutputInterface,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def evaluator_run(
+        self,
+        evaluator_key: str,
+        *,
+        inputs: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = OMIT,
+        settings: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = OMIT,
+        credentials: typing.Optional[
+            typing.Dict[str, typing.Optional[typing.Any]]
+        ] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorOutputInterface:
+        """
+        Endpoint to evaluate LLM app run
+
+        Args:
+        request (Request): The request object.
+        evaluator_key (str): The key of the evaluator.
+        payload (EvaluatorInputInterface): The payload containing the request data.
+
+        Returns:
+        result: EvaluatorOutputInterface object containing the outputs.
+
+        Parameters
+        ----------
+        evaluator_key : str
+
+        inputs : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        settings : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        credentials : typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorOutputInterface
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+            base_url="https://yourhost.com/path/to/api",
+        )
+
+
+        async def main() -> None:
+            await client.evaluators.evaluator_run(
+                evaluator_key="evaluator_key",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"evaluators/{jsonable_encoder(evaluator_key)}/run",
+            method="POST",
+            json={
+                "inputs": inputs,
+                "settings": settings,
+                "credentials": credentials,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    EvaluatorOutputInterface,
+                    parse_obj_as(
+                        type_=EvaluatorOutputInterface,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    )
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     async def get_evaluator_configs(
         self, *, app_id: str, request_options: typing.Optional[RequestOptions] = None
     ) -> typing.List[EvaluatorConfig]:

agenta/client/backend/observability/client.py

@@ -10,6 +10,7 @@ from json.decoder import JSONDecodeError
 from ..core.api_error import ApiError
 from ..types.create_span import CreateSpan
 from ..types.create_trace_response import CreateTraceResponse
+from ..core.serialization import convert_and_respect_annotation_metadata
 from ..types.with_pagination import WithPagination
 from ..types.trace_detail import TraceDetail
 from ..core.jsonable_encoder import jsonable_encoder
@@ -155,7 +156,11 @@ class ObservabilityClient:
             method="POST",
             json={
                 "trace": trace,
-                "spans": spans,
+                "spans": convert_and_respect_annotation_metadata(
+                    object_=spans,
+                    annotation=typing.Sequence[CreateSpan],
+                    direction="write",
+                ),
             },
             request_options=request_options,
             omit=OMIT,
@@ -764,7 +769,11 @@ class AsyncObservabilityClient:
             method="POST",
             json={
                 "trace": trace,
-                "spans": spans,
+                "spans": convert_and_respect_annotation_metadata(
+                    object_=spans,
+                    annotation=typing.Sequence[CreateSpan],
+                    direction="write",
+                ),
             },
             request_options=request_options,
             omit=OMIT,

agenta/client/backend/testsets/client.py

@@ -26,9 +26,9 @@ class TestsetsClient:
         self,
         *,
         file: core.File,
-        upload_type: typing.Optional[str] = None,
-        testset_name: typing.Optional[str] = None,
-        app_id: typing.Optional[str] = None,
+        upload_type: typing.Optional[str] = OMIT,
+        testset_name: typing.Optional[str] = OMIT,
+        app_id: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> TestSetSimpleResponse:
         """
@@ -261,10 +261,10 @@ class TestsetsClient:
         request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
         """
-        Fetch a specific testset in a MongoDB collection using its id.
+        Fetch a specific testset in a MongoDB collection using its \_id.
 
         Args:
-        testset_id (str): The id of the testset to fetch.
+        testset_id (str): The \_id of the testset to fetch.
 
         Returns:
         The requested testset if found, else an HTTPException.
@@ -555,9 +555,9 @@ class AsyncTestsetsClient:
         self,
         *,
         file: core.File,
-        upload_type: typing.Optional[str] = None,
-        testset_name: typing.Optional[str] = None,
-        app_id: typing.Optional[str] = None,
+        upload_type: typing.Optional[str] = OMIT,
+        testset_name: typing.Optional[str] = OMIT,
+        app_id: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> TestSetSimpleResponse:
         """
@@ -814,10 +814,10 @@ class AsyncTestsetsClient:
         request_options: typing.Optional[RequestOptions] = None,
     ) -> typing.Optional[typing.Any]:
         """
-        Fetch a specific testset in a MongoDB collection using its id.
+        Fetch a specific testset in a MongoDB collection using its \_id.
 
         Args:
-        testset_id (str): The id of the testset to fetch.
+        testset_id (str): The \_id of the testset to fetch.
 
         Returns:
         The requested testset if found, else an HTTPException.

agenta/client/backend/types/__init__.py

@@ -8,6 +8,8 @@ from .app_variant_revision import AppVariantRevision
 from .base_output import BaseOutput
 from .body_import_testset import BodyImportTestset
 from .config_db import ConfigDb
+from .config_dto import ConfigDto
+from .config_response_model import ConfigResponseModel
 from .correct_answer import CorrectAnswer
 from .create_app_output import CreateAppOutput
 from .create_span import CreateSpan
@@ -27,6 +29,8 @@ from .evaluation_status_enum import EvaluationStatusEnum
 from .evaluation_type import EvaluationType
 from .evaluator import Evaluator
 from .evaluator_config import EvaluatorConfig
+from .evaluator_mapping_output_interface import EvaluatorMappingOutputInterface
+from .evaluator_output_interface import EvaluatorOutputInterface
 from .get_config_response import GetConfigResponse
 from .http_validation_error import HttpValidationError
 from .human_evaluation import HumanEvaluation
@@ -37,6 +41,7 @@ from .human_evaluation_scenario_update import HumanEvaluationScenarioUpdate
 from .human_evaluation_update import HumanEvaluationUpdate
 from .image import Image
 from .invite_request import InviteRequest
+from .lifecycle_dto import LifecycleDto
 from .list_api_keys_response import ListApiKeysResponse
 from .llm_run_rate_limit import LlmRunRateLimit
 from .llm_tokens import LlmTokens
@@ -47,6 +52,8 @@ from .organization import Organization
 from .organization_output import OrganizationOutput
 from .outputs import Outputs
 from .permission import Permission
+from .reference_dto import ReferenceDto
+from .reference_request_model import ReferenceRequestModel
 from .result import Result
 from .score import Score
 from .simple_evaluation_output import SimpleEvaluationOutput
@@ -81,6 +88,8 @@ __all__ = [
     "BaseOutput",
     "BodyImportTestset",
     "ConfigDb",
+    "ConfigDto",
+    "ConfigResponseModel",
     "CorrectAnswer",
     "CreateAppOutput",
     "CreateSpan",
@@ -100,6 +109,8 @@ __all__ = [
     "EvaluationType",
     "Evaluator",
     "EvaluatorConfig",
+    "EvaluatorMappingOutputInterface",
+    "EvaluatorOutputInterface",
     "GetConfigResponse",
     "HttpValidationError",
     "HumanEvaluation",
@@ -110,6 +121,7 @@ __all__ = [
     "HumanEvaluationUpdate",
     "Image",
     "InviteRequest",
+    "LifecycleDto",
     "ListApiKeysResponse",
     "LlmRunRateLimit",
     "LlmTokens",
@@ -120,6 +132,8 @@ __all__ = [
     "OrganizationOutput",
     "Outputs",
     "Permission",
+    "ReferenceDto",
+    "ReferenceRequestModel",
     "Result",
     "Score",
     "SimpleEvaluationOutput",

agenta/client/backend/types/app.py

@@ -9,6 +9,7 @@ import pydantic
 class App(UniversalBaseModel):
     app_id: str
     app_name: str
+    updated_at: str
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(

agenta/client/backend/types/app_variant_response.py

@@ -11,8 +11,8 @@ class AppVariantResponse(UniversalBaseModel):
     app_name: str
     variant_id: str
     variant_name: str
-    parameters: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
     project_id: str
+    parameters: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
     base_name: str
     base_id: str
     config_name: str
@@ -21,6 +21,8 @@ class AppVariantResponse(UniversalBaseModel):
     created_at: typing.Optional[str] = None
     updated_at: typing.Optional[str] = None
     modified_by_id: typing.Optional[str] = None
+    organization_id: typing.Optional[str] = None
+    workspace_id: typing.Optional[str] = None
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(

agenta/client/backend/types/config_dto.py

@@ -0,0 +1,32 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from ..core.pydantic_utilities import UniversalBaseModel
+import typing
+from .reference_dto import ReferenceDto
+from .lifecycle_dto import LifecycleDto
+from ..core.pydantic_utilities import IS_PYDANTIC_V2
+import pydantic
+
+
+class ConfigDto(UniversalBaseModel):
+    params: typing.Dict[str, typing.Optional[typing.Any]]
+    url: typing.Optional[str] = None
+    application_ref: typing.Optional[ReferenceDto] = None
+    service_ref: typing.Optional[ReferenceDto] = None
+    variant_ref: typing.Optional[ReferenceDto] = None
+    environment_ref: typing.Optional[ReferenceDto] = None
+    application_lifecycle: typing.Optional[LifecycleDto] = None
+    service_lifecycle: typing.Optional[LifecycleDto] = None
+    variant_lifecycle: typing.Optional[LifecycleDto] = None
+    environment_lifecycle: typing.Optional[LifecycleDto] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
+            extra="allow", frozen=True
+        )  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

agenta/client/backend/types/config_response_model.py

@@ -0,0 +1,32 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from ..core.pydantic_utilities import UniversalBaseModel
+import typing
+from .reference_dto import ReferenceDto
+from .lifecycle_dto import LifecycleDto
+from ..core.pydantic_utilities import IS_PYDANTIC_V2
+import pydantic
+
+
+class ConfigResponseModel(UniversalBaseModel):
+    params: typing.Dict[str, typing.Optional[typing.Any]]
+    url: typing.Optional[str] = None
+    application_ref: typing.Optional[ReferenceDto] = None
+    service_ref: typing.Optional[ReferenceDto] = None
+    variant_ref: typing.Optional[ReferenceDto] = None
+    environment_ref: typing.Optional[ReferenceDto] = None
+    application_lifecycle: typing.Optional[LifecycleDto] = None
+    service_lifecycle: typing.Optional[LifecycleDto] = None
+    variant_lifecycle: typing.Optional[LifecycleDto] = None
+    environment_lifecycle: typing.Optional[LifecycleDto] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
+            extra="allow", frozen=True
+        )  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

agenta/client/backend/types/create_span.py

@@ -12,6 +12,7 @@ import pydantic
 class CreateSpan(UniversalBaseModel):
     id: str
     app_id: str
+    project_id: typing.Optional[str] = None
     variant_id: typing.Optional[str] = None
     variant_name: typing.Optional[str] = None
     inputs: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
@@ -34,11 +35,11 @@ class CreateSpan(UniversalBaseModel):
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
-            extra="allow", frozen=False
+            extra="allow", frozen=True
         )  # type: ignore # Pydantic v2
     else:
 
         class Config:
-            frozen = False
+            frozen = True
             smart_union = True
             extra = pydantic.Extra.allow

agenta/client/backend/types/environment_output.py

@@ -9,6 +9,7 @@ import pydantic
 class EnvironmentOutput(UniversalBaseModel):
     name: str
     app_id: str
+    project_id: str
     deployed_app_variant_id: typing.Optional[str] = None
     deployed_variant_name: typing.Optional[str] = None
     deployed_app_variant_revision_id: typing.Optional[str] = None

agenta/client/backend/types/environment_output_extended.py

@@ -10,6 +10,7 @@ import pydantic
 class EnvironmentOutputExtended(UniversalBaseModel):
     name: str
     app_id: str
+    project_id: str
     deployed_app_variant_id: typing.Optional[str] = None
     deployed_variant_name: typing.Optional[str] = None
     deployed_app_variant_revision_id: typing.Optional[str] = None

agenta/client/backend/types/evaluation.py

@@ -12,8 +12,7 @@ import pydantic
 class Evaluation(UniversalBaseModel):
     id: str
     app_id: str
-    user_id: str
-    user_username: str
+    project_id: str
     variant_ids: typing.List[str]
     variant_names: typing.List[str]
     variant_revision_ids: typing.List[str]

agenta/client/backend/types/evaluator.py

@@ -13,6 +13,8 @@ class Evaluator(UniversalBaseModel):
     settings_template: typing.Dict[str, typing.Optional[typing.Any]]
     description: typing.Optional[str] = None
     oss: typing.Optional[bool] = None
+    requires_llm_api_keys: typing.Optional[bool] = None
+    tags: typing.List[str]
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(

agenta/client/backend/types/evaluator_config.py

@@ -9,6 +9,7 @@ import pydantic
 class EvaluatorConfig(UniversalBaseModel):
     id: str
     name: str
+    project_id: str
     evaluator_key: str
     settings_values: typing.Optional[
         typing.Dict[str, typing.Optional[typing.Any]]