scale-gp-beta 0.1.0a11__py3-none-any.whl → 0.1.0a13__py3-none-any.whl

scale_gp_beta/types/evaluation_task.py

@@ -1,10 +1,14 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+from __future__ import annotations
+
 from typing import TYPE_CHECKING, Dict, List, Union, Optional
 from typing_extensions import Literal, Annotated, TypeAlias
 
 from .._utils import PropertyInfo
+from .._compat import PYDANTIC_V2
 from .._models import BaseModel
+from .item_locator import ItemLocator
 
 __all__ = [
     "EvaluationTask",
@@ -21,63 +25,76 @@ __all__ = [
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace",
+    "MetricEvaluationTask",
+    "MetricEvaluationTaskConfiguration",
+    "MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
+    "AutoEvaluationQuestionTask",
+    "AutoEvaluationQuestionTaskConfiguration",
+    "ContributorEvaluationQuestionTask",
+    "ContributorEvaluationQuestionTaskConfiguration",
 ]
 
 
 class ChatCompletionEvaluationTaskConfiguration(BaseModel):
-    messages: Union[List[Dict[str, object]], str]
+    messages: Union[List[Dict[str, object]], ItemLocator]
 
     model: str
 
-    audio: Union[Dict[str, object], str, None] = None
+    audio: Union[Dict[str, object], ItemLocator, None] = None
 
-    frequency_penalty: Union[float, str, None] = None
+    frequency_penalty: Union[float, ItemLocator, None] = None
 
-    function_call: Union[Dict[str, object], str, None] = None
+    function_call: Union[Dict[str, object], ItemLocator, None] = None
 
-    functions: Union[List[Dict[str, object]], str, None] = None
+    functions: Union[List[Dict[str, object]], ItemLocator, None] = None
 
-    logit_bias: Union[Dict[str, int], str, None] = None
+    logit_bias: Union[Dict[str, int], ItemLocator, None] = None
 
-    logprobs: Union[bool, str, None] = None
+    logprobs: Union[bool, ItemLocator, None] = None
 
-    max_completion_tokens: Union[int, str, None] = None
+    max_completion_tokens: Union[int, ItemLocator, None] = None
 
-    max_tokens: Union[int, str, None] = None
+    max_tokens: Union[int, ItemLocator, None] = None
 
-    metadata: Union[Dict[str, str], str, None] = None
+    metadata: Union[Dict[str, str], ItemLocator, None] = None
 
-    modalities: Union[List[str], str, None] = None
+    modalities: Union[List[str], ItemLocator, None] = None
 
-    n: Union[int, str, None] = None
+    n: Union[int, ItemLocator, None] = None
 
-    parallel_tool_calls: Union[bool, str, None] = None
+    parallel_tool_calls: Union[bool, ItemLocator, None] = None
 
-    prediction: Union[Dict[str, object], str, None] = None
+    prediction: Union[Dict[str, object], ItemLocator, None] = None
 
-    presence_penalty: Union[float, str, None] = None
+    presence_penalty: Union[float, ItemLocator, None] = None
 
     reasoning_effort: Optional[str] = None
 
-    response_format: Union[Dict[str, object], str, None] = None
+    response_format: Union[Dict[str, object], ItemLocator, None] = None
 
-    seed: Union[int, str, None] = None
+    seed: Union[int, ItemLocator, None] = None
 
     stop: Optional[str] = None
 
-    store: Union[bool, str, None] = None
+    store: Union[bool, ItemLocator, None] = None
 
-    temperature: Union[float, str, None] = None
+    temperature: Union[float, ItemLocator, None] = None
 
     tool_choice: Optional[str] = None
 
-    tools: Union[List[Dict[str, object]], str, None] = None
+    tools: Union[List[Dict[str, object]], ItemLocator, None] = None
 
-    top_k: Union[int, str, None] = None
+    top_k: Union[int, ItemLocator, None] = None
 
-    top_logprobs: Union[int, str, None] = None
+    top_logprobs: Union[int, ItemLocator, None] = None
 
-    top_p: Union[float, str, None] = None
+    top_p: Union[float, ItemLocator, None] = None
 
     if TYPE_CHECKING:
         # Stub to indicate that arbitrary properties are accepted.
@@ -90,7 +107,7 @@ class ChatCompletionEvaluationTask(BaseModel):
     configuration: ChatCompletionEvaluationTaskConfiguration
 
     alias: Optional[str] = None
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `chat_completion`"""
 
     task_type: Optional[Literal["chat_completion"]] = None
 
@@ -102,14 +119,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
 
 
 GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
-    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, str
+    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
 ]
 
 
 class GenericInferenceEvaluationTaskConfiguration(BaseModel):
     model: str
 
-    args: Union[Dict[str, object], str, None] = None
+    args: Union[Dict[str, object], ItemLocator, None] = None
 
     inference_configuration: Optional[GenericInferenceEvaluationTaskConfigurationInferenceConfiguration] = None
 
@@ -118,7 +135,7 @@ class GenericInferenceEvaluationTask(BaseModel):
     configuration: GenericInferenceEvaluationTaskConfiguration
 
     alias: Optional[str] = None
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `inference`"""
 
     task_type: Optional[Literal["inference"]] = None
 
@@ -169,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
         List[ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace]
     ] = None
 
+    return_span: Optional[bool] = None
+
     use_channels: Optional[bool] = None
 
 
 ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
-    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, str
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
 ]
 
 
 class ApplicationVariantV1EvaluationTaskConfiguration(BaseModel):
     application_variant_id: str
 
-    inputs: Union[Dict[str, object], str]
+    inputs: Union[Dict[str, object], ItemLocator]
 
-    history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], str, None] = None
+    history: Union[List[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator, None] = None
 
-    operation_metadata: Union[Dict[str, object], str, None] = None
+    operation_metadata: Union[Dict[str, object], ItemLocator, None] = None
 
     overrides: Optional[ApplicationVariantV1EvaluationTaskConfigurationOverrides] = None
     """Execution override options for agentic applications"""
@@ -194,12 +213,192 @@ class ApplicationVariantV1EvaluationTask(BaseModel):
     configuration: ApplicationVariantV1EvaluationTaskConfiguration
 
     alias: Optional[str] = None
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `application_variant`"""
 
     task_type: Optional[Literal["application_variant"]] = None
 
 
+class MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["bleu"]
+
+
+class MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["meteor"]
+
+
+class MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["cosine_similarity"]
+
+
+class MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["f1"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rouge1"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rouge2"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rougeL"]
+
+
+MetricEvaluationTaskConfiguration: TypeAlias = Annotated[
+    Union[
+        MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator,
+    ],
+    PropertyInfo(discriminator="type"),
+]
+
+
+class MetricEvaluationTask(BaseModel):
+    configuration: MetricEvaluationTaskConfiguration
+
+    alias: Optional[str] = None
+    """Alias to title the results column.
+
+    Defaults to the metric type specified in the configuration
+    """
+
+    task_type: Optional[Literal["metric"]] = None
+
+
+class AutoEvaluationQuestionTaskConfiguration(BaseModel):
+    model: str
+    """model specified as `model_vendor/model_name`"""
+
+    prompt: str
+
+    question_id: str
+    """question to be evaluated"""
+
+
+class AutoEvaluationQuestionTask(BaseModel):
+    configuration: AutoEvaluationQuestionTaskConfiguration
+
+    alias: Optional[str] = None
+    """Alias to title the results column. Defaults to the `auto_evaluation_question`"""
+
+    task_type: Optional[Literal["auto_evaluation.question"]] = None
+
+
+class ContributorEvaluationQuestionTaskConfiguration(BaseModel):
+    layout: "Container"
+
+    question_id: str
+
+    queue_id: Optional[str] = None
+    """The contributor annotation queue to include this task in. Defaults to `default`"""
+
+
+class ContributorEvaluationQuestionTask(BaseModel):
+    configuration: ContributorEvaluationQuestionTaskConfiguration
+
+    alias: Optional[str] = None
+    """Alias to title the results column.
+
+    Defaults to the `contributor_evaluation_question`
+    """
+
+    task_type: Optional[Literal["contributor_evaluation.question"]] = None
+
+
 EvaluationTask: TypeAlias = Annotated[
-    Union[ChatCompletionEvaluationTask, GenericInferenceEvaluationTask, ApplicationVariantV1EvaluationTask],
+    Union[
+        ChatCompletionEvaluationTask,
+        GenericInferenceEvaluationTask,
+        ApplicationVariantV1EvaluationTask,
+        MetricEvaluationTask,
+        AutoEvaluationQuestionTask,
+        ContributorEvaluationQuestionTask,
+    ],
     PropertyInfo(discriminator="task_type"),
 ]
+
+from .container import Container
+
+if PYDANTIC_V2:
+    ChatCompletionEvaluationTask.model_rebuild()
+    ChatCompletionEvaluationTaskConfiguration.model_rebuild()
+    GenericInferenceEvaluationTask.model_rebuild()
+    GenericInferenceEvaluationTaskConfiguration.model_rebuild()
+    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.model_rebuild()
+    ApplicationVariantV1EvaluationTask.model_rebuild()
+    ApplicationVariantV1EvaluationTaskConfiguration.model_rebuild()
+    ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.model_rebuild()
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.model_rebuild()
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.model_rebuild()
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.model_rebuild()
+    MetricEvaluationTask.model_rebuild()
+    MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.model_rebuild()
+    MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.model_rebuild()
+    AutoEvaluationQuestionTask.model_rebuild()
+    AutoEvaluationQuestionTaskConfiguration.model_rebuild()
+    ContributorEvaluationQuestionTask.model_rebuild()
+    ContributorEvaluationQuestionTaskConfiguration.model_rebuild()
+else:
+    ChatCompletionEvaluationTask.update_forward_refs()  # type: ignore
+    ChatCompletionEvaluationTaskConfiguration.update_forward_refs()  # type: ignore
+    GenericInferenceEvaluationTask.update_forward_refs()  # type: ignore
+    GenericInferenceEvaluationTaskConfiguration.update_forward_refs()  # type: ignore
+    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTask.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTaskConfiguration.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState.update_forward_refs()  # type: ignore
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace.update_forward_refs()  # type: ignore
+    MetricEvaluationTask.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator.update_forward_refs()  # type: ignore
+    MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator.update_forward_refs()  # type: ignore
+    AutoEvaluationQuestionTask.update_forward_refs()  # type: ignore
+    AutoEvaluationQuestionTaskConfiguration.update_forward_refs()  # type: ignore
+    ContributorEvaluationQuestionTask.update_forward_refs()  # type: ignore
+    ContributorEvaluationQuestionTaskConfiguration.update_forward_refs()  # type: ignore

scale_gp_beta/types/evaluation_task_param.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 from typing import Dict, List, Union, Iterable
 from typing_extensions import Literal, Required, TypeAlias, TypedDict
 
+from .item_locator import ItemLocator
+
 __all__ = [
     "EvaluationTaskParam",
     "ChatCompletionEvaluationTask",
@@ -20,63 +22,76 @@ __all__ = [
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace",
+    "MetricEvaluationTask",
+    "MetricEvaluationTaskConfiguration",
+    "MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
+    "AutoEvaluationQuestionTask",
+    "AutoEvaluationQuestionTaskConfiguration",
+    "ContributorEvaluationQuestionTask",
+    "ContributorEvaluationQuestionTaskConfiguration",
 ]
 
 
 class ChatCompletionEvaluationTaskConfigurationTyped(TypedDict, total=False):
-    messages: Required[Union[Iterable[Dict[str, object]], str]]
+    messages: Required[Union[Iterable[Dict[str, object]], ItemLocator]]
 
     model: Required[str]
 
-    audio: Union[Dict[str, object], str]
+    audio: Union[Dict[str, object], ItemLocator]
 
-    frequency_penalty: Union[float, str]
+    frequency_penalty: Union[float, ItemLocator]
 
-    function_call: Union[Dict[str, object], str]
+    function_call: Union[Dict[str, object], ItemLocator]
 
-    functions: Union[Iterable[Dict[str, object]], str]
+    functions: Union[Iterable[Dict[str, object]], ItemLocator]
 
-    logit_bias: Union[Dict[str, int], str]
+    logit_bias: Union[Dict[str, int], ItemLocator]
 
-    logprobs: Union[bool, str]
+    logprobs: Union[bool, ItemLocator]
 
-    max_completion_tokens: Union[int, str]
+    max_completion_tokens: Union[int, ItemLocator]
 
-    max_tokens: Union[int, str]
+    max_tokens: Union[int, ItemLocator]
 
-    metadata: Union[Dict[str, str], str]
+    metadata: Union[Dict[str, str], ItemLocator]
 
-    modalities: Union[List[str], str]
+    modalities: Union[List[str], ItemLocator]
 
-    n: Union[int, str]
+    n: Union[int, ItemLocator]
 
-    parallel_tool_calls: Union[bool, str]
+    parallel_tool_calls: Union[bool, ItemLocator]
 
-    prediction: Union[Dict[str, object], str]
+    prediction: Union[Dict[str, object], ItemLocator]
 
-    presence_penalty: Union[float, str]
+    presence_penalty: Union[float, ItemLocator]
 
     reasoning_effort: str
 
-    response_format: Union[Dict[str, object], str]
+    response_format: Union[Dict[str, object], ItemLocator]
 
-    seed: Union[int, str]
+    seed: Union[int, ItemLocator]
 
     stop: str
 
-    store: Union[bool, str]
+    store: Union[bool, ItemLocator]
 
-    temperature: Union[float, str]
+    temperature: Union[float, ItemLocator]
 
     tool_choice: str
 
-    tools: Union[Iterable[Dict[str, object]], str]
+    tools: Union[Iterable[Dict[str, object]], ItemLocator]
 
-    top_k: Union[int, str]
+    top_k: Union[int, ItemLocator]
 
-    top_logprobs: Union[int, str]
+    top_logprobs: Union[int, ItemLocator]
 
-    top_p: Union[float, str]
+    top_p: Union[float, ItemLocator]
 
 
 ChatCompletionEvaluationTaskConfiguration: TypeAlias = Union[
@@ -88,7 +103,7 @@ class ChatCompletionEvaluationTask(TypedDict, total=False):
     configuration: Required[ChatCompletionEvaluationTaskConfiguration]
 
     alias: str
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `chat_completion`"""
 
     task_type: Literal["chat_completion"]
 
@@ -102,14 +117,14 @@ class GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInf
 
 
 GenericInferenceEvaluationTaskConfigurationInferenceConfiguration: TypeAlias = Union[
-    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, str
+    GenericInferenceEvaluationTaskConfigurationInferenceConfigurationLaunchInferenceConfiguration, ItemLocator
 ]
 
 
 class GenericInferenceEvaluationTaskConfiguration(TypedDict, total=False):
     model: Required[str]
 
-    args: Union[Dict[str, object], str]
+    args: Union[Dict[str, object], ItemLocator]
 
     inference_configuration: GenericInferenceEvaluationTaskConfigurationInferenceConfiguration
 
@@ -118,7 +133,7 @@ class GenericInferenceEvaluationTask(TypedDict, total=False):
     configuration: Required[GenericInferenceEvaluationTaskConfiguration]
 
     alias: str
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `inference`"""
 
     task_type: Literal["inference"]
 
@@ -171,22 +186,24 @@ class ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplication
         ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace
     ]
 
+    return_span: bool
+
     use_channels: bool
 
 
 ApplicationVariantV1EvaluationTaskConfigurationOverrides: TypeAlias = Union[
-    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, str
+    ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides, ItemLocator
 ]
 
 
 class ApplicationVariantV1EvaluationTaskConfiguration(TypedDict, total=False):
     application_variant_id: Required[str]
 
-    inputs: Required[Union[Dict[str, object], str]]
+    inputs: Required[Union[Dict[str, object], ItemLocator]]
 
-    history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], str]
+    history: Union[Iterable[ApplicationVariantV1EvaluationTaskConfigurationHistoryUnionMember0], ItemLocator]
 
-    operation_metadata: Union[Dict[str, object], str]
+    operation_metadata: Union[Dict[str, object], ItemLocator]
 
     overrides: ApplicationVariantV1EvaluationTaskConfigurationOverrides
     """Execution override options for agentic applications"""
@@ -196,11 +213,137 @@ class ApplicationVariantV1EvaluationTask(TypedDict, total=False):
     configuration: Required[ApplicationVariantV1EvaluationTaskConfiguration]
 
     alias: str
-    """Alias to title the results column. Defaults to the `task_type`"""
+    """Alias to title the results column. Defaults to the `application_variant`"""
 
     task_type: Literal["application_variant"]
 
 
+class MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["bleu"]]
+
+
+class MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["meteor"]]
+
+
+class MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["cosine_similarity"]]
+
+
+class MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["f1"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rouge1"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rouge2"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rougeL"]]
+
+
+MetricEvaluationTaskConfiguration: TypeAlias = Union[
+    MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator,
+]
+
+
+class MetricEvaluationTask(TypedDict, total=False):
+    configuration: Required[MetricEvaluationTaskConfiguration]
+
+    alias: str
+    """Alias to title the results column.
+
+    Defaults to the metric type specified in the configuration
+    """
+
+    task_type: Literal["metric"]
+
+
+class AutoEvaluationQuestionTaskConfiguration(TypedDict, total=False):
+    model: Required[str]
+    """model specified as `model_vendor/model_name`"""
+
+    prompt: Required[str]
+
+    question_id: Required[str]
+    """question to be evaluated"""
+
+
+class AutoEvaluationQuestionTask(TypedDict, total=False):
+    configuration: Required[AutoEvaluationQuestionTaskConfiguration]
+
+    alias: str
+    """Alias to title the results column. Defaults to the `auto_evaluation_question`"""
+
+    task_type: Literal["auto_evaluation.question"]
+
+
+class ContributorEvaluationQuestionTaskConfiguration(TypedDict, total=False):
+    layout: Required["ContainerParam"]
+
+    question_id: Required[str]
+
+    queue_id: str
+    """The contributor annotation queue to include this task in. Defaults to `default`"""
+
+
+class ContributorEvaluationQuestionTask(TypedDict, total=False):
+    configuration: Required[ContributorEvaluationQuestionTaskConfiguration]
+
+    alias: str
+    """Alias to title the results column.
+
+    Defaults to the `contributor_evaluation_question`
+    """
+
+    task_type: Literal["contributor_evaluation.question"]
+
+
 EvaluationTaskParam: TypeAlias = Union[
-    ChatCompletionEvaluationTask, GenericInferenceEvaluationTask, ApplicationVariantV1EvaluationTask
+    ChatCompletionEvaluationTask,
+    GenericInferenceEvaluationTask,
+    ApplicationVariantV1EvaluationTask,
+    MetricEvaluationTask,
+    AutoEvaluationQuestionTask,
+    ContributorEvaluationQuestionTask,
 ]
+
+from .container_param import ContainerParam