scale-gp-beta 0.1.0a11__py3-none-any.whl → 0.1.0a12__py3-none-any.whl

scale_gp_beta/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "scale_gp_beta"
-__version__ = "0.1.0-alpha.11"  # x-release-please-version
+__version__ = "0.1.0-alpha.12"  # x-release-please-version

scale_gp_beta/types/evaluation_task.py

@@ -21,6 +21,15 @@ __all__ = [
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace",
+    "MetricEvaluationTask",
+    "MetricEvaluationTaskConfiguration",
+    "MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
 ]
 
 
@@ -199,7 +208,91 @@ class ApplicationVariantV1EvaluationTask(BaseModel):
     task_type: Optional[Literal["application_variant"]] = None
 
 
+class MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["bleu"]
+
+
+class MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["meteor"]
+
+
+class MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["cosine_similarity"]
+
+
+class MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["f1"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rouge1"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rouge2"]
+
+
+class MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator(BaseModel):
+    candidate: str
+
+    reference: str
+
+    type: Literal["rougeL"]
+
+
+MetricEvaluationTaskConfiguration: TypeAlias = Annotated[
+    Union[
+        MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator,
+        MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator,
+    ],
+    PropertyInfo(discriminator="type"),
+]
+
+
+class MetricEvaluationTask(BaseModel):
+    configuration: MetricEvaluationTaskConfiguration
+
+    alias: Optional[str] = None
+    """Alias to title the results column. Defaults to the `task_type`"""
+
+    task_type: Optional[Literal["metric"]] = None
+
+
 EvaluationTask: TypeAlias = Annotated[
-    Union[ChatCompletionEvaluationTask, GenericInferenceEvaluationTask, ApplicationVariantV1EvaluationTask],
+    Union[
+        ChatCompletionEvaluationTask,
+        GenericInferenceEvaluationTask,
+        ApplicationVariantV1EvaluationTask,
+        MetricEvaluationTask,
+    ],
     PropertyInfo(discriminator="task_type"),
 ]

scale_gp_beta/types/evaluation_task_param.py

@@ -20,6 +20,15 @@ __all__ = [
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverrides",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesInitialState",
     "ApplicationVariantV1EvaluationTaskConfigurationOverridesAgenticApplicationOverridesPartialTrace",
+    "MetricEvaluationTask",
+    "MetricEvaluationTaskConfiguration",
+    "MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator",
+    "MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator",
 ]
 
 
@@ -201,6 +210,85 @@ class ApplicationVariantV1EvaluationTask(TypedDict, total=False):
     task_type: Literal["application_variant"]
 
 
+class MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["bleu"]]
+
+
+class MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["meteor"]]
+
+
+class MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["cosine_similarity"]]
+
+
+class MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["f1"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rouge1"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rouge2"]]
+
+
+class MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator(TypedDict, total=False):
+    candidate: Required[str]
+
+    reference: Required[str]
+
+    type: Required[Literal["rougeL"]]
+
+
+MetricEvaluationTaskConfiguration: TypeAlias = Union[
+    MetricEvaluationTaskConfigurationBleuScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationMeteorScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationCosineSimilarityScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationF1ScorerConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorer1ConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorer2ConfigWithItemLocator,
+    MetricEvaluationTaskConfigurationRougeScorerLConfigWithItemLocator,
+]
+
+
+class MetricEvaluationTask(TypedDict, total=False):
+    configuration: Required[MetricEvaluationTaskConfiguration]
+
+    alias: str
+    """Alias to title the results column. Defaults to the `task_type`"""
+
+    task_type: Literal["metric"]
+
+
 EvaluationTaskParam: TypeAlias = Union[
-    ChatCompletionEvaluationTask, GenericInferenceEvaluationTask, ApplicationVariantV1EvaluationTask
+    ChatCompletionEvaluationTask,
+    GenericInferenceEvaluationTask,
+    ApplicationVariantV1EvaluationTask,
+    MetricEvaluationTask,
 ]

{scale_gp_beta-0.1.0a11.dist-info → scale_gp_beta-0.1.0a12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scale-gp-beta
-Version: 0.1.0a11
+Version: 0.1.0a12
 Summary: The official Python library for the Scale GP API
 Project-URL: Homepage, https://github.com/scaleapi/sgp-python-beta
 Project-URL: Repository, https://github.com/scaleapi/sgp-python-beta

{scale_gp_beta-0.1.0a11.dist-info → scale_gp_beta-0.1.0a12.dist-info}/RECORD

@@ -11,7 +11,7 @@ scale_gp_beta/_resource.py,sha256=siZly_U6D0AOVLAzaOsqUdEFFzVMbWRj-ml30nvRp7E,11
 scale_gp_beta/_response.py,sha256=GemuybPk0uemovTlGHyHkj-ScYTTDJA0jqH5FQqIPwQ,28852
 scale_gp_beta/_streaming.py,sha256=fcCSGXslmi2SmmkM05g2SACXHk2Mj7k1X5uMBu6U5s8,10112
 scale_gp_beta/_types.py,sha256=ScQhVBaKbtJrER3NkXbjokWE9DqSqREMIw9LE0NrFfA,6150
-scale_gp_beta/_version.py,sha256=mODcLQ7YXuAUUS-DJtjwA5JDqLi0iX-NsPL3W6BhJLY,174
+scale_gp_beta/_version.py,sha256=8Knp4MLFQyXpPmUI1ikQFJejcnJ6hlWsrVpiUf6jLlA,174
 scale_gp_beta/pagination.py,sha256=6AAa8_V0wARlMd1MIXijugYbG1mILGc2tHVKbUQbZyQ,2595
 scale_gp_beta/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scale_gp_beta/_utils/__init__.py,sha256=PNZ_QJuzZEgyYXqkO1HVhGkj5IU9bglVUcw7H-Knjzw,2062
@@ -63,8 +63,8 @@ scale_gp_beta/types/evaluation_item_list_params.py,sha256=LquF3dWIU6b7O_Sy_b0R2F
 scale_gp_beta/types/evaluation_item_retrieve_params.py,sha256=UYEKIAQ4dy92ZOSV1tWDZcvXG7_0BSpOND5Ehzs7QM4,296
 scale_gp_beta/types/evaluation_list_params.py,sha256=bAYktX3x-rEqNKbBocDyJ0Iqwwz2rZLPK9bHxlU4wYQ,443
 scale_gp_beta/types/evaluation_retrieve_params.py,sha256=_YuT-E2VO-f_SvHaIe24KBbhTNoK8T-3tVB6Ov6cqfg,356
-scale_gp_beta/types/evaluation_task.py,sha256=2ariEI9Hg63xvMo7AlVv74vKFnxxQkuRgm6ODcP_hew,6461
-scale_gp_beta/types/evaluation_task_param.py,sha256=8C43Kt2077v4XzU-QAKJe1qigmdsW1_L8PwB2t21QBo,5958
+scale_gp_beta/types/evaluation_task.py,sha256=B6W-7Q9LXJlWcMimCA-FQAFfa6txg7LmGqx8Hterhms,9114
+scale_gp_beta/types/evaluation_task_param.py,sha256=cJz1MyC4CSBuu1R6rGpHNZGrZaEjGdUPqhrGb5eRxDw,8785
 scale_gp_beta/types/file.py,sha256=Xkha0eSr1q6hkwjE9e2XNgk8kuHNoTEe1LXNhz6o-1k,528
 scale_gp_beta/types/file_create_params.py,sha256=KpXv6JCbd8BlgceTmBTewxOky2JTJaTW3mcGiVVU7wE,317
 scale_gp_beta/types/file_delete_response.py,sha256=lOsiaw8qrUOnH7smxb27-n7M4D1chfXlAUaMTRmdldY,336
@@ -91,7 +91,7 @@ scale_gp_beta/types/chat/chat_completion_chunk.py,sha256=57-i6LyOk6IX2HZvXsoUC26
 scale_gp_beta/types/chat/completion_create_params.py,sha256=Y7vJNvNM4Sov77l55aS5YtyRnrf7isediu3nKr6YE-A,4505
 scale_gp_beta/types/chat/completion_create_response.py,sha256=0OhfoJW8azVRrZdXRRMuiJ7kEEeMDnKScxrr3sayzDo,374
 scale_gp_beta/types/files/__init__.py,sha256=OKfJYcKb4NObdiRObqJV_dOyDQ8feXekDUge2o_4pXQ,122
-scale_gp_beta-0.1.0a11.dist-info/METADATA,sha256=SdvTUqYYe0ZNeAhUEiyFLYpEGXUmiq5lZ7mHgEjkPts,16942
-scale_gp_beta-0.1.0a11.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-scale_gp_beta-0.1.0a11.dist-info/licenses/LICENSE,sha256=x49Bj8r_ZpqfzThbmfHyZ_bE88XvHdIMI_ANyLHFFRE,11338
-scale_gp_beta-0.1.0a11.dist-info/RECORD,,
+scale_gp_beta-0.1.0a12.dist-info/METADATA,sha256=GBvVuBH4tEVKba9R-Drxp0nXTkMQlYRUYxQrLL7A5Mk,16942
+scale_gp_beta-0.1.0a12.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+scale_gp_beta-0.1.0a12.dist-info/licenses/LICENSE,sha256=x49Bj8r_ZpqfzThbmfHyZ_bE88XvHdIMI_ANyLHFFRE,11338
+scale_gp_beta-0.1.0a12.dist-info/RECORD,,