vellum-ai 0.3.19__py3-none-any.whl → 0.3.21__py3-none-any.whl

vellum/__init__.py

@@ -148,6 +148,7 @@ from .types import (
     MetadataFilterConfigRequest,
     MetadataFilterRuleCombinator,
     MetadataFilterRuleRequest,
+    MlModelUsage,
     ModelVersionBuildConfig,
     ModelVersionExecConfig,
     ModelVersionExecConfigParameters,
@@ -210,6 +211,7 @@ from .types import (
     PaginatedSlimDocumentList,
     PaginatedSlimWorkflowDeploymentList,
     PaginatedTestSuiteRunExecutionList,
+    PaginatedTestSuiteTestCaseList,
     ProcessingFailureReasonEnum,
     ProcessingStateEnum,
     PromptDeploymentExpandMetaRequestRequest,
@@ -372,6 +374,9 @@ from .types import (
     TestSuiteRunMetricOutput,
     TestSuiteRunMetricOutput_Error,
     TestSuiteRunMetricOutput_Number,
+    TestSuiteRunMetricOutput_String,
+    TestSuiteRunMetricStringOutput,
+    TestSuiteRunMetricStringOutputTypeEnum,
     TestSuiteRunRead,
     TestSuiteRunState,
     TestSuiteRunTestSuite,
@@ -637,6 +642,7 @@ __all__ = [
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
+    "MlModelUsage",
     "ModelVersionBuildConfig",
     "ModelVersionExecConfig",
     "ModelVersionExecConfigParameters",
@@ -700,6 +706,7 @@ __all__ = [
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
     "PaginatedTestSuiteRunExecutionList",
+    "PaginatedTestSuiteTestCaseList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -862,6 +869,9 @@ __all__ = [
     "TestSuiteRunMetricOutput",
     "TestSuiteRunMetricOutput_Error",
     "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunMetricOutput_String",
+    "TestSuiteRunMetricStringOutput",
+    "TestSuiteRunMetricStringOutputTypeEnum",
     "TestSuiteRunRead",
     "TestSuiteRunState",
     "TestSuiteRunTestSuite",

vellum/client.py

@@ -167,6 +167,7 @@ class Vellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -301,6 +302,7 @@ class Vellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -1123,6 +1125,7 @@ class AsyncVellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -1257,6 +1260,7 @@ class AsyncVellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},

vellum/core/client_wrapper.py

@@ -18,7 +18,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.3.19",
+            "X-Fern-SDK-Version": "0.3.21",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

vellum/resources/test_suites/client.py

@@ -10,6 +10,7 @@ from ...core.jsonable_encoder import jsonable_encoder
 from ...core.remove_none_from_dict import remove_none_from_dict
 from ...core.request_options import RequestOptions
 from ...types.named_test_case_variable_value_request import NamedTestCaseVariableValueRequest
+from ...types.paginated_test_suite_test_case_list import PaginatedTestSuiteTestCaseList
 from ...types.test_suite_test_case import TestSuiteTestCase
 
 try:
@@ -25,6 +26,76 @@ class TestSuitesClient:
     def __init__(self, *, client_wrapper: SyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    def list_test_suite_test_cases(
+        self,
+        id: str,
+        *,
+        limit: typing.Optional[int] = None,
+        offset: typing.Optional[int] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> PaginatedTestSuiteTestCaseList:
+        """
+        List the Test Cases associated with a Test Suite
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.test_suites.list_test_suite_test_cases(
+            id="id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/test-suites/{jsonable_encoder(id)}/test-cases",
+            ),
+            params=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        "limit": limit,
+                        "offset": offset,
+                        **(
+                            request_options.get("additional_query_parameters", {})
+                            if request_options is not None
+                            else {}
+                        ),
+                    }
+                )
+            ),
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteTestCaseList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     def upsert_test_suite_test_case(
         self,
         id: str,
@@ -169,6 +240,76 @@ class AsyncTestSuitesClient:
     def __init__(self, *, client_wrapper: AsyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    async def list_test_suite_test_cases(
+        self,
+        id: str,
+        *,
+        limit: typing.Optional[int] = None,
+        offset: typing.Optional[int] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> PaginatedTestSuiteTestCaseList:
+        """
+        List the Test Cases associated with a Test Suite
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+        await client.test_suites.list_test_suite_test_cases(
+            id="id",
+        )
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/test-suites/{jsonable_encoder(id)}/test-cases",
+            ),
+            params=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        "limit": limit,
+                        "offset": offset,
+                        **(
+                            request_options.get("additional_query_parameters", {})
+                            if request_options is not None
+                            else {}
+                        ),
+                    }
+                )
+            ),
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteTestCaseList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     async def upsert_test_suite_test_case(
         self,
         id: str,

vellum/types/__init__.py

@@ -165,6 +165,7 @@ from .logprobs_enum import LogprobsEnum
 from .metadata_filter_config_request import MetadataFilterConfigRequest
 from .metadata_filter_rule_combinator import MetadataFilterRuleCombinator
 from .metadata_filter_rule_request import MetadataFilterRuleRequest
+from .ml_model_usage import MlModelUsage
 from .model_version_build_config import ModelVersionBuildConfig
 from .model_version_exec_config import ModelVersionExecConfig
 from .model_version_exec_config_parameters import ModelVersionExecConfigParameters
@@ -233,6 +234,7 @@ from .paginated_slim_deployment_read_list import PaginatedSlimDeploymentReadList
 from .paginated_slim_document_list import PaginatedSlimDocumentList
 from .paginated_slim_workflow_deployment_list import PaginatedSlimWorkflowDeploymentList
 from .paginated_test_suite_run_execution_list import PaginatedTestSuiteRunExecutionList
+from .paginated_test_suite_test_case_list import PaginatedTestSuiteTestCaseList
 from .processing_failure_reason_enum import ProcessingFailureReasonEnum
 from .processing_state_enum import ProcessingStateEnum
 from .prompt_deployment_expand_meta_request_request import PromptDeploymentExpandMetaRequestRequest
@@ -418,7 +420,10 @@ from .test_suite_run_metric_output import (
     TestSuiteRunMetricOutput,
     TestSuiteRunMetricOutput_Error,
     TestSuiteRunMetricOutput_Number,
+    TestSuiteRunMetricOutput_String,
 )
+from .test_suite_run_metric_string_output import TestSuiteRunMetricStringOutput
+from .test_suite_run_metric_string_output_type_enum import TestSuiteRunMetricStringOutputTypeEnum
 from .test_suite_run_read import TestSuiteRunRead
 from .test_suite_run_state import TestSuiteRunState
 from .test_suite_run_test_suite import TestSuiteRunTestSuite
@@ -669,6 +674,7 @@ __all__ = [
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
+    "MlModelUsage",
     "ModelVersionBuildConfig",
     "ModelVersionExecConfig",
     "ModelVersionExecConfigParameters",
@@ -731,6 +737,7 @@ __all__ = [
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
     "PaginatedTestSuiteRunExecutionList",
+    "PaginatedTestSuiteTestCaseList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -893,6 +900,9 @@ __all__ = [
     "TestSuiteRunMetricOutput",
     "TestSuiteRunMetricOutput_Error",
     "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunMetricOutput_String",
+    "TestSuiteRunMetricStringOutput",
+    "TestSuiteRunMetricStringOutputTypeEnum",
     "TestSuiteRunRead",
     "TestSuiteRunState",
     "TestSuiteRunTestSuite",

vellum/types/fulfilled_prompt_execution_meta.py

@@ -5,6 +5,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .finish_reason_enum import FinishReasonEnum
+from .ml_model_usage import MlModelUsage
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -19,6 +20,7 @@ class FulfilledPromptExecutionMeta(pydantic.BaseModel):
 
     latency: typing.Optional[int] = None
     finish_reason: typing.Optional[FinishReasonEnum] = None
+    usage: typing.Optional[MlModelUsage] = None
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/ml_model_usage.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class MlModelUsage(pydantic.BaseModel):
+    output_token_count: typing.Optional[int] = None
+    input_token_count: typing.Optional[int] = None
+    input_char_count: typing.Optional[int] = None
+    output_char_count: typing.Optional[int] = None
+    compute_nanos: typing.Optional[int] = None
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/paginated_test_suite_test_case_list.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_test_case import TestSuiteTestCase
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class PaginatedTestSuiteTestCaseList(pydantic.BaseModel):
+    count: int
+    next: typing.Optional[str] = None
+    previous: typing.Optional[str] = None
+    results: typing.List[TestSuiteTestCase]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/prompt_deployment_expand_meta_request_request.py

@@ -37,6 +37,11 @@ class PromptDeploymentExpandMetaRequestRequest(pydantic.BaseModel):
     If enabled, the response will include the reason provided by the model for why the execution finished.
     """
 
+    usage: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    If enabled, the response will include model host usage tracking. This may increase latency for some model hosts.
+    """
+
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
         return super().json(**kwargs_with_defaults)

vellum/types/prompt_execution_meta.py

@@ -5,6 +5,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .finish_reason_enum import FinishReasonEnum
+from .ml_model_usage import MlModelUsage
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -17,6 +18,7 @@ class PromptExecutionMeta(pydantic.BaseModel):
     The subset of the metadata tracked by Vellum during prompt execution that the request opted into with `expand_meta`.
     """
 
+    usage: typing.Optional[MlModelUsage] = None
     model_name: typing.Optional[str] = None
     latency: typing.Optional[int] = None
     deployment_release_tag: typing.Optional[str] = None

vellum/types/test_suite_run_metric_output.py

@@ -6,6 +6,17 @@ import typing
 
 from .test_suite_run_metric_error_output import TestSuiteRunMetricErrorOutput
 from .test_suite_run_metric_number_output import TestSuiteRunMetricNumberOutput
+from .test_suite_run_metric_string_output import TestSuiteRunMetricStringOutput
+
+
+class TestSuiteRunMetricOutput_String(TestSuiteRunMetricStringOutput):
+    type: typing.Literal["STRING"] = "STRING"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
 
 
 class TestSuiteRunMetricOutput_Number(TestSuiteRunMetricNumberOutput):
@@ -28,4 +39,6 @@ class TestSuiteRunMetricOutput_Error(TestSuiteRunMetricErrorOutput):
         populate_by_name = True
 
 
-TestSuiteRunMetricOutput = typing.Union[TestSuiteRunMetricOutput_Number, TestSuiteRunMetricOutput_Error]
+TestSuiteRunMetricOutput = typing.Union[
+    TestSuiteRunMetricOutput_String, TestSuiteRunMetricOutput_Number, TestSuiteRunMetricOutput_Error
+]

vellum/types/test_suite_run_metric_string_output.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunMetricStringOutput(pydantic.BaseModel):
+    """
+    Output for a test suite run metric that is of type STRING
+    """
+
+    value: str
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_metric_string_output_type_enum.py

@@ -0,0 +1,5 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+TestSuiteRunMetricStringOutputTypeEnum = typing.Literal["STRING"]

{vellum_ai-0.3.19.dist-info → vellum_ai-0.3.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vellum-ai
-Version: 0.3.19
+Version: 0.3.21
 Summary: 
 Requires-Python: >=3.8,<4.0
 Classifier: Programming Language :: Python :: 3

{vellum_ai-0.3.19.dist-info → vellum_ai-0.3.21.dist-info}/RECORD

@@ -1,8 +1,8 @@
-vellum/__init__.py,sha256=17WXNkvq4OC3V1VqGWc64nJklzsPgBH0D_uux72OYng,33627
-vellum/client.py,sha256=XB_tltKpWwPoZpsm4g0ftLPKFYjZSLjLAteJwp-xSWA,96636
+vellum/__init__.py,sha256=h2QBLHkAs9OTsNI_W0fYfaUrQUmLd37wNbkK63TQcGY,33979
+vellum/client.py,sha256=ViOMnbiP_kKm3ayXW6DY1wC7Ge8tnI389qC8XVffyB8,96748
 vellum/core/__init__.py,sha256=RWfyDqkzWsf8e3VGc3NV60MovfJbg5XWzNFGB2DZ0hA,790
 vellum/core/api_error.py,sha256=RE8LELok2QCjABadECTvtDp7qejA1VmINCh6TbqPwSE,426
-vellum/core/client_wrapper.py,sha256=g4ugtTY_3bC7PtOub1v7_JStUsGSQkAytTegUNwUuWY,1698
+vellum/core/client_wrapper.py,sha256=zOXh9ZnRKMmvfCIpadlblmkfFSVgnNdLnEaRVillKps,1698
 vellum/core/datetime_utils.py,sha256=nBys2IsYrhPdszxGKCNRPSOCwa-5DWOHG95FB8G9PKo,1047
 vellum/core/file.py,sha256=sy1RUGZ3aJYuw998bZytxxo6QdgKmlnlgBaMvwEKCGg,1480
 vellum/core/http_client.py,sha256=5ok6hqgZDJhg57EHvMnr0BBaHdG50QxFPKaCZ9aVWTc,5059
@@ -39,12 +39,12 @@ vellum/resources/sandboxes/client.py,sha256=gW_DTrJ3w6BfaSATZxB0Z-bxVUXQ0JtPAjL6
 vellum/resources/test_suite_runs/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
 vellum/resources/test_suite_runs/client.py,sha256=T5vF1S5jDfQLxKOg-2Q2XYCF9g5W-dos6Ff6Av21gr8,17120
 vellum/resources/test_suites/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
-vellum/resources/test_suites/client.py,sha256=zB8Ne4rUqwyazytyBWNKuI449C8MB7S5dugRg5NBjd4,13316
+vellum/resources/test_suites/client.py,sha256=7PmEjJ_2TlvqJoP3C4XkdA8SyYMHVQwREB4T8LowZbM,18966
 vellum/resources/workflow_deployments/__init__.py,sha256=-5BCA0kSmW6WUh4gqLuQtHv4zFdt9lccuDwMU5YvEu4,173
 vellum/resources/workflow_deployments/client.py,sha256=DUY1h8mvXqwP-zilmwoek_XbLNTFeeVbxgd7M-k2IzA,11164
 vellum/resources/workflow_deployments/types/__init__.py,sha256=rmS_4dtbgLHGNQJ_pOloygrjl4sNbKZjTEKBxbMyz6E,208
 vellum/resources/workflow_deployments/types/workflow_deployments_list_request_status.py,sha256=8-G1SalBR6-AfRnb6POOR9M3tvZa3CGwFIs1ArZb6uw,542
-vellum/types/__init__.py,sha256=59p1m_erR_1kg_kWmTUWaBafV5zzijV_5EZ7yJXIRvI,44795
+vellum/types/__init__.py,sha256=aXYiDbcBGEypul3Ae4DcgwNT55UtGftQsVxTPRpVAZU,45312
 vellum/types/api_node_result.py,sha256=aLOooryBCfglTeZwOuyyqRoHz5oPTja2mYYY7Gj8XxA,1040
 vellum/types/api_node_result_data.py,sha256=cQcZiwI1hjRvQFlkVEGxyaD360vEklIfjBmhBAeI75U,1215
 vellum/types/array_chat_message_content.py,sha256=Z7nKH8RNgot0i4-159w95OHywLjc-376gBPOovbw2hk,1087
@@ -108,7 +108,7 @@ vellum/types/fulfilled_execute_prompt_event.py,sha256=2_jNUOaSg12WKggFiczulVoHFM
 vellum/types/fulfilled_execute_prompt_response.py,sha256=LyVpUlphfBrqSWzo1QyIzdTjAJ8cgW60U98VnMd83ws,1508
 vellum/types/fulfilled_execute_workflow_workflow_result_event.py,sha256=ROLCpeU2FoqyisPDvgumUlYFDUbwOr93pEfII-ST3NI,1146
 vellum/types/fulfilled_function_call.py,sha256=wIk8JVy9HyjSbY7rPOhrXPh3OuKy3mtpUvPSOx2FcUc,1056
-vellum/types/fulfilled_prompt_execution_meta.py,sha256=_cDf859bAtbKHRt3cvdLHeuQi44CUqjUSYWITLBeubk,1196
+vellum/types/fulfilled_prompt_execution_meta.py,sha256=FHivWXQbtMMoud9vMC4rc_ik1JWPZG6HfJOkuevCnCs,1285
 vellum/types/fulfilled_workflow_node_result_event.py,sha256=MGxts5L-VBaYP7xbL5ojjtKWEXUv45XSPqT7Y2RtvjI,1454
 vellum/types/function_call.py,sha256=U1_fLhZ_VGnHYPX8rb7iqyyUhBzzbRAGR-ai4wLDrs4,813
 vellum/types/function_call_chat_message_content.py,sha256=z9bTybrPXuhztOUw65PXGws-obJkCJWhc0Zfv1ZwA-k,1122
@@ -144,6 +144,7 @@ vellum/types/logprobs_enum.py,sha256=cw9yroKgrZzpPHniPV4hg7_21PM9_FSX1GArHHC_-_o
 vellum/types/metadata_filter_config_request.py,sha256=JH5EBNES30DuRIhgad6IYwXZcuAW6bRajSnWUqs4hG4,1412
 vellum/types/metadata_filter_rule_combinator.py,sha256=hmfQ3drGlGYqRA78iZr7ek8qZTFN-WENrwL3_OlexMM,528
 vellum/types/metadata_filter_rule_request.py,sha256=JGPkjXzMS9I3dlEQUAJ0HllJ3F-Znp-0x-4hp06miPs,1428
+vellum/types/ml_model_usage.py,sha256=2o6Yy9lMDTqRHQNpUzQSoQMfA5_my79LA1pBZDcGLV0,1143
 vellum/types/model_version_build_config.py,sha256=0dOGfizN30eYJtjs-fJc32TUBGdpYuMFo57a_tEutSg,1400
 vellum/types/model_version_exec_config.py,sha256=tJJ1Kb8rBdbkD4z9fVwP94bjR9gee4I9N-i9o9w8vVs,1677
 vellum/types/model_version_exec_config_parameters.py,sha256=jF4wm_GIeITFavMNGSvBd-fa5v9fEjWxoHaZhVFx0CI,1326
@@ -184,11 +185,12 @@ vellum/types/paginated_slim_deployment_read_list.py,sha256=4pMZYAVl7pLI8d9-CE3NY
 vellum/types/paginated_slim_document_list.py,sha256=Zvr_VzHcUEEiqAXEitoaM5YcPcaTKgEx8fziy3SH1CU,1127
 vellum/types/paginated_slim_workflow_deployment_list.py,sha256=dCew3Xy72Mo5MHIBdCbcuDaH0v2yeCRNBDt4m-8tb0A,1168
 vellum/types/paginated_test_suite_run_execution_list.py,sha256=pwT-FeIxOBnP_S3kpqlAUOtlXEYfFgfViRGxeFTebRI,1117
+vellum/types/paginated_test_suite_test_case_list.py,sha256=0XlghGkiSubI41oR1UGSdYOKDUM4XLCgmcElKaYFdkA,1101
 vellum/types/processing_failure_reason_enum.py,sha256=MDj2vNyO1Y-2WHuolkrGez8F1cZqS6ultfsqvGI4Fg8,752
 vellum/types/processing_state_enum.py,sha256=rMhw5oLZNfhR4QkIgFfLMWRSLXgHt9qhiguEqWQNz5k,962
-vellum/types/prompt_deployment_expand_meta_request_request.py,sha256=jvi3NW0nJxyLwNEhk35oEdAm_aDOuy48eNy5LqvkkRs,1885
+vellum/types/prompt_deployment_expand_meta_request_request.py,sha256=bpA4eO3wIwg5fZK89eoC8B8QsRrKFddIN69o04YEBPk,2083
 vellum/types/prompt_deployment_input_request.py,sha256=o8PCi7TsitddNxiAfDblxaXkJ1-WjuDTAW0LnKbMDY4,1233
-vellum/types/prompt_execution_meta.py,sha256=ix_1Wscqv3kthyyirFEjwY6xXYD5wuw0sdypyfb8hAo,1338
+vellum/types/prompt_execution_meta.py,sha256=4RNkaZ2QjUtZwGUc9574Y4KuoZnzW2UCWj2RNiNiGKQ,1427
 vellum/types/prompt_node_result.py,sha256=s6jusFXMD1calmDVdL0lWO0NIqILWpdLP8xFRqf-5-k,1054
 vellum/types/prompt_node_result_data.py,sha256=q6y32EyaaAoIxM1od6-5RQAmPTitBQ-Hlkw5QNdeo6g,1045
 vellum/types/prompt_output.py,sha256=Df5UnLyBRf79_oUnYiMl9vi_h41yKEDLw1KgkgiX1Uk,1442
@@ -301,7 +303,9 @@ vellum/types/test_suite_run_metric_error_output.py,sha256=N0ZRTO1QV4Bobrh9u1-k5q
 vellum/types/test_suite_run_metric_error_output_type_enum.py,sha256=ZPKy0jkcgfuMhI2hG_3UUcw2sYH1BlV_ze8Axlq5nIU,144
 vellum/types/test_suite_run_metric_number_output.py,sha256=vedeqnSpDl7Y23DdgAlcl16Gxd1WlNo4m4PQ2SGI8Tk,1020
 vellum/types/test_suite_run_metric_number_output_type_enum.py,sha256=sXoES0zoliicpd66SwlF6xsVUjWSezGNkuxp6CthO2s,146
-vellum/types/test_suite_run_metric_output.py,sha256=O_v0dGiAnXXRc_MpEZT3La5p6iBPPRNfNWJesGIHe9M,907
+vellum/types/test_suite_run_metric_output.py,sha256=QkHRKO1tnQWoeCq7xc7TfSiAjC927Ha2hdePBJeEn0A,1291
+vellum/types/test_suite_run_metric_string_output.py,sha256=FxScPpKm8kj6pGTTQ9slJgkb1PQZn5LBa0ZFOnCUxJw,1018
+vellum/types/test_suite_run_metric_string_output_type_enum.py,sha256=6OKwFyk0SUq4IKp6yg_FAc3ADM0jVCK-chq5JR9iwqo,146
 vellum/types/test_suite_run_read.py,sha256=bb6kgQaMQpkshETe_dXiMNVCx3r05t12ajaiqTAnvHg,1551
 vellum/types/test_suite_run_state.py,sha256=myD2yLYpEiTUK4AyRZwl3MrXOKYkDA8k4fpqnDnALAY,1111
 vellum/types/test_suite_run_test_suite.py,sha256=jsSjV803xCQ4zQ29MCvCM06rH1PCviCZ7G8KzR-LiOA,953
@@ -359,7 +363,7 @@ vellum/types/workflow_result_event_output_data_search_results.py,sha256=wYc76-DW
 vellum/types/workflow_result_event_output_data_string.py,sha256=7SR46tKt-I_cGd_eVeKU8ymQ7R3vKkA_dFHCgAU1GcQ,1533
 vellum/types/workflow_stream_event.py,sha256=KA6Bkk_XA6AIPWR-1vKnwF1A8l_Bm5y0arQCWWWRpsk,911
 vellum/version.py,sha256=neLt8HBHHUtDF9M5fsyUzHT-pKooEPvceaLDqqIGb0s,77
-vellum_ai-0.3.19.dist-info/LICENSE,sha256=CcaljEIoOBaU-wItPH4PmM_mDCGpyuUY0Er1BGu5Ti8,1073
-vellum_ai-0.3.19.dist-info/METADATA,sha256=F_XSE4YdnDaMpkP7fdqrL6I8THooOux4LK9VkieCbEI,3550
-vellum_ai-0.3.19.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-vellum_ai-0.3.19.dist-info/RECORD,,
+vellum_ai-0.3.21.dist-info/LICENSE,sha256=CcaljEIoOBaU-wItPH4PmM_mDCGpyuUY0Er1BGu5Ti8,1073
+vellum_ai-0.3.21.dist-info/METADATA,sha256=BPMxESrgGDvdc1CRhBOdYCRugknRd_NwOR5D5NzmRI4,3550
+vellum_ai-0.3.21.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+vellum_ai-0.3.21.dist-info/RECORD,,