vellum-ai 0.3.12__py3-none-any.whl → 0.3.14__py3-none-any.whl

vellum/__init__.py

@@ -203,6 +203,7 @@ from .types import (
     PaginatedSlimDeploymentReadList,
     PaginatedSlimDocumentList,
     PaginatedSlimWorkflowDeploymentList,
+    PaginatedTestSuiteRunExecutionList,
     ProcessingFailureReasonEnum,
     ProcessingStateEnum,
     PromptDeploymentExpandMetaRequestRequest,
@@ -328,6 +329,47 @@ from .types import (
     TestCaseVariableValue_Number,
     TestCaseVariableValue_SearchResults,
     TestCaseVariableValue_String,
+    TestSuiteRunDeploymentReleaseTagExecConfig,
+    TestSuiteRunDeploymentReleaseTagExecConfigData,
+    TestSuiteRunDeploymentReleaseTagExecConfigDataRequest,
+    TestSuiteRunDeploymentReleaseTagExecConfigRequest,
+    TestSuiteRunDeploymentReleaseTagExecConfigTypeEnum,
+    TestSuiteRunExecConfig,
+    TestSuiteRunExecConfigRequest,
+    TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
+    TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_WorkflowReleaseTag,
+    TestSuiteRunExecution,
+    TestSuiteRunExecutionChatHistoryOutput,
+    TestSuiteRunExecutionErrorOutput,
+    TestSuiteRunExecutionJsonOutput,
+    TestSuiteRunExecutionMetricResult,
+    TestSuiteRunExecutionNumberOutput,
+    TestSuiteRunExecutionOutput,
+    TestSuiteRunExecutionOutput_ChatHistory,
+    TestSuiteRunExecutionOutput_Error,
+    TestSuiteRunExecutionOutput_Json,
+    TestSuiteRunExecutionOutput_Number,
+    TestSuiteRunExecutionOutput_SearchResults,
+    TestSuiteRunExecutionOutput_String,
+    TestSuiteRunExecutionSearchResultsOutput,
+    TestSuiteRunExecutionStringOutput,
+    TestSuiteRunMetricErrorOutput,
+    TestSuiteRunMetricErrorOutputTypeEnum,
+    TestSuiteRunMetricNumberOutput,
+    TestSuiteRunMetricNumberOutputTypeEnum,
+    TestSuiteRunMetricOutput,
+    TestSuiteRunMetricOutput_Error,
+    TestSuiteRunMetricOutput_Number,
+    TestSuiteRunRead,
+    TestSuiteRunState,
+    TestSuiteRunTestSuite,
+    TestSuiteRunWorkflowReleaseTagExecConfig,
+    TestSuiteRunWorkflowReleaseTagExecConfigData,
+    TestSuiteRunWorkflowReleaseTagExecConfigDataRequest,
+    TestSuiteRunWorkflowReleaseTagExecConfigRequest,
+    TestSuiteRunWorkflowReleaseTagExecConfigTypeEnum,
     TestSuiteTestCase,
     UploadDocumentErrorResponse,
     UploadDocumentResponse,
@@ -424,6 +466,7 @@ from .resources import (
     model_versions,
     registered_prompts,
     sandboxes,
+    test_suite_runs,
     test_suites,
     workflow_deployments,
 )
@@ -639,6 +682,7 @@ __all__ = [
     "PaginatedSlimDeploymentReadList",
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
+    "PaginatedTestSuiteRunExecutionList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -764,6 +808,47 @@ __all__ = [
     "TestCaseVariableValue_Number",
     "TestCaseVariableValue_SearchResults",
     "TestCaseVariableValue_String",
+    "TestSuiteRunDeploymentReleaseTagExecConfig",
+    "TestSuiteRunDeploymentReleaseTagExecConfigData",
+    "TestSuiteRunDeploymentReleaseTagExecConfigDataRequest",
+    "TestSuiteRunDeploymentReleaseTagExecConfigRequest",
+    "TestSuiteRunDeploymentReleaseTagExecConfigTypeEnum",
+    "TestSuiteRunExecConfig",
+    "TestSuiteRunExecConfigRequest",
+    "TestSuiteRunExecConfigRequest_DeploymentReleaseTag",
+    "TestSuiteRunExecConfigRequest_WorkflowReleaseTag",
+    "TestSuiteRunExecConfig_DeploymentReleaseTag",
+    "TestSuiteRunExecConfig_WorkflowReleaseTag",
+    "TestSuiteRunExecution",
+    "TestSuiteRunExecutionChatHistoryOutput",
+    "TestSuiteRunExecutionErrorOutput",
+    "TestSuiteRunExecutionJsonOutput",
+    "TestSuiteRunExecutionMetricResult",
+    "TestSuiteRunExecutionNumberOutput",
+    "TestSuiteRunExecutionOutput",
+    "TestSuiteRunExecutionOutput_ChatHistory",
+    "TestSuiteRunExecutionOutput_Error",
+    "TestSuiteRunExecutionOutput_Json",
+    "TestSuiteRunExecutionOutput_Number",
+    "TestSuiteRunExecutionOutput_SearchResults",
+    "TestSuiteRunExecutionOutput_String",
+    "TestSuiteRunExecutionSearchResultsOutput",
+    "TestSuiteRunExecutionStringOutput",
+    "TestSuiteRunMetricErrorOutput",
+    "TestSuiteRunMetricErrorOutputTypeEnum",
+    "TestSuiteRunMetricNumberOutput",
+    "TestSuiteRunMetricNumberOutputTypeEnum",
+    "TestSuiteRunMetricOutput",
+    "TestSuiteRunMetricOutput_Error",
+    "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunRead",
+    "TestSuiteRunState",
+    "TestSuiteRunTestSuite",
+    "TestSuiteRunWorkflowReleaseTagExecConfig",
+    "TestSuiteRunWorkflowReleaseTagExecConfigData",
+    "TestSuiteRunWorkflowReleaseTagExecConfigDataRequest",
+    "TestSuiteRunWorkflowReleaseTagExecConfigRequest",
+    "TestSuiteRunWorkflowReleaseTagExecConfigTypeEnum",
     "TestSuiteTestCase",
     "UploadDocumentErrorResponse",
     "UploadDocumentResponse",
@@ -856,6 +941,7 @@ __all__ = [
     "model_versions",
     "registered_prompts",
     "sandboxes",
+    "test_suite_runs",
     "test_suites",
     "workflow_deployments",
 ]

vellum/client.py

@@ -22,6 +22,7 @@ from .resources.folder_entities.client import AsyncFolderEntitiesClient, FolderE
 from .resources.model_versions.client import AsyncModelVersionsClient, ModelVersionsClient
 from .resources.registered_prompts.client import AsyncRegisteredPromptsClient, RegisteredPromptsClient
 from .resources.sandboxes.client import AsyncSandboxesClient, SandboxesClient
+from .resources.test_suite_runs.client import AsyncTestSuiteRunsClient, TestSuiteRunsClient
 from .resources.test_suites.client import AsyncTestSuitesClient, TestSuitesClient
 from .resources.workflow_deployments.client import AsyncWorkflowDeploymentsClient, WorkflowDeploymentsClient
 from .types.execute_prompt_event import ExecutePromptEvent
@@ -72,6 +73,7 @@ class Vellum:
         self.model_versions = ModelVersionsClient(client_wrapper=self._client_wrapper)
         self.registered_prompts = RegisteredPromptsClient(client_wrapper=self._client_wrapper)
         self.sandboxes = SandboxesClient(client_wrapper=self._client_wrapper)
+        self.test_suite_runs = TestSuiteRunsClient(client_wrapper=self._client_wrapper)
         self.test_suites = TestSuitesClient(client_wrapper=self._client_wrapper)
         self.workflow_deployments = WorkflowDeploymentsClient(client_wrapper=self._client_wrapper)
 
@@ -655,6 +657,7 @@ class AsyncVellum:
         self.model_versions = AsyncModelVersionsClient(client_wrapper=self._client_wrapper)
         self.registered_prompts = AsyncRegisteredPromptsClient(client_wrapper=self._client_wrapper)
         self.sandboxes = AsyncSandboxesClient(client_wrapper=self._client_wrapper)
+        self.test_suite_runs = AsyncTestSuiteRunsClient(client_wrapper=self._client_wrapper)
         self.test_suites = AsyncTestSuitesClient(client_wrapper=self._client_wrapper)
         self.workflow_deployments = AsyncWorkflowDeploymentsClient(client_wrapper=self._client_wrapper)
 

vellum/core/client_wrapper.py

@@ -16,7 +16,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.3.12",
+            "X-Fern-SDK-Version": "0.3.14",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

vellum/resources/__init__.py

@@ -8,6 +8,7 @@ from . import (
     model_versions,
     registered_prompts,
     sandboxes,
+    test_suite_runs,
     test_suites,
     workflow_deployments,
 )
@@ -26,6 +27,7 @@ __all__ = [
     "model_versions",
     "registered_prompts",
     "sandboxes",
+    "test_suite_runs",
     "test_suites",
     "workflow_deployments",
 ]

vellum/resources/test_suite_runs/__init__.py

@@ -0,0 +1,2 @@
+# This file was auto-generated by Fern from our API Definition.
+

vellum/resources/test_suite_runs/client.py

@@ -0,0 +1,223 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+import urllib.parse
+from json.decoder import JSONDecodeError
+
+from ...core.api_error import ApiError
+from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ...core.jsonable_encoder import jsonable_encoder
+from ...core.remove_none_from_dict import remove_none_from_dict
+from ...types.paginated_test_suite_run_execution_list import PaginatedTestSuiteRunExecutionList
+from ...types.test_suite_run_exec_config_request import TestSuiteRunExecConfigRequest
+from ...types.test_suite_run_read import TestSuiteRunRead
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class TestSuiteRunsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    def create(
+        self, *, test_suite_id: typing.Optional[str] = OMIT, exec_config: TestSuiteRunExecConfigRequest
+    ) -> TestSuiteRunRead:
+        """
+        Trigger a Test Suite and create a new Test Suite Run
+
+        Parameters:
+            - test_suite_id: typing.Optional[str]. The ID of the Test Suite to run
+
+            - exec_config: TestSuiteRunExecConfigRequest. Configuration that defines how the Test Suite should be run
+        """
+        _request: typing.Dict[str, typing.Any] = {"exec_config": exec_config}
+        if test_suite_id is not OMIT:
+            _request["test_suite_id"] = test_suite_id
+        _response = self._client_wrapper.httpx_client.request(
+            "POST",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_environment().default}/", "v1/test-suite-runs"),
+            json=jsonable_encoder(_request),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(TestSuiteRunRead, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def retrieve(self, id: str) -> TestSuiteRunRead:
+        """
+        Retrieve a specific Test Suite Run by ID
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite run.
+        ---
+        from vellum.client import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.test_suite_runs.retrieve(
+            id="id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_environment().default}/", f"v1/test-suite-runs/{id}"),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(TestSuiteRunRead, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    def list_test_suite_run_executions(
+        self, id: str, *, limit: typing.Optional[int] = None, offset: typing.Optional[int] = None
+    ) -> PaginatedTestSuiteRunExecutionList:
+        """
+        Parameters:
+            - id: str. A UUID string identifying this test suite run.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+        ---
+        from vellum.client import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.test_suite_runs.list_test_suite_run_executions(
+            id="id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/", f"v1/test-suite-runs/{id}/executions"
+            ),
+            params=remove_none_from_dict({"limit": limit, "offset": offset}),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteRunExecutionList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+
+class AsyncTestSuiteRunsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    async def create(
+        self, *, test_suite_id: typing.Optional[str] = OMIT, exec_config: TestSuiteRunExecConfigRequest
+    ) -> TestSuiteRunRead:
+        """
+        Trigger a Test Suite and create a new Test Suite Run
+
+        Parameters:
+            - test_suite_id: typing.Optional[str]. The ID of the Test Suite to run
+
+            - exec_config: TestSuiteRunExecConfigRequest. Configuration that defines how the Test Suite should be run
+        """
+        _request: typing.Dict[str, typing.Any] = {"exec_config": exec_config}
+        if test_suite_id is not OMIT:
+            _request["test_suite_id"] = test_suite_id
+        _response = await self._client_wrapper.httpx_client.request(
+            "POST",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_environment().default}/", "v1/test-suite-runs"),
+            json=jsonable_encoder(_request),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(TestSuiteRunRead, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def retrieve(self, id: str) -> TestSuiteRunRead:
+        """
+        Retrieve a specific Test Suite Run by ID
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite run.
+        ---
+        from vellum.client import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+        await client.test_suite_runs.retrieve(
+            id="id",
+        )
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_environment().default}/", f"v1/test-suite-runs/{id}"),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(TestSuiteRunRead, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def list_test_suite_run_executions(
+        self, id: str, *, limit: typing.Optional[int] = None, offset: typing.Optional[int] = None
+    ) -> PaginatedTestSuiteRunExecutionList:
+        """
+        Parameters:
+            - id: str. A UUID string identifying this test suite run.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+        ---
+        from vellum.client import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+        await client.test_suite_runs.list_test_suite_run_executions(
+            id="id",
+        )
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/", f"v1/test-suite-runs/{id}/executions"
+            ),
+            params=remove_none_from_dict({"limit": limit, "offset": offset}),
+            headers=self._client_wrapper.get_headers(),
+            timeout=None,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteRunExecutionList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)

vellum/types/__init__.py

@@ -226,6 +226,7 @@ from .paginated_document_index_read_list import PaginatedDocumentIndexReadList
 from .paginated_slim_deployment_read_list import PaginatedSlimDeploymentReadList
 from .paginated_slim_document_list import PaginatedSlimDocumentList
 from .paginated_slim_workflow_deployment_list import PaginatedSlimWorkflowDeploymentList
+from .paginated_test_suite_run_execution_list import PaginatedTestSuiteRunExecutionList
 from .processing_failure_reason_enum import ProcessingFailureReasonEnum
 from .processing_state_enum import ProcessingStateEnum
 from .prompt_deployment_expand_meta_request_request import PromptDeploymentExpandMetaRequestRequest
@@ -363,6 +364,61 @@ from .test_case_variable_value import (
     TestCaseVariableValue_SearchResults,
     TestCaseVariableValue_String,
 )
+from .test_suite_run_deployment_release_tag_exec_config import TestSuiteRunDeploymentReleaseTagExecConfig
+from .test_suite_run_deployment_release_tag_exec_config_data import TestSuiteRunDeploymentReleaseTagExecConfigData
+from .test_suite_run_deployment_release_tag_exec_config_data_request import (
+    TestSuiteRunDeploymentReleaseTagExecConfigDataRequest,
+)
+from .test_suite_run_deployment_release_tag_exec_config_request import TestSuiteRunDeploymentReleaseTagExecConfigRequest
+from .test_suite_run_deployment_release_tag_exec_config_type_enum import (
+    TestSuiteRunDeploymentReleaseTagExecConfigTypeEnum,
+)
+from .test_suite_run_exec_config import (
+    TestSuiteRunExecConfig,
+    TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_WorkflowReleaseTag,
+)
+from .test_suite_run_exec_config_request import (
+    TestSuiteRunExecConfigRequest,
+    TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
+)
+from .test_suite_run_execution import TestSuiteRunExecution
+from .test_suite_run_execution_chat_history_output import TestSuiteRunExecutionChatHistoryOutput
+from .test_suite_run_execution_error_output import TestSuiteRunExecutionErrorOutput
+from .test_suite_run_execution_json_output import TestSuiteRunExecutionJsonOutput
+from .test_suite_run_execution_metric_result import TestSuiteRunExecutionMetricResult
+from .test_suite_run_execution_number_output import TestSuiteRunExecutionNumberOutput
+from .test_suite_run_execution_output import (
+    TestSuiteRunExecutionOutput,
+    TestSuiteRunExecutionOutput_ChatHistory,
+    TestSuiteRunExecutionOutput_Error,
+    TestSuiteRunExecutionOutput_Json,
+    TestSuiteRunExecutionOutput_Number,
+    TestSuiteRunExecutionOutput_SearchResults,
+    TestSuiteRunExecutionOutput_String,
+)
+from .test_suite_run_execution_search_results_output import TestSuiteRunExecutionSearchResultsOutput
+from .test_suite_run_execution_string_output import TestSuiteRunExecutionStringOutput
+from .test_suite_run_metric_error_output import TestSuiteRunMetricErrorOutput
+from .test_suite_run_metric_error_output_type_enum import TestSuiteRunMetricErrorOutputTypeEnum
+from .test_suite_run_metric_number_output import TestSuiteRunMetricNumberOutput
+from .test_suite_run_metric_number_output_type_enum import TestSuiteRunMetricNumberOutputTypeEnum
+from .test_suite_run_metric_output import (
+    TestSuiteRunMetricOutput,
+    TestSuiteRunMetricOutput_Error,
+    TestSuiteRunMetricOutput_Number,
+)
+from .test_suite_run_read import TestSuiteRunRead
+from .test_suite_run_state import TestSuiteRunState
+from .test_suite_run_test_suite import TestSuiteRunTestSuite
+from .test_suite_run_workflow_release_tag_exec_config import TestSuiteRunWorkflowReleaseTagExecConfig
+from .test_suite_run_workflow_release_tag_exec_config_data import TestSuiteRunWorkflowReleaseTagExecConfigData
+from .test_suite_run_workflow_release_tag_exec_config_data_request import (
+    TestSuiteRunWorkflowReleaseTagExecConfigDataRequest,
+)
+from .test_suite_run_workflow_release_tag_exec_config_request import TestSuiteRunWorkflowReleaseTagExecConfigRequest
+from .test_suite_run_workflow_release_tag_exec_config_type_enum import TestSuiteRunWorkflowReleaseTagExecConfigTypeEnum
 from .test_suite_test_case import TestSuiteTestCase
 from .upload_document_error_response import UploadDocumentErrorResponse
 from .upload_document_response import UploadDocumentResponse
@@ -658,6 +714,7 @@ __all__ = [
     "PaginatedSlimDeploymentReadList",
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
+    "PaginatedTestSuiteRunExecutionList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -783,6 +840,47 @@ __all__ = [
     "TestCaseVariableValue_Number",
     "TestCaseVariableValue_SearchResults",
     "TestCaseVariableValue_String",
+    "TestSuiteRunDeploymentReleaseTagExecConfig",
+    "TestSuiteRunDeploymentReleaseTagExecConfigData",
+    "TestSuiteRunDeploymentReleaseTagExecConfigDataRequest",
+    "TestSuiteRunDeploymentReleaseTagExecConfigRequest",
+    "TestSuiteRunDeploymentReleaseTagExecConfigTypeEnum",
+    "TestSuiteRunExecConfig",
+    "TestSuiteRunExecConfigRequest",
+    "TestSuiteRunExecConfigRequest_DeploymentReleaseTag",
+    "TestSuiteRunExecConfigRequest_WorkflowReleaseTag",
+    "TestSuiteRunExecConfig_DeploymentReleaseTag",
+    "TestSuiteRunExecConfig_WorkflowReleaseTag",
+    "TestSuiteRunExecution",
+    "TestSuiteRunExecutionChatHistoryOutput",
+    "TestSuiteRunExecutionErrorOutput",
+    "TestSuiteRunExecutionJsonOutput",
+    "TestSuiteRunExecutionMetricResult",
+    "TestSuiteRunExecutionNumberOutput",
+    "TestSuiteRunExecutionOutput",
+    "TestSuiteRunExecutionOutput_ChatHistory",
+    "TestSuiteRunExecutionOutput_Error",
+    "TestSuiteRunExecutionOutput_Json",
+    "TestSuiteRunExecutionOutput_Number",
+    "TestSuiteRunExecutionOutput_SearchResults",
+    "TestSuiteRunExecutionOutput_String",
+    "TestSuiteRunExecutionSearchResultsOutput",
+    "TestSuiteRunExecutionStringOutput",
+    "TestSuiteRunMetricErrorOutput",
+    "TestSuiteRunMetricErrorOutputTypeEnum",
+    "TestSuiteRunMetricNumberOutput",
+    "TestSuiteRunMetricNumberOutputTypeEnum",
+    "TestSuiteRunMetricOutput",
+    "TestSuiteRunMetricOutput_Error",
+    "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunRead",
+    "TestSuiteRunState",
+    "TestSuiteRunTestSuite",
+    "TestSuiteRunWorkflowReleaseTagExecConfig",
+    "TestSuiteRunWorkflowReleaseTagExecConfigData",
+    "TestSuiteRunWorkflowReleaseTagExecConfigDataRequest",
+    "TestSuiteRunWorkflowReleaseTagExecConfigRequest",
+    "TestSuiteRunWorkflowReleaseTagExecConfigTypeEnum",
     "TestSuiteTestCase",
     "UploadDocumentErrorResponse",
     "UploadDocumentResponse",

vellum/types/paginated_test_suite_run_execution_list.py

@@ -0,0 +1,32 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_execution import TestSuiteRunExecution
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class PaginatedTestSuiteRunExecutionList(pydantic.BaseModel):
+    count: typing.Optional[int]
+    next: typing.Optional[str]
+    previous: typing.Optional[str]
+    results: typing.Optional[typing.List[TestSuiteRunExecution]]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_deployment_release_tag_exec_config.py

@@ -0,0 +1,36 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_deployment_release_tag_exec_config_data import TestSuiteRunDeploymentReleaseTagExecConfigData
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunDeploymentReleaseTagExecConfig(pydantic.BaseModel):
+    """
+    Execution configuration for running a Test Suite against a Prompt Deployment
+    """
+
+    data: TestSuiteRunDeploymentReleaseTagExecConfigData
+    test_case_ids: typing.Optional[typing.List[str]] = pydantic.Field(
+        description="Optionally specify a subset of test case ids to run. If not provided, all test cases within the test suite will be run by default."
+    )
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_deployment_release_tag_exec_config_data.py

@@ -0,0 +1,31 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunDeploymentReleaseTagExecConfigData(pydantic.BaseModel):
+    deployment_id: str = pydantic.Field(description="The ID of the Prompt Deployment to run the Test Suite against.")
+    tag: typing.Optional[str] = pydantic.Field(
+        description="A tag identifying which release of the Prompt Deployment to run the Test Suite against. Useful for testing past versions of the Prompt Deployment"
+    )
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_deployment_release_tag_exec_config_data_request.py

@@ -0,0 +1,31 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunDeploymentReleaseTagExecConfigDataRequest(pydantic.BaseModel):
+    deployment_id: str = pydantic.Field(description="The ID of the Prompt Deployment to run the Test Suite against.")
+    tag: typing.Optional[str] = pydantic.Field(
+        description="A tag identifying which release of the Prompt Deployment to run the Test Suite against. Useful for testing past versions of the Prompt Deployment"
+    )
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        json_encoders = {dt.datetime: serialize_datetime}