vellum-ai 0.6.9__py3-none-any.whl → 0.7.1__py3-none-any.whl

vellum/__init__.py

@@ -205,6 +205,7 @@ from .types import (
     MapNodeResultData,
     MergeEnum,
     MergeNodeResult,
+    MergeNodeResultData,
     MetadataFilterConfigRequest,
     MetadataFilterRuleCombinator,
     MetadataFilterRuleRequest,
@@ -454,6 +455,7 @@ from .types import (
     TestSuiteRunExecConfig_External,
     TestSuiteRunExecConfig_WorkflowReleaseTag,
     TestSuiteRunExecution,
+    TestSuiteRunExecutionArrayOutput,
     TestSuiteRunExecutionChatHistoryOutput,
     TestSuiteRunExecutionErrorOutput,
     TestSuiteRunExecutionFunctionCallOutput,
@@ -462,6 +464,7 @@ from .types import (
     TestSuiteRunExecutionMetricResult,
     TestSuiteRunExecutionNumberOutput,
     TestSuiteRunExecutionOutput,
+    TestSuiteRunExecutionOutput_Array,
     TestSuiteRunExecutionOutput_ChatHistory,
     TestSuiteRunExecutionOutput_Error,
     TestSuiteRunExecutionOutput_FunctionCall,
@@ -499,6 +502,7 @@ from .types import (
     TestSuiteTestCaseBulkOperationRequest_Create,
     TestSuiteTestCaseBulkOperationRequest_Delete,
     TestSuiteTestCaseBulkOperationRequest_Replace,
+    TestSuiteTestCaseBulkOperationRequest_Upsert,
     TestSuiteTestCaseBulkResult,
     TestSuiteTestCaseBulkResult_Created,
     TestSuiteTestCaseBulkResult_Deleted,
@@ -515,6 +519,7 @@ from .types import (
     TestSuiteTestCaseReplaceBulkOperationRequest,
     TestSuiteTestCaseReplacedBulkResult,
     TestSuiteTestCaseReplacedBulkResultData,
+    TestSuiteTestCaseUpsertBulkOperationRequest,
     TextEmbedding3LargeEnum,
     TextEmbedding3SmallEnum,
     TextEmbeddingAda002Enum,
@@ -525,6 +530,8 @@ from .types import (
     TokenOverlappingWindowChunkingRequest,
     UploadDocumentErrorResponse,
     UploadDocumentResponse,
+    UpsertEnum,
+    UpsertTestSuiteTestCaseRequest,
     VellumError,
     VellumErrorCodeEnum,
     VellumErrorRequest,
@@ -839,6 +846,7 @@ __all__ = [
     "MapNodeResultData",
     "MergeEnum",
     "MergeNodeResult",
+    "MergeNodeResultData",
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
@@ -1089,6 +1097,7 @@ __all__ = [
     "TestSuiteRunExecConfig_External",
     "TestSuiteRunExecConfig_WorkflowReleaseTag",
     "TestSuiteRunExecution",
+    "TestSuiteRunExecutionArrayOutput",
     "TestSuiteRunExecutionChatHistoryOutput",
     "TestSuiteRunExecutionErrorOutput",
     "TestSuiteRunExecutionFunctionCallOutput",
@@ -1097,6 +1106,7 @@ __all__ = [
     "TestSuiteRunExecutionMetricResult",
     "TestSuiteRunExecutionNumberOutput",
     "TestSuiteRunExecutionOutput",
+    "TestSuiteRunExecutionOutput_Array",
     "TestSuiteRunExecutionOutput_ChatHistory",
     "TestSuiteRunExecutionOutput_Error",
     "TestSuiteRunExecutionOutput_FunctionCall",
@@ -1134,6 +1144,7 @@ __all__ = [
     "TestSuiteTestCaseBulkOperationRequest_Create",
     "TestSuiteTestCaseBulkOperationRequest_Delete",
     "TestSuiteTestCaseBulkOperationRequest_Replace",
+    "TestSuiteTestCaseBulkOperationRequest_Upsert",
     "TestSuiteTestCaseBulkResult",
     "TestSuiteTestCaseBulkResult_Created",
     "TestSuiteTestCaseBulkResult_Deleted",
@@ -1150,6 +1161,7 @@ __all__ = [
     "TestSuiteTestCaseReplaceBulkOperationRequest",
     "TestSuiteTestCaseReplacedBulkResult",
     "TestSuiteTestCaseReplacedBulkResultData",
+    "TestSuiteTestCaseUpsertBulkOperationRequest",
     "TextEmbedding3LargeEnum",
     "TextEmbedding3SmallEnum",
     "TextEmbeddingAda002Enum",
@@ -1160,6 +1172,8 @@ __all__ = [
     "TokenOverlappingWindowChunkingRequest",
     "UploadDocumentErrorResponse",
     "UploadDocumentResponse",
+    "UpsertEnum",
+    "UpsertTestSuiteTestCaseRequest",
     "VellumEnvironment",
     "VellumError",
     "VellumErrorCodeEnum",

vellum/client.py

@@ -121,23 +121,23 @@ class Vellum:
         Executes a deployed Prompt and returns the result.
 
         Parameters:
-            - inputs: typing.Sequence[PromptDeploymentInputRequest]. The list of inputs defined in the Prompt's deployment with their corresponding values.
+            - inputs: typing.Sequence[PromptDeploymentInputRequest]. A list consisting of the Prompt Deployment's input variables and their values.
 
             - prompt_deployment_id: typing.Optional[str]. The ID of the Prompt Deployment. Must provide either this or prompt_deployment_name.
 
-            - prompt_deployment_name: typing.Optional[str]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - prompt_deployment_name: typing.Optional[str]. The unique name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
 
             - release_tag: typing.Optional[str]. Optionally specify a release tag if you want to pin to a specific release of the Prompt Deployment
 
-            - external_id: typing.Optional[str]. "Optionally include a unique identifier for tracking purposes. Must be unique for a given prompt deployment.
+            - external_id: typing.Optional[str]. Optionally include a unique identifier for tracking purposes. Must be unique within a given Prompt Deployment.
 
-            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. An optionally specified configuration used to opt in to including additional metadata about this prompt execution in the API response. Corresponding values will be returned under the `meta` key of the API response.
 
-            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest].
+            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest]. Overrides for the raw API request sent to the model host. Combined with `expand_raw`, it can be used to access new features from models.
 
-            - expand_raw: typing.Optional[typing.Sequence[str]]. Returns the raw API response data sent from the model host. Combined with `raw_overrides`, it can be used to access new features from models.
+            - expand_raw: typing.Optional[typing.Sequence[str]]. A list of keys whose values you'd like to directly return from the JSON response of the model provider. Useful if you need lower-level info returned by model providers that Vellum would otherwise omit. Corresponding key/value pairs will be returned under the `raw` key of the API response.
 
-            - metadata: typing.Optional[typing.Dict[str, typing.Any]].
+            - metadata: typing.Optional[typing.Dict[str, typing.Any]]. Arbitrary JSON metadata associated with this request. Can be used to capture additional monitoring data such as user id, session id, etc. for future analysis.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---
@@ -256,23 +256,23 @@ class Vellum:
         Executes a deployed Prompt and streams back the results.
 
         Parameters:
-            - inputs: typing.Sequence[PromptDeploymentInputRequest]. The list of inputs defined in the Prompt's deployment with their corresponding values.
+            - inputs: typing.Sequence[PromptDeploymentInputRequest]. A list consisting of the Prompt Deployment's input variables and their values.
 
             - prompt_deployment_id: typing.Optional[str]. The ID of the Prompt Deployment. Must provide either this or prompt_deployment_name.
 
-            - prompt_deployment_name: typing.Optional[str]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - prompt_deployment_name: typing.Optional[str]. The unique name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
 
             - release_tag: typing.Optional[str]. Optionally specify a release tag if you want to pin to a specific release of the Prompt Deployment
 
-            - external_id: typing.Optional[str]. "Optionally include a unique identifier for tracking purposes. Must be unique for a given prompt deployment.
+            - external_id: typing.Optional[str]. Optionally include a unique identifier for tracking purposes. Must be unique within a given Prompt Deployment.
 
-            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. An optionally specified configuration used to opt in to including additional metadata about this prompt execution in the API response. Corresponding values will be returned under the `meta` key of the API response.
 
-            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest].
+            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest]. Overrides for the raw API request sent to the model host. Combined with `expand_raw`, it can be used to access new features from models.
 
-            - expand_raw: typing.Optional[typing.Sequence[str]]. Returns the raw API response data sent from the model host. Combined with `raw_overrides`, it can be used to access new features from models.
+            - expand_raw: typing.Optional[typing.Sequence[str]]. A list of keys whose values you'd like to directly return from the JSON response of the model provider. Useful if you need lower-level info returned by model providers that Vellum would otherwise omit. Corresponding key/value pairs will be returned under the `raw` key of the API response.
 
-            - metadata: typing.Optional[typing.Dict[str, typing.Any]].
+            - metadata: typing.Optional[typing.Dict[str, typing.Any]]. Arbitrary JSON metadata associated with this request. Can be used to capture additional monitoring data such as user id, session id, etc. for future analysis.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---
@@ -1082,23 +1082,23 @@ class AsyncVellum:
         Executes a deployed Prompt and returns the result.
 
         Parameters:
-            - inputs: typing.Sequence[PromptDeploymentInputRequest]. The list of inputs defined in the Prompt's deployment with their corresponding values.
+            - inputs: typing.Sequence[PromptDeploymentInputRequest]. A list consisting of the Prompt Deployment's input variables and their values.
 
             - prompt_deployment_id: typing.Optional[str]. The ID of the Prompt Deployment. Must provide either this or prompt_deployment_name.
 
-            - prompt_deployment_name: typing.Optional[str]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - prompt_deployment_name: typing.Optional[str]. The unique name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
 
             - release_tag: typing.Optional[str]. Optionally specify a release tag if you want to pin to a specific release of the Prompt Deployment
 
-            - external_id: typing.Optional[str]. "Optionally include a unique identifier for tracking purposes. Must be unique for a given prompt deployment.
+            - external_id: typing.Optional[str]. Optionally include a unique identifier for tracking purposes. Must be unique within a given Prompt Deployment.
 
-            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. An optionally specified configuration used to opt in to including additional metadata about this prompt execution in the API response. Corresponding values will be returned under the `meta` key of the API response.
 
-            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest].
+            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest]. Overrides for the raw API request sent to the model host. Combined with `expand_raw`, it can be used to access new features from models.
 
-            - expand_raw: typing.Optional[typing.Sequence[str]]. Returns the raw API response data sent from the model host. Combined with `raw_overrides`, it can be used to access new features from models.
+            - expand_raw: typing.Optional[typing.Sequence[str]]. A list of keys whose values you'd like to directly return from the JSON response of the model provider. Useful if you need lower-level info returned by model providers that Vellum would otherwise omit. Corresponding key/value pairs will be returned under the `raw` key of the API response.
 
-            - metadata: typing.Optional[typing.Dict[str, typing.Any]].
+            - metadata: typing.Optional[typing.Dict[str, typing.Any]]. Arbitrary JSON metadata associated with this request. Can be used to capture additional monitoring data such as user id, session id, etc. for future analysis.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---
@@ -1217,23 +1217,23 @@ class AsyncVellum:
         Executes a deployed Prompt and streams back the results.
 
         Parameters:
-            - inputs: typing.Sequence[PromptDeploymentInputRequest]. The list of inputs defined in the Prompt's deployment with their corresponding values.
+            - inputs: typing.Sequence[PromptDeploymentInputRequest]. A list consisting of the Prompt Deployment's input variables and their values.
 
             - prompt_deployment_id: typing.Optional[str]. The ID of the Prompt Deployment. Must provide either this or prompt_deployment_name.
 
-            - prompt_deployment_name: typing.Optional[str]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - prompt_deployment_name: typing.Optional[str]. The unique name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
 
             - release_tag: typing.Optional[str]. Optionally specify a release tag if you want to pin to a specific release of the Prompt Deployment
 
-            - external_id: typing.Optional[str]. "Optionally include a unique identifier for tracking purposes. Must be unique for a given prompt deployment.
+            - external_id: typing.Optional[str]. Optionally include a unique identifier for tracking purposes. Must be unique within a given Prompt Deployment.
 
-            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. The name of the Prompt Deployment. Must provide either this or prompt_deployment_id.
+            - expand_meta: typing.Optional[PromptDeploymentExpandMetaRequestRequest]. An optionally specified configuration used to opt in to including additional metadata about this prompt execution in the API response. Corresponding values will be returned under the `meta` key of the API response.
 
-            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest].
+            - raw_overrides: typing.Optional[RawPromptExecutionOverridesRequest]. Overrides for the raw API request sent to the model host. Combined with `expand_raw`, it can be used to access new features from models.
 
-            - expand_raw: typing.Optional[typing.Sequence[str]]. Returns the raw API response data sent from the model host. Combined with `raw_overrides`, it can be used to access new features from models.
+            - expand_raw: typing.Optional[typing.Sequence[str]]. A list of keys whose values you'd like to directly return from the JSON response of the model provider. Useful if you need lower-level info returned by model providers that Vellum would otherwise omit. Corresponding key/value pairs will be returned under the `raw` key of the API response.
 
-            - metadata: typing.Optional[typing.Dict[str, typing.Any]].
+            - metadata: typing.Optional[typing.Dict[str, typing.Any]]. Arbitrary JSON metadata associated with this request. Can be used to capture additional monitoring data such as user id, session id, etc. for future analysis.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---

vellum/core/client_wrapper.py

@@ -18,7 +18,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.6.9",
+            "X-Fern-SDK-Version": "0.7.1",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

vellum/lib/test_suites/resources.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import time
 from functools import cached_property
-from typing import Callable, Generator, List, Any
+from typing import Callable, Generator, List, cast, Iterable
 
 from vellum import TestSuiteRunRead, TestSuiteRunMetricOutput_Number
 from vellum.client import Vellum
@@ -174,7 +174,7 @@ class VellumTestSuiteRunResults:
         self,
         metric_identifier: str | None = None,
         output_identifier: str | None = None,
-    ) -> List[float]:
+    ) -> List[float | None]:
         """Returns the values of a numeric metric output that match the given criteria."""
 
         metric_outputs: list[TestSuiteRunMetricOutput_Number] = []
@@ -198,7 +198,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return sum(output_values) / len(output_values)
+        return sum(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values))) / len(output_values)
 
     def get_min_metric_output(
         self, metric_identifier: str | None = None, output_identifier: str | None = None
@@ -207,7 +207,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return min(output_values)
+        return min(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values)))
 
     def get_max_metric_output(
         self, metric_identifier: str | None = None, output_identifier: str | None = None
@@ -216,7 +216,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return max(output_values)
+        return max(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values)))
 
     def wait_until_complete(self) -> None:
         """Wait until the Test Suite Run is no longer in a QUEUED or RUNNING state."""

vellum/resources/document_indexes/client.py

@@ -458,6 +458,63 @@ class DocumentIndexesClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    def add_document(
+        self, document_id: str, id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> None:
+        """
+        Adds a previously uploaded Document to the specified Document Index.
+
+        Parameters:
+            - document_id: str. Either the Vellum-generated ID or the originally supplied external_id that uniquely identifies the Document you'd like to add.
+
+            - id: str. Either the Vellum-generated ID or the originally specified name that uniquely identifies the Document Index to which you'd like to add the Document.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.document_indexes.add_document(
+            document_id="document_id",
+            id="id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            method="POST",
+            url=urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/document-indexes/{jsonable_encoder(id)}/documents/{jsonable_encoder(document_id)}",
+            ),
+            params=jsonable_encoder(
+                request_options.get("additional_query_parameters") if request_options is not None else None
+            ),
+            json=jsonable_encoder(remove_none_from_dict(request_options.get("additional_body_parameters", {})))
+            if request_options is not None
+            else None,
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     def remove_document(
         self, document_id: str, id: str, *, request_options: typing.Optional[RequestOptions] = None
     ) -> None:
@@ -950,6 +1007,63 @@ class AsyncDocumentIndexesClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    async def add_document(
+        self, document_id: str, id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> None:
+        """
+        Adds a previously uploaded Document to the specified Document Index.
+
+        Parameters:
+            - document_id: str. Either the Vellum-generated ID or the originally supplied external_id that uniquely identifies the Document you'd like to add.
+
+            - id: str. Either the Vellum-generated ID or the originally specified name that uniquely identifies the Document Index to which you'd like to add the Document.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+        await client.document_indexes.add_document(
+            document_id="document_id",
+            id="id",
+        )
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            method="POST",
+            url=urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/document-indexes/{jsonable_encoder(id)}/documents/{jsonable_encoder(document_id)}",
+            ),
+            params=jsonable_encoder(
+                request_options.get("additional_query_parameters") if request_options is not None else None
+            ),
+            json=jsonable_encoder(remove_none_from_dict(request_options.get("additional_body_parameters", {})))
+            if request_options is not None
+            else None,
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     async def remove_document(
         self, document_id: str, id: str, *, request_options: typing.Optional[RequestOptions] = None
     ) -> None:

vellum/resources/test_suites/client.py

@@ -11,11 +11,11 @@ from ...core.jsonable_encoder import jsonable_encoder
 from ...core.pydantic_utilities import pydantic_v1
 from ...core.remove_none_from_dict import remove_none_from_dict
 from ...core.request_options import RequestOptions
-from ...types.named_test_case_variable_value_request import NamedTestCaseVariableValueRequest
 from ...types.paginated_test_suite_test_case_list import PaginatedTestSuiteTestCaseList
 from ...types.test_suite_test_case import TestSuiteTestCase
 from ...types.test_suite_test_case_bulk_operation_request import TestSuiteTestCaseBulkOperationRequest
 from ...types.test_suite_test_case_bulk_result import TestSuiteTestCaseBulkResult
+from ...types.upsert_test_suite_test_case_request import UpsertTestSuiteTestCaseRequest
 
 # this is used as the default value for optional parameters
 OMIT = typing.cast(typing.Any, ...)
@@ -99,11 +99,7 @@ class TestSuitesClient:
         self,
         id: str,
         *,
-        upsert_test_suite_test_case_request_id: typing.Optional[str] = OMIT,
-        external_id: typing.Optional[str] = OMIT,
-        label: typing.Optional[str] = OMIT,
-        input_values: typing.Sequence[NamedTestCaseVariableValueRequest],
-        evaluation_values: typing.Sequence[NamedTestCaseVariableValueRequest],
+        request: UpsertTestSuiteTestCaseRequest,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> TestSuiteTestCase:
         """
@@ -118,18 +114,11 @@ class TestSuitesClient:
         Parameters:
             - id: str. A UUID string identifying this test suite.
 
-            - upsert_test_suite_test_case_request_id: typing.Optional[str]. The Vellum-generated ID of an existing Test Case whose data you'd like to replace. If specified and no Test Case exists with this ID, a 404 will be returned.
-
-            - external_id: typing.Optional[str]. An ID external to Vellum that uniquely identifies the Test Case that you'd like to create/update. If there's a match on a Test Case that was previously created with the same external_id, it will be updated. Otherwise, a new Test Case will be created with this value as its external_id. If no external_id is specified, then a new Test Case will always be created.
-
-            - label: typing.Optional[str]. A human-readable label used to convey the intention of this Test Case
-
-            - input_values: typing.Sequence[NamedTestCaseVariableValueRequest]. Values for each of the Test Case's input variables
-
-            - evaluation_values: typing.Sequence[NamedTestCaseVariableValueRequest]. Values for each of the Test Case's evaluation variables
+            - request: UpsertTestSuiteTestCaseRequest.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---
+        from vellum import UpsertTestSuiteTestCaseRequest
         from vellum.client import Vellum
 
         client = Vellum(
@@ -137,17 +126,12 @@ class TestSuitesClient:
         )
         client.test_suites.upsert_test_suite_test_case(
             id="id",
-            input_values=[],
-            evaluation_values=[],
+            request=UpsertTestSuiteTestCaseRequest(
+                input_values=[],
+                evaluation_values=[],
+            ),
         )
         """
-        _request: typing.Dict[str, typing.Any] = {"input_values": input_values, "evaluation_values": evaluation_values}
-        if upsert_test_suite_test_case_request_id is not OMIT:
-            _request["id"] = upsert_test_suite_test_case_request_id
-        if external_id is not OMIT:
-            _request["external_id"] = external_id
-        if label is not OMIT:
-            _request["label"] = label
         _response = self._client_wrapper.httpx_client.request(
             method="POST",
             url=urllib.parse.urljoin(
@@ -157,10 +141,10 @@ class TestSuitesClient:
             params=jsonable_encoder(
                 request_options.get("additional_query_parameters") if request_options is not None else None
             ),
-            json=jsonable_encoder(_request)
+            json=jsonable_encoder(request)
             if request_options is None or request_options.get("additional_body_parameters") is None
             else {
-                **jsonable_encoder(_request),
+                **jsonable_encoder(request),
                 **(jsonable_encoder(remove_none_from_dict(request_options.get("additional_body_parameters", {})))),
             },
             headers=jsonable_encoder(
@@ -402,11 +386,7 @@ class AsyncTestSuitesClient:
         self,
         id: str,
         *,
-        upsert_test_suite_test_case_request_id: typing.Optional[str] = OMIT,
-        external_id: typing.Optional[str] = OMIT,
-        label: typing.Optional[str] = OMIT,
-        input_values: typing.Sequence[NamedTestCaseVariableValueRequest],
-        evaluation_values: typing.Sequence[NamedTestCaseVariableValueRequest],
+        request: UpsertTestSuiteTestCaseRequest,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> TestSuiteTestCase:
         """
@@ -421,18 +401,11 @@ class AsyncTestSuitesClient:
         Parameters:
             - id: str. A UUID string identifying this test suite.
 
-            - upsert_test_suite_test_case_request_id: typing.Optional[str]. The Vellum-generated ID of an existing Test Case whose data you'd like to replace. If specified and no Test Case exists with this ID, a 404 will be returned.
-
-            - external_id: typing.Optional[str]. An ID external to Vellum that uniquely identifies the Test Case that you'd like to create/update. If there's a match on a Test Case that was previously created with the same external_id, it will be updated. Otherwise, a new Test Case will be created with this value as its external_id. If no external_id is specified, then a new Test Case will always be created.
-
-            - label: typing.Optional[str]. A human-readable label used to convey the intention of this Test Case
-
-            - input_values: typing.Sequence[NamedTestCaseVariableValueRequest]. Values for each of the Test Case's input variables
-
-            - evaluation_values: typing.Sequence[NamedTestCaseVariableValueRequest]. Values for each of the Test Case's evaluation variables
+            - request: UpsertTestSuiteTestCaseRequest.
 
             - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
         ---
+        from vellum import UpsertTestSuiteTestCaseRequest
         from vellum.client import AsyncVellum
 
         client = AsyncVellum(
@@ -440,17 +413,12 @@ class AsyncTestSuitesClient:
         )
         await client.test_suites.upsert_test_suite_test_case(
             id="id",
-            input_values=[],
-            evaluation_values=[],
+            request=UpsertTestSuiteTestCaseRequest(
+                input_values=[],
+                evaluation_values=[],
+            ),
         )
         """
-        _request: typing.Dict[str, typing.Any] = {"input_values": input_values, "evaluation_values": evaluation_values}
-        if upsert_test_suite_test_case_request_id is not OMIT:
-            _request["id"] = upsert_test_suite_test_case_request_id
-        if external_id is not OMIT:
-            _request["external_id"] = external_id
-        if label is not OMIT:
-            _request["label"] = label
         _response = await self._client_wrapper.httpx_client.request(
             method="POST",
             url=urllib.parse.urljoin(
@@ -460,10 +428,10 @@ class AsyncTestSuitesClient:
             params=jsonable_encoder(
                 request_options.get("additional_query_parameters") if request_options is not None else None
             ),
-            json=jsonable_encoder(_request)
+            json=jsonable_encoder(request)
             if request_options is None or request_options.get("additional_body_parameters") is None
             else {
-                **jsonable_encoder(_request),
+                **jsonable_encoder(request),
                 **(jsonable_encoder(remove_none_from_dict(request_options.get("additional_body_parameters", {})))),
             },
             headers=jsonable_encoder(

vellum/types/__init__.py

@@ -242,6 +242,7 @@ from .map_node_result import MapNodeResult
 from .map_node_result_data import MapNodeResultData
 from .merge_enum import MergeEnum
 from .merge_node_result import MergeNodeResult
+from .merge_node_result_data import MergeNodeResultData
 from .metadata_filter_config_request import MetadataFilterConfigRequest
 from .metadata_filter_rule_combinator import MetadataFilterRuleCombinator
 from .metadata_filter_rule_request import MetadataFilterRuleRequest
@@ -517,6 +518,7 @@ from .test_suite_run_exec_config_request import (
     TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
 )
 from .test_suite_run_execution import TestSuiteRunExecution
+from .test_suite_run_execution_array_output import TestSuiteRunExecutionArrayOutput
 from .test_suite_run_execution_chat_history_output import TestSuiteRunExecutionChatHistoryOutput
 from .test_suite_run_execution_error_output import TestSuiteRunExecutionErrorOutput
 from .test_suite_run_execution_function_call_output import TestSuiteRunExecutionFunctionCallOutput
@@ -526,6 +528,7 @@ from .test_suite_run_execution_metric_result import TestSuiteRunExecutionMetricR
 from .test_suite_run_execution_number_output import TestSuiteRunExecutionNumberOutput
 from .test_suite_run_execution_output import (
     TestSuiteRunExecutionOutput,
+    TestSuiteRunExecutionOutput_Array,
     TestSuiteRunExecutionOutput_ChatHistory,
     TestSuiteRunExecutionOutput_Error,
     TestSuiteRunExecutionOutput_FunctionCall,
@@ -569,6 +572,7 @@ from .test_suite_test_case_bulk_operation_request import (
     TestSuiteTestCaseBulkOperationRequest_Create,
     TestSuiteTestCaseBulkOperationRequest_Delete,
     TestSuiteTestCaseBulkOperationRequest_Replace,
+    TestSuiteTestCaseBulkOperationRequest_Upsert,
 )
 from .test_suite_test_case_bulk_result import (
     TestSuiteTestCaseBulkResult,
@@ -588,6 +592,7 @@ from .test_suite_test_case_rejected_bulk_result import TestSuiteTestCaseRejected
 from .test_suite_test_case_replace_bulk_operation_request import TestSuiteTestCaseReplaceBulkOperationRequest
 from .test_suite_test_case_replaced_bulk_result import TestSuiteTestCaseReplacedBulkResult
 from .test_suite_test_case_replaced_bulk_result_data import TestSuiteTestCaseReplacedBulkResultData
+from .test_suite_test_case_upsert_bulk_operation_request import TestSuiteTestCaseUpsertBulkOperationRequest
 from .text_embedding_3_large_enum import TextEmbedding3LargeEnum
 from .text_embedding_3_small_enum import TextEmbedding3SmallEnum
 from .text_embedding_ada_002_enum import TextEmbeddingAda002Enum
@@ -598,6 +603,8 @@ from .token_overlapping_window_chunking import TokenOverlappingWindowChunking
 from .token_overlapping_window_chunking_request import TokenOverlappingWindowChunkingRequest
 from .upload_document_error_response import UploadDocumentErrorResponse
 from .upload_document_response import UploadDocumentResponse
+from .upsert_enum import UpsertEnum
+from .upsert_test_suite_test_case_request import UpsertTestSuiteTestCaseRequest
 from .vellum_error import VellumError
 from .vellum_error_code_enum import VellumErrorCodeEnum
 from .vellum_error_request import VellumErrorRequest
@@ -897,6 +904,7 @@ __all__ = [
     "MapNodeResultData",
     "MergeEnum",
     "MergeNodeResult",
+    "MergeNodeResultData",
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
@@ -1146,6 +1154,7 @@ __all__ = [
     "TestSuiteRunExecConfig_External",
     "TestSuiteRunExecConfig_WorkflowReleaseTag",
     "TestSuiteRunExecution",
+    "TestSuiteRunExecutionArrayOutput",
     "TestSuiteRunExecutionChatHistoryOutput",
     "TestSuiteRunExecutionErrorOutput",
     "TestSuiteRunExecutionFunctionCallOutput",
@@ -1154,6 +1163,7 @@ __all__ = [
     "TestSuiteRunExecutionMetricResult",
     "TestSuiteRunExecutionNumberOutput",
     "TestSuiteRunExecutionOutput",
+    "TestSuiteRunExecutionOutput_Array",
     "TestSuiteRunExecutionOutput_ChatHistory",
     "TestSuiteRunExecutionOutput_Error",
     "TestSuiteRunExecutionOutput_FunctionCall",
@@ -1191,6 +1201,7 @@ __all__ = [
     "TestSuiteTestCaseBulkOperationRequest_Create",
     "TestSuiteTestCaseBulkOperationRequest_Delete",
     "TestSuiteTestCaseBulkOperationRequest_Replace",
+    "TestSuiteTestCaseBulkOperationRequest_Upsert",
     "TestSuiteTestCaseBulkResult",
     "TestSuiteTestCaseBulkResult_Created",
     "TestSuiteTestCaseBulkResult_Deleted",
@@ -1207,6 +1218,7 @@ __all__ = [
     "TestSuiteTestCaseReplaceBulkOperationRequest",
     "TestSuiteTestCaseReplacedBulkResult",
     "TestSuiteTestCaseReplacedBulkResultData",
+    "TestSuiteTestCaseUpsertBulkOperationRequest",
     "TextEmbedding3LargeEnum",
     "TextEmbedding3SmallEnum",
     "TextEmbeddingAda002Enum",
@@ -1217,6 +1229,8 @@ __all__ = [
     "TokenOverlappingWindowChunkingRequest",
     "UploadDocumentErrorResponse",
     "UploadDocumentResponse",
+    "UpsertEnum",
+    "UpsertTestSuiteTestCaseRequest",
     "VellumError",
     "VellumErrorCodeEnum",
     "VellumErrorRequest",