vellum-ai 0.3.22__py3-none-any.whl → 0.3.23__py3-none-any.whl

vellum/__init__.py

@@ -104,6 +104,8 @@ from .types import (
     ExecutionVellumValue_Number,
     ExecutionVellumValue_SearchResults,
     ExecutionVellumValue_String,
+    ExternalTestCaseExecution,
+    ExternalTestCaseExecutionRequest,
     FinishReasonEnum,
     FulfilledEnum,
     FulfilledExecutePromptEvent,
@@ -155,12 +157,19 @@ from .types import (
     ModelVersionRead,
     ModelVersionReadStatusEnum,
     ModelVersionSandboxSnapshot,
+    NamedTestCaseChatHistoryVariableValue,
     NamedTestCaseChatHistoryVariableValueRequest,
+    NamedTestCaseErrorVariableValue,
     NamedTestCaseErrorVariableValueRequest,
+    NamedTestCaseJsonVariableValue,
     NamedTestCaseJsonVariableValueRequest,
+    NamedTestCaseNumberVariableValue,
     NamedTestCaseNumberVariableValueRequest,
+    NamedTestCaseSearchResultsVariableValue,
     NamedTestCaseSearchResultsVariableValueRequest,
+    NamedTestCaseStringVariableValue,
     NamedTestCaseStringVariableValueRequest,
+    NamedTestCaseVariableValue,
     NamedTestCaseVariableValueRequest,
     NamedTestCaseVariableValueRequest_ChatHistory,
     NamedTestCaseVariableValueRequest_Error,
@@ -168,6 +177,12 @@ from .types import (
     NamedTestCaseVariableValueRequest_Number,
     NamedTestCaseVariableValueRequest_SearchResults,
     NamedTestCaseVariableValueRequest_String,
+    NamedTestCaseVariableValue_ChatHistory,
+    NamedTestCaseVariableValue_Error,
+    NamedTestCaseVariableValue_Json,
+    NamedTestCaseVariableValue_Number,
+    NamedTestCaseVariableValue_SearchResults,
+    NamedTestCaseVariableValue_String,
     NodeInputCompiledArrayValue,
     NodeInputCompiledChatHistoryValue,
     NodeInputCompiledErrorValue,
@@ -350,8 +365,10 @@ from .types import (
     TestSuiteRunExecConfig,
     TestSuiteRunExecConfigRequest,
     TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_External,
     TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
     TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_External,
     TestSuiteRunExecConfig_WorkflowReleaseTag,
     TestSuiteRunExecution,
     TestSuiteRunExecutionChatHistoryOutput,
@@ -369,6 +386,11 @@ from .types import (
     TestSuiteRunExecutionOutput_String,
     TestSuiteRunExecutionSearchResultsOutput,
     TestSuiteRunExecutionStringOutput,
+    TestSuiteRunExternalExecConfig,
+    TestSuiteRunExternalExecConfigData,
+    TestSuiteRunExternalExecConfigDataRequest,
+    TestSuiteRunExternalExecConfigRequest,
+    TestSuiteRunExternalExecConfigTypeEnum,
     TestSuiteRunMetricErrorOutput,
     TestSuiteRunMetricErrorOutputTypeEnum,
     TestSuiteRunMetricNumberOutput,
@@ -598,6 +620,8 @@ __all__ = [
     "ExecutionVellumValue_Number",
     "ExecutionVellumValue_SearchResults",
     "ExecutionVellumValue_String",
+    "ExternalTestCaseExecution",
+    "ExternalTestCaseExecutionRequest",
     "FinishReasonEnum",
     "ForbiddenError",
     "FulfilledEnum",
@@ -651,12 +675,19 @@ __all__ = [
     "ModelVersionRead",
     "ModelVersionReadStatusEnum",
     "ModelVersionSandboxSnapshot",
+    "NamedTestCaseChatHistoryVariableValue",
     "NamedTestCaseChatHistoryVariableValueRequest",
+    "NamedTestCaseErrorVariableValue",
     "NamedTestCaseErrorVariableValueRequest",
+    "NamedTestCaseJsonVariableValue",
     "NamedTestCaseJsonVariableValueRequest",
+    "NamedTestCaseNumberVariableValue",
     "NamedTestCaseNumberVariableValueRequest",
+    "NamedTestCaseSearchResultsVariableValue",
     "NamedTestCaseSearchResultsVariableValueRequest",
+    "NamedTestCaseStringVariableValue",
     "NamedTestCaseStringVariableValueRequest",
+    "NamedTestCaseVariableValue",
     "NamedTestCaseVariableValueRequest",
     "NamedTestCaseVariableValueRequest_ChatHistory",
     "NamedTestCaseVariableValueRequest_Error",
@@ -664,6 +695,12 @@ __all__ = [
     "NamedTestCaseVariableValueRequest_Number",
     "NamedTestCaseVariableValueRequest_SearchResults",
     "NamedTestCaseVariableValueRequest_String",
+    "NamedTestCaseVariableValue_ChatHistory",
+    "NamedTestCaseVariableValue_Error",
+    "NamedTestCaseVariableValue_Json",
+    "NamedTestCaseVariableValue_Number",
+    "NamedTestCaseVariableValue_SearchResults",
+    "NamedTestCaseVariableValue_String",
     "NodeInputCompiledArrayValue",
     "NodeInputCompiledChatHistoryValue",
     "NodeInputCompiledErrorValue",
@@ -847,8 +884,10 @@ __all__ = [
     "TestSuiteRunExecConfig",
     "TestSuiteRunExecConfigRequest",
     "TestSuiteRunExecConfigRequest_DeploymentReleaseTag",
+    "TestSuiteRunExecConfigRequest_External",
     "TestSuiteRunExecConfigRequest_WorkflowReleaseTag",
     "TestSuiteRunExecConfig_DeploymentReleaseTag",
+    "TestSuiteRunExecConfig_External",
     "TestSuiteRunExecConfig_WorkflowReleaseTag",
     "TestSuiteRunExecution",
     "TestSuiteRunExecutionChatHistoryOutput",
@@ -866,6 +905,11 @@ __all__ = [
     "TestSuiteRunExecutionOutput_String",
     "TestSuiteRunExecutionSearchResultsOutput",
     "TestSuiteRunExecutionStringOutput",
+    "TestSuiteRunExternalExecConfig",
+    "TestSuiteRunExternalExecConfigData",
+    "TestSuiteRunExternalExecConfigDataRequest",
+    "TestSuiteRunExternalExecConfigRequest",
+    "TestSuiteRunExternalExecConfigTypeEnum",
     "TestSuiteRunMetricErrorOutput",
     "TestSuiteRunMetricErrorOutputTypeEnum",
     "TestSuiteRunMetricNumberOutput",

vellum/core/client_wrapper.py

@@ -18,7 +18,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.3.22",
+            "X-Fern-SDK-Version": "0.3.23",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

vellum/types/__init__.py

@@ -123,6 +123,8 @@ from .execution_vellum_value import (
     ExecutionVellumValue_SearchResults,
     ExecutionVellumValue_String,
 )
+from .external_test_case_execution import ExternalTestCaseExecution
+from .external_test_case_execution_request import ExternalTestCaseExecutionRequest
 from .finish_reason_enum import FinishReasonEnum
 from .fulfilled_enum import FulfilledEnum
 from .fulfilled_execute_prompt_event import FulfilledExecutePromptEvent
@@ -172,12 +174,27 @@ from .model_version_exec_config_parameters import ModelVersionExecConfigParamete
 from .model_version_read import ModelVersionRead
 from .model_version_read_status_enum import ModelVersionReadStatusEnum
 from .model_version_sandbox_snapshot import ModelVersionSandboxSnapshot
+from .named_test_case_chat_history_variable_value import NamedTestCaseChatHistoryVariableValue
 from .named_test_case_chat_history_variable_value_request import NamedTestCaseChatHistoryVariableValueRequest
+from .named_test_case_error_variable_value import NamedTestCaseErrorVariableValue
 from .named_test_case_error_variable_value_request import NamedTestCaseErrorVariableValueRequest
+from .named_test_case_json_variable_value import NamedTestCaseJsonVariableValue
 from .named_test_case_json_variable_value_request import NamedTestCaseJsonVariableValueRequest
+from .named_test_case_number_variable_value import NamedTestCaseNumberVariableValue
 from .named_test_case_number_variable_value_request import NamedTestCaseNumberVariableValueRequest
+from .named_test_case_search_results_variable_value import NamedTestCaseSearchResultsVariableValue
 from .named_test_case_search_results_variable_value_request import NamedTestCaseSearchResultsVariableValueRequest
+from .named_test_case_string_variable_value import NamedTestCaseStringVariableValue
 from .named_test_case_string_variable_value_request import NamedTestCaseStringVariableValueRequest
+from .named_test_case_variable_value import (
+    NamedTestCaseVariableValue,
+    NamedTestCaseVariableValue_ChatHistory,
+    NamedTestCaseVariableValue_Error,
+    NamedTestCaseVariableValue_Json,
+    NamedTestCaseVariableValue_Number,
+    NamedTestCaseVariableValue_SearchResults,
+    NamedTestCaseVariableValue_String,
+)
 from .named_test_case_variable_value_request import (
     NamedTestCaseVariableValueRequest,
     NamedTestCaseVariableValueRequest_ChatHistory,
@@ -389,11 +406,13 @@ from .test_suite_run_deployment_release_tag_exec_config_type_enum import (
 from .test_suite_run_exec_config import (
     TestSuiteRunExecConfig,
     TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_External,
     TestSuiteRunExecConfig_WorkflowReleaseTag,
 )
 from .test_suite_run_exec_config_request import (
     TestSuiteRunExecConfigRequest,
     TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_External,
     TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
 )
 from .test_suite_run_execution import TestSuiteRunExecution
@@ -414,6 +433,11 @@ from .test_suite_run_execution_output import (
 )
 from .test_suite_run_execution_search_results_output import TestSuiteRunExecutionSearchResultsOutput
 from .test_suite_run_execution_string_output import TestSuiteRunExecutionStringOutput
+from .test_suite_run_external_exec_config import TestSuiteRunExternalExecConfig
+from .test_suite_run_external_exec_config_data import TestSuiteRunExternalExecConfigData
+from .test_suite_run_external_exec_config_data_request import TestSuiteRunExternalExecConfigDataRequest
+from .test_suite_run_external_exec_config_request import TestSuiteRunExternalExecConfigRequest
+from .test_suite_run_external_exec_config_type_enum import TestSuiteRunExternalExecConfigTypeEnum
 from .test_suite_run_metric_error_output import TestSuiteRunMetricErrorOutput
 from .test_suite_run_metric_error_output_type_enum import TestSuiteRunMetricErrorOutputTypeEnum
 from .test_suite_run_metric_number_output import TestSuiteRunMetricNumberOutput
@@ -632,6 +656,8 @@ __all__ = [
     "ExecutionVellumValue_Number",
     "ExecutionVellumValue_SearchResults",
     "ExecutionVellumValue_String",
+    "ExternalTestCaseExecution",
+    "ExternalTestCaseExecutionRequest",
     "FinishReasonEnum",
     "FulfilledEnum",
     "FulfilledExecutePromptEvent",
@@ -683,12 +709,19 @@ __all__ = [
     "ModelVersionRead",
     "ModelVersionReadStatusEnum",
     "ModelVersionSandboxSnapshot",
+    "NamedTestCaseChatHistoryVariableValue",
     "NamedTestCaseChatHistoryVariableValueRequest",
+    "NamedTestCaseErrorVariableValue",
     "NamedTestCaseErrorVariableValueRequest",
+    "NamedTestCaseJsonVariableValue",
     "NamedTestCaseJsonVariableValueRequest",
+    "NamedTestCaseNumberVariableValue",
     "NamedTestCaseNumberVariableValueRequest",
+    "NamedTestCaseSearchResultsVariableValue",
     "NamedTestCaseSearchResultsVariableValueRequest",
+    "NamedTestCaseStringVariableValue",
     "NamedTestCaseStringVariableValueRequest",
+    "NamedTestCaseVariableValue",
     "NamedTestCaseVariableValueRequest",
     "NamedTestCaseVariableValueRequest_ChatHistory",
     "NamedTestCaseVariableValueRequest_Error",
@@ -696,6 +729,12 @@ __all__ = [
     "NamedTestCaseVariableValueRequest_Number",
     "NamedTestCaseVariableValueRequest_SearchResults",
     "NamedTestCaseVariableValueRequest_String",
+    "NamedTestCaseVariableValue_ChatHistory",
+    "NamedTestCaseVariableValue_Error",
+    "NamedTestCaseVariableValue_Json",
+    "NamedTestCaseVariableValue_Number",
+    "NamedTestCaseVariableValue_SearchResults",
+    "NamedTestCaseVariableValue_String",
     "NodeInputCompiledArrayValue",
     "NodeInputCompiledChatHistoryValue",
     "NodeInputCompiledErrorValue",
@@ -878,8 +917,10 @@ __all__ = [
     "TestSuiteRunExecConfig",
     "TestSuiteRunExecConfigRequest",
     "TestSuiteRunExecConfigRequest_DeploymentReleaseTag",
+    "TestSuiteRunExecConfigRequest_External",
     "TestSuiteRunExecConfigRequest_WorkflowReleaseTag",
     "TestSuiteRunExecConfig_DeploymentReleaseTag",
+    "TestSuiteRunExecConfig_External",
     "TestSuiteRunExecConfig_WorkflowReleaseTag",
     "TestSuiteRunExecution",
     "TestSuiteRunExecutionChatHistoryOutput",
@@ -897,6 +938,11 @@ __all__ = [
     "TestSuiteRunExecutionOutput_String",
     "TestSuiteRunExecutionSearchResultsOutput",
     "TestSuiteRunExecutionStringOutput",
+    "TestSuiteRunExternalExecConfig",
+    "TestSuiteRunExternalExecConfigData",
+    "TestSuiteRunExternalExecConfigDataRequest",
+    "TestSuiteRunExternalExecConfigRequest",
+    "TestSuiteRunExternalExecConfigTypeEnum",
     "TestSuiteRunMetricErrorOutput",
     "TestSuiteRunMetricErrorOutputTypeEnum",
     "TestSuiteRunMetricNumberOutput",

vellum/types/external_test_case_execution.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .named_test_case_variable_value import NamedTestCaseVariableValue
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class ExternalTestCaseExecution(pydantic.BaseModel):
+    outputs: typing.List[NamedTestCaseVariableValue] = pydantic.Field()
+    """
+    The output values of a callable that was executed against a Test Case outside of Vellum
+    """
+
+    test_case_id: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/external_test_case_execution_request.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .named_test_case_variable_value_request import NamedTestCaseVariableValueRequest
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class ExternalTestCaseExecutionRequest(pydantic.BaseModel):
+    outputs: typing.List[NamedTestCaseVariableValueRequest] = pydantic.Field()
+    """
+    The output values of a callable that was executed against a Test Case outside of Vellum
+    """
+
+    test_case_id: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_chat_history_variable_value.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .chat_message import ChatMessage
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseChatHistoryVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type CHAT_HISTORY
+    """
+
+    value: typing.Optional[typing.List[ChatMessage]] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_error_variable_value.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .vellum_error import VellumError
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseErrorVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type ERROR
+    """
+
+    value: typing.Optional[VellumError] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_json_variable_value.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseJsonVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type JSON
+    """
+
+    value: typing.Optional[typing.Dict[str, typing.Any]] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_number_variable_value.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseNumberVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type NUMBER
+    """
+
+    value: typing.Optional[float] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_search_results_variable_value.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .search_result import SearchResult
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseSearchResultsVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type SEARCH_RESULTS
+    """
+
+    value: typing.Optional[typing.List[SearchResult]] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_string_variable_value.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseStringVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type STRING
+    """
+
+    value: typing.Optional[str] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_variable_value.py

@@ -0,0 +1,82 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from __future__ import annotations
+
+import typing
+
+from .named_test_case_chat_history_variable_value import NamedTestCaseChatHistoryVariableValue
+from .named_test_case_error_variable_value import NamedTestCaseErrorVariableValue
+from .named_test_case_json_variable_value import NamedTestCaseJsonVariableValue
+from .named_test_case_number_variable_value import NamedTestCaseNumberVariableValue
+from .named_test_case_search_results_variable_value import NamedTestCaseSearchResultsVariableValue
+from .named_test_case_string_variable_value import NamedTestCaseStringVariableValue
+
+
+class NamedTestCaseVariableValue_String(NamedTestCaseStringVariableValue):
+    type: typing.Literal["STRING"] = "STRING"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Number(NamedTestCaseNumberVariableValue):
+    type: typing.Literal["NUMBER"] = "NUMBER"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Json(NamedTestCaseJsonVariableValue):
+    type: typing.Literal["JSON"] = "JSON"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_ChatHistory(NamedTestCaseChatHistoryVariableValue):
+    type: typing.Literal["CHAT_HISTORY"] = "CHAT_HISTORY"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_SearchResults(NamedTestCaseSearchResultsVariableValue):
+    type: typing.Literal["SEARCH_RESULTS"] = "SEARCH_RESULTS"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Error(NamedTestCaseErrorVariableValue):
+    type: typing.Literal["ERROR"] = "ERROR"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+NamedTestCaseVariableValue = typing.Union[
+    NamedTestCaseVariableValue_String,
+    NamedTestCaseVariableValue_Number,
+    NamedTestCaseVariableValue_Json,
+    NamedTestCaseVariableValue_ChatHistory,
+    NamedTestCaseVariableValue_SearchResults,
+    NamedTestCaseVariableValue_Error,
+]

vellum/types/sandbox_scenario.py

@@ -13,6 +13,10 @@ except ImportError:
 
 
 class SandboxScenario(pydantic.BaseModel):
+    """
+    Sandbox Scenario
+    """
+
     label: typing.Optional[str] = None
     inputs: typing.List[ScenarioInput] = pydantic.Field()
     """

vellum/types/test_case_chat_history_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseChatHistoryVariableValue(pydantic.BaseModel):
+    """
+    A chat history value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.List[ChatMessage]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_error_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseErrorVariableValue(pydantic.BaseModel):
+    """
+    An error value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[VellumError] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_json_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseJsonVariableValue(pydantic.BaseModel):
+    """
+    A JSON value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.Dict[str, typing.Any]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_number_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseNumberVariableValue(pydantic.BaseModel):
+    """
+    A numerical value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[float] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_search_results_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseSearchResultsVariableValue(pydantic.BaseModel):
+    """
+    A search results value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.List[SearchResult]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_string_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseStringVariableValue(pydantic.BaseModel):
+    """
+    A string value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[str] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_suite_run_exec_config.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import typing
 
 from .test_suite_run_deployment_release_tag_exec_config import TestSuiteRunDeploymentReleaseTagExecConfig
+from .test_suite_run_external_exec_config import TestSuiteRunExternalExecConfig
 from .test_suite_run_workflow_release_tag_exec_config import TestSuiteRunWorkflowReleaseTagExecConfig
 
 
@@ -28,6 +29,18 @@ class TestSuiteRunExecConfig_WorkflowReleaseTag(TestSuiteRunWorkflowReleaseTagEx
         populate_by_name = True
 
 
+class TestSuiteRunExecConfig_External(TestSuiteRunExternalExecConfig):
+    type: typing.Literal["EXTERNAL"] = "EXTERNAL"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
 TestSuiteRunExecConfig = typing.Union[
-    TestSuiteRunExecConfig_DeploymentReleaseTag, TestSuiteRunExecConfig_WorkflowReleaseTag
+    TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_WorkflowReleaseTag,
+    TestSuiteRunExecConfig_External,
 ]

vellum/types/test_suite_run_exec_config_request.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import typing
 
 from .test_suite_run_deployment_release_tag_exec_config_request import TestSuiteRunDeploymentReleaseTagExecConfigRequest
+from .test_suite_run_external_exec_config_request import TestSuiteRunExternalExecConfigRequest
 from .test_suite_run_workflow_release_tag_exec_config_request import TestSuiteRunWorkflowReleaseTagExecConfigRequest
 
 
@@ -28,6 +29,18 @@ class TestSuiteRunExecConfigRequest_WorkflowReleaseTag(TestSuiteRunWorkflowRelea
         populate_by_name = True
 
 
+class TestSuiteRunExecConfigRequest_External(TestSuiteRunExternalExecConfigRequest):
+    type: typing.Literal["EXTERNAL"] = "EXTERNAL"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
 TestSuiteRunExecConfigRequest = typing.Union[
-    TestSuiteRunExecConfigRequest_DeploymentReleaseTag, TestSuiteRunExecConfigRequest_WorkflowReleaseTag
+    TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
+    TestSuiteRunExecConfigRequest_External,
 ]

vellum/types/test_suite_run_external_exec_config.py

@@ -0,0 +1,38 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_external_exec_config_data import TestSuiteRunExternalExecConfigData
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfig(pydantic.BaseModel):
+    """
+    Execution configuration for running a Vellum Test Suite against an external callable
+    """
+
+    data: TestSuiteRunExternalExecConfigData
+    test_case_ids: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
+    """
+    Optionally specify a subset of test case ids to run. If not provided, all test cases within the test suite will be run by default.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_data.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .external_test_case_execution import ExternalTestCaseExecution
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfigData(pydantic.BaseModel):
+    executions: typing.List[ExternalTestCaseExecution] = pydantic.Field()
+    """
+    The executions of some callable external to Vellum whose outputs you would like to evaluate.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_data_request.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .external_test_case_execution_request import ExternalTestCaseExecutionRequest
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfigDataRequest(pydantic.BaseModel):
+    executions: typing.List[ExternalTestCaseExecutionRequest] = pydantic.Field()
+    """
+    The executions of some callable external to Vellum whose outputs you would like to evaluate.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_request.py

@@ -0,0 +1,38 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_external_exec_config_data_request import TestSuiteRunExternalExecConfigDataRequest
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfigRequest(pydantic.BaseModel):
+    """
+    Execution configuration for running a Vellum Test Suite against an external callable
+    """
+
+    data: TestSuiteRunExternalExecConfigDataRequest
+    test_case_ids: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
+    """
+    Optionally specify a subset of test case ids to run. If not provided, all test cases within the test suite will be run by default.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_type_enum.py

@@ -0,0 +1,5 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+TestSuiteRunExternalExecConfigTypeEnum = typing.Literal["EXTERNAL"]

{vellum_ai-0.3.22.dist-info → vellum_ai-0.3.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vellum-ai
-Version: 0.3.22
+Version: 0.3.23
 Summary: 
 Requires-Python: >=3.8,<4.0
 Classifier: Programming Language :: Python :: 3
@@ -12,6 +12,7 @@ Requires-Dist: cdktf (>=0.20.5,<0.21.0)
 Requires-Dist: httpx (>=0.21.2)
 Requires-Dist: publication (==0.0.3)
 Requires-Dist: pydantic (>=1.9.2)
+Requires-Dist: requests_mock (>=1.11.0,<2.0.0)
 Requires-Dist: typing_extensions (>=4.0.0)
 Description-Content-Type: text/markdown
 

{vellum_ai-0.3.22.dist-info → vellum_ai-0.3.23.dist-info}/RECORD

@@ -1,8 +1,8 @@
-vellum/__init__.py,sha256=mUd1ylk7-rmszIiaOrYjlUXY0x4Mm12RfoWjbGBiHGg,34129
+vellum/__init__.py,sha256=zm-CWI_Au1C6g1pl0fpSNhkcxxAKqAm_ItDmOEbVOaE,35917
 vellum/client.py,sha256=ViOMnbiP_kKm3ayXW6DY1wC7Ge8tnI389qC8XVffyB8,96748
 vellum/core/__init__.py,sha256=RWfyDqkzWsf8e3VGc3NV60MovfJbg5XWzNFGB2DZ0hA,790
 vellum/core/api_error.py,sha256=RE8LELok2QCjABadECTvtDp7qejA1VmINCh6TbqPwSE,426
-vellum/core/client_wrapper.py,sha256=Qz__lng9uFpAK3GgV8prOkVIO4N5IUlsxO3umwr0wdY,1698
+vellum/core/client_wrapper.py,sha256=SO-9_qHhPtQgGR2hRQ59d9BqN1YH2UEfVsqghxfNUOc,1698
 vellum/core/datetime_utils.py,sha256=nBys2IsYrhPdszxGKCNRPSOCwa-5DWOHG95FB8G9PKo,1047
 vellum/core/file.py,sha256=sy1RUGZ3aJYuw998bZytxxo6QdgKmlnlgBaMvwEKCGg,1480
 vellum/core/http_client.py,sha256=5ok6hqgZDJhg57EHvMnr0BBaHdG50QxFPKaCZ9aVWTc,5059
@@ -53,7 +53,7 @@ vellum/terraform/document_index/__init__.py,sha256=qq2zENI22bUvqGk_a1lmsoTr5O_xC
 vellum/terraform/provider/__init__.py,sha256=K1yLlTZkYBxhD4bhUV1v23hxDGgbfsAIGsSyeB54dNQ,10298
 vellum/terraform/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 vellum/terraform/versions.json,sha256=STW6Mg3BKDacFmbWHXziHxE90GWncZf4AIzCLiXm_7o,56
-vellum/types/__init__.py,sha256=h34Y5qA32WVk6x7y54Mp_dqG2ydbnKh1X0MvlVWghcE,45549
+vellum/types/__init__.py,sha256=gHhZPDj0jEl6wl1-aSQQYeigJUFcmLulwwZiYdjIpho,48010
 vellum/types/api_node_result.py,sha256=aLOooryBCfglTeZwOuyyqRoHz5oPTja2mYYY7Gj8XxA,1040
 vellum/types/api_node_result_data.py,sha256=cQcZiwI1hjRvQFlkVEGxyaD360vEklIfjBmhBAeI75U,1215
 vellum/types/array_chat_message_content.py,sha256=Z7nKH8RNgot0i4-159w95OHywLjc-376gBPOovbw2hk,1087
@@ -111,6 +111,8 @@ vellum/types/execution_number_vellum_value.py,sha256=N3lcnG1cc21T8b4pO7O8ccnc5T9
 vellum/types/execution_search_results_vellum_value.py,sha256=tj3hPVHyTJ9RrYyOJ8w0R2vw7Mf_mazjJHSURiSsgk8,1129
 vellum/types/execution_string_vellum_value.py,sha256=B9Vvtmvz9KHynnomuHl9vwLur2fGMs6PQma4wsFv55w,1060
 vellum/types/execution_vellum_value.py,sha256=aEkd5eaFhGrI3POUFii5w3WZkvUgpB47KIFaAKS9_cQ,3144
+vellum/types/external_test_case_execution.py,sha256=gnTaatDxnxDzCjV6DIBiW1dX1vcHTvtI7fk8CcHaUyI,1179
+vellum/types/external_test_case_execution_request.py,sha256=pi-T5ODv1pnCpVDs3qXpw5BCMoNt4LBcVE1L6vHsvcA,1208
 vellum/types/finish_reason_enum.py,sha256=zAvOLiq2ngmDILEftziwCGYb4z-wlih0XHRNOo35FzM,726
 vellum/types/fulfilled_enum.py,sha256=MenrGneRMBvsq-n8tkMXAx0wppGWLJgkqdKnfklGP9U,124
 vellum/types/fulfilled_execute_prompt_event.py,sha256=2_jNUOaSg12WKggFiczulVoHFM7i3yIAIZbL99tVf2M,1292
@@ -160,12 +162,19 @@ vellum/types/model_version_exec_config_parameters.py,sha256=jF4wm_GIeITFavMNGSvB
 vellum/types/model_version_read.py,sha256=Ize9tyNNnR2oMXQSKwrEfuJilR5O3qSssfrJQwmSFqw,2425
 vellum/types/model_version_read_status_enum.py,sha256=uQCzeBoMqn29-V2b6fiq5rv9G3yBApaHWQLMMRRL-Hs,1032
 vellum/types/model_version_sandbox_snapshot.py,sha256=bXe_7Fp-CSQ_WOvjX8GJfO7ZbV0i99nbkv5yiqdFjeI,1309
+vellum/types/named_test_case_chat_history_variable_value.py,sha256=5wH7HIREWrDevirBMYqD276snHoAQzZ1H6GqHTzIECU,1101
 vellum/types/named_test_case_chat_history_variable_value_request.py,sha256=pYXfGF4VX_PcKdow8ttZo0Afv91vVZD3r0ZLIPmYE4w,1130
+vellum/types/named_test_case_error_variable_value.py,sha256=a17xJ5OjHhcLJeiC572_RIUdBDjIGJOLZUHDHW-V22E,1075
 vellum/types/named_test_case_error_variable_value_request.py,sha256=Hlihlz1rd9XMD_WPJKhuq8u8cTwAW-RM4JIlN9mrqak,1104
+vellum/types/named_test_case_json_variable_value.py,sha256=QrAD4HK3vrqV9P3k9-toDbEv_TDazBEyHD4MxQrG4nw,1052
 vellum/types/named_test_case_json_variable_value_request.py,sha256=eYUPPF1meDi5aIoQUvCmUVzh9u7MDt2lFyRK6NWI8m8,1059
+vellum/types/named_test_case_number_variable_value.py,sha256=3kcix5uYk2K9rxe9l17YuzjuFMt2kqxF6LkVO23tDAU,1033
 vellum/types/named_test_case_number_variable_value_request.py,sha256=RfKzbPCKINiCaMwul-XXxOb7QaAYqBzud_5cFFINY7E,1040
+vellum/types/named_test_case_search_results_variable_value.py,sha256=4Qb2imPy1nk7hbahG314XJbmrginXbsRnhUzYLhIU6s,1108
 vellum/types/named_test_case_search_results_variable_value_request.py,sha256=25mLk7D6u3btNhI8q9xHdpZlJJTLFZCIjZscFFCz1l0,1137
+vellum/types/named_test_case_string_variable_value.py,sha256=h5t9JDk6R8s73iw3bCsTfVtBjXtqIIZQMqvWHIOcYgI,1031
 vellum/types/named_test_case_string_variable_value_request.py,sha256=Mch56tlzPnlAuDrkuqLpPYehsca245u1QpYEnOz9LmU,1038
+vellum/types/named_test_case_variable_value.py,sha256=VQEAojwa2aSdLayEMn6oCBXMYY5Zr5dTZxfVJ_oJJTs,2584
 vellum/types/named_test_case_variable_value_request.py,sha256=hCNEq0Ubqod6kOELWZcbizCocKvLGbPwgaFAo1sWOuw,2807
 vellum/types/node_input_compiled_array_value.py,sha256=Zw78U4629WGprdmBVsnsthUsDNiJcC_7CecOs3B1M4M,1076
 vellum/types/node_input_compiled_chat_history_value.py,sha256=aOY49Id1sawCnfSnkP_UDoHMTUp3I1JwHrSUMtt57J8,1047
@@ -229,7 +238,7 @@ vellum/types/rejected_execute_workflow_workflow_result_event.py,sha256=l3tAFlDWB
 vellum/types/rejected_function_call.py,sha256=LeTe__cO3r7TqqiLRUDIoraetIMPbUZF52lDx3tTNEg,1095
 vellum/types/rejected_prompt_execution_meta.py,sha256=qlkDcHDCoJM1e4S_Dm9pdB3Mqy-g4rDFWuWx6xO9ezY,1195
 vellum/types/rejected_workflow_node_result_event.py,sha256=2HrFDp3H1sH4a03enG4E6HgWHp5w0yrs4kVYkxbJwfY,1350
-vellum/types/sandbox_scenario.py,sha256=8uxTOne5AUFm61cTjI1-UysM0ZqTkozsM8G0Q2eqvZ8,1157
+vellum/types/sandbox_scenario.py,sha256=XJwq_VrO5I5aEmL4ce9R2u3qmmyyQSj48PfMiPeItDc,1195
 vellum/types/scenario_input.py,sha256=UqdiHUjPCcj-evU5jYL7TNdctXaslpvuNeps2mimslk,1166
 vellum/types/scenario_input_request.py,sha256=Ir9rXTN4hmAAdW-CRR75V-gDi4ddebq-qXX5tEKkcnI,1195
 vellum/types/scenario_input_type_enum.py,sha256=S16TuGQ_Ifge1QLKem_ZlHSsqr3_lfLGZpEAj2IYrQk,580
@@ -286,20 +295,20 @@ vellum/types/terminal_node_result_data.py,sha256=i4Oa1itlpyFKDwXPAOtfu--vqyM6UG_
 vellum/types/terminal_node_result_output.py,sha256=lKXaYst9-JuY0rk0tjDV4mD2s_G-7gZm5FTKbW9xYo8,3164
 vellum/types/terminal_node_search_results_result.py,sha256=PZ-PsvMnI6_CdkbwfhN0KqwxcsBECoG5C3oZaSN84BU,1172
 vellum/types/terminal_node_string_result.py,sha256=38fufuCtk8ifqcBqTxT9wT3B8eulWfYGPZGRHE3x4RE,1103
-vellum/types/test_case_chat_history_variable_value.py,sha256=MyDknq8L3TUL5VsLD7bMrlYa9OMJf9GWgLpPDpJrPjs,1031
-vellum/types/test_case_error_variable_value.py,sha256=sbrmxd-Jc2X-D4MaqOgXF2r1LFEOGhZy_PiH-eUFWzI,1012
-vellum/types/test_case_json_variable_value.py,sha256=IR_WGLnW8QzAd-mIb6DhUUxz6T4UM4ffnC03LXN4t0M,990
-vellum/types/test_case_number_variable_value.py,sha256=YrR7TllXXOJAJMInADf0FswIxYESxKWDrR3yTGSWTwA,969
-vellum/types/test_case_search_results_variable_value.py,sha256=sjT-aoUzK0dJCi4FlSpYBQggoxkJXmeDkHagc6pSIj8,1036
-vellum/types/test_case_string_variable_value.py,sha256=5RKbSlS_mQoiv70QJkLVqi1W3mgQLrMzd8isaWZQKgE,967
+vellum/types/test_case_chat_history_variable_value.py,sha256=sFqH6lVDNK1oCH7F_WL3POv00yVj5HSXkdGzt5jYW7A,1118
+vellum/types/test_case_error_variable_value.py,sha256=7BYEk3YQH4XAR2BEMIJmq7RRuTgg1s-ehJGd2V476Oo,1093
+vellum/types/test_case_json_variable_value.py,sha256=2rrCqJLIRNK1Ld23eAF4UZ3yMbvvQSrRF0JF_ODDepU,1069
+vellum/types/test_case_number_variable_value.py,sha256=C4kb2mdgaRLVBfG965hYIaemtwVCglEQlKykvoNCI7E,1053
+vellum/types/test_case_search_results_variable_value.py,sha256=B_9E4ZxrOVeyjHaYp2LJkFmTkV0Ym8ycWkHPBLPly34,1125
+vellum/types/test_case_string_variable_value.py,sha256=afj2U-uuikqPGf7tcNCvHWScAosZ5VtdEF4PecUGE-I,1048
 vellum/types/test_case_variable_value.py,sha256=bE_lqnjrZMYFedIdd8Dqt1jlZ1wLdeuhCviHrawELEQ,2423
 vellum/types/test_suite_run_deployment_release_tag_exec_config.py,sha256=dixH2anYonGi3INTDnCtQgQ1VLtSLIcsWOX-qpBhR1c,1427
 vellum/types/test_suite_run_deployment_release_tag_exec_config_data.py,sha256=h1KBU0bSZZ9qDeq4wEFLgBwXC9aLSBRaxbvadjKQaD0,1279
 vellum/types/test_suite_run_deployment_release_tag_exec_config_data_request.py,sha256=XwWET1EsPwKszS3r5h2m5ZMxH5CyryBBtwQ7R5sbBio,1286
 vellum/types/test_suite_run_deployment_release_tag_exec_config_request.py,sha256=yTK-3MlF81q0NiQ0SO7VC2dnjaVX8fBDlJUqQzGArUg,1465
 vellum/types/test_suite_run_deployment_release_tag_exec_config_type_enum.py,sha256=P49aTblFExVUA90soNhC-_S0q8xAHGCWOnm9uRiqgp0,174
-vellum/types/test_suite_run_exec_config.py,sha256=I5MlYJBynyPouRDWe3pgsyIuyAx-CSiedLyF7UQEQXY,1092
-vellum/types/test_suite_run_exec_config_request.py,sha256=hspIHnkklKLBg58Rq2uWEjd04o_zLiBcJIzGHIATrso,1171
+vellum/types/test_suite_run_exec_config.py,sha256=ilF_6HuSMOt1csjBLGd8zzKB4BxbF1KL8okpD79xhag,1483
+vellum/types/test_suite_run_exec_config_request.py,sha256=cV_Yq4cbHK8MZxPp6RQqA6HerbTSyDPcZCyBULxELF0,1598
 vellum/types/test_suite_run_execution.py,sha256=Gj0jFE9-VZDQCxdzM9xe5-r3XX726WUgaVwoAxZQZ84,1215
 vellum/types/test_suite_run_execution_chat_history_output.py,sha256=54-Tq87GTSgHeS7ntFg6nQsYi2X4ppIcrZotRFEfE5s,1172
 vellum/types/test_suite_run_execution_error_output.py,sha256=fqI3Saza7PEsHHUAeVENzgT-WcS5oJMyW1H7D1Em-gg,1146
@@ -310,6 +319,11 @@ vellum/types/test_suite_run_execution_number_output.py,sha256=OiUI84wzKTuxx4XODC
 vellum/types/test_suite_run_execution_output.py,sha256=vOm3PiblQDdybmDUzP5KUzgkZxSrk31MBmJog-iW3Pk,2615
 vellum/types/test_suite_run_execution_search_results_output.py,sha256=9KJlNGazc74AkXEzg8oqycd08CO8ib5gJ96qpaaGmIw,1179
 vellum/types/test_suite_run_execution_string_output.py,sha256=JdfHP1icwQDRFFaim0wFQHuIC3yg2_t8946vH3MwdY4,1102
+vellum/types/test_suite_run_external_exec_config.py,sha256=J5DAkJlFNENbDw4qOQj9bfduQpA1pP_DP8mXh01ZxNU,1385
+vellum/types/test_suite_run_external_exec_config_data.py,sha256=1VEFnB6GDwiR0Kf_onQylBxrTMHUE8pGQ5xuwfZkMMU,1169
+vellum/types/test_suite_run_external_exec_config_data_request.py,sha256=rj5irhCWV5BltCXnq0yRkLdbTZfnumrPLyFjNQw2zjc,1198
+vellum/types/test_suite_run_external_exec_config_request.py,sha256=zx8gUKZ5lzV--oPPzEkR2U1vlZyIfma-AVOY6DD2dUw,1414
+vellum/types/test_suite_run_external_exec_config_type_enum.py,sha256=yX-VwukrI1rPH7_R2D7sRlBioduW6q791zXWiODMlo0,148
 vellum/types/test_suite_run_metric_error_output.py,sha256=N0ZRTO1QV4Bobrh9u1-k5qBMiFh8CF2NLiElGA94oIo,1062
 vellum/types/test_suite_run_metric_error_output_type_enum.py,sha256=ZPKy0jkcgfuMhI2hG_3UUcw2sYH1BlV_ze8Axlq5nIU,144
 vellum/types/test_suite_run_metric_number_output.py,sha256=vedeqnSpDl7Y23DdgAlcl16Gxd1WlNo4m4PQ2SGI8Tk,1020
@@ -374,7 +388,7 @@ vellum/types/workflow_result_event_output_data_search_results.py,sha256=wYc76-DW
 vellum/types/workflow_result_event_output_data_string.py,sha256=7SR46tKt-I_cGd_eVeKU8ymQ7R3vKkA_dFHCgAU1GcQ,1533
 vellum/types/workflow_stream_event.py,sha256=KA6Bkk_XA6AIPWR-1vKnwF1A8l_Bm5y0arQCWWWRpsk,911
 vellum/version.py,sha256=neLt8HBHHUtDF9M5fsyUzHT-pKooEPvceaLDqqIGb0s,77
-vellum_ai-0.3.22.dist-info/LICENSE,sha256=CcaljEIoOBaU-wItPH4PmM_mDCGpyuUY0Er1BGu5Ti8,1073
-vellum_ai-0.3.22.dist-info/METADATA,sha256=0x43xw8oDjeEy-VY7XfF8-0eLLikv5ag2z6dX5y-jQw,3550
-vellum_ai-0.3.22.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-vellum_ai-0.3.22.dist-info/RECORD,,
+vellum_ai-0.3.23.dist-info/LICENSE,sha256=CcaljEIoOBaU-wItPH4PmM_mDCGpyuUY0Er1BGu5Ti8,1073
+vellum_ai-0.3.23.dist-info/METADATA,sha256=ue4qundj5MNtdoq_lrm-kPpsbb7w_uAQRUub6hgX-Xc,3597
+vellum_ai-0.3.23.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+vellum_ai-0.3.23.dist-info/RECORD,,