vellum-ai 0.3.21__py3-none-any.whl → 0.3.23__py3-none-any.whl

vellum/types/named_test_case_number_variable_value_request.py

@@ -12,8 +12,12 @@ except ImportError:
 
 
 class NamedTestCaseNumberVariableValueRequest(pydantic.BaseModel):
-    name: str
+    """
+    Named Test Case value that is of type NUMBER
+    """
+
     value: typing.Optional[float] = None
+    name: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/named_test_case_search_results_variable_value.py

@@ -0,0 +1,35 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .search_result import SearchResult
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseSearchResultsVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type SEARCH_RESULTS
+    """
+
+    value: typing.Optional[typing.List[SearchResult]] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_search_results_variable_value_request.py

@@ -13,8 +13,12 @@ except ImportError:
 
 
 class NamedTestCaseSearchResultsVariableValueRequest(pydantic.BaseModel):
-    name: str
+    """
+    Named Test Case value that is of type SEARCH_RESULTS
+    """
+
     value: typing.Optional[typing.List[SearchResultRequest]] = None
+    name: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/named_test_case_string_variable_value.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class NamedTestCaseStringVariableValue(pydantic.BaseModel):
+    """
+    Named Test Case value that is of type STRING
+    """
+
+    value: typing.Optional[str] = None
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/named_test_case_string_variable_value_request.py

@@ -12,8 +12,12 @@ except ImportError:
 
 
 class NamedTestCaseStringVariableValueRequest(pydantic.BaseModel):
-    name: str
+    """
+    Named Test Case value that is of type STRING
+    """
+
     value: typing.Optional[str] = None
+    name: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/named_test_case_variable_value.py

@@ -0,0 +1,82 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from __future__ import annotations
+
+import typing
+
+from .named_test_case_chat_history_variable_value import NamedTestCaseChatHistoryVariableValue
+from .named_test_case_error_variable_value import NamedTestCaseErrorVariableValue
+from .named_test_case_json_variable_value import NamedTestCaseJsonVariableValue
+from .named_test_case_number_variable_value import NamedTestCaseNumberVariableValue
+from .named_test_case_search_results_variable_value import NamedTestCaseSearchResultsVariableValue
+from .named_test_case_string_variable_value import NamedTestCaseStringVariableValue
+
+
+class NamedTestCaseVariableValue_String(NamedTestCaseStringVariableValue):
+    type: typing.Literal["STRING"] = "STRING"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Number(NamedTestCaseNumberVariableValue):
+    type: typing.Literal["NUMBER"] = "NUMBER"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Json(NamedTestCaseJsonVariableValue):
+    type: typing.Literal["JSON"] = "JSON"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_ChatHistory(NamedTestCaseChatHistoryVariableValue):
+    type: typing.Literal["CHAT_HISTORY"] = "CHAT_HISTORY"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_SearchResults(NamedTestCaseSearchResultsVariableValue):
+    type: typing.Literal["SEARCH_RESULTS"] = "SEARCH_RESULTS"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+class NamedTestCaseVariableValue_Error(NamedTestCaseErrorVariableValue):
+    type: typing.Literal["ERROR"] = "ERROR"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
+NamedTestCaseVariableValue = typing.Union[
+    NamedTestCaseVariableValue_String,
+    NamedTestCaseVariableValue_Number,
+    NamedTestCaseVariableValue_Json,
+    NamedTestCaseVariableValue_ChatHistory,
+    NamedTestCaseVariableValue_SearchResults,
+    NamedTestCaseVariableValue_Error,
+]

vellum/types/prompt_template_block.py

@@ -7,6 +7,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .block_type_enum import BlockTypeEnum
+from .prompt_template_block_state import PromptTemplateBlockState
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -18,6 +19,7 @@ class PromptTemplateBlock(pydantic.BaseModel):
     id: str
     block_type: BlockTypeEnum
     properties: PromptTemplateBlockProperties
+    state: typing.Optional[PromptTemplateBlockState] = None
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/prompt_template_block_request.py

@@ -7,6 +7,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .block_type_enum import BlockTypeEnum
+from .prompt_template_block_state import PromptTemplateBlockState
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -18,6 +19,7 @@ class PromptTemplateBlockRequest(pydantic.BaseModel):
     id: str
     block_type: BlockTypeEnum
     properties: PromptTemplateBlockPropertiesRequest
+    state: typing.Optional[PromptTemplateBlockState] = None
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/prompt_template_block_state.py

@@ -0,0 +1,22 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import enum
+import typing
+
+T_Result = typing.TypeVar("T_Result")
+
+
+class PromptTemplateBlockState(str, enum.Enum):
+    """
+    - `ENABLED` - ENABLED
+    - `DISABLED` - DISABLED
+    """
+
+    ENABLED = "ENABLED"
+    DISABLED = "DISABLED"
+
+    def visit(self, enabled: typing.Callable[[], T_Result], disabled: typing.Callable[[], T_Result]) -> T_Result:
+        if self is PromptTemplateBlockState.ENABLED:
+            return enabled()
+        if self is PromptTemplateBlockState.DISABLED:
+            return disabled()

vellum/types/sandbox_scenario.py

@@ -13,6 +13,10 @@ except ImportError:
 
 
 class SandboxScenario(pydantic.BaseModel):
+    """
+    Sandbox Scenario
+    """
+
     label: typing.Optional[str] = None
     inputs: typing.List[ScenarioInput] = pydantic.Field()
     """

vellum/types/test_case_chat_history_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseChatHistoryVariableValue(pydantic.BaseModel):
+    """
+    A chat history value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.List[ChatMessage]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_error_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseErrorVariableValue(pydantic.BaseModel):
+    """
+    An error value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[VellumError] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_json_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseJsonVariableValue(pydantic.BaseModel):
+    """
+    A JSON value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.Dict[str, typing.Any]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_number_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseNumberVariableValue(pydantic.BaseModel):
+    """
+    A numerical value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[float] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_search_results_variable_value.py

@@ -13,7 +13,12 @@ except ImportError:
 
 
 class TestCaseSearchResultsVariableValue(pydantic.BaseModel):
+    """
+    A search results value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[typing.List[SearchResult]] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_case_string_variable_value.py

@@ -12,7 +12,12 @@ except ImportError:
 
 
 class TestCaseStringVariableValue(pydantic.BaseModel):
+    """
+    A string value for a variable in a Test Case.
+    """
+
     variable_id: str
+    name: str
     value: typing.Optional[str] = None
 
     def json(self, **kwargs: typing.Any) -> str:

vellum/types/test_suite_run_exec_config.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import typing
 
 from .test_suite_run_deployment_release_tag_exec_config import TestSuiteRunDeploymentReleaseTagExecConfig
+from .test_suite_run_external_exec_config import TestSuiteRunExternalExecConfig
 from .test_suite_run_workflow_release_tag_exec_config import TestSuiteRunWorkflowReleaseTagExecConfig
 
 
@@ -28,6 +29,18 @@ class TestSuiteRunExecConfig_WorkflowReleaseTag(TestSuiteRunWorkflowReleaseTagEx
         populate_by_name = True
 
 
+class TestSuiteRunExecConfig_External(TestSuiteRunExternalExecConfig):
+    type: typing.Literal["EXTERNAL"] = "EXTERNAL"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
 TestSuiteRunExecConfig = typing.Union[
-    TestSuiteRunExecConfig_DeploymentReleaseTag, TestSuiteRunExecConfig_WorkflowReleaseTag
+    TestSuiteRunExecConfig_DeploymentReleaseTag,
+    TestSuiteRunExecConfig_WorkflowReleaseTag,
+    TestSuiteRunExecConfig_External,
 ]

vellum/types/test_suite_run_exec_config_request.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import typing
 
 from .test_suite_run_deployment_release_tag_exec_config_request import TestSuiteRunDeploymentReleaseTagExecConfigRequest
+from .test_suite_run_external_exec_config_request import TestSuiteRunExternalExecConfigRequest
 from .test_suite_run_workflow_release_tag_exec_config_request import TestSuiteRunWorkflowReleaseTagExecConfigRequest
 
 
@@ -28,6 +29,18 @@ class TestSuiteRunExecConfigRequest_WorkflowReleaseTag(TestSuiteRunWorkflowRelea
         populate_by_name = True
 
 
+class TestSuiteRunExecConfigRequest_External(TestSuiteRunExternalExecConfigRequest):
+    type: typing.Literal["EXTERNAL"] = "EXTERNAL"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
+
+
 TestSuiteRunExecConfigRequest = typing.Union[
-    TestSuiteRunExecConfigRequest_DeploymentReleaseTag, TestSuiteRunExecConfigRequest_WorkflowReleaseTag
+    TestSuiteRunExecConfigRequest_DeploymentReleaseTag,
+    TestSuiteRunExecConfigRequest_WorkflowReleaseTag,
+    TestSuiteRunExecConfigRequest_External,
 ]

vellum/types/test_suite_run_execution_chat_history_output.py

@@ -13,8 +13,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionChatHistoryOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type CHAT_HISTORY
+    """
+
+    name: str
     value: typing.Optional[typing.List[ChatMessage]] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_error_output.py

@@ -13,8 +13,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionErrorOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type ERROR
+    """
+
+    name: str
     value: typing.Optional[VellumError] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_json_output.py

@@ -12,8 +12,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionJsonOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type JSON
+    """
+
+    name: str
     value: typing.Optional[typing.Dict[str, typing.Any]] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_metric_definition.py

@@ -0,0 +1,31 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExecutionMetricDefinition(pydantic.BaseModel):
+    id: typing.Optional[str] = None
+    label: typing.Optional[str] = None
+    name: typing.Optional[str] = None
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_execution_metric_result.py

@@ -4,6 +4,7 @@ import datetime as dt
 import typing
 
 from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_execution_metric_definition import TestSuiteRunExecutionMetricDefinition
 from .test_suite_run_metric_output import TestSuiteRunMetricOutput
 
 try:
@@ -15,6 +16,8 @@ except ImportError:
 class TestSuiteRunExecutionMetricResult(pydantic.BaseModel):
     metric_id: str
     outputs: typing.List[TestSuiteRunMetricOutput]
+    metric_label: typing.Optional[str] = None
+    metric_definition: typing.Optional[TestSuiteRunExecutionMetricDefinition] = None
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_number_output.py

@@ -12,8 +12,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionNumberOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type NUMBER
+    """
+
+    name: str
     value: typing.Optional[float] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_search_results_output.py

@@ -13,8 +13,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionSearchResultsOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type SEARCH_RESULTS
+    """
+
+    name: str
     value: typing.Optional[typing.List[SearchResult]] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_execution_string_output.py

@@ -12,8 +12,13 @@ except ImportError:
 
 
 class TestSuiteRunExecutionStringOutput(pydantic.BaseModel):
-    output_variable_id: str
+    """
+    Execution output of an entity evaluated during a Test Suite Run that is of type STRING
+    """
+
+    name: str
     value: typing.Optional[str] = None
+    output_variable_id: str
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum/types/test_suite_run_external_exec_config.py

@@ -0,0 +1,38 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_run_external_exec_config_data import TestSuiteRunExternalExecConfigData
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfig(pydantic.BaseModel):
+    """
+    Execution configuration for running a Vellum Test Suite against an external callable
+    """
+
+    data: TestSuiteRunExternalExecConfigData
+    test_case_ids: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
+    """
+    Optionally specify a subset of test case ids to run. If not provided, all test cases within the test suite will be run by default.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_data.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .external_test_case_execution import ExternalTestCaseExecution
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfigData(pydantic.BaseModel):
+    executions: typing.List[ExternalTestCaseExecution] = pydantic.Field()
+    """
+    The executions of some callable external to Vellum whose outputs you would like to evaluate.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum/types/test_suite_run_external_exec_config_data_request.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .external_test_case_execution_request import ExternalTestCaseExecutionRequest
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunExternalExecConfigDataRequest(pydantic.BaseModel):
+    executions: typing.List[ExternalTestCaseExecutionRequest] = pydantic.Field()
+    """
+    The executions of some callable external to Vellum whose outputs you would like to evaluate.
+    """
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}