vellum-ai 0.3.19__tar.gz → 0.3.21__tar.gz

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vellum-ai
-Version: 0.3.19
+Version: 0.3.21
 Summary: 
 Requires-Python: >=3.8,<4.0
 Classifier: Programming Language :: Python :: 3

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vellum-ai"
-version = "0.3.19"
+version = "0.3.21"
 description = ""
 readme = "README.md"
 authors = []

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/__init__.py

@@ -148,6 +148,7 @@ from .types import (
     MetadataFilterConfigRequest,
     MetadataFilterRuleCombinator,
     MetadataFilterRuleRequest,
+    MlModelUsage,
     ModelVersionBuildConfig,
     ModelVersionExecConfig,
     ModelVersionExecConfigParameters,
@@ -210,6 +211,7 @@ from .types import (
     PaginatedSlimDocumentList,
     PaginatedSlimWorkflowDeploymentList,
     PaginatedTestSuiteRunExecutionList,
+    PaginatedTestSuiteTestCaseList,
     ProcessingFailureReasonEnum,
     ProcessingStateEnum,
     PromptDeploymentExpandMetaRequestRequest,
@@ -372,6 +374,9 @@ from .types import (
     TestSuiteRunMetricOutput,
     TestSuiteRunMetricOutput_Error,
     TestSuiteRunMetricOutput_Number,
+    TestSuiteRunMetricOutput_String,
+    TestSuiteRunMetricStringOutput,
+    TestSuiteRunMetricStringOutputTypeEnum,
     TestSuiteRunRead,
     TestSuiteRunState,
     TestSuiteRunTestSuite,
@@ -637,6 +642,7 @@ __all__ = [
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
+    "MlModelUsage",
     "ModelVersionBuildConfig",
     "ModelVersionExecConfig",
     "ModelVersionExecConfigParameters",
@@ -700,6 +706,7 @@ __all__ = [
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
     "PaginatedTestSuiteRunExecutionList",
+    "PaginatedTestSuiteTestCaseList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -862,6 +869,9 @@ __all__ = [
     "TestSuiteRunMetricOutput",
     "TestSuiteRunMetricOutput_Error",
     "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunMetricOutput_String",
+    "TestSuiteRunMetricStringOutput",
+    "TestSuiteRunMetricStringOutputTypeEnum",
     "TestSuiteRunRead",
     "TestSuiteRunState",
     "TestSuiteRunTestSuite",

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/client.py

@@ -167,6 +167,7 @@ class Vellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -301,6 +302,7 @@ class Vellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -1123,6 +1125,7 @@ class AsyncVellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},
@@ -1257,6 +1260,7 @@ class AsyncVellum:
                 deployment_release_tag=True,
                 prompt_version_id=True,
                 finish_reason=True,
+                usage=True,
             ),
             raw_overrides=RawPromptExecutionOverridesRequest(
                 body={"string": {"key": "value"}},

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/core/client_wrapper.py

@@ -18,7 +18,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.3.19",
+            "X-Fern-SDK-Version": "0.3.21",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/resources/test_suites/client.py

@@ -10,6 +10,7 @@ from ...core.jsonable_encoder import jsonable_encoder
 from ...core.remove_none_from_dict import remove_none_from_dict
 from ...core.request_options import RequestOptions
 from ...types.named_test_case_variable_value_request import NamedTestCaseVariableValueRequest
+from ...types.paginated_test_suite_test_case_list import PaginatedTestSuiteTestCaseList
 from ...types.test_suite_test_case import TestSuiteTestCase
 
 try:
@@ -25,6 +26,76 @@ class TestSuitesClient:
     def __init__(self, *, client_wrapper: SyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    def list_test_suite_test_cases(
+        self,
+        id: str,
+        *,
+        limit: typing.Optional[int] = None,
+        offset: typing.Optional[int] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> PaginatedTestSuiteTestCaseList:
+        """
+        List the Test Cases associated with a Test Suite
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.test_suites.list_test_suite_test_cases(
+            id="id",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/test-suites/{jsonable_encoder(id)}/test-cases",
+            ),
+            params=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        "limit": limit,
+                        "offset": offset,
+                        **(
+                            request_options.get("additional_query_parameters", {})
+                            if request_options is not None
+                            else {}
+                        ),
+                    }
+                )
+            ),
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteTestCaseList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     def upsert_test_suite_test_case(
         self,
         id: str,
@@ -169,6 +240,76 @@ class AsyncTestSuitesClient:
     def __init__(self, *, client_wrapper: AsyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    async def list_test_suite_test_cases(
+        self,
+        id: str,
+        *,
+        limit: typing.Optional[int] = None,
+        offset: typing.Optional[int] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> PaginatedTestSuiteTestCaseList:
+        """
+        List the Test Cases associated with a Test Suite
+
+        Parameters:
+            - id: str. A UUID string identifying this test suite.
+
+            - limit: typing.Optional[int]. Number of results to return per page.
+
+            - offset: typing.Optional[int]. The initial index from which to return the results.
+
+            - request_options: typing.Optional[RequestOptions]. Request-specific configuration.
+        ---
+        from vellum.client import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+        await client.test_suites.list_test_suite_test_cases(
+            id="id",
+        )
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(
+                f"{self._client_wrapper.get_environment().default}/",
+                f"v1/test-suites/{jsonable_encoder(id)}/test-cases",
+            ),
+            params=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        "limit": limit,
+                        "offset": offset,
+                        **(
+                            request_options.get("additional_query_parameters", {})
+                            if request_options is not None
+                            else {}
+                        ),
+                    }
+                )
+            ),
+            headers=jsonable_encoder(
+                remove_none_from_dict(
+                    {
+                        **self._client_wrapper.get_headers(),
+                        **(request_options.get("additional_headers", {}) if request_options is not None else {}),
+                    }
+                )
+            ),
+            timeout=request_options.get("timeout_in_seconds")
+            if request_options is not None and request_options.get("timeout_in_seconds") is not None
+            else self._client_wrapper.get_timeout(),
+            retries=0,
+            max_retries=request_options.get("max_retries") if request_options is not None else 0,  # type: ignore
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(PaginatedTestSuiteTestCaseList, _response.json())  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     async def upsert_test_suite_test_case(
         self,
         id: str,

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/types/__init__.py

@@ -165,6 +165,7 @@ from .logprobs_enum import LogprobsEnum
 from .metadata_filter_config_request import MetadataFilterConfigRequest
 from .metadata_filter_rule_combinator import MetadataFilterRuleCombinator
 from .metadata_filter_rule_request import MetadataFilterRuleRequest
+from .ml_model_usage import MlModelUsage
 from .model_version_build_config import ModelVersionBuildConfig
 from .model_version_exec_config import ModelVersionExecConfig
 from .model_version_exec_config_parameters import ModelVersionExecConfigParameters
@@ -233,6 +234,7 @@ from .paginated_slim_deployment_read_list import PaginatedSlimDeploymentReadList
 from .paginated_slim_document_list import PaginatedSlimDocumentList
 from .paginated_slim_workflow_deployment_list import PaginatedSlimWorkflowDeploymentList
 from .paginated_test_suite_run_execution_list import PaginatedTestSuiteRunExecutionList
+from .paginated_test_suite_test_case_list import PaginatedTestSuiteTestCaseList
 from .processing_failure_reason_enum import ProcessingFailureReasonEnum
 from .processing_state_enum import ProcessingStateEnum
 from .prompt_deployment_expand_meta_request_request import PromptDeploymentExpandMetaRequestRequest
@@ -418,7 +420,10 @@ from .test_suite_run_metric_output import (
     TestSuiteRunMetricOutput,
     TestSuiteRunMetricOutput_Error,
     TestSuiteRunMetricOutput_Number,
+    TestSuiteRunMetricOutput_String,
 )
+from .test_suite_run_metric_string_output import TestSuiteRunMetricStringOutput
+from .test_suite_run_metric_string_output_type_enum import TestSuiteRunMetricStringOutputTypeEnum
 from .test_suite_run_read import TestSuiteRunRead
 from .test_suite_run_state import TestSuiteRunState
 from .test_suite_run_test_suite import TestSuiteRunTestSuite
@@ -669,6 +674,7 @@ __all__ = [
     "MetadataFilterConfigRequest",
     "MetadataFilterRuleCombinator",
     "MetadataFilterRuleRequest",
+    "MlModelUsage",
     "ModelVersionBuildConfig",
     "ModelVersionExecConfig",
     "ModelVersionExecConfigParameters",
@@ -731,6 +737,7 @@ __all__ = [
     "PaginatedSlimDocumentList",
     "PaginatedSlimWorkflowDeploymentList",
     "PaginatedTestSuiteRunExecutionList",
+    "PaginatedTestSuiteTestCaseList",
     "ProcessingFailureReasonEnum",
     "ProcessingStateEnum",
     "PromptDeploymentExpandMetaRequestRequest",
@@ -893,6 +900,9 @@ __all__ = [
     "TestSuiteRunMetricOutput",
     "TestSuiteRunMetricOutput_Error",
     "TestSuiteRunMetricOutput_Number",
+    "TestSuiteRunMetricOutput_String",
+    "TestSuiteRunMetricStringOutput",
+    "TestSuiteRunMetricStringOutputTypeEnum",
     "TestSuiteRunRead",
     "TestSuiteRunState",
     "TestSuiteRunTestSuite",

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/types/fulfilled_prompt_execution_meta.py

@@ -5,6 +5,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .finish_reason_enum import FinishReasonEnum
+from .ml_model_usage import MlModelUsage
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -19,6 +20,7 @@ class FulfilledPromptExecutionMeta(pydantic.BaseModel):
 
     latency: typing.Optional[int] = None
     finish_reason: typing.Optional[FinishReasonEnum] = None
+    usage: typing.Optional[MlModelUsage] = None
 
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}

vellum_ai-0.3.21/src/vellum/types/ml_model_usage.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class MlModelUsage(pydantic.BaseModel):
+    output_token_count: typing.Optional[int] = None
+    input_token_count: typing.Optional[int] = None
+    input_char_count: typing.Optional[int] = None
+    output_char_count: typing.Optional[int] = None
+    compute_nanos: typing.Optional[int] = None
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum_ai-0.3.21/src/vellum/types/paginated_test_suite_test_case_list.py

@@ -0,0 +1,33 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+from .test_suite_test_case import TestSuiteTestCase
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class PaginatedTestSuiteTestCaseList(pydantic.BaseModel):
+    count: int
+    next: typing.Optional[str] = None
+    previous: typing.Optional[str] = None
+    results: typing.List[TestSuiteTestCase]
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/types/prompt_deployment_expand_meta_request_request.py

@@ -37,6 +37,11 @@ class PromptDeploymentExpandMetaRequestRequest(pydantic.BaseModel):
     If enabled, the response will include the reason provided by the model for why the execution finished.
     """
 
+    usage: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    If enabled, the response will include model host usage tracking. This may increase latency for some model hosts.
+    """
+
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
         return super().json(**kwargs_with_defaults)

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/types/prompt_execution_meta.py

@@ -5,6 +5,7 @@ import typing
 
 from ..core.datetime_utils import serialize_datetime
 from .finish_reason_enum import FinishReasonEnum
+from .ml_model_usage import MlModelUsage
 
 try:
     import pydantic.v1 as pydantic  # type: ignore
@@ -17,6 +18,7 @@ class PromptExecutionMeta(pydantic.BaseModel):
     The subset of the metadata tracked by Vellum during prompt execution that the request opted into with `expand_meta`.
     """
 
+    usage: typing.Optional[MlModelUsage] = None
     model_name: typing.Optional[str] = None
     latency: typing.Optional[int] = None
     deployment_release_tag: typing.Optional[str] = None

{vellum_ai-0.3.19 → vellum_ai-0.3.21}/src/vellum/types/test_suite_run_metric_output.py

@@ -6,6 +6,17 @@ import typing
 
 from .test_suite_run_metric_error_output import TestSuiteRunMetricErrorOutput
 from .test_suite_run_metric_number_output import TestSuiteRunMetricNumberOutput
+from .test_suite_run_metric_string_output import TestSuiteRunMetricStringOutput
+
+
+class TestSuiteRunMetricOutput_String(TestSuiteRunMetricStringOutput):
+    type: typing.Literal["STRING"] = "STRING"
+
+    class Config:
+        frozen = True
+        smart_union = True
+        allow_population_by_field_name = True
+        populate_by_name = True
 
 
 class TestSuiteRunMetricOutput_Number(TestSuiteRunMetricNumberOutput):
@@ -28,4 +39,6 @@ class TestSuiteRunMetricOutput_Error(TestSuiteRunMetricErrorOutput):
         populate_by_name = True
 
 
-TestSuiteRunMetricOutput = typing.Union[TestSuiteRunMetricOutput_Number, TestSuiteRunMetricOutput_Error]
+TestSuiteRunMetricOutput = typing.Union[
+    TestSuiteRunMetricOutput_String, TestSuiteRunMetricOutput_Number, TestSuiteRunMetricOutput_Error
+]

vellum_ai-0.3.21/src/vellum/types/test_suite_run_metric_string_output.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import datetime as dt
+import typing
+
+from ..core.datetime_utils import serialize_datetime
+
+try:
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class TestSuiteRunMetricStringOutput(pydantic.BaseModel):
+    """
+    Output for a test suite run metric that is of type STRING
+    """
+
+    value: str
+    name: str
+
+    def json(self, **kwargs: typing.Any) -> str:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().json(**kwargs_with_defaults)
+
+    def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
+        kwargs_with_defaults: typing.Any = {"by_alias": True, "exclude_unset": True, **kwargs}
+        return super().dict(**kwargs_with_defaults)
+
+    class Config:
+        frozen = True
+        smart_union = True
+        extra = pydantic.Extra.allow
+        json_encoders = {dt.datetime: serialize_datetime}

vellum_ai-0.3.21/src/vellum/types/test_suite_run_metric_string_output_type_enum.py

@@ -0,0 +1,5 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+TestSuiteRunMetricStringOutputTypeEnum = typing.Literal["STRING"]