vellum-ai 0.9.16__py3-none-any.whl → 0.9.16rc4__py3-none-any.whl

vellum/environment.py

@@ -1,17 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release.
 
-from __future__ import annotations
-
-
-class VellumEnvironment:
-    PRODUCTION: VellumEnvironment
-
-    def __init__(self, *, default: str, documents: str, predict: str):
-        self.default = default
-        self.documents = documents
-        self.predict = predict
-
-
-VellumEnvironment.PRODUCTION = VellumEnvironment(
-    default="https://api.vellum.ai", documents="https://documents.vellum.ai", predict="https://predict.vellum.ai"
-)
+from vellum.client.environment import VellumEnvironment

vellum/errors/__init__.py

@@ -1,8 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
 
-from .bad_request_error import BadRequestError
-from .forbidden_error import ForbiddenError
-from .internal_server_error import InternalServerError
-from .not_found_error import NotFoundError
-
-__all__ = ["BadRequestError", "ForbiddenError", "InternalServerError", "NotFoundError"]
+from vellum.client.errors import *

vellum/errors/bad_request_error.py

@@ -1,9 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
 
-from ..core.api_error import ApiError
-import typing
-
-
-class BadRequestError(ApiError):
-    def __init__(self, body: typing.Optional[typing.Any]):
-        super().__init__(status_code=400, body=body)
+from vellum.client.errors.bad_request_error import *

vellum/errors/forbidden_error.py

@@ -1,9 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
 
-from ..core.api_error import ApiError
-import typing
-
-
-class ForbiddenError(ApiError):
-    def __init__(self, body: typing.Optional[typing.Any]):
-        super().__init__(status_code=403, body=body)
+from vellum.client.errors.forbidden_error import *

vellum/errors/internal_server_error.py

@@ -1,9 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
 
-from ..core.api_error import ApiError
-import typing
-
-
-class InternalServerError(ApiError):
-    def __init__(self, body: typing.Optional[typing.Any]):
-        super().__init__(status_code=500, body=body)
+from vellum.client.errors.internal_server_error import *

vellum/errors/not_found_error.py

@@ -1,9 +1,3 @@
-# This file was auto-generated by Fern from our API Definition.
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
 
-from ..core.api_error import ApiError
-import typing
-
-
-class NotFoundError(ApiError):
-    def __init__(self, body: typing.Optional[typing.Any]):
-        super().__init__(status_code=404, body=body)
+from vellum.client.errors.not_found_error import *

vellum/{lib/test_suites → evaluations}/resources.py

@@ -8,15 +8,15 @@ from uuid import UUID
 
 from vellum import TestSuiteRunRead, TestSuiteRunMetricNumberOutput
 from vellum.client import Vellum, OMIT
-from vellum.lib.test_suites.constants import (
+from vellum.evaluations.constants import (
     DEFAULT_MAX_POLLING_DURATION_MS,
     DEFAULT_POLLING_INTERVAL_MS,
 )
-from vellum.lib.test_suites.exceptions import TestSuiteRunResultsException
-from vellum.lib.utils.env import get_api_key
-from vellum.lib.utils.paginator import PaginatedResults, get_all_results
-from vellum.lib.utils.typing import cast_not_optional
-from vellum.lib.utils.uuid import is_valid_uuid
+from vellum.evaluations.exceptions import TestSuiteRunResultsException
+from vellum.evaluations.utils.env import get_api_key
+from vellum.evaluations.utils.paginator import PaginatedResults, get_all_results
+from vellum.evaluations.utils.typing import cast_not_optional
+from vellum.evaluations.utils.uuid import is_valid_uuid
 from vellum.types import (
     ExternalTestCaseExecutionRequest,
     NamedTestCaseVariableValueRequest,
@@ -61,14 +61,10 @@ class VellumTestSuiteRunExecution(TestSuiteRunExecution):
         ]
 
         if len(filtered_metric_outputs) == 0:
-            raise TestSuiteRunResultsException(
-                f"No metric outputs found with identifier: {output_identifier}"
-            )
+            raise TestSuiteRunResultsException(f"No metric outputs found with identifier: {output_identifier}")
 
         if len(filtered_metric_outputs) > 1:
-            raise TestSuiteRunResultsException(
-                f"Multiple metric outputs found with identifier: {output_identifier}"
-            )
+            raise TestSuiteRunResultsException(f"Multiple metric outputs found with identifier: {output_identifier}")
 
         return filtered_metric_outputs[0]
 
@@ -87,29 +83,16 @@ class VellumTestSuiteRunExecution(TestSuiteRunExecution):
             if metric_identifier is None
             or (metric_output.metric_id == metric_identifier)
             or (metric_output.metric_label == metric_identifier)
-            or (
-                metric_output.metric_definition
-                and metric_output.metric_definition.id == metric_identifier
-            )
-            or (
-                metric_output.metric_definition
-                and metric_output.metric_definition.name == metric_identifier
-            )
-            or (
-                metric_output.metric_definition
-                and metric_output.metric_definition.label == metric_identifier
-            )
+            or (metric_output.metric_definition and metric_output.metric_definition.id == metric_identifier)
+            or (metric_output.metric_definition and metric_output.metric_definition.name == metric_identifier)
+            or (metric_output.metric_definition and metric_output.metric_definition.label == metric_identifier)
         ]
 
         if len(filtered_metric_results) == 0:
-            raise TestSuiteRunResultsException(
-                f"No metric results found with identifier: {metric_identifier}"
-            )
+            raise TestSuiteRunResultsException(f"No metric results found with identifier: {metric_identifier}")
 
         if len(filtered_metric_results) > 1:
-            raise TestSuiteRunResultsException(
-                f"Multiple metric results found with identifier: {metric_identifier}"
-            )
+            raise TestSuiteRunResultsException(f"Multiple metric results found with identifier: {metric_identifier}")
 
         return filtered_metric_results[0].outputs
 
@@ -129,9 +112,7 @@ class VellumTestSuiteRunResults:
         self._client = client or Vellum(
             api_key=get_api_key(),
         )
-        self._executions: Generator[VellumTestSuiteRunExecution, None, None] | None = (
-            None
-        )
+        self._executions: Generator[VellumTestSuiteRunExecution, None, None] | None = None
         self._polling_interval = polling_interval
         self._max_polling_duration = max_polling_duration
 
@@ -149,9 +130,7 @@ class VellumTestSuiteRunResults:
         """Retrieve a metric's output across all executions by providing the info needed to uniquely identify it."""
 
         return [
-            execution.get_metric_output(
-                metric_identifier=metric_identifier, output_identifier=output_identifier
-            )
+            execution.get_metric_output(metric_identifier=metric_identifier, output_identifier=output_identifier)
             for execution in self.all_executions
         ]
 
@@ -201,9 +180,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return sum(
-            cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values))
-        ) / len(output_values)
+        return sum(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values))) / len(output_values)
 
     def get_min_metric_output(
         self, metric_identifier: str | None = None, output_identifier: str | None = None
@@ -212,9 +189,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return min(
-            cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values))
-        )
+        return min(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values)))
 
     def get_max_metric_output(
         self, metric_identifier: str | None = None, output_identifier: str | None = None
@@ -223,9 +198,7 @@ class VellumTestSuiteRunResults:
         output_values = self.get_numeric_metric_output_values(
             metric_identifier=metric_identifier, output_identifier=output_identifier
         )
-        return max(
-            cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values))
-        )
+        return max(cast(Iterable[float], filter(lambda o: isinstance(o, float), output_values)))
 
     def wait_until_complete(self) -> None:
         """Wait until the Test Suite Run is no longer in a QUEUED or RUNNING state."""
@@ -239,9 +212,7 @@ class VellumTestSuiteRunResults:
 
             current_time = time.time_ns()
             if ((current_time - start_time) / 1e6) > self._max_polling_duration:
-                raise TestSuiteRunResultsException(
-                    "Test suite run timed out polling for executions"
-                )
+                raise TestSuiteRunResultsException("Test suite run timed out polling for executions")
 
             time.sleep(self._polling_interval / 1000.0)
 
@@ -313,9 +284,7 @@ class VellumTestSuite:
 
     def run_external(
         self,
-        executable: Callable[
-            [List[TestCaseVariableValue]], List[NamedTestCaseVariableValueRequest]
-        ],
+        executable: Callable[[List[TestCaseVariableValue]], List[NamedTestCaseVariableValueRequest]],
     ) -> VellumTestSuiteRunResults:
         """
         Runs this Vellum Test Suite on any executable function defined external to Vellum.

vellum/plugins/pydantic.py

@@ -0,0 +1,74 @@
+from typing import Any, Dict, Literal, Optional, Tuple, Union
+
+from pydantic.plugin import (
+    PydanticPluginProtocol,
+    SchemaKind,
+    SchemaTypePath,
+    ValidateJsonHandlerProtocol,
+    ValidatePythonHandlerProtocol,
+    ValidateStringsHandlerProtocol,
+)
+from pydantic_core import CoreSchema
+
+from vellum.workflows.descriptors.base import BaseDescriptor
+
+
+# https://docs.pydantic.dev/2.8/concepts/plugins/#build-a-plugin
+class OnValidatePython(ValidatePythonHandlerProtocol):
+    tracked_descriptors: Dict[str, BaseDescriptor] = {}
+
+    def on_enter(
+        self,
+        input: Any,
+        *,
+        strict: Optional[bool] = None,
+        from_attributes: Optional[bool] = None,
+        context: Optional[Dict[str, Any]] = None,
+        self_instance: Optional[Any] = None,
+        allow_partial: Union[bool, Literal["off", "on", "trailing-strings"]] = False,
+    ) -> None:
+        if not isinstance(input, dict):
+            return
+
+        self.tracked_descriptors = {}
+
+        for key, value in input.items():
+            if isinstance(value, BaseDescriptor):
+                self.tracked_descriptors[key] = value
+                # TODO: This does not yet work for descriptors that map to more complex types
+                # https://app.shortcut.com/vellum/story/4636
+                input[key] = value.types[0]()
+
+    def on_success(self, result: Any) -> None:
+        if self.tracked_descriptors:
+            frozen = result.model_config.get("frozen")
+            if frozen:
+                result.model_config["frozen"] = False
+
+            for key, value in self.tracked_descriptors.items():
+                setattr(result, key, value)
+
+            if frozen:
+                result.model_config["frozen"] = True
+
+            self.tracked_descriptors = {}
+
+
+class VellumPydanticPlugin(PydanticPluginProtocol):
+    def new_schema_validator(
+        self,
+        schema: CoreSchema,
+        schema_type: Any,
+        schema_type_path: SchemaTypePath,
+        schema_kind: SchemaKind,
+        config: Any,
+        plugin_settings: Dict[str, Any],
+    ) -> Tuple[
+        Union[ValidatePythonHandlerProtocol, None],
+        Union[ValidateJsonHandlerProtocol, None],
+        Union[ValidateStringsHandlerProtocol, None],
+    ]:
+        return OnValidatePython(), None, None
+
+
+pydantic_plugin = VellumPydanticPlugin()

vellum/plugins/utils.py

@@ -0,0 +1,19 @@
+from pydantic.plugin import _loader as _pydantic_plugin_loader
+
+from vellum.plugins.pydantic import pydantic_plugin
+
+_loaded = False
+
+
+def load_runtime_plugins() -> None:
+    global _loaded
+    if _loaded:
+        return
+    _loaded = True
+
+    # TODO: This is a hack to get the Vellum plugin to load. We're supposed to use
+    # pyproject.toml, but I couldn't figure out after an hour
+    # https://app.shortcut.com/vellum/story/4635
+    _pydantic_plugin_loader.get_plugins()
+    if _pydantic_plugin_loader._plugins is not None:
+        _pydantic_plugin_loader._plugins["vellum_plugin"] = pydantic_plugin