vellum-ai 0.14.35__py3-none-any.whl → 0.14.37__py3-none-any.whl

vellum/__init__.py

@@ -300,6 +300,9 @@ from .types import (
     PromptDeploymentExpandMetaRequest,
     PromptDeploymentInputRequest,
     PromptDeploymentParentContext,
+    PromptDeploymentRelease,
+    PromptDeploymentReleasePromptDeployment,
+    PromptDeploymentReleasePromptVersion,
     PromptExecConfig,
     PromptExecutionMeta,
     PromptNodeExecutionMeta,
@@ -922,6 +925,9 @@ __all__ = [
     "PromptDeploymentExpandMetaRequest",
     "PromptDeploymentInputRequest",
     "PromptDeploymentParentContext",
+    "PromptDeploymentRelease",
+    "PromptDeploymentReleasePromptDeployment",
+    "PromptDeploymentReleasePromptVersion",
     "PromptExecConfig",
     "PromptExecutionMeta",
     "PromptNodeExecutionMeta",

vellum/client/__init__.py

@@ -7,6 +7,7 @@ from .core.client_wrapper import SyncClientWrapper
 from .resources.ad_hoc.client import AdHocClient
 from .resources.container_images.client import ContainerImagesClient
 from .resources.deployments.client import DeploymentsClient
+from .resources.release_reviews.client import ReleaseReviewsClient
 from .resources.document_indexes.client import DocumentIndexesClient
 from .resources.documents.client import DocumentsClient
 from .resources.folder_entities.client import FolderEntitiesClient
@@ -18,7 +19,6 @@ from .resources.sandboxes.client import SandboxesClient
 from .resources.test_suite_runs.client import TestSuiteRunsClient
 from .resources.test_suites.client import TestSuitesClient
 from .resources.workflow_deployments.client import WorkflowDeploymentsClient
-from .resources.release_reviews.client import ReleaseReviewsClient
 from .resources.workflow_sandboxes.client import WorkflowSandboxesClient
 from .resources.workflows.client import WorkflowsClient
 from .resources.workspace_secrets.client import WorkspaceSecretsClient
@@ -65,6 +65,7 @@ from .core.client_wrapper import AsyncClientWrapper
 from .resources.ad_hoc.client import AsyncAdHocClient
 from .resources.container_images.client import AsyncContainerImagesClient
 from .resources.deployments.client import AsyncDeploymentsClient
+from .resources.release_reviews.client import AsyncReleaseReviewsClient
 from .resources.document_indexes.client import AsyncDocumentIndexesClient
 from .resources.documents.client import AsyncDocumentsClient
 from .resources.folder_entities.client import AsyncFolderEntitiesClient
@@ -76,7 +77,6 @@ from .resources.sandboxes.client import AsyncSandboxesClient
 from .resources.test_suite_runs.client import AsyncTestSuiteRunsClient
 from .resources.test_suites.client import AsyncTestSuitesClient
 from .resources.workflow_deployments.client import AsyncWorkflowDeploymentsClient
-from .resources.release_reviews.client import AsyncReleaseReviewsClient
 from .resources.workflow_sandboxes.client import AsyncWorkflowSandboxesClient
 from .resources.workflows.client import AsyncWorkflowsClient
 from .resources.workspace_secrets.client import AsyncWorkspaceSecretsClient
@@ -143,6 +143,7 @@ class Vellum:
         self.ad_hoc = AdHocClient(client_wrapper=self._client_wrapper)
         self.container_images = ContainerImagesClient(client_wrapper=self._client_wrapper)
         self.deployments = DeploymentsClient(client_wrapper=self._client_wrapper)
+        self.release_reviews = ReleaseReviewsClient(client_wrapper=self._client_wrapper)
         self.document_indexes = DocumentIndexesClient(client_wrapper=self._client_wrapper)
         self.documents = DocumentsClient(client_wrapper=self._client_wrapper)
         self.folder_entities = FolderEntitiesClient(client_wrapper=self._client_wrapper)
@@ -154,7 +155,6 @@ class Vellum:
         self.test_suite_runs = TestSuiteRunsClient(client_wrapper=self._client_wrapper)
         self.test_suites = TestSuitesClient(client_wrapper=self._client_wrapper)
         self.workflow_deployments = WorkflowDeploymentsClient(client_wrapper=self._client_wrapper)
-        self.release_reviews = ReleaseReviewsClient(client_wrapper=self._client_wrapper)
         self.workflow_sandboxes = WorkflowSandboxesClient(client_wrapper=self._client_wrapper)
         self.workflows = WorkflowsClient(client_wrapper=self._client_wrapper)
         self.workspace_secrets = WorkspaceSecretsClient(client_wrapper=self._client_wrapper)
@@ -1486,6 +1486,7 @@ class AsyncVellum:
         self.ad_hoc = AsyncAdHocClient(client_wrapper=self._client_wrapper)
         self.container_images = AsyncContainerImagesClient(client_wrapper=self._client_wrapper)
         self.deployments = AsyncDeploymentsClient(client_wrapper=self._client_wrapper)
+        self.release_reviews = AsyncReleaseReviewsClient(client_wrapper=self._client_wrapper)
         self.document_indexes = AsyncDocumentIndexesClient(client_wrapper=self._client_wrapper)
         self.documents = AsyncDocumentsClient(client_wrapper=self._client_wrapper)
         self.folder_entities = AsyncFolderEntitiesClient(client_wrapper=self._client_wrapper)
@@ -1497,7 +1498,6 @@ class AsyncVellum:
         self.test_suite_runs = AsyncTestSuiteRunsClient(client_wrapper=self._client_wrapper)
         self.test_suites = AsyncTestSuitesClient(client_wrapper=self._client_wrapper)
         self.workflow_deployments = AsyncWorkflowDeploymentsClient(client_wrapper=self._client_wrapper)
-        self.release_reviews = AsyncReleaseReviewsClient(client_wrapper=self._client_wrapper)
         self.workflow_sandboxes = AsyncWorkflowSandboxesClient(client_wrapper=self._client_wrapper)
         self.workflows = AsyncWorkflowsClient(client_wrapper=self._client_wrapper)
         self.workspace_secrets = AsyncWorkspaceSecretsClient(client_wrapper=self._client_wrapper)

vellum/client/core/client_wrapper.py

@@ -18,7 +18,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "vellum-ai",
-            "X-Fern-SDK-Version": "0.14.35",
+            "X-Fern-SDK-Version": "0.14.37",
         }
         headers["X_API_KEY"] = self.api_key
         return headers

vellum/client/resources/release_reviews/client.py

@@ -3,11 +3,12 @@
 from ...core.client_wrapper import SyncClientWrapper
 import typing
 from ...core.request_options import RequestOptions
-from ...types.workflow_deployment_release import WorkflowDeploymentRelease
+from ...types.prompt_deployment_release import PromptDeploymentRelease
 from ...core.jsonable_encoder import jsonable_encoder
 from ...core.pydantic_utilities import parse_obj_as
 from json.decoder import JSONDecodeError
 from ...core.api_error import ApiError
+from ...types.workflow_deployment_release import WorkflowDeploymentRelease
 from ...core.client_wrapper import AsyncClientWrapper
 
 
@@ -15,6 +16,60 @@ class ReleaseReviewsClient:
     def __init__(self, *, client_wrapper: SyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    def retrieve_prompt_deployment_release(
+        self, id: str, release_id_or_release_tag: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> PromptDeploymentRelease:
+        """
+        Retrieve a specific Prompt Deployment Release by either its UUID or the name of a Release Tag that points to it.
+
+        Parameters
+        ----------
+        id : str
+            A UUID string identifying this deployment.
+
+        release_id_or_release_tag : str
+            Either the UUID of Prompt Deployment Release you'd like to retrieve, or the name of a Release Tag that's pointing to the Prompt Deployment Release you'd like to retrieve.
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        PromptDeploymentRelease
+
+
+        Examples
+        --------
+        from vellum import Vellum
+
+        client = Vellum(
+            api_key="YOUR_API_KEY",
+        )
+        client.release_reviews.retrieve_prompt_deployment_release(
+            id="id",
+            release_id_or_release_tag="release_id_or_release_tag",
+        )
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"v1/deployments/{jsonable_encoder(id)}/releases/{jsonable_encoder(release_id_or_release_tag)}",
+            base_url=self._client_wrapper.get_environment().default,
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    PromptDeploymentRelease,
+                    parse_obj_as(
+                        type_=PromptDeploymentRelease,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     def retrieve_workflow_deployment_release(
         self, id: str, release_id_or_release_tag: str, *, request_options: typing.Optional[RequestOptions] = None
     ) -> WorkflowDeploymentRelease:
@@ -74,6 +129,68 @@ class AsyncReleaseReviewsClient:
     def __init__(self, *, client_wrapper: AsyncClientWrapper):
         self._client_wrapper = client_wrapper
 
+    async def retrieve_prompt_deployment_release(
+        self, id: str, release_id_or_release_tag: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> PromptDeploymentRelease:
+        """
+        Retrieve a specific Prompt Deployment Release by either its UUID or the name of a Release Tag that points to it.
+
+        Parameters
+        ----------
+        id : str
+            A UUID string identifying this deployment.
+
+        release_id_or_release_tag : str
+            Either the UUID of Prompt Deployment Release you'd like to retrieve, or the name of a Release Tag that's pointing to the Prompt Deployment Release you'd like to retrieve.
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        PromptDeploymentRelease
+
+
+        Examples
+        --------
+        import asyncio
+
+        from vellum import AsyncVellum
+
+        client = AsyncVellum(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.release_reviews.retrieve_prompt_deployment_release(
+                id="id",
+                release_id_or_release_tag="release_id_or_release_tag",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"v1/deployments/{jsonable_encoder(id)}/releases/{jsonable_encoder(release_id_or_release_tag)}",
+            base_url=self._client_wrapper.get_environment().default,
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                return typing.cast(
+                    PromptDeploymentRelease,
+                    parse_obj_as(
+                        type_=PromptDeploymentRelease,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
     async def retrieve_workflow_deployment_release(
         self, id: str, release_id_or_release_tag: str, *, request_options: typing.Optional[RequestOptions] = None
     ) -> WorkflowDeploymentRelease:

vellum/client/types/__init__.py

@@ -308,6 +308,9 @@ from .prompt_block_state import PromptBlockState
 from .prompt_deployment_expand_meta_request import PromptDeploymentExpandMetaRequest
 from .prompt_deployment_input_request import PromptDeploymentInputRequest
 from .prompt_deployment_parent_context import PromptDeploymentParentContext
+from .prompt_deployment_release import PromptDeploymentRelease
+from .prompt_deployment_release_prompt_deployment import PromptDeploymentReleasePromptDeployment
+from .prompt_deployment_release_prompt_version import PromptDeploymentReleasePromptVersion
 from .prompt_exec_config import PromptExecConfig
 from .prompt_execution_meta import PromptExecutionMeta
 from .prompt_node_execution_meta import PromptNodeExecutionMeta
@@ -902,6 +905,9 @@ __all__ = [
     "PromptDeploymentExpandMetaRequest",
     "PromptDeploymentInputRequest",
     "PromptDeploymentParentContext",
+    "PromptDeploymentRelease",
+    "PromptDeploymentReleasePromptDeployment",
+    "PromptDeploymentReleasePromptVersion",
     "PromptExecConfig",
     "PromptExecutionMeta",
     "PromptNodeExecutionMeta",

vellum/client/types/logical_operator.py

@@ -29,6 +29,7 @@ LogicalOperator = typing.Union[
         "parseJson",
         "and",
         "or",
+        "isError",
     ],
     typing.Any,
 ]

vellum/client/types/prompt_deployment_release.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from ..core.pydantic_utilities import UniversalBaseModel
+import datetime as dt
+from .release_environment import ReleaseEnvironment
+import typing
+from .release_created_by import ReleaseCreatedBy
+from .prompt_deployment_release_prompt_version import PromptDeploymentReleasePromptVersion
+from .prompt_deployment_release_prompt_deployment import PromptDeploymentReleasePromptDeployment
+from .release_release_tag import ReleaseReleaseTag
+from .slim_release_review import SlimReleaseReview
+from ..core.pydantic_utilities import IS_PYDANTIC_V2
+import pydantic
+
+
+class PromptDeploymentRelease(UniversalBaseModel):
+    id: str
+    created: dt.datetime
+    environment: ReleaseEnvironment
+    created_by: typing.Optional[ReleaseCreatedBy] = None
+    prompt_version: PromptDeploymentReleasePromptVersion
+    deployment: PromptDeploymentReleasePromptDeployment
+    description: typing.Optional[str] = None
+    release_tags: typing.List[ReleaseReleaseTag]
+    reviews: typing.List[SlimReleaseReview]
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

vellum/client/types/prompt_deployment_release_prompt_deployment.py

@@ -0,0 +1,19 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from ..core.pydantic_utilities import UniversalBaseModel
+from ..core.pydantic_utilities import IS_PYDANTIC_V2
+import typing
+import pydantic
+
+
+class PromptDeploymentReleasePromptDeployment(UniversalBaseModel):
+    name: str
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

vellum/client/types/prompt_deployment_release_prompt_version.py

@@ -0,0 +1,19 @@
+# This file was auto-generated by Fern from our API Definition.
+
+from ..core.pydantic_utilities import UniversalBaseModel
+from ..core.pydantic_utilities import IS_PYDANTIC_V2
+import typing
+import pydantic
+
+
+class PromptDeploymentReleasePromptVersion(UniversalBaseModel):
+    id: str
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

vellum/types/prompt_deployment_release.py

@@ -0,0 +1,3 @@
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
+
+from vellum.client.types.prompt_deployment_release import *

vellum/types/prompt_deployment_release_prompt_deployment.py

@@ -0,0 +1,3 @@
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
+
+from vellum.client.types.prompt_deployment_release_prompt_deployment import *

vellum/types/prompt_deployment_release_prompt_version.py

@@ -0,0 +1,3 @@
+# WARNING: This file will be removed in a future release. Please import from "vellum.client" instead.
+
+from vellum.client.types.prompt_deployment_release_prompt_version import *

vellum/workflows/inputs/base.py

@@ -4,6 +4,7 @@ from typing_extensions import dataclass_transform
 from pydantic import GetCoreSchemaHandler
 from pydantic_core import core_schema
 
+from vellum.workflows.constants import undefined
 from vellum.workflows.errors.types import WorkflowErrorCode
 from vellum.workflows.exceptions import WorkflowInitializationException
 from vellum.workflows.references import ExternalInputReference, WorkflowInputReference
@@ -15,7 +16,7 @@ from vellum.workflows.types.utils import get_class_attr_names, infer_types
 class _BaseInputsMeta(type):
     def __getattribute__(cls, name: str) -> Any:
         if not name.startswith("_") and name in cls.__annotations__ and issubclass(cls, BaseInputs):
-            instance = vars(cls).get(name)
+            instance = vars(cls).get(name, undefined)
             types = infer_types(cls, name)
 
             if getattr(cls, "__descriptor_class__", None) is ExternalInputReference:

vellum/workflows/nodes/displayable/bases/prompt_deployment_node.py

@@ -38,6 +38,7 @@ class BasePromptDeploymentNode(BasePromptNode, Generic[StateType]):
     expand_raw: Optional[Sequence[str]] - Expandable raw fields to include in the response
     metadata: Optional[Dict[str, Optional[Any]]] - The metadata to use for the Prompt Execution
     request_options: Optional[RequestOptions] - The request options to use for the Prompt Execution
+    ml_model_fallback: Optional[Sequence[str]] - ML model fallbacks to use
     """
 
     # Either the Prompt Deployment's UUID or its name.
@@ -50,6 +51,7 @@ class BasePromptDeploymentNode(BasePromptNode, Generic[StateType]):
     raw_overrides: Optional[RawPromptExecutionOverridesRequest] = OMIT
     expand_raw: Optional[Sequence[str]] = OMIT
     metadata: Optional[Dict[str, Optional[Any]]] = OMIT
+    ml_model_fallbacks: Optional[Sequence[str]] = OMIT
 
     class Trigger(BasePromptNode.Trigger):
         merge_behavior = MergeBehavior.AWAIT_ANY

vellum/workflows/nodes/displayable/guardrail_node/node.py

@@ -2,6 +2,7 @@ from uuid import UUID
 from typing import Any, ClassVar, Dict, Generic, List, Optional, Union, cast
 
 from vellum import ChatHistoryInput, ChatMessage, JsonInput, MetricDefinitionInput, NumberInput, StringInput
+from vellum.client import ApiError
 from vellum.core import RequestOptions
 from vellum.workflows.constants import LATEST_RELEASE_TAG
 from vellum.workflows.errors.types import WorkflowErrorCode
@@ -34,28 +35,50 @@ class GuardrailNode(BaseNode[StateType], Generic[StateType]):
 
     class Outputs(BaseOutputs):
         score: float
+        normalized_score: Optional[float]
         log: Optional[str]
 
     def run(self) -> Outputs:
-        metric_execution = self._context.vellum_client.metric_definitions.execute_metric_definition(
-            self.metric_definition if isinstance(self.metric_definition, str) else str(self.metric_definition),
-            inputs=self._compile_metric_inputs(),
-            release_tag=self.release_tag,
-            request_options=self.request_options,
-        )
+        try:
+            metric_execution = self._context.vellum_client.metric_definitions.execute_metric_definition(
+                self.metric_definition if isinstance(self.metric_definition, str) else str(self.metric_definition),
+                inputs=self._compile_metric_inputs(),
+                release_tag=self.release_tag,
+                request_options=self.request_options,
+            )
+
+        except ApiError:
+            raise NodeException(
+                code=WorkflowErrorCode.NODE_EXECUTION,
+                message="Failed to execute metric definition",
+            )
 
         metric_outputs = {output.name: output.value for output in metric_execution.outputs}
 
-        score = metric_outputs.get("score")
+        SCORE_KEY = "score"
+        NORMALIZED_SCORE_KEY = "normalized_score"
+        LOG_KEY = "log"
+
+        score = metric_outputs.get(SCORE_KEY)
         if not isinstance(score, float):
             raise NodeException(
-                message="Metric execution must have one output named 'score' with type 'float'",
+                message=f"Metric execution must have one output named '{SCORE_KEY}' with type 'float'",
                 code=WorkflowErrorCode.INVALID_OUTPUTS,
             )
-        metric_outputs.pop("score")
+        metric_outputs.pop(SCORE_KEY)
+
+        if NORMALIZED_SCORE_KEY in metric_outputs:
+            normalized_score = metric_outputs.pop(NORMALIZED_SCORE_KEY)
+            if not isinstance(normalized_score, float):
+                raise NodeException(
+                    message=f"Metric execution must have one output named '{NORMALIZED_SCORE_KEY}' with type 'float'",
+                    code=WorkflowErrorCode.INVALID_OUTPUTS,
+                )
+        else:
+            normalized_score = None
 
-        if "log" in metric_outputs:
-            log = metric_outputs.pop("log") or ""
+        if LOG_KEY in metric_outputs:
+            log = metric_outputs.pop(LOG_KEY) or ""
             if not isinstance(log, str):
                 raise NodeException(
                     message="Metric execution log output must be of type 'str'",
@@ -64,7 +87,7 @@ class GuardrailNode(BaseNode[StateType], Generic[StateType]):
         else:
             log = None
 
-        return self.Outputs(score=score, log=log, **metric_outputs)
+        return self.Outputs(score=score, normalized_score=normalized_score, log=log, **metric_outputs)
 
     def _compile_metric_inputs(self) -> List[MetricDefinitionInput]:
         # TODO: We may want to consolidate with prompt deployment input compilation

vellum/workflows/nodes/displayable/guardrail_node/test_node.py

@@ -1,8 +1,11 @@
 import pytest
 
 from vellum import TestSuiteRunMetricNumberOutput
+from vellum.client import ApiError
 from vellum.client.types.metric_definition_execution import MetricDefinitionExecution
 from vellum.client.types.test_suite_run_metric_string_output import TestSuiteRunMetricStringOutput
+from vellum.workflows.errors import WorkflowErrorCode
+from vellum.workflows.exceptions import NodeException
 from vellum.workflows.nodes.displayable.guardrail_node.node import GuardrailNode
 
 
@@ -36,3 +39,88 @@ def test_run_guardrail_node__empty_log(vellum_client, log_value):
     # THEN the workflow should have completed successfully
     assert outputs.score == 0.6
     assert outputs.log == ""
+
+
+def test_run_guardrail_node__normalized_score(vellum_client):
+    """Confirm that we can successfully invoke a Guardrail Node"""
+
+    # GIVEN a Guardrail Node
+    class MyGuard(GuardrailNode):
+        metric_definition = "example_metric_definition"
+        metric_inputs = {}
+
+    # AND we know that the guardrail node will return a normalized score
+    mock_metric_execution = MetricDefinitionExecution(
+        outputs=[
+            TestSuiteRunMetricNumberOutput(
+                name="score",
+                value=0.6,
+            ),
+            TestSuiteRunMetricNumberOutput(
+                name="normalized_score",
+                value=1.0,
+            ),
+        ],
+    )
+    vellum_client.metric_definitions.execute_metric_definition.return_value = mock_metric_execution
+
+    # WHEN we run the Guardrail Node
+    outputs = MyGuard().run()
+
+    # THEN the workflow should have completed successfully
+    assert outputs.score == 0.6
+    assert outputs.normalized_score == 1.0
+
+
+def test_run_guardrail_node__normalized_score_null(vellum_client):
+    # GIVEN a Guardrail Node
+    class MyGuard(GuardrailNode):
+        metric_definition = "example_metric_definition"
+        metric_inputs = {}
+
+    # AND we know that the guardrail node will return a normalized score that is None
+    mock_metric_execution = MetricDefinitionExecution(
+        outputs=[
+            TestSuiteRunMetricNumberOutput(
+                name="score",
+                value=0.6,
+            ),
+            TestSuiteRunMetricNumberOutput(
+                name="normalized_score",
+                value=None,
+            ),
+        ],
+    )
+    vellum_client.metric_definitions.execute_metric_definition.return_value = mock_metric_execution
+
+    # WHEN we run the Guardrail Node
+    with pytest.raises(NodeException) as exc_info:
+        MyGuard().run()
+
+    # THEN we get an exception
+    assert exc_info.value.message == "Metric execution must have one output named 'normalized_score' with type 'float'"
+    assert exc_info.value.code == WorkflowErrorCode.INVALID_OUTPUTS
+
+
+def test_run_guardrail_node__api_error(vellum_client):
+    # GIVEN a Guardrail Node
+    class MyGuard(GuardrailNode):
+        metric_definition = "example_metric_definition"
+        metric_inputs = {}
+
+    # AND the API client raises an ApiError when called
+    api_error = ApiError(status_code=503)
+    vellum_client.metric_definitions.execute_metric_definition.side_effect = api_error
+
+    # WHEN we run the Guardrail Node
+    with pytest.raises(NodeException) as exc_info:
+        MyGuard().run()
+
+    # THEN we get a NodeException with the appropriate error code
+    assert exc_info.value.code == WorkflowErrorCode.NODE_EXECUTION
+    assert "Failed to execute metric definition" in exc_info.value.message
+
+    # Verify the mock was called with the expected arguments
+    vellum_client.metric_definitions.execute_metric_definition.assert_called_once_with(
+        "example_metric_definition", inputs=[], release_tag="LATEST", request_options=None
+    )

vellum/workflows/nodes/displayable/prompt_deployment_node/node.py

@@ -1,6 +1,7 @@
 import json
-from typing import Iterator
+from typing import Any, Dict, Iterator, Type, Union
 
+from vellum.workflows.constants import undefined
 from vellum.workflows.errors import WorkflowErrorCode
 from vellum.workflows.exceptions import NodeException
 from vellum.workflows.nodes.displayable.bases import BasePromptDeploymentNode as BasePromptDeploymentNode
@@ -11,7 +12,7 @@ from vellum.workflows.types.generics import StateType
 
 class PromptDeploymentNode(BasePromptDeploymentNode[StateType]):
     """
-    Used to execute a Prompt Deployment and surface a string output for convenience.
+    Used to execute a Prompt Deployment and surface a string output and json output if applicable for convenience.
 
     prompt_inputs: EntityInputsInterface - The inputs for the Prompt
     deployment: Union[UUID, str] - Either the Prompt Deployment's UUID or its name.
@@ -33,9 +34,11 @@ class PromptDeploymentNode(BasePromptDeploymentNode[StateType]):
         The outputs of the PromptDeploymentNode.
 
         text: str - The result of the Prompt Execution
+        json: Optional[Dict[Any, Any]] - The result of the Prompt Execution in JSON format
         """
 
         text: str
+        json: Union[Dict[Any, Any], Type[undefined]] = undefined
 
     def run(self) -> Iterator[BaseOutput]:
         outputs = yield from self._process_prompt_event_stream()
@@ -46,12 +49,18 @@ class PromptDeploymentNode(BasePromptDeploymentNode[StateType]):
             )
 
         string_outputs = []
+        json_output = None
+
         for output in outputs:
             if output.value is None:
                 continue
 
             if output.type == "STRING":
                 string_outputs.append(output.value)
+                try:
+                    json_output = json.loads(output.value)
+                except (json.JSONDecodeError, TypeError):
+                    pass
             elif output.type == "JSON":
                 string_outputs.append(json.dumps(output.value, indent=4))
             elif output.type == "FUNCTION_CALL":
@@ -61,3 +70,6 @@ class PromptDeploymentNode(BasePromptDeploymentNode[StateType]):
 
         value = "\n".join(string_outputs)
         yield BaseOutput(name="text", value=value)
+
+        if json_output:
+            yield BaseOutput(name="json", value=json_output)