flyte 0.2.0b12__py3-none-any.whl → 0.2.0b14__py3-none-any.whl

flyte/_protos/workflow/state_service_pb2_grpc.py

@@ -15,15 +15,15 @@ class StateServiceStub(object):
         Args:
             channel: A grpc.Channel.
         """
-        self.Store = channel.unary_unary(
-                '/cloudidl.workflow.StateService/Store',
-                request_serializer=workflow_dot_state__service__pb2.StoreRequest.SerializeToString,
-                response_deserializer=workflow_dot_state__service__pb2.StoreResponse.FromString,
+        self.Put = channel.stream_stream(
+                '/cloudidl.workflow.StateService/Put',
+                request_serializer=workflow_dot_state__service__pb2.PutRequest.SerializeToString,
+                response_deserializer=workflow_dot_state__service__pb2.PutResponse.FromString,
                 )
-        self.Load = channel.unary_unary(
-                '/cloudidl.workflow.StateService/Load',
-                request_serializer=workflow_dot_state__service__pb2.LoadRequest.SerializeToString,
-                response_deserializer=workflow_dot_state__service__pb2.LoadResponse.FromString,
+        self.Get = channel.stream_stream(
+                '/cloudidl.workflow.StateService/Get',
+                request_serializer=workflow_dot_state__service__pb2.GetRequest.SerializeToString,
+                response_deserializer=workflow_dot_state__service__pb2.GetResponse.FromString,
                 )
         self.Watch = channel.unary_stream(
                 '/cloudidl.workflow.StateService/Watch',
@@ -36,15 +36,15 @@ class StateServiceServicer(object):
     """provides an interface for managing the state of actions.
     """
 
-    def Store(self, request, context):
-        """store the state of an action.
+    def Put(self, request_iterator, context):
+        """put the state of an action.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def Load(self, request, context):
-        """load the state of an action.
+    def Get(self, request_iterator, context):
+        """get the state of an action.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -60,15 +60,15 @@ class StateServiceServicer(object):
 
 def add_StateServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
-            'Store': grpc.unary_unary_rpc_method_handler(
-                    servicer.Store,
-                    request_deserializer=workflow_dot_state__service__pb2.StoreRequest.FromString,
-                    response_serializer=workflow_dot_state__service__pb2.StoreResponse.SerializeToString,
+            'Put': grpc.stream_stream_rpc_method_handler(
+                    servicer.Put,
+                    request_deserializer=workflow_dot_state__service__pb2.PutRequest.FromString,
+                    response_serializer=workflow_dot_state__service__pb2.PutResponse.SerializeToString,
             ),
-            'Load': grpc.unary_unary_rpc_method_handler(
-                    servicer.Load,
-                    request_deserializer=workflow_dot_state__service__pb2.LoadRequest.FromString,
-                    response_serializer=workflow_dot_state__service__pb2.LoadResponse.SerializeToString,
+            'Get': grpc.stream_stream_rpc_method_handler(
+                    servicer.Get,
+                    request_deserializer=workflow_dot_state__service__pb2.GetRequest.FromString,
+                    response_serializer=workflow_dot_state__service__pb2.GetResponse.SerializeToString,
             ),
             'Watch': grpc.unary_stream_rpc_method_handler(
                     servicer.Watch,
@@ -87,7 +87,7 @@ class StateService(object):
     """
 
     @staticmethod
-    def Store(request,
+    def Put(request_iterator,
             target,
             options=(),
             channel_credentials=None,
@@ -97,14 +97,14 @@ class StateService(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/cloudidl.workflow.StateService/Store',
-            workflow_dot_state__service__pb2.StoreRequest.SerializeToString,
-            workflow_dot_state__service__pb2.StoreResponse.FromString,
+        return grpc.experimental.stream_stream(request_iterator, target, '/cloudidl.workflow.StateService/Put',
+            workflow_dot_state__service__pb2.PutRequest.SerializeToString,
+            workflow_dot_state__service__pb2.PutResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def Load(request,
+    def Get(request_iterator,
             target,
             options=(),
             channel_credentials=None,
@@ -114,9 +114,9 @@ class StateService(object):
             wait_for_ready=None,
             timeout=None,
             metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/cloudidl.workflow.StateService/Load',
-            workflow_dot_state__service__pb2.LoadRequest.SerializeToString,
-            workflow_dot_state__service__pb2.LoadResponse.FromString,
+        return grpc.experimental.stream_stream(request_iterator, target, '/cloudidl.workflow.StateService/Get',
+            workflow_dot_state__service__pb2.GetRequest.SerializeToString,
+            workflow_dot_state__service__pb2.GetResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 

flyte/_run.py

@@ -63,6 +63,7 @@ class _Runner:
         raw_data_path: str | None = None,
         metadata_path: str | None = None,
         run_base_dir: str | None = None,
+        overwrite_cache: bool = False,
     ):
         init_config = _get_init_config()
         client = init_config.client if init_config else None
@@ -81,6 +82,7 @@ class _Runner:
         self._raw_data_path = raw_data_path
         self._metadata_path = metadata_path or "/tmp"
         self._run_base_dir = run_base_dir or "/tmp/base"
+        self._overwrite_cache = overwrite_cache
 
     @requires_initialization
     async def _run_remote(self, obj: TaskTemplate[P, R] | LazyEntity, *args: P.args, **kwargs: P.kwargs) -> Run:
@@ -182,6 +184,9 @@ class _Runner:
                         project_id=project_id,
                         task_spec=task_spec,
                         inputs=inputs.proto_inputs,
+                        run_spec=run_definition_pb2.RunSpec(
+                            overwrite_cache=self._overwrite_cache,
+                        ),
                     ),
                 )
                 return Run(pb2=resp.run)
@@ -414,6 +419,7 @@ def with_runcontext(
     interactive_mode: bool | None = None,
     raw_data_path: str | None = None,
     run_base_dir: str | None = None,
+    overwrite_cache: bool = False,
 ) -> _Runner:
     """
     Launch a new run with the given parameters as the context.

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b12'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b12')
+__version__ = version = '0.2.0b14'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b14')

flyte/cli/_common.py

@@ -137,8 +137,8 @@ class InvokeBaseMixin:
             if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
                 raise click.ClickException(f"Invalid argument provided. Please check your input. Error: {e.details()}")
             raise click.ClickException(f"RPC error invoking command: {e!s}") from e
-        except flyte.errors.InitializationError:
-            raise click.ClickException("Initialize the CLI with a remote configuration. For example, pass --endpoint")
+        except flyte.errors.InitializationError as e:
+            raise click.ClickException(f"Initialization failed. Pass remote config for CLI. (Reason: {e})")
         except flyte.errors.BaseRuntimeError as e:
             raise click.ClickException(f"{e.kind} failure, {e.code}. {e}") from e
         except click.exceptions.Exit as e:

flyte/cli/_delete.py

@@ -10,7 +10,7 @@ def delete():
     """
 
 
-@click.command(cls=common.CommandBase)
+@delete.command(cls=common.CommandBase)
 @click.argument("name", type=str, required=True)
 @click.pass_obj
 def secret(cfg: common.CLIConfig, name: str, project: str | None = None, domain: str | None = None):

flyte/cli/_deploy.py

@@ -147,6 +147,6 @@ deploy = EnvFiles(
     name="deploy",
     help="""
     Deploy one or more environments from a python file.
-    The deploy command will create or update environments in the Flyte system.
+    This command will create or update environments in the Flyte system.
     """,
 )

flyte/cli/_get.py

@@ -20,7 +20,7 @@ def get():
     Using a `get` subcommand without any arguments will retrieve a list of available resources to get.
     For example:
 
-    * `get project` (without specifiying aproject), will list all projects.
+    * `get project` (without specifying a project), will list all projects.
     * `get project my_project` will return the details of the project named `my_project`.
 
     In some cases, a partially specified command will act as a filter and return available further parameters.
@@ -143,7 +143,7 @@ def action(
     "--pretty",
     is_flag=True,
     default=False,
-    help="Show logs in a auto scrolling box, where number of lines is limited to `--lines`",
+    help="Show logs in an auto-scrolling box, where number of lines is limited to `--lines`",
 )
 @click.option(
     "--attempt", "-a", type=int, default=None, help="Attempt number to show logs for, defaults to the latest attempt."

flyte/cli/_run.py

@@ -66,7 +66,7 @@ class RunArguments:
                 ["--follow", "-f"],
                 is_flag=True,
                 default=False,
-                help="Wait and watch logs for the parent action. If not provided, the cli will exit after "
+                help="Wait and watch logs for the parent action. If not provided, the CLI will exit after "
                 "successfully launching a remote execution with a link to the UI.",
             )
         },
@@ -99,8 +99,6 @@ class RunTaskCommand(click.Command):
 
             obj = CLIConfig(flyte.config.auto(), ctx)
 
-        if not self.run_args.local:
-            assert obj.endpoint, "CLI Config should have an endpoint"
         obj.init(self.run_args.project, self.run_args.domain)
 
         async def _run():
@@ -108,7 +106,6 @@ class RunTaskCommand(click.Command):
 
             r = flyte.with_runcontext(
                 copy_style=self.run_args.copy_style,
-                version=self.run_args.copy_style,
                 mode="local" if self.run_args.local else "remote",
                 name=self.run_args.name,
             ).run(self.obj, **ctx.params)

flyte/cli/main.py

@@ -5,31 +5,31 @@ from flyte._logging import initialize_logger, logger
 from ._abort import abort
 from ._common import CLIConfig
 from ._create import create
+from ._delete import delete
 from ._deploy import deploy
 from ._gen import gen
 from ._get import get
 from ._run import run
 
-click.rich_click.COMMAND_GROUPS = {
-    "flyte": [
-        {
-            "name": "Running workflows",
-            "commands": ["run", "abort"],
-        },
-        {
-            "name": "Management",
-            "commands": ["create", "deploy", "get"],
-        },
-        {
-            "name": "Documentation generation",
-            "commands": ["gen"],
-        },
-    ]
-}
-
 help_config = click.RichHelpConfiguration(
     use_markdown=True,
     use_markdown_emoji=True,
+    command_groups={
+        "flyte": [
+            {
+                "name": "Run and stop tasks",
+                "commands": ["run", "abort"],
+            },
+            {
+                "name": "Management",
+                "commands": ["create", "deploy", "get", "delete"],
+            },
+            {
+                "name": "Documentation generation",
+                "commands": ["gen"],
+            },
+        ]
+    },
 )
 
 
@@ -103,7 +103,7 @@ def main(
     config_file: str | None,
 ):
     """
-    The Flyte CLI is the the command line interface for working with the Flyte SDK and backend.
+    The Flyte CLI is the command line interface for working with the Flyte SDK and backend.
 
     It follows a simple verb/noun structure,
     where the top-level commands are verbs that describe the action to be taken,
@@ -163,3 +163,4 @@ main.add_command(get)  # type: ignore
 main.add_command(create)  # type: ignore
 main.add_command(abort)  # type: ignore
 main.add_command(gen)  # type: ignore
+main.add_command(delete)  # type: ignore

flyte/remote/_client/auth/_channel.py

@@ -1,3 +1,4 @@
+import os
 import ssl
 import typing
 
@@ -15,6 +16,11 @@ from ._authenticators.factory import (
     get_async_proxy_authenticator,
 )
 
+# Set environment variables for gRPC, this reduces log spew and avoids unnecessary warnings
+if "GRPC_VERBOSITY" not in os.environ:
+    os.environ["GRPC_VERBOSITY"] = "ERROR"
+    os.environ["GRPC_CPP_MIN_LOG_LEVEL"] = "ERROR"
+
 # Initialize gRPC AIO early enough so it can be used in the main thread
 init_grpc_aio()
 

flyte/remote/_data.py

@@ -16,7 +16,7 @@ from flyteidl.service import dataproxy_pb2
 from google.protobuf import duration_pb2
 
 from flyte._initialize import CommonInit, ensure_client, get_client, get_common_config
-from flyte.errors import RuntimeSystemError
+from flyte.errors import InitializationError, RuntimeSystemError
 
 _UPLOAD_EXPIRES_IN = timedelta(seconds=60)
 
@@ -83,6 +83,8 @@ async def _upload_single_file(
             raise RuntimeSystemError(
                 "PermissionDenied", f"Failed to get signed url for {fp}, please check your permissions."
             )
+        elif e.code() == grpc.StatusCode.UNAVAILABLE:
+            raise InitializationError("EndpointUnavailable", "user", "Service is unavailable.")
         else:
             raise RuntimeSystemError(e.code().value, f"Failed to get signed url for {fp}.")
     except Exception as e:

flyte/syncify/_api.py

@@ -5,6 +5,7 @@ import atexit
 import concurrent.futures
 import functools
 import inspect
+import logging
 import threading
 from typing import (
     Any,
@@ -21,6 +22,8 @@ from typing import (
     overload,
 )
 
+from flyte._logging import logger
+
 P = ParamSpec("P")
 R_co = TypeVar("R_co", covariant=True)
 T = TypeVar("T")
@@ -50,7 +53,7 @@ class SyncGenFunction(Protocol[P, R_co]):
 
 class _BackgroundLoop:
     """
-    A background event loop that runs in a separate thread and used the the Syncify decorator to run asynchronous
+    A background event loop that runs in a separate thread and used the `Syncify` decorator to run asynchronous
     functions or methods synchronously.
     """
 
@@ -105,6 +108,19 @@ class _BackgroundLoop:
                 yield future.result()
             except (StopAsyncIteration, StopIteration):
                 break
+            except Exception as e:
+                if logger.getEffectiveLevel() > logging.DEBUG:
+                    # If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the
+                    # user
+                    # This is because the stack trace will include the Syncify wrapper and the background loop thread
+                    tb = e.__traceback__
+                    while tb and tb.tb_next:
+                        if tb.tb_frame.f_code.co_name == "":
+                            break
+                        tb = tb.tb_next
+                    raise e.with_traceback(tb)
+                # If the log level is DEBUG, we will keep the extra stack frames to help with debugging
+                raise e
 
     def call_in_loop_sync(self, coro: Coroutine[Any, Any, R_co]) -> R_co | Iterator[R_co]:
         """
@@ -144,6 +160,19 @@ class _BackgroundLoop:
                 yield v
             except StopAsyncIteration:
                 break
+            except Exception as e:
+                if logger.getEffectiveLevel() > logging.DEBUG:
+                    # If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the
+                    # user.
+                    # This is because the stack trace will include the Syncify wrapper and the background loop thread
+                    tb = e.__traceback__
+                    while tb and tb.tb_next:
+                        if tb.tb_frame.f_code.co_name == "":
+                            break
+                        tb = tb.tb_next
+                    raise e.with_traceback(tb)
+                # If the log level is DEBUG, we will keep the extra stack frames to help with debugging
+                raise e
 
     async def aio(self, coro: Coroutine[Any, Any, R_co]) -> R_co:
         """
@@ -152,12 +181,25 @@ class _BackgroundLoop:
         if self.is_in_loop():
             # If we are already in the background loop, just run the coroutine
             return await coro
-        # Otherwise, run it in the background loop and wait for the result
-        future: concurrent.futures.Future[R_co] = asyncio.run_coroutine_threadsafe(coro, self.loop)
-        # Wrap the future in an asyncio Future to await it in an async context
-        aio_future: asyncio.Future[R_co] = asyncio.wrap_future(future)
-        # await for the future to complete and return its result
-        return await aio_future
+        try:
+            # Otherwise, run it in the background loop and wait for the result
+            future: concurrent.futures.Future[R_co] = asyncio.run_coroutine_threadsafe(coro, self.loop)
+            # Wrap the future in an asyncio Future to await it in an async context
+            aio_future: asyncio.Future[R_co] = asyncio.wrap_future(future)
+            # await for the future to complete and return its result
+            return await aio_future
+        except Exception as e:
+            if logger.getEffectiveLevel() > logging.DEBUG:
+                # If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
+                # This is because the stack trace will include the Syncify wrapper and the background loop thread
+                tb = e.__traceback__
+                while tb and tb.tb_next:
+                    if tb.tb_frame.f_code.co_name == "":
+                        break
+                    tb = tb.tb_next
+                raise e.with_traceback(tb)
+            # If the log level is DEBUG, we will keep the extra stack frames to help with debugging
+            raise e
 
 
 class _SyncWrapper:
@@ -175,23 +217,6 @@ class _SyncWrapper:
         self._bg_loop = bg_loop
         self._underlying_obj = underlying_obj
 
-    def __call__(self, *args: Any, **kwargs: Any) -> Any:
-        if threading.current_thread().name == self._bg_loop.thread.name:
-            # If we are already in the background loop thread, we can call the function directly
-            raise AssertionError(
-                f"Deadlock detected: blocking call used in syncify thread {self._bg_loop.thread.name} "
-                f"when calling function {self.fn}, use .aio() if in an async call."
-            )
-        # bind method if needed
-        coro_fn = self.fn
-
-        if inspect.isasyncgenfunction(coro_fn):
-            # Handle async iterator by converting to sync iterator
-            async_gen = coro_fn(*args, **kwargs)
-            return self._bg_loop.iterate_in_loop_sync(async_gen)
-        else:
-            return self._bg_loop.call_in_loop_sync(coro_fn(*args, **kwargs))
-
     def __get__(self, instance: Any, owner: Any) -> Any:
         """
         This method is called when the wrapper is accessed as a method of a class instance.
@@ -212,20 +237,63 @@ class _SyncWrapper:
         functools.update_wrapper(wrapper, self.fn)
         return wrapper
 
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        if threading.current_thread().name == self._bg_loop.thread.name:
+            # If we are already in the background loop thread, we can call the function directly
+            raise AssertionError(
+                f"Deadlock detected: blocking call used in syncify thread {self._bg_loop.thread.name} "
+                f"when calling function {self.fn}, use .aio() if in an async call."
+            )
+        try:
+            # bind method if needed
+            coro_fn = self.fn
+
+            if inspect.isasyncgenfunction(coro_fn):
+                # Handle async iterator by converting to sync iterator
+                async_gen = coro_fn(*args, **kwargs)
+                return self._bg_loop.iterate_in_loop_sync(async_gen)
+            else:
+                return self._bg_loop.call_in_loop_sync(coro_fn(*args, **kwargs))
+        except Exception as e:
+            if logger.getEffectiveLevel() > logging.DEBUG:
+                # If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
+                # This is because the stack trace will include the Syncify wrapper and the background loop thread
+                tb = e.__traceback__
+                while tb and tb.tb_next:
+                    if tb.tb_frame.f_code.co_name == self.fn.__name__:
+                        break
+                    tb = tb.tb_next
+                raise e.with_traceback(tb)
+            # If the log level is DEBUG, we will keep the extra stack frames to help with debugging
+            raise e
+
     def aio(self, *args: Any, **kwargs: Any) -> Any:
         fn = self.fn
 
-        if inspect.isasyncgenfunction(fn):
-            # If the function is an async generator, we need to handle it differently
-            async_iter = fn(*args, **kwargs)
-            return self._bg_loop.iterate_in_loop(async_iter)
-        else:
-            # If we are already in the background loop, just return the coroutine
-            coro = fn(*args, **kwargs)
-            if hasattr(coro, "__aiter__"):
-                # If the coroutine is an async iterator, we need to handle it differently
-                return self._bg_loop.iterate_in_loop(coro)
-            return self._bg_loop.aio(coro)
+        try:
+            if inspect.isasyncgenfunction(fn):
+                # If the function is an async generator, we need to handle it differently
+                async_iter = fn(*args, **kwargs)
+                return self._bg_loop.iterate_in_loop(async_iter)
+            else:
+                # If we are already in the background loop, just return the coroutine
+                coro = fn(*args, **kwargs)
+                if hasattr(coro, "__aiter__"):
+                    # If the coroutine is an async iterator, we need to handle it differently
+                    return self._bg_loop.iterate_in_loop(coro)
+                return self._bg_loop.aio(coro)
+        except Exception as e:
+            if logger.getEffectiveLevel() > logging.DEBUG:
+                # If the log level is not DEBUG, we will remove the extra stack frames to avoid confusion for the user
+                # This is because the stack trace will include the Syncify wrapper and the background loop thread
+                tb = e.__traceback__
+                while tb and tb.tb_next:
+                    if tb.tb_frame.f_code.co_name == self.fn.__name__:
+                        break
+                    tb = tb.tb_next
+                raise e.with_traceback(tb)
+            # If the log level is DEBUG, we will keep the extra stack frames to help with debugging
+            raise e
 
 
 class Syncify:

flyte/types/_type_engine.py

@@ -35,6 +35,7 @@ from mashumaro.jsonschema.models import Context, JSONSchema
 from mashumaro.jsonschema.plugins import BasePlugin
 from mashumaro.jsonschema.schema import Instance
 from mashumaro.mixins.json import DataClassJSONMixin
+from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin
 
 import flyte.storage as storage
@@ -352,6 +353,79 @@ class RestrictedTypeTransformer(TypeTransformer[T], ABC):
         raise RestrictedTypeError(f"Transformer for type {self.python_type} is restricted currently")
 
 
+class PydanticTransformer(TypeTransformer[BaseModel]):
+    def __init__(self):
+        super().__init__("Pydantic Transformer", BaseModel, enable_type_assertions=False)
+
+    def get_literal_type(self, t: Type[BaseModel]) -> LiteralType:
+        schema = t.model_json_schema()
+        fields = t.__annotations__.items()
+
+        literal_type = {}
+        for name, python_type in fields:
+            try:
+                literal_type[name] = TypeEngine.to_literal_type(python_type)
+            except Exception as e:
+                logger.warning(
+                    "Field {} of type {} cannot be converted to a literal type. Error: {}".format(name, python_type, e)
+                )
+
+        # This is for attribute access in FlytePropeller.
+        ts = TypeStructure(tag="", dataclass_type=literal_type)
+
+        meta_struct = struct_pb2.Struct()
+        meta_struct.update(
+            {
+                CACHE_KEY_METADATA: {
+                    SERIALIZATION_FORMAT: MESSAGEPACK,
+                }
+            }
+        )
+
+        return LiteralType(
+            simple=SimpleType.STRUCT,
+            metadata=schema,
+            structure=ts,
+            annotation=TypeAnnotation(annotations=meta_struct),
+        )
+
+    async def to_literal(
+        self,
+        python_val: BaseModel,
+        python_type: Type[BaseModel],
+        expected: LiteralType,
+    ) -> Literal:
+        json_str = python_val.model_dump_json()
+        dict_obj = json.loads(json_str)
+        msgpack_bytes = msgpack.dumps(dict_obj)
+        return Literal(scalar=Scalar(binary=Binary(value=msgpack_bytes, tag=MESSAGEPACK)))
+
+    def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[BaseModel]) -> BaseModel:
+        if binary_idl_object.tag == MESSAGEPACK:
+            dict_obj = msgpack.loads(binary_idl_object.value, strict_map_key=False)
+            json_str = json.dumps(dict_obj)
+            python_val = expected_python_type.model_validate_json(
+                json_data=json_str, strict=False, context={"deserialize": True}
+            )
+            return python_val
+        else:
+            raise TypeTransformerFailedError(f"Unsupported binary format: `{binary_idl_object.tag}`")
+
+    async def to_python_value(self, lv: Literal, expected_python_type: Type[BaseModel]) -> BaseModel:
+        """
+        There are two kinds of literal values to handle:
+        1. Protobuf Structs (from the UI)
+        2. Binary scalars (from other sources)
+        We need to account for both cases accordingly.
+        """
+        if lv and lv.HasField("scalar") and lv.scalar.HasField("binary"):
+            return self.from_binary_idl(lv.scalar.binary, expected_python_type)  # type: ignore
+
+        json_str = _json_format.MessageToJson(lv.scalar.generic)
+        python_val = expected_python_type.model_validate_json(json_str, strict=False, context={"deserialize": True})
+        return python_val
+
+
 class PydanticSchemaPlugin(BasePlugin):
     """This allows us to generate proper schemas for Pydantic models."""
 
@@ -562,9 +636,8 @@ class DataclassTransformer(TypeTransformer[object]):
 
         # This is for attribute access in FlytePropeller.
         ts = TypeStructure(tag="", dataclass_type=literal_type)
-        from google.protobuf.struct_pb2 import Struct
 
-        meta_struct = Struct()
+        meta_struct = struct_pb2.Struct()
         meta_struct.update(
             {
                 CACHE_KEY_METADATA: {
@@ -627,7 +700,7 @@ class DataclassTransformer(TypeTransformer[object]):
                 field.type = self._get_origin_type_in_annotation(cast(type, field.type))
         return python_type
 
-    async def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[T]) -> T:
+    def from_binary_idl(self, binary_idl_object: Binary, expected_python_type: Type[T]) -> T:
         if binary_idl_object.tag == MESSAGEPACK:
             if issubclass(expected_python_type, DataClassJSONMixin):
                 dict_obj = msgpack.loads(binary_idl_object.value, strict_map_key=False)
@@ -652,9 +725,10 @@ class DataclassTransformer(TypeTransformer[object]):
                 "user defined datatypes in Flytekit"
             )
 
-        if lv.scalar and lv.scalar.binary:
-            return await self.from_binary_idl(lv.scalar.binary, expected_python_type)  # type: ignore
+        if lv.HasField("scalar") and lv.scalar.HasField("binary"):
+            return self.from_binary_idl(lv.scalar.binary, expected_python_type)  # type: ignore
 
+        # todo: revisit this, it should always be a binary in v2.
         json_str = _json_format.MessageToJson(lv.scalar.generic)
 
         # The `from_json` function is provided from mashumaro's `DataClassJSONMixin`.
@@ -970,11 +1044,10 @@ class TypeEngine(typing.Generic[T]):
                 return cls._REGISTRY[python_type.__origin__]
 
         # Handling UnionType specially - PEP 604
-        if sys.version_info >= (3, 10):
-            import types
+        import types
 
-            if isinstance(python_type, types.UnionType):
-                return cls._REGISTRY[types.UnionType]
+        if isinstance(python_type, types.UnionType):
+            return cls._REGISTRY[types.UnionType]
 
         if python_type in cls._REGISTRY:
             return cls._REGISTRY[python_type]
@@ -2041,6 +2114,7 @@ def _register_default_type_transformers():
     TypeEngine.register(DictTransformer())
     TypeEngine.register(EnumTransformer())
     TypeEngine.register(ProtobufTransformer())
+    TypeEngine.register(PydanticTransformer())
 
     # inner type is. Also unsupported are typing's Tuples. Even though you can look inside them, Flyte's type system
     # doesn't support these currently.