flyte 0.2.0b14__py3-none-any.whl → 0.2.0b15__py3-none-any.whl

flyte/_internal/runtime/convert.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import base64
 import hashlib
 import inspect
@@ -7,12 +8,11 @@ from dataclasses import dataclass
 from types import NoneType
 from typing import Any, Dict, List, Tuple, Union, get_args
 
-from flyteidl.core import execution_pb2, literals_pb2
-from flyteidl.core.interface_pb2 import TypedInterface
+from flyteidl.core import execution_pb2, interface_pb2, literals_pb2
 
 import flyte.errors
 import flyte.storage as storage
-from flyte._protos.workflow import run_definition_pb2, task_definition_pb2
+from flyte._protos.workflow import common_pb2, run_definition_pb2, task_definition_pb2
 from flyte.models import ActionID, NativeInterface, TaskContext
 from flyte.types import TypeEngine, TypeTransformerFailedError
 
@@ -60,6 +60,40 @@ async def convert_inputs_to_native(inputs: Inputs, python_interface: NativeInter
     return native_vals
 
 
+async def convert_upload_default_inputs(interface: NativeInterface) -> List[common_pb2.NamedParameter]:
+    """
+    Converts the default inputs of a NativeInterface to a list of NamedParameters for upload.
+    This is used to upload default inputs to the Flyte backend.
+    """
+    if not interface.inputs:
+        return []
+
+    vars = []
+    literal_coros = []
+    for input_name, (input_type, default_value) in interface.inputs.items():
+        if default_value is not inspect.Parameter.empty:
+            lt = TypeEngine.to_literal_type(input_type)
+            literal_coros.append(TypeEngine.to_literal(default_value, input_type, lt))
+            vars.append((input_name, lt))
+
+    literals: List[literals_pb2.Literal] = await asyncio.gather(*literal_coros)
+    named_params = []
+    for (name, lt), literal in zip(vars, literals):
+        param = interface_pb2.Parameter(
+            var=interface_pb2.Variable(
+                type=lt,
+            ),
+            default=literal,
+        )
+        named_params.append(
+            common_pb2.NamedParameter(
+                name=name,
+                parameter=param,
+            ),
+        )
+    return named_params
+
+
 def is_optional_type(tp) -> bool:
     """
     True if the *annotation* `tp` is equivalent to Optional[…].
@@ -70,22 +104,42 @@ def is_optional_type(tp) -> bool:
 
 async def convert_from_native_to_inputs(interface: NativeInterface, *args, **kwargs) -> Inputs:
     kwargs = interface.convert_to_kwargs(*args, **kwargs)
-    if len(kwargs) == 0:
+
+    if len(kwargs) < interface.num_required_inputs():
+        raise ValueError(
+            f"Received {len(kwargs)} inputs but interface has {interface.num_required_inputs()} required inputs. "
+            f"Please provide all required inputs. Inputs received: {kwargs}, interface: {interface}"
+        )
+
+    if len(interface.inputs) == 0:
         return Inputs.empty()
 
     # fill in defaults if missing
+    type_hints: Dict[str, type] = {}
+    already_converted_kwargs: Dict[str, literals_pb2.Literal] = {}
     for input_name, (input_type, default_value) in interface.inputs.items():
-        if input_name not in kwargs:
-            if (default_value is not None and default_value is not inspect.Signature.empty) or (
-                default_value is None and is_optional_type(input_type)
-            ):
+        if input_name in kwargs:
+            type_hints[input_name] = input_type
+        elif (default_value is not None and default_value is not inspect.Signature.empty) or (
+            default_value is None and is_optional_type(input_type)
+        ):
+            if default_value == NativeInterface.has_default:
+                if interface._remote_defaults is None or input_name not in interface._remote_defaults:
+                    raise ValueError(f"Input '{input_name}' has a default value but it is not set in the interface.")
+                already_converted_kwargs[input_name] = interface._remote_defaults[input_name]
+            else:
                 kwargs[input_name] = default_value
-    if len(kwargs) < len(interface.inputs):
-        raise ValueError(
-            f"Received {len(kwargs)} inputs but interface has {len(interface.inputs)}. "
-            f"Please provide all required inputs."
-        )
-    literal_map = await TypeEngine.dict_to_literal_map(kwargs, interface.get_input_types())
+                type_hints[input_name] = input_type
+
+    literal_map = await TypeEngine.dict_to_literal_map(kwargs, type_hints)
+    if len(already_converted_kwargs) > 0:
+        copied_literals: Dict[str, literals_pb2.Literal] = {}
+        for k, v in literal_map.literals.items():
+            copied_literals[k] = v
+        # Add the already converted kwargs to the literal map
+        for k, v in already_converted_kwargs.items():
+            copied_literals[k] = v
+        literal_map = literals_pb2.LiteralMap(literals=copied_literals)
     # Make sure we the interface, not literal_map or kwargs, because those may have a different order
     return Inputs(
         proto_inputs=run_definition_pb2.Inputs(
@@ -228,7 +282,7 @@ def generate_inputs_hash(serialized_inputs: str | bytes) -> str:
 def generate_cache_key_hash(
     task_name: str,
     inputs_hash: str,
-    task_interface: TypedInterface,
+    task_interface: interface_pb2.TypedInterface,
     cache_version: str,
     ignored_input_vars: List[str],
     proto_inputs: run_definition_pb2.Inputs,

flyte/_internal/runtime/task_serde.py

@@ -16,7 +16,7 @@ import flyte.errors
 from flyte._cache.cache import VersionParameters, cache_from_request
 from flyte._logging import logger
 from flyte._pod import _PRIMARY_CONTAINER_NAME_FIELD, PodTemplate
-from flyte._protos.workflow import task_definition_pb2
+from flyte._protos.workflow import common_pb2, environment_pb2, task_definition_pb2
 from flyte._secret import SecretRequest, secrets_from_request
 from flyte._task import AsyncFunctionTaskTemplate, TaskTemplate
 from flyte.models import CodeBundle, SerializationContext
@@ -26,6 +26,9 @@ from ..._timeout import TimeoutType, timeout_from_request
 from .resources_serde import get_proto_extended_resources, get_proto_resources
 from .types_serde import transform_native_to_typed_interface
 
+_MAX_ENV_NAME_LENGTH = 63  # Maximum length for environment names
+_MAX_TASK_SHORT_NAME_LENGTH = 63  # Maximum length for task short names
+
 
 def load_class(qualified_name) -> Type:
     """
@@ -52,17 +55,31 @@ def load_task(resolver: str, *resolver_args: str) -> TaskTemplate:
 
 
 def translate_task_to_wire(
-    task: TaskTemplate, serialization_context: SerializationContext
+    task: TaskTemplate,
+    serialization_context: SerializationContext,
+    default_inputs: Optional[typing.List[common_pb2.NamedParameter]] = None,
 ) -> task_definition_pb2.TaskSpec:
     """
     Translate a task to a wire format. This is a placeholder function.
 
     :param task: The task to translate.
     :param serialization_context: The serialization context to use for the translation.
+    :param default_inputs: Optional list of default inputs for the task.
 
     :return: The translated task.
     """
-    return get_proto_task(task, serialization_context)
+    tt = get_proto_task(task, serialization_context)
+    env: environment_pb2.Environment | None = None
+    if task.parent_env and task.parent_env():
+        _env = task.parent_env()
+        if _env:
+            env = environment_pb2.Environment(name=_env.name[:_MAX_ENV_NAME_LENGTH])
+    return task_definition_pb2.TaskSpec(
+        task_template=tt,
+        default_inputs=default_inputs,
+        short_name=task.friendly_name[:_MAX_TASK_SHORT_NAME_LENGTH],
+        environment=env,
+    )
 
 
 def get_security_context(secrets: Optional[SecretRequest]) -> Optional[security_pb2.SecurityContext]:
@@ -111,7 +128,7 @@ def get_proto_timeout(timeout: TimeoutType | None) -> Optional[duration_pb2.Dura
     return duration_pb2.Duration(seconds=max_runtime_timeout.seconds)
 
 
-def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext) -> task_definition_pb2.TaskSpec:
+def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext) -> tasks_pb2.TaskTemplate:
     task_id = identifier_pb2.Identifier(
         resource_type=identifier_pb2.ResourceType.TASK,
         project=serialize_context.project,
@@ -158,7 +175,7 @@ def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext)
     else:
         logger.debug(f"Cache disabled for task {task.name}")
 
-    tt = tasks_pb2.TaskTemplate(
+    return tasks_pb2.TaskTemplate(
         id=task_id,
         type=task.task_type,
         metadata=tasks_pb2.TaskMetadata(
@@ -166,12 +183,16 @@ def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext)
             discovery_version=cache_version,
             cache_serializable=task_cache.serialize,
             cache_ignore_input_vars=task_cache.get_ignored_inputs() if cache_enabled else None,
-            runtime=tasks_pb2.RuntimeMetadata(),
+            runtime=tasks_pb2.RuntimeMetadata(
+                version=flyte.version(),
+                type=tasks_pb2.RuntimeMetadata.RuntimeType.FLYTE_SDK,
+                flavor="python",
+            ),
             retries=get_proto_retry_strategy(task.retries),
             timeout=get_proto_timeout(task.timeout),
             pod_template_name=task.pod_template if task.pod_template and isinstance(task.pod_template, str) else None,
             interruptible=task.interruptable,
-            generates_deck=wrappers_pb2.BoolValue(value=False),  # TODO add support for reports
+            generates_deck=wrappers_pb2.BoolValue(value=task.report),
         ),
         interface=transform_native_to_typed_interface(task.native_interface),
         custom=custom,
@@ -183,7 +204,6 @@ def get_proto_task(task: TaskTemplate, serialize_context: SerializationContext)
         sql=sql,
         extended_resources=get_proto_extended_resources(task.resources),
     )
-    return task_definition_pb2.TaskSpec(task_template=tt)
 
 
 def _get_urun_container(

flyte/_logging.py

@@ -9,6 +9,17 @@ from ._tools import ipython_check, is_in_cluster
 DEFAULT_LOG_LEVEL = logging.WARNING
 
 
+def make_hyperlink(label: str, url: str):
+    """
+    Create a hyperlink in the terminal output.
+    """
+    BLUE = "\033[94m"
+    RESET = "\033[0m"
+    OSC8_BEGIN = f"\033]8;;{url}\033\\"
+    OSC8_END = "\033]8;;\033\\"
+    return f"{BLUE}{OSC8_BEGIN}{label}{RESET}{OSC8_END}"
+
+
 def is_rich_logging_disabled() -> bool:
     """
     Check if rich logging is enabled

flyte/_run.py

@@ -70,7 +70,7 @@ class _Runner:
         if not force_mode and client is not None:
             force_mode = "remote"
         force_mode = force_mode or "local"
-        logger.debug(f"Effective run mode: {force_mode}, client configured: {client is not None}")
+        logger.debug(f"Effective run mode: `{force_mode}`, client configured: `{client is not None}`")
         self._mode = force_mode
         self._name = name
         self._service_account = service_account

flyte/_task.py

@@ -83,6 +83,7 @@ class TaskTemplate(Generic[P, R]):
 
     name: str
     interface: NativeInterface
+    friendly_name: str = ""
     task_type: str = "python"
     task_type_version: int = 0
     image: Union[str, Image, Literal["auto"]] = "auto"
@@ -108,9 +109,9 @@ class TaskTemplate(Generic[P, R]):
     def __post_init__(self):
         # Auto set the image based on the image request
         if self.image == "auto":
-            self.image = Image.auto()
+            self.image = Image.from_debian_base()
         elif isinstance(self.image, str):
-            self.image = Image.from_prebuilt(str(self.image))
+            self.image = Image.from_base(str(self.image))
 
         # Auto set cache based on the cache request
         if isinstance(self.cache, str):
@@ -126,6 +127,10 @@ class TaskTemplate(Generic[P, R]):
         if isinstance(self.retries, int):
             self.retries = RetryStrategy(count=self.retries)
 
+        if self.friendly_name == "":
+            # If friendly_name is not set, use the name of the task
+            self.friendly_name = self.name
+
     def __getstate__(self):
         """
         This method is called when the object is pickled. We need to remove the parent_env reference

flyte/_task_environment.py

@@ -54,7 +54,7 @@ class TaskEnvironment(Environment):
     :param resources: Resources to allocate for the environment.
     :param env: Environment variables to set for the environment.
     :param secrets: Secrets to inject into the environment.
-    :param env_dep_hints: Environment dependencies to hint, so when you deploy the environment, the dependencies are
+    :param depends_on: Environment dependencies to hint, so when you deploy the environment, the dependencies are
         also deployed. This is useful when you have a set of environments that depend on each other.
     :param cache: Cache policy for the environment.
     :param reusable: Reuse policy for the environment, if set, a python process may be reused for multiple tasks.
@@ -74,7 +74,7 @@ class TaskEnvironment(Environment):
         resources: Optional[Resources] = None,
         env: Optional[Dict[str, str]] = None,
         secrets: Optional[SecretRequest] = None,
-        env_dep_hints: Optional[List[Environment]] = None,
+        depends_on: Optional[List[Environment]] = None,
         **kwargs: Any,
     ) -> TaskEnvironment:
         """
@@ -103,8 +103,8 @@ class TaskEnvironment(Environment):
             kwargs["reusable"] = reusable
         if secrets is not None:
             kwargs["secrets"] = secrets
-        if env_dep_hints is not None:
-            kwargs["env_dep_hints"] = env_dep_hints
+        if depends_on is not None:
+            kwargs["depends_on"] = depends_on
         return replace(self, **kwargs)
 
     def task(
@@ -122,7 +122,7 @@ class TaskEnvironment(Environment):
     ) -> Union[AsyncFunctionTaskTemplate, Callable[P, R]]:
         """
         :param _func: Optional The function to decorate. If not provided, the decorator will return a callable that
-        :param name: Optional The name of the task (defaults to the function name)
+        :param name: Optional A friendly name for the task (defaults to the function name)
         :param cache: Optional The cache policy for the task, defaults to auto, which will cache the results of the
         task.
         :param retries: Optional The number of retries for the task, defaults to 0, which means no retries.
@@ -138,8 +138,8 @@ class TaskEnvironment(Environment):
                 raise ValueError("Cannot set pod_template when environment is reusable.")
 
         def decorator(func: FunctionTypes) -> AsyncFunctionTaskTemplate[P, R]:
-            task_name = name or func.__name__
-            task_name = self.name + "." + task_name
+            friendly_name = name or func.__name__
+            task_name = self.name + "." + func.__name__
 
             tmpl: AsyncFunctionTaskTemplate = AsyncFunctionTaskTemplate(
                 func=func,
@@ -157,6 +157,7 @@ class TaskEnvironment(Environment):
                 parent_env=weakref.ref(self),
                 interface=NativeInterface.from_callable(func),
                 report=report,
+                friendly_name=friendly_name,
             )
             self._tasks[task_name] = tmpl
             return tmpl

flyte/_utils/coro_management.py

@@ -13,6 +13,9 @@ async def run_coros(*coros: typing.Coroutine, return_when: str = asyncio.FIRST_C
     """
     tasks: typing.List[asyncio.Task[typing.Never]] = [asyncio.create_task(c) for c in coros]
     done, pending = await asyncio.wait(tasks, return_when=return_when)
+    # TODO we might want to handle asyncio.CancelledError here, for cases when the `action` is cancelled
+    # and we want to propagate it to all tasks. Though the backend will handle it anyway,
+    # so this is not strictly necessary.
 
     for t in pending:  # type: asyncio.Task
         t.cancel()  # Cancel all tasks that didn't finish first

flyte/_utils/helpers.py

@@ -52,7 +52,33 @@ def base36_encode(byte_data: bytes) -> str:
     return "".join(reversed(base36))
 
 
-# does not work at all in the setuptools case. see old flytekit editable installs
+def _iter_editable():
+    """
+    Yield (project_name, source_path) for every editable distribution
+    visible to the current interpreter
+    """
+    import json
+    import pathlib
+    from importlib.metadata import distributions
+
+    for dist in distributions():
+        # PEP-610 / PEP-660 (preferred, wheel-style editables)
+        direct = dist.read_text("direct_url.json")
+        if direct:
+            data = json.loads(direct)
+            if data.get("dir_info", {}).get("editable"):  # spec key
+                # todo: will need testing on windows
+                yield dist.metadata["Name"], pathlib.Path(data["url"][7:])  # strip file://
+                continue
+
+        # Legacy setuptools-develop / pip-e (egg-link)
+        for file in dist.files or ():  # importlib.metadata 3.8+
+            if file.suffix == ".egg-link":
+                with open(dist.locate_file(file), "r") as f:
+                    line = f.readline()
+                    yield dist.metadata["Name"], pathlib.Path(line.strip())
+
+
 def get_cwd_editable_install() -> typing.Optional[Path]:
     """
     This helper function is incomplete since it hasn't been tested with all the package managers out there,
@@ -65,28 +91,13 @@ def get_cwd_editable_install() -> typing.Optional[Path]:
 
     :return:
     """
-    import site
 
     from flyte._logging import logger
 
-    egg_links = [Path(p) for p in Path(site.getsitepackages()[0]).glob("*.egg-link")]
-    pth_files = [Path(p) for p in Path(site.getsitepackages()[0]).glob("*.pth")]
-
-    if not egg_links and not pth_files:
-        logger.debug("No editable installs found.")
-        return None
-
     editable_installs = []
-    egg_links.extend(pth_files)
-    for file in egg_links:
-        with open(file, "r") as f:
-            line = f.readline()
-            if line:
-                # Check if the first line is a directory
-                p = Path(line)
-                if p.is_dir():
-                    editable_installs.append(p)
-    logger.debug(f"Editable installs: {editable_installs}")
+    for name, path in _iter_editable():
+        logger.debug(f"Detected editable install: {name} at {path}")
+        editable_installs.append(path)
 
     # check to see if the current working directory is in any of the editable installs
     # including if the current folder is the root folder, one level up from the src and contains

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b14'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b14')
+__version__ = version = '0.2.0b15'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b15')

flyte/cli/_common.py

@@ -19,7 +19,6 @@ from rich.table import Table
 from rich.traceback import Traceback
 
 import flyte.errors
-from flyte._logging import logger
 from flyte.config import Config
 
 PREFERRED_BORDER_COLOR = "dim cyan"
@@ -104,8 +103,7 @@ class CLIConfig:
 
         updated_config = self.config.with_params(platform_cfg, task_cfg)
 
-        logger.debug(f"Initializing CLI with config: {updated_config}")
-        flyte.init_from_config(updated_config)
+        flyte.init_from_config(updated_config, log_level=self.log_level)
 
 
 class InvokeBaseMixin:
@@ -316,7 +314,7 @@ def get_table(title: str, vals: Iterable[Any]) -> Table:
             has_rich_repr = True
         elif not isinstance(p, (list, tuple)):
             raise ValueError("Expected a list or tuple of values, or an object with __rich_repr__ method.")
-        o = p.__rich_repr__() if has_rich_repr else p
+        o = list(p.__rich_repr__()) if has_rich_repr else p
         if headers is None:
             headers = [k for k, _ in o]
             for h in headers:

flyte/cli/_deploy.py

@@ -111,7 +111,7 @@ class EnvPerFileGroup(common.ObjectsPerFileGroup):
             name=obj_name,
             obj_name=obj_name,
             obj=obj,
-            help=obj.description,
+            help=f"{obj.name}" + (f": {obj.description}" if obj.description else ""),
             deploy_args=self.deploy_args,
         )
 

flyte/cli/main.py

@@ -144,7 +144,8 @@ def main(
         initialize_logger(log_level)
 
     cfg = config.auto(config_file=config_file)
-    logger.debug(f"Using config file discovered at location {cfg.source}")
+    if cfg.source:
+        logger.debug(f"Using config file discovered at location `{cfg.source.absolute()}`")
 
     ctx.obj = CLIConfig(
         log_level=log_level,
@@ -154,7 +155,6 @@ def main(
         config=cfg,
         ctx=ctx,
     )
-    logger.debug(f"Final materialized Cli config: {ctx.obj}")
 
 
 main.add_command(run)

flyte/errors.py

@@ -161,3 +161,12 @@ class RuntimeDataValidationError(RuntimeUserError):
         super().__init__(
             "DataValiationError", f"In task {task_name} variable {var}, failed to serialize/deserialize because {e}"
         )
+
+
+class DeploymentError(RuntimeUserError):
+    """
+    This error is raised when the deployment of a task fails, or some preconditions for deployment are not met.
+    """
+
+    def __init__(self, message: str):
+        super().__init__("DeploymentError", message, "user")

flyte/extras/_container.py

@@ -78,9 +78,9 @@ class ContainerTask(TaskTemplate):
         self._image = image
         if isinstance(image, str):
             if image == "auto":
-                self._image = Image.auto()
+                self._image = Image.from_debian_base()
             else:
-                self._image = Image.from_prebuilt(image)
+                self._image = Image.from_base(image)
         self._cmd = command
         self._args = arguments
         self._input_data_dir = input_data_dir

flyte/io/_structured_dataset/basic_dfs.py

@@ -27,19 +27,12 @@ else:
 T = TypeVar("T")
 
 
-# pr: add back after storage
-def get_pandas_storage_options(uri: str, data_config=None, anonymous: bool = False) -> typing.Optional[typing.Dict]:
+def get_pandas_storage_options(uri: str, anonymous: bool = False) -> typing.Optional[typing.Dict]:
     from pandas.io.common import is_fsspec_url  # type: ignore
 
     if is_fsspec_url(uri):
         if uri.startswith("s3"):
-            # pr: after storage, replace with real call to get_fsspec_storage_options
-            return {
-                "cache_regions": True,
-                "client_kwargs": {"endpoint_url": "http://localhost:30002"},
-                "key": "minio",
-                "secret": "miniostorage",
-            }
+            return storage.get_configured_fsspec_kwargs("s3", anonymous=anonymous)
         return {}
 
     # Pandas does not allow storage_options for non-fsspec paths e.g. local.
@@ -70,7 +63,7 @@ class PandasToCSVEncodingHandler(StructuredDatasetEncoder):
         df.to_csv(
             path,
             index=False,
-            storage_options=get_pandas_storage_options(uri=path, data_config=None),
+            storage_options=get_pandas_storage_options(uri=path),
         )
         structured_dataset_type.format = CSV
         return literals_pb2.StructuredDataset(
@@ -87,20 +80,21 @@ class CSVToPandasDecodingHandler(StructuredDatasetDecoder):
         proto_value: literals_pb2.StructuredDataset,
         current_task_metadata: literals_pb2.StructuredDatasetMetadata,
     ) -> "pd.DataFrame":
-        from botocore.exceptions import NoCredentialsError
-
         uri = proto_value.uri
         columns = None
-        kwargs = get_pandas_storage_options(uri=uri, data_config=None)
+        kwargs = get_pandas_storage_options(uri=uri)
         path = os.path.join(uri, ".csv")
         if current_task_metadata.structured_dataset_type and current_task_metadata.structured_dataset_type.columns:
             columns = [c.name for c in current_task_metadata.structured_dataset_type.columns]
         try:
             return pd.read_csv(path, usecols=columns, storage_options=kwargs)
-        except NoCredentialsError:
-            logger.debug("S3 source detected, attempting anonymous S3 access")
-            kwargs = get_pandas_storage_options(uri=uri, data_config=None, anonymous=True)
-            return pd.read_csv(path, usecols=columns, storage_options=kwargs)
+        except Exception as exc:
+            if exc.__class__.__name__ == "NoCredentialsError":
+                logger.debug("S3 source detected, attempting anonymous S3 access")
+                kwargs = get_pandas_storage_options(uri=uri, anonymous=True)
+                return pd.read_csv(path, usecols=columns, storage_options=kwargs)
+            else:
+                raise
 
 
 class PandasToParquetEncodingHandler(StructuredDatasetEncoder):
@@ -128,7 +122,7 @@ class PandasToParquetEncodingHandler(StructuredDatasetEncoder):
             path,
             coerce_timestamps="us",
             allow_truncated_timestamps=False,
-            storage_options=get_pandas_storage_options(uri=path, data_config=None),
+            storage_options=get_pandas_storage_options(uri=path),
         )
         structured_dataset_type.format = PARQUET
         return literals_pb2.StructuredDataset(
@@ -145,19 +139,20 @@ class ParquetToPandasDecodingHandler(StructuredDatasetDecoder):
         flyte_value: literals_pb2.StructuredDataset,
         current_task_metadata: literals_pb2.StructuredDatasetMetadata,
     ) -> "pd.DataFrame":
-        from botocore.exceptions import NoCredentialsError
-
         uri = flyte_value.uri
         columns = None
-        kwargs = get_pandas_storage_options(uri=uri, data_config=None)
+        kwargs = get_pandas_storage_options(uri=uri)
         if current_task_metadata.structured_dataset_type and current_task_metadata.structured_dataset_type.columns:
             columns = [c.name for c in current_task_metadata.structured_dataset_type.columns]
         try:
             return pd.read_parquet(uri, columns=columns, storage_options=kwargs)
-        except NoCredentialsError:
-            logger.debug("S3 source detected, attempting anonymous S3 access")
-            kwargs = get_pandas_storage_options(uri=uri, data_config=None, anonymous=True)
-            return pd.read_parquet(uri, columns=columns, storage_options=kwargs)
+        except Exception as exc:
+            if exc.__class__.__name__ == "NoCredentialsError":
+                logger.debug("S3 source detected, attempting anonymous S3 access")
+                kwargs = get_pandas_storage_options(uri=uri, anonymous=True)
+                return pd.read_parquet(uri, columns=columns, storage_options=kwargs)
+            else:
+                raise
 
 
 class ArrowToParquetEncodingHandler(StructuredDatasetEncoder):
@@ -199,7 +194,6 @@ class ParquetToArrowDecodingHandler(StructuredDatasetDecoder):
         current_task_metadata: literals_pb2.StructuredDatasetMetadata,
     ) -> "pa.Table":
         import pyarrow.parquet as pq
-        from botocore.exceptions import NoCredentialsError
 
         uri = proto_value.uri
         if not storage.is_remote(uri):
@@ -211,9 +205,11 @@ class ParquetToArrowDecodingHandler(StructuredDatasetDecoder):
             columns = [c.name for c in current_task_metadata.structured_dataset_type.columns]
         try:
             return pq.read_table(path, columns=columns)
-        except NoCredentialsError as e:
-            logger.debug("S3 source detected, attempting anonymous S3 access")
-            fs = storage.get_underlying_filesystem(path=uri, anonymous=True)
-            if fs is not None:
-                return pq.read_table(path, filesystem=fs, columns=columns)
-            raise e
+        except Exception as exc:
+            if exc.__class__.__name__ == "NoCredentialsError":
+                logger.debug("S3 source detected, attempting anonymous S3 access")
+                fs = storage.get_underlying_filesystem(path=uri, anonymous=True)
+                if fs is not None:
+                    return pq.read_table(path, filesystem=fs, columns=columns)
+            else:
+                raise