flyte 0.2.0b9__py3-none-any.whl → 0.2.0b10__py3-none-any.whl

flyte/__init__.py

@@ -38,7 +38,7 @@ __all__ = [
     "deploy",
     "group",
     "init",
-    "init_auto_from_config",
+    "init_from_config",
     "map",
     "run",
     "trace",
@@ -51,7 +51,7 @@ from ._deploy import deploy
 from ._environment import Environment
 from ._group import group
 from ._image import Image
-from ._initialize import init, init_auto_from_config
+from ._initialize import init, init_from_config
 from ._map import map
 from ._resources import GPU, TPU, Device, Resources
 from ._retry import RetryStrategy

flyte/_deploy.py

@@ -128,6 +128,9 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
     else:
         code_bundle = await build_code_bundle(from_dir=cfg.root_dir, dryrun=dryrun, copy_style=copy_style)
         deployment.version = code_bundle.computed_version
+        # TODO we should update the version to include the image cache digest and code bundle digest. This is
+        # to ensure that changes in image dependencies, cause an update to the deployment version.
+        # TODO Also hash the environment and tasks to ensure that changes in the environment or tasks
 
     sc = SerializationContext(
         project=cfg.project,

flyte/_initialize.py

@@ -138,14 +138,13 @@ async def init(
     :param project: Optional project name (not used in this implementation)
     :param domain: Optional domain name (not used in this implementation)
     :param root_dir: Optional root directory from which to determine how to load files, and find paths to files.
+      This is useful for determining the root directory for the current project, and for locating files like config etc.
+      also use to determine all the code that needs to be copied to the remote location.
       defaults to the editable install directory if the cwd is in a Python editable install, else just the cwd.
     :param log_level: Optional logging level for the logger, default is set using the default initialization policies
     :param api_key: Optional API key for authentication
     :param endpoint: Optional API endpoint URL
     :param headless: Optional Whether to run in headless mode
-    :param mode: Optional execution model (local, remote). Default is local. When local is used,
-           the execution will be done locally. When remote is used, the execution will be sent to a remote server,
-           In the remote case, the endpoint or api_key must be set.
     :param insecure_skip_verify: Whether to skip SSL certificate verification
     :param auth_client_config: Optional client configuration for authentication
     :param auth_type: The authentication type to use (Pkce, ClientSecret, ExternalCommand, DeviceFlow)
@@ -177,6 +176,9 @@ async def init(
 
     global _init_config  # noqa: PLW0603
 
+    if endpoint and "://" not in endpoint:
+        endpoint = f"dns:///{endpoint}"
+
     with _init_lock:
         client = None
         if endpoint or api_key:
@@ -209,12 +211,16 @@ async def init(
 
 
 @syncify
-async def init_auto_from_config(path_or_config: str | Config | None = None) -> None:
+async def init_from_config(path_or_config: str | Config | None = None, root_dir: Path | None = None) -> None:
     """
     Initialize the Flyte system using a configuration file or Config object. This method should be called before any
     other Flyte remote API methods are called. Thread-safe implementation.
 
     :param path_or_config: Path to the configuration file or Config object
+    :param root_dir: Optional root directory from which to determine how to load files, and find paths to
+        files like config etc. For example if one uses the copy-style=="all", it is essential to determine the
+        root directory for the current project. If not provided, it defaults to the editable install directory or
+        if not available, the current working directory.
     :return: None
     """
     import flyte.config as config
@@ -222,7 +228,10 @@ async def init_auto_from_config(path_or_config: str | Config | None = None) -> N
     cfg: config.Config
     if path_or_config is None or isinstance(path_or_config, str):
         # If a string is passed, treat it as a path to the config file
-        cfg = config.auto(path_or_config)
+        if root_dir and path_or_config:
+            cfg = config.auto(str(root_dir / path_or_config))
+        else:
+            cfg = config.auto(path_or_config)
     else:
         # If a Config object is passed, use it directly
         cfg = path_or_config
@@ -241,6 +250,7 @@ async def init_auto_from_config(path_or_config: str | Config | None = None) -> N
         proxy_command=cfg.platform.proxy_command,
         client_id=cfg.platform.client_id,
         client_credentials_secret=cfg.platform.client_credentials_secret,
+        root_dir=root_dir,
     )
 
 

flyte/_internal/controllers/_local_controller.py

@@ -14,7 +14,7 @@ from flyte._logging import log, logger
 from flyte._protos.workflow import task_definition_pb2
 from flyte._task import TaskTemplate
 from flyte._utils.helpers import _selector_policy
-from flyte.models import ActionID, NativeInterface, RawDataPath
+from flyte.models import ActionID, NativeInterface
 
 R = TypeVar("R")
 
@@ -86,7 +86,7 @@ class LocalController:
         sub_action_id, sub_action_output_path = convert.generate_sub_action_id_and_output_path(
             tctx, _task.name, serialized_inputs, 0
         )
-        sub_action_raw_data_path = RawDataPath(path=sub_action_output_path)
+        sub_action_raw_data_path = tctx.raw_data_path
 
         out, err = await direct_dispatch(
             _task,

flyte/_run.py

@@ -5,6 +5,7 @@ import pathlib
 import uuid
 from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union, cast
 
+import flyte.errors
 from flyte.errors import InitializationError
 from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath, SerializationContext, TaskContext
 from flyte.syncify import syncify
@@ -25,6 +26,7 @@ from ._tools import ipython_check
 
 if TYPE_CHECKING:
     from flyte.remote import Run
+    from flyte.remote._task import LazyEntity
 
     from ._code_bundle import CopyFiles
 
@@ -81,8 +83,11 @@ class _Runner:
         self._run_base_dir = run_base_dir or "/tmp/base"
 
     @requires_initialization
-    async def _run_remote(self, obj: TaskTemplate[P, R], *args: P.args, **kwargs: P.kwargs) -> Run:
+    async def _run_remote(self, obj: TaskTemplate[P, R] | LazyEntity, *args: P.args, **kwargs: P.kwargs) -> Run:
+        import grpc
+
         from flyte.remote import Run
+        from flyte.remote._task import LazyEntity
 
         from ._code_bundle import build_code_bundle, build_pkl_bundle
         from ._deploy import build_images, plan_deploy
@@ -93,37 +98,47 @@ class _Runner:
 
         cfg = get_common_config()
 
-        if obj.parent_env is None:
-            raise ValueError("Task is not attached to an environment. Please attach the task to an environment.")
+        if isinstance(obj, LazyEntity):
+            task = await obj.fetch.aio()
+            task_spec = task.pb2.spec
+            inputs = await convert_from_native_to_inputs(task.interface, *args, **kwargs)
+            version = task.pb2.task_id.version
+            code_bundle = None
+        else:
+            if obj.parent_env is None:
+                raise ValueError("Task is not attached to an environment. Please attach the task to an environment")
 
-        deploy_plan = plan_deploy(cast(Environment, obj.parent_env()))
-        image_cache = await build_images(deploy_plan)
+            deploy_plan = plan_deploy(cast(Environment, obj.parent_env()))
+            image_cache = await build_images(deploy_plan)
 
-        if self._interactive_mode:
-            code_bundle = await build_pkl_bundle(
-                obj, upload_to_controlplane=not self._dry_run, copy_bundle_to=self._copy_bundle_to
-            )
-        else:
-            if self._copy_files != "none":
-                code_bundle = await build_code_bundle(
-                    from_dir=cfg.root_dir, dryrun=self._dry_run, copy_bundle_to=self._copy_bundle_to
+            if self._interactive_mode:
+                code_bundle = await build_pkl_bundle(
+                    obj, upload_to_controlplane=not self._dry_run, copy_bundle_to=self._copy_bundle_to
                 )
             else:
-                code_bundle = None
+                if self._copy_files != "none":
+                    code_bundle = await build_code_bundle(
+                        from_dir=cfg.root_dir,
+                        dryrun=self._dry_run,
+                        copy_bundle_to=self._copy_bundle_to,
+                        copy_style=self._copy_files,
+                    )
+                else:
+                    code_bundle = None
 
-        version = self._version or (
-            code_bundle.computed_version if code_bundle and code_bundle.computed_version else None
-        )
-        if not version:
-            raise ValueError("Version is required when running a task")
-        s_ctx = SerializationContext(
-            code_bundle=code_bundle,
-            version=version,
-            image_cache=image_cache,
-            root_dir=cfg.root_dir,
-        )
-        task_spec = translate_task_to_wire(obj, s_ctx)
-        inputs = await convert_from_native_to_inputs(obj.native_interface, *args, **kwargs)
+            version = self._version or (
+                code_bundle.computed_version if code_bundle and code_bundle.computed_version else None
+            )
+            if not version:
+                raise ValueError("Version is required when running a task")
+            s_ctx = SerializationContext(
+                code_bundle=code_bundle,
+                version=version,
+                image_cache=image_cache,
+                root_dir=cfg.root_dir,
+            )
+            task_spec = translate_task_to_wire(obj, s_ctx)
+            inputs = await convert_from_native_to_inputs(obj.native_interface, *args, **kwargs)
 
         if not self._dry_run:
             if get_client() is None:
@@ -160,15 +175,35 @@ class _Runner:
             if task_spec.task_template.id.version == "":
                 task_spec.task_template.id.version = version
 
-            resp = await get_client().run_service.CreateRun(
-                run_service_pb2.CreateRunRequest(
-                    run_id=run_id,
-                    project_id=project_id,
-                    task_spec=task_spec,
-                    inputs=inputs.proto_inputs,
-                ),
-            )
-            return Run(pb2=resp.run)
+            try:
+                resp = await get_client().run_service.CreateRun(
+                    run_service_pb2.CreateRunRequest(
+                        run_id=run_id,
+                        project_id=project_id,
+                        task_spec=task_spec,
+                        inputs=inputs.proto_inputs,
+                    ),
+                )
+                return Run(pb2=resp.run)
+            except grpc.aio.AioRpcError as e:
+                if e.code() == grpc.StatusCode.UNAVAILABLE:
+                    raise flyte.errors.RuntimeSystemError(
+                        "SystemUnavailableError",
+                        "Flyte system is currently unavailable. check your configuration, or the service status.",
+                    ) from e
+                elif e.code() == grpc.StatusCode.INVALID_ARGUMENT:
+                    raise flyte.errors.RuntimeUserError("InvalidArgumentError", e.details())
+                elif e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                    # TODO maybe this should be a pass and return existing run?
+                    raise flyte.errors.RuntimeUserError(
+                        "RunAlreadyExistsError",
+                        f"A run with the name '{self._name}' already exists. Please choose a different name.",
+                    )
+                else:
+                    raise flyte.errors.RuntimeSystemError(
+                        "RunCreationError",
+                        f"Failed to create run: {e.details()}",
+                    ) from e
 
         class DryRun(Run):
             def __init__(self, _task_spec, _inputs, _code_bundle):
@@ -225,7 +260,10 @@ class _Runner:
             else:
                 if self._copy_files != "none":
                     code_bundle = await build_code_bundle(
-                        from_dir=cfg.root_dir, dryrun=self._dry_run, copy_bundle_to=self._copy_bundle_to
+                        from_dir=cfg.root_dir,
+                        dryrun=self._dry_run,
+                        copy_bundle_to=self._copy_bundle_to,
+                        copy_style=self._copy_files,
                     )
                 else:
                     code_bundle = None
@@ -313,7 +351,7 @@ class _Runner:
             report=Report(name=action.name),
             mode="local",
         )
-        async with ctx.replace_task_context(tctx):
+        with ctx.replace_task_context(tctx):
             # make the local version always runs on a different thread, returns a wrapped future.
             if obj._call_as_synchronous:
                 fut = controller.submit_sync(obj, *args, **kwargs)
@@ -323,7 +361,9 @@ class _Runner:
                 return await controller.submit(obj, *args, **kwargs)
 
     @syncify
-    async def run(self, task: TaskTemplate[P, Union[R, Run]], *args: P.args, **kwargs: P.kwargs) -> Union[R, Run]:
+    async def run(
+        self, task: TaskTemplate[P, Union[R, Run]] | LazyEntity, *args: P.args, **kwargs: P.kwargs
+    ) -> Union[R, Run]:
         """
         Run an async `@env.task` or `TaskTemplate` instance. The existing async context will be used.
 
@@ -345,8 +385,13 @@ class _Runner:
         :param kwargs: Keyword arguments to pass to the Task
         :return: Run instance or the result of the task
         """
+        from flyte.remote._task import LazyEntity
+
+        if isinstance(task, LazyEntity) and self._mode != "remote":
+            raise ValueError("Remote task can only be run in remote mode.")
         if self._mode == "remote":
             return await self._run_remote(task, *args, **kwargs)
+        task = cast(TaskTemplate, task)
         if self._mode == "hybrid":
             return await self._run_hybrid(task, *args, **kwargs)
 

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b9'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b9')
+__version__ = version = '0.2.0b10'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b10')

flyte/cli/_common.py

@@ -32,7 +32,7 @@ PROJECT_OPTION = click.Option(
     required=False,
     type=str,
     default=None,
-    help="Project to operate on",
+    help="Project to which this command applies.",
     show_default=True,
 )
 
@@ -41,7 +41,7 @@ DOMAIN_OPTION = click.Option(
     required=False,
     type=str,
     default=None,
-    help="Domain to operate on",
+    help="Domain to which this command applies.",
     show_default=True,
 )
 
@@ -51,7 +51,7 @@ DRY_RUN_OPTION = click.Option(
     type=bool,
     is_flag=True,
     default=False,
-    help="Dry run, do not actually call the backend service.",
+    help="Dry run. Do not actually call the backend service.",
     show_default=True,
 )
 
@@ -105,7 +105,7 @@ class CLIConfig:
         updated_config = self.config.with_params(platform_cfg, task_cfg)
 
         logger.debug(f"Initializing CLI with config: {updated_config}")
-        flyte.init_auto_from_config(updated_config)
+        flyte.init_from_config(updated_config)
 
 
 class InvokeBaseMixin:

flyte/cli/_create.py

@@ -70,7 +70,7 @@ def secret(
     "--org",
     type=str,
     required=False,
-    help="Organization to use, this will override the organization in the configusraion file.",
+    help="Organization to use. This will override the organization in the configuration file.",
 )
 @click.option(
     "-o",

flyte/cli/_params.py

@@ -23,7 +23,7 @@ from mashumaro.codecs.json import JSONEncoder
 
 from flyte._logging import logger
 from flyte.io import Dir, File
-from flyte.io.pickle.transformer import FlytePickleTransformer
+from flyte.types._pickle import FlytePickleTransformer
 
 
 class StructuredDataset:

flyte/cli/main.py

@@ -90,7 +90,7 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "config_file",
     required=False,
     type=click.Path(exists=True),
-    help="Path to the configuration file to use. If not specified, the default configuration file is used."
+    help="Path to the configuration file to use. If not specified, the default configuration file is used.",
 )
 @click.rich_config(help_config=help_config)
 @click.pass_context

flyte/extras/_container.py

@@ -12,17 +12,14 @@ from flyte.models import NativeInterface, SerializationContext
 _PRIMARY_CONTAINER_NAME_FIELD = "primary_container_name"
 
 
-def _extract_command_key(cmd: str, **kwargs) -> Any:
+def _extract_command_key(cmd: str, **kwargs) -> List[Any] | None:
     """
     Extract the key from the command using regex.
     """
     import re
 
-    input_regex = r"^\{\{\s*\.inputs\.(.*?)\s*\}\}$"
-    match = re.match(input_regex, cmd)
-    if match:
-        return match.group(1)
-    return None
+    input_regex = r"\{\{\.inputs\.([a-zA-Z0-9_]+)\}\}"
+    return re.findall(input_regex, cmd)
 
 
 def _extract_path_command_key(cmd: str, input_data_dir: Optional[str]) -> Optional[str]:
@@ -70,7 +67,7 @@ class ContainerTask(TaskTemplate):
         input_data_dir: str | pathlib.Path = "/var/inputs",
         output_data_dir: str | pathlib.Path = "/var/outputs",
         metadata_format: MetadataFormat = "JSON",
-        local_logs: bool = False,
+        local_logs: bool = True,
         **kwargs,
     ):
         super().__init__(
@@ -106,34 +103,33 @@ class ContainerTask(TaskTemplate):
         For FlyteFile and FlyteDirectory commands, e.g., "/var/inputs/inputs", we extract the key from strings that
          begin with the specified `input_data_dir`.
         """
-        # from flytekit.types.directory import FlyteDirectory
-        # from flytekit.types.file import FlyteFile
+        from flyte.io import Dir, File
 
         volume_binding: Dict[str, Dict[str, str]] = {}
         path_k = _extract_path_command_key(cmd, str(self._input_data_dir))
-        k = path_k if path_k else _extract_command_key(cmd)
-
-        if k:
-            input_val = kwargs.get(k)
-            # TODO: Add support file and directory transformer first
-            # if type(input_val) in [FlyteFile, FlyteDirectory]:
-            #     if not path_k:
-            #         raise AssertionError(
-            #             "FlyteFile and FlyteDirectory commands should not use the template syntax like this:
-            #             {{.inputs.infile}}\n"
-            #             "Please use a path-like syntax, such as: /var/inputs/infile.\n"
-            #             "This requirement is due to how Flyte Propeller processes template syntax inputs."
-            #         )
-            #     local_flyte_file_or_dir_path = str(input_val)
-            #     remote_flyte_file_or_dir_path = os.path.join(self._input_data_dir, k)  # type: ignore
-            #     volume_binding[local_flyte_file_or_dir_path] = {
-            #         "bind": remote_flyte_file_or_dir_path,
-            #         "mode": "rw",
-            #     }
-            #     command = remote_flyte_file_or_dir_path
-            command = str(input_val)
-        else:
-            command = cmd
+        keys = path_k if path_k else _extract_command_key(cmd)
+
+        if keys:
+            for k in keys:
+                input_val = kwargs.get(k)
+                # TODO: Add support file and directory transformer first
+                if type(input_val) in [File, Dir]:
+                    if not path_k:
+                        raise AssertionError(
+                            "File and Directory commands should not use the template syntax "
+                            "like this: {{.inputs.infile}}\n"
+                            "Please use a path-like syntax, such as: /var/inputs/infile.\n"
+                            "This requirement is due to how Flyte Propeller processes template syntax inputs."
+                        )
+                    local_flyte_file_or_dir_path = str(input_val)
+                    remote_flyte_file_or_dir_path = os.path.join(self._input_data_dir, k)  # type: ignore
+                    volume_binding[local_flyte_file_or_dir_path] = {
+                        "bind": remote_flyte_file_or_dir_path,
+                        "mode": "rw",
+                    }
+                    command = remote_flyte_file_or_dir_path
+                else:
+                    command = cmd
 
         return command, volume_binding
 
@@ -235,6 +231,7 @@ class ContainerTask(TaskTemplate):
             raise AssertionError(f"Only Image objects are supported, not strings. Got {self._image} instead.")
         uri = self._image.uri
         self._pull_image_if_not_exists(client, uri)
+        print(f"Command: {commands!r}")
 
         container = client.containers.run(uri, command=commands, remove=True, volumes=volume_bindings, detach=True)
 

flyte/io/__init__.py

@@ -3,9 +3,25 @@
 
 This package contains additional data types beyond the primitive data types in python to abstract data flow
 of large datasets in Union.
+
 """
 
-__all__ = ["Dir", "File"]
+__all__ = [
+    "Dir",
+    "File",
+    "StructuredDataset",
+    "StructuredDatasetDecoder",
+    "StructuredDatasetEncoder",
+    "StructuredDatasetTransformerEngine",
+    "lazy_import_structured_dataset_handler",
+]
 
 from ._dir import Dir
 from ._file import File
+from ._structured_dataset import (
+    StructuredDataset,
+    StructuredDatasetDecoder,
+    StructuredDatasetEncoder,
+    StructuredDatasetTransformerEngine,
+    lazy_import_structured_dataset_handler,
+)

flyte/io/_file.py

@@ -232,6 +232,8 @@ class File(BaseModel, Generic[T], SerializableType):
             # This code is broadly similar to what storage.get_stream does, but without actually reading from the stream
             file_handle = None
             try:
+                if "b" not in mode:
+                    raise ValueError("Mode must include 'b' for binary access, when using remote files.")
                 if isinstance(fs, AsyncFileSystem):
                     file_handle = await fs.open_async(self.path, mode)
                     yield file_handle

flyte/io/{structured_dataset → _structured_dataset}/basic_dfs.py

@@ -9,7 +9,7 @@ from fsspec.core import split_protocol, strip_protocol
 import flyte.storage as storage
 from flyte._logging import logger
 from flyte._utils import lazy_module
-from flyte.io.structured_dataset.structured_dataset import (
+from flyte.io._structured_dataset.structured_dataset import (
     CSV,
     PARQUET,
     StructuredDataset,

flyte/io/{structured_dataset → _structured_dataset}/structured_dataset.py

@@ -168,7 +168,7 @@ class StructuredDataset(SerializableType, DataClassJSONMixin):
         return self._literal_sd
 
     def open(self, dataframe_type: Type[DF]):
-        from flyte.io.structured_dataset import lazy_import_structured_dataset_handler
+        from flyte.io._structured_dataset import lazy_import_structured_dataset_handler
 
         """
         Load the handler if needed. For the use case like:

flyte/models.py

@@ -95,6 +95,7 @@ class RawDataPath:
                 # Create a temporary directory for data storage
                 p = tempfile.mkdtemp()
                 logger.debug(f"Creating temporary directory for data storage: {p}")
+                pathlib.Path(p).mkdir(parents=True, exist_ok=True)
                 return RawDataPath(path=p)
             case str():
                 return RawDataPath(path=local_folder)

flyte/types/__init__.py

@@ -1,9 +1,32 @@
+"""
+# Flyte Type System
+
+The Flyte type system provides a way to define, transform, and manipulate types in Flyte workflows.
+Since the data flowing through Flyte has to often cross process, container and langauge boundaries, the type system
+is designed to be serializable to a universal format that can be understood across different environments. This
+universal format is based on Protocol Buffers. The types are called LiteralTypes and the runtime
+representation of data is called Literals.
+
+The type system includes:
+- **TypeEngine**: The core engine that manages type transformations and serialization. This is the main entry point for
+  for all the internal type transformations and serialization logic.
+- **TypeTransformer**: A class that defines how to transform one type to another. This is extensible
+    allowing users to define custom types and transformations.
+- **Renderable**: An interface for types that can be rendered as HTML, that can be outputted to a flyte.report.
+
+It is always possible to bypass the type system and use the `FlytePickle` type to serialize any python object
+ into a pickle format. The pickle format is not human-readable, but can be passed between flyte tasks that are
+ written in python. The Pickled objects cannot be represented in the UI, and may be in-efficient for large datasets.
+"""
+
 from ._interface import guess_interface
+from ._pickle import FlytePickle
 from ._renderer import Renderable
 from ._string_literals import literal_string_repr
 from ._type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError
 
 __all__ = [
+    "FlytePickle",
     "Renderable",
     "TypeEngine",
     "TypeTransformer",

flyte/{io/pickle/transformer.py → types/_pickle.py}

@@ -8,7 +8,8 @@ import cloudpickle
 from flyteidl.core import literals_pb2, types_pb2
 
 import flyte.storage as storage
-from flyte.types import TypeEngine, TypeTransformer
+
+from ._type_engine import TypeEngine, TypeTransformer
 
 T = typing.TypeVar("T")
 

flyte/types/_type_engine.py

@@ -1008,7 +1008,7 @@ class TypeEngine(typing.Generic[T]):
             return cls._DATACLASS_TRANSFORMER
 
         display_pickle_warning(str(python_type))
-        from flyte.io.pickle.transformer import FlytePickleTransformer
+        from flyte.types._pickle import FlytePickleTransformer
 
         return FlytePickleTransformer()
 
@@ -1021,7 +1021,7 @@ class TypeEngine(typing.Generic[T]):
             # Avoid a race condition where concurrent threads may exit lazy_import_transformers before the transformers
             # have been imported. This could be implemented without a lock if you assume python assignments are atomic
             # and re-registering transformers is acceptable, but I decided to play it safe.
-            from flyte.io.structured_dataset import lazy_import_structured_dataset_handler
+            from flyte.io import lazy_import_structured_dataset_handler
 
             # todo: bring in extras transformers (pytorch, etc.)
             lazy_import_structured_dataset_handler()
@@ -1657,7 +1657,7 @@ class DictTransformer(TypeTransformer[dict]):
         Converts a Python dictionary to a Flyte-specific ``Literal`` using MessagePack encoding.
         Falls back to Pickle if encoding fails and `allow_pickle` is True.
         """
-        from flyte.io.pickle.transformer import FlytePickle
+        from flyte.types._pickle import FlytePickle
 
         try:
             # Handle dictionaries with non-string keys (e.g., Dict[int, Type])
@@ -1763,7 +1763,7 @@ class DictTransformer(TypeTransformer[dict]):
         # pr: han-ru is this part still necessary?
         if lv and lv.HasField("scalar") and lv.scalar.HasField("generic"):
             if lv.metadata and lv.metadata.get("format", None) == "pickle":
-                from flyte.io.pickle.transformer import FlytePickle
+                from flyte.types._pickle import FlytePickle
 
                 uri = json.loads(_json_format.MessageToJson(lv.scalar.generic)).get("pickle_file")
                 return await FlytePickle.from_pickle(uri)

{flyte-0.2.0b9.dist-info → flyte-0.2.0b10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 0.2.0b9
+Version: 0.2.0b10
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10
@@ -61,7 +61,6 @@ Note that the `--follow` command is optional. Use this to stream updates to the
 3. look at examples/... for various examples. 
 4. For a single script recommend using uv run scripts with metadata headers.
 
-
 ```python
 import flyte
 
@@ -80,14 +79,14 @@ async def say_hello_nested(data: str) -> str:
 
 if __name__ == "__main__":
     import asyncio
-    
+
     # to run pure python - the SDK is not invoked at all
     asyncio.run(say_hello_nested("test"))
-    
+
     # To run locally, but run through type system etc
     flyte.init()
     flyte.run(say_hello_nested, "World")
-    
+
     # To run remote
     flyte.init(endpoint="dns:///localhost:8090", insecure=True)
     flyte.run(say_hello_nested, "World")
@@ -95,7 +94,7 @@ if __name__ == "__main__":
     flyte.with_runcontext(mode="local").run(...)  # this will run locally only
 
     # To run remote with a config
-    flyte.init_auto_from_config("config.yaml")
+    flyte.init_from_config("config.yaml")
 ```
 
 # CLI

{flyte-0.2.0b9.dist-info → flyte-0.2.0b10.dist-info}/RECORD

@@ -1,30 +1,30 @@
-flyte/__init__.py,sha256=-gRsCKXTW5kOSQPq-MLdJjiPDXO1bjxFssa_7_RnywU,1425
+flyte/__init__.py,sha256=QzKbvypcEAJCF7g8oqpIZhpQfYOsqz6hEvT2OwK0IAc,1415
 flyte/_build.py,sha256=MkgfLAPeL56YeVrGRNZUCZgbwzlEzVP3wLbl5Qru4yk,578
 flyte/_context.py,sha256=K0-TCt-_pHOoE5Xni87_8uIe2vCBOhfNQEtjGT4Hu4k,5239
-flyte/_deploy.py,sha256=hHZKLU3U7t1ZF_8x6LykkPu_KSDyaHL3f2WzyjLj9BQ,7723
+flyte/_deploy.py,sha256=NApDDNQRGauhZP_eBYUnVSQilooSLpmmHQNhqNa5jmk,8034
 flyte/_doc.py,sha256=_OPCf3t_git6UT7kSJISFaWO9cfNzJhhoe6JjVdyCJo,706
 flyte/_docstring.py,sha256=SsG0Ab_YMAwy2ABJlEo3eBKlyC3kwPdnDJ1FIms-ZBQ,1127
 flyte/_environment.py,sha256=BkChtdVhWB3SwMSvetDZ-KiNBgRFlAXgq69PHT4zyG0,2942
 flyte/_group.py,sha256=7o1j16sZyUmYB50mOiq1ui4TBAKhRpDqLakV8Ya1kw4,803
 flyte/_hash.py,sha256=Of_Zl_DzzzF2jp4ZsLm-3o-xJFCCJ8_GubmLI1htx78,504
 flyte/_image.py,sha256=NeBvjCdwFAVGd666ufi1q-YOvhwdTEzAeJl5YBfl0bI,29043
-flyte/_initialize.py,sha256=ihTIvoMHs67UKbtFLR_zy9M1e7OK26ywoc_yMfLYwMw,16499
+flyte/_initialize.py,sha256=N7q2n22iHSWmOB86ZLfgMd1En_XU6dz3OusB-t5nUwg,17073
 flyte/_interface.py,sha256=MP5o_qpIwfBNtAc7zo_cLSjMugsPyanuO6EgUSk4fBE,3644
 flyte/_logging.py,sha256=FQvF3W1kkFypbARcOQ7WZVXO0XJasXp8EhozF6E6-aQ,3379
 flyte/_map.py,sha256=efPd8O-JKUg1OY3_MzU3KGbhsGYDVRNBwWr0ceNIXhQ,7444
 flyte/_resources.py,sha256=UOLyEVhdxolvrHhddiBbYdJuE1RkM_l7xeS9G1abe6M,7583
 flyte/_retry.py,sha256=rfLv0MvWxzPByKESTglEmjPsytEAKiIvvmzlJxXwsfE,941
 flyte/_reusable_environment.py,sha256=P4FBATVKAYcIKpdFN98sI8acPyKy8eIGx6V0kUb9YdM,1289
-flyte/_run.py,sha256=gMcQ6km5jm34BAfubJkp7SChTACblBH4M6m8FfZQeP8,17723
+flyte/_run.py,sha256=ePlFrABDR0ud_qxY55Nk4eATDialHGHsxNdTiVWYQkw,19919
 flyte/_secret.py,sha256=SqIHs6mi8hEkIIBZe3bI9jJsPt65Mt6dV5uh9_op1ME,2392
 flyte/_task.py,sha256=Gq34LbELz5dIc4dvYjgRXyzAnBvs8ED3iU1FWVuAFRE,18391
 flyte/_task_environment.py,sha256=J1LFH9Zz1nPhlsrc_rYny1SS3QC1b55X7tRYoTG7Vk4,6815
 flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
 flyte/_tools.py,sha256=JewkQZBR_M85tS6QY8e4xXue75jbOE48nID4ZHnc9jY,632
 flyte/_trace.py,sha256=7OQtQNosIlycTwaMjdc3GW4h3T3N0bYTsY6og4clPl8,5234
-flyte/_version.py,sha256=6yaYOwx6af-xfDGkZHMO3lXrEv1xpPMopHstyivLlUQ,519
+flyte/_version.py,sha256=mlXpANdUuYfy0QPXXxhosF8MwMVypx0OlvhibSGO2P8,521
 flyte/errors.py,sha256=m2JUNqLC6anVW6UiDK_ihuA06q_Hkw1mIUMDskb2OW8,4289
-flyte/models.py,sha256=8ZqWar7n-u9ZYyxct88T_0-41LXjSjstoQvDfiDjp64,12841
+flyte/models.py,sha256=A85HnTLqInJiMmQKhpl2IXb2Uh6_46tdRrwW2TTzD9o,12908
 flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/_bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/_bin/runtime.py,sha256=Q1vajoQ913KUfCQSP0u8Qoj-AYgfvU2i2wI19ydqnDw,4849
@@ -39,7 +39,7 @@ flyte/_code_bundle/_utils.py,sha256=b0s3ZVKSRwaa_2CMTCqt2iRrUvTTW3FmlyqCD9k5BS0,
 flyte/_code_bundle/bundle.py,sha256=8T0gcXck6dmg-8L2-0G3B2iNjC-Xwydu806iyKneMMY,8789
 flyte/_internal/__init__.py,sha256=vjXgGzAAjy609YFkAy9_RVPuUlslsHSJBXCLNTVnqOY,136
 flyte/_internal/controllers/__init__.py,sha256=5CBnS9lb1VFMzZuRXUiaPhlN3G9qh7Aq9kTwxW5hsRw,4301
-flyte/_internal/controllers/_local_controller.py,sha256=Nsqy1Grrw6JggKwiQn7lsChjL2gT5f-bHyaqrT2vjnA,7111
+flyte/_internal/controllers/_local_controller.py,sha256=LDwVxwm365L6-OEpwo5FcWe6abxK8L1231Hf54pYLZ8,7076
 flyte/_internal/controllers/_trace.py,sha256=Ga2b65sn9q2IoOwHBZV2inMYyO6-CSDwzN7E3pDxsEI,1126
 flyte/_internal/controllers/remote/__init__.py,sha256=9_azH1eHLqY6VULpDugXi7Kf1kK1ODqEnsQ_3wM6IqU,1919
 flyte/_internal/controllers/remote/_action.py,sha256=w6vE1vPz1BwxvwfotDWjTNbDXfGEPrRBA8N3UVQ6P0w,4905
@@ -137,31 +137,28 @@ flyte/_utils/lazy_module.py,sha256=fvXPjvZLzCfcI8Vzs4pKedUDdY0U_RQ1ZVrp9b8qBQY,1
 flyte/_utils/uv_script_parser.py,sha256=PxqD8lSMi6xv0uDd1s8LKB2IPZr4ttZJCUweqlyMTKk,1483
 flyte/cli/__init__.py,sha256=M02O-UGqQlA8JJ_jyWRiwQhTNc5CJJ7x9J7fNxTxBT0,52
 flyte/cli/_abort.py,sha256=lTftDmVXEIrFz1XAASCqWbXQEQDqRdTCJqY7izk2USI,593
-flyte/cli/_common.py,sha256=UkhKgjoti-5iLGczUkcwN5QmhkWk5uFsoL9C2gHoiCo,10766
-flyte/cli/_create.py,sha256=fPyme_o8-Qf1Cy7DSspXgwYcBDO4G8nPmoUpSDprfz4,4087
+flyte/cli/_common.py,sha256=dwZ5OPz3x8-8uVtMiV1HpqA1MaoBJZH8yBON-axFSZs,10795
+flyte/cli/_create.py,sha256=DFdNemGqNVdlgDxcIqMmZJBF1zPtAHeWO1XeT2caEPs,4087
 flyte/cli/_delete.py,sha256=qq5A9d6vXdYvYz-SECXiC6LU5rAzahNTZKiKacOtcZ4,545
 flyte/cli/_deploy.py,sha256=owGe_RVwC73p3dOPWYSo442OVuZyagF8LKTvC57xubQ,4466
 flyte/cli/_gen.py,sha256=vlE5l8UR1zz4RSdaRyUfYFvGR0TLxGcTYcP4dhA3Pvg,5458
 flyte/cli/_get.py,sha256=Pl9nHVHzRQT8IG3k_VAMJ2Xvaq9URsovm96op_153os,9843
-flyte/cli/_params.py,sha256=X3GpuftXmtfIsYQ7vBilD4kmlkXTc7_AxpaxohRjSuY,19458
+flyte/cli/_params.py,sha256=fs6fNBRGYXZMmjjcySi3U1O_pQNgCnW9ko89KCcS6nU,19450
 flyte/cli/_run.py,sha256=zRu4DTO__BVsLgpSLUbYR1Twvzc_2zivUhcviymCRf0,7884
-flyte/cli/main.py,sha256=V_gXj3YDr43igWQtugcPt8CbJincoTE9g1uG52JdHgk,4449
+flyte/cli/main.py,sha256=PJUQ6slm1CcwIfeP6pbJKlPuyOqXxb_K8PjnZb8sbhw,4450
 flyte/config/__init__.py,sha256=MiwEYK5Iv7MRR22z61nzbsbvZ9Q6MdmAU_g9If1Pmb8,144
 flyte/config/_config.py,sha256=QE3T0W8xOULjJaqDMdMF90f9gFVjGR6h8QPOLsyqjYw,9831
 flyte/config/_internal.py,sha256=Bj0uzn3PYgxKbzM-q2GKXxp7Y6cyzhPzUB-Y2i6cQKo,2836
 flyte/config/_reader.py,sha256=c16jm0_IYxwEAjXENtllLeO_sT5Eg2RNLG4UjnAv_x4,7157
 flyte/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/extras/__init__.py,sha256=FhB0uK7H1Yo5De9vOuF7UGnezTKncj3u2Wo5uQdWN0g,74
-flyte/extras/_container.py,sha256=R7tMIy2EuprEGowYhjPMwnszSutgWIzAeKPeXNc2nmI,11255
-flyte/io/__init__.py,sha256=e2wHVEoZ84TGOtOPrtTg6hJpeuxiYI56Sg011yq6nUQ,236
-flyte/io/_dataframe.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+flyte/extras/_container.py,sha256=kMrYPmfnDFxPmFhI9cZeoBKzQ96jMCO4l4FpwV1b0PA,11202
+flyte/io/__init__.py,sha256=F7hlpin_1JJjsdFZSn7_jQgltPzsjETX1DCYGz-ELqI,629
 flyte/io/_dir.py,sha256=rih9CY1YjNX05bcAu5LG62Xoyij5GXAlv7jLyVF0je8,15310
-flyte/io/_file.py,sha256=eL48TwJjPDyN9tPG9Hfs22P4qqc-GBsTm3E5e-JmfdY,15430
-flyte/io/pickle/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flyte/io/pickle/transformer.py,sha256=b7vAxLHgi-HMHy3vOG0sGMeDf6LimFmU_jAcvNOZI1o,4047
-flyte/io/structured_dataset/__init__.py,sha256=69ixVV9OEXiLiQ6SV2S8tEC7dVQe7YTt9NV1OotlG64,4524
-flyte/io/structured_dataset/basic_dfs.py,sha256=lrcALNYke_gSmmAhIiUN5afqhA-W5bSuKJUCoW6S4CE,8223
-flyte/io/structured_dataset/structured_dataset.py,sha256=DrRIHA3zbkfLBekw3pPTF_SM0Rbn_BGBp1YJPyd9zY0,52644
+flyte/io/_file.py,sha256=kp5700SKPy5htmMhm4hE2ybb99Ykny1b0Kwm3huCWXs,15572
+flyte/io/_structured_dataset/__init__.py,sha256=69ixVV9OEXiLiQ6SV2S8tEC7dVQe7YTt9NV1OotlG64,4524
+flyte/io/_structured_dataset/basic_dfs.py,sha256=77aYFrFigPC7cjM6UjCbU26REtXmwIBBapFN6KGYOO8,8224
+flyte/io/_structured_dataset/structured_dataset.py,sha256=ddRjz36RhNxIy0gakzdLStBzoo4cAOgXbNqiqt5YhMI,52645
 flyte/remote/__init__.py,sha256=zBWV88VF-L8430xVrOyk07EmLsOKhOUMVBsqFUDtO6Q,565
 flyte/remote/_console.py,sha256=avmELJPx8nQMAVPrHlh6jEIRPjrMwFpdZjJsWOOa9rE,660
 flyte/remote/_data.py,sha256=DPK85gB6M71RjxqIh1Q5PdZ9xcJ0m1w_3cT2lAO0r7w,5795
@@ -201,14 +198,15 @@ flyte/storage/_storage.py,sha256=mBy7MKII2M1UTVm_EUUDwVb7uT1_AOPzQr2wCJ-fgW0,987
 flyte/storage/_utils.py,sha256=8oLCM-7D7JyJhzUi1_Q1NFx8GBUPRfou0T_5tPBmPbE,309
 flyte/syncify/__init__.py,sha256=WgTk-v-SntULnI55CsVy71cxGJ9Q6pxpTrhbPFuouJ0,1974
 flyte/syncify/_api.py,sha256=x37A7OtUS0WVUB1wkoN0i5iu6HQPDYlPc1uwzVFOrfM,10927
-flyte/types/__init__.py,sha256=xMIYOolT3Vq0qXy7unw90IVdYztdMDpKg0oG0XAPC9o,364
+flyte/types/__init__.py,sha256=9310PRtVrwJePwEPeoUO0HPyIkgaja7-Dar_QlE_MUI,1745
 flyte/types/_interface.py,sha256=mY7mb8v2hJPGk7AU99gdOWl4_jArA1VFtjYGlE31SK0,953
+flyte/types/_pickle.py,sha256=PjdR66OTDMZ3OYq6GvM_Ua0cIo5t2XQaIjmpJ9xo4Ys,4050
 flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
 flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
-flyte/types/_type_engine.py,sha256=QxyoDWRG_whfLCz88YqEVVoTTnca0FZv9eHeLLT0_-s,93645
+flyte/types/_type_engine.py,sha256=3PiRPYtE0UmEpTfwwcFWaiVmG1XOfXQl3Pj4v2zClNk,93602
 flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
-flyte-0.2.0b9.dist-info/METADATA,sha256=4iN_NmCj__wWn6iVgYBAaYotKg6posAg0V34TzisufQ,4828
-flyte-0.2.0b9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flyte-0.2.0b9.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
-flyte-0.2.0b9.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
-flyte-0.2.0b9.dist-info/RECORD,,
+flyte-0.2.0b10.dist-info/METADATA,sha256=XALFwwiy3iCSCTVj80DPGhfL8gfPIVUOsyAMmbpNzac,4811
+flyte-0.2.0b10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flyte-0.2.0b10.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
+flyte-0.2.0b10.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
+flyte-0.2.0b10.dist-info/RECORD,,