flyte 0.2.0b9__py3-none-any.whl → 0.2.0b11__py3-none-any.whl

flyte/_utils/org_discovery.py

@@ -0,0 +1,31 @@
+def hostname_from_url(url: str) -> str:
+    """Parse a URL and return the hostname part."""
+
+    # Handle dns:/// format specifically (gRPC convention)
+    if url.startswith("dns:///"):
+        return url[7:]  # Skip the "dns:///" prefix
+
+    # Handle standard URL formats
+    import urllib.parse
+
+    parsed = urllib.parse.urlparse(url)
+    return parsed.netloc or parsed.path.lstrip("/").rsplit("/")[0]
+
+
+def org_from_endpoint(endpoint: str | None) -> str | None:
+    """
+    Extracts the organization from the endpoint URL. The organization is assumed to be the first part of the domain.
+    This is temporary until we have a proper organization discovery mechanism through APIs.
+
+    :param endpoint: The endpoint URL
+    :return: The organization name or None if not found
+    """
+    if not endpoint:
+        return None
+
+    hostname = hostname_from_url(endpoint)
+    domain_parts = hostname.split(".")
+    if len(domain_parts) > 2:
+        # Assuming the organization is the first part of the domain
+        return domain_parts[0]
+    return None

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b9'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b9')
+__version__ = version = '0.2.0b11'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b11')

flyte/cli/_common.py

@@ -32,7 +32,7 @@ PROJECT_OPTION = click.Option(
     required=False,
     type=str,
     default=None,
-    help="Project to operate on",
+    help="Project to which this command applies.",
     show_default=True,
 )
 
@@ -41,7 +41,7 @@ DOMAIN_OPTION = click.Option(
     required=False,
     type=str,
     default=None,
-    help="Domain to operate on",
+    help="Domain to which this command applies.",
     show_default=True,
 )
 
@@ -51,7 +51,7 @@ DRY_RUN_OPTION = click.Option(
     type=bool,
     is_flag=True,
     default=False,
-    help="Dry run, do not actually call the backend service.",
+    help="Dry run. Do not actually call the backend service.",
     show_default=True,
 )
 
@@ -78,7 +78,7 @@ class CLIConfig:
     log_level: int | None = logging.ERROR
     endpoint: str | None = None
     insecure: bool = False
-    org_override: str | None = None
+    org: str | None = None
 
     def replace(self, **kwargs) -> CLIConfig:
         """
@@ -90,7 +90,7 @@ class CLIConfig:
         from flyte.config._config import TaskConfig
 
         task_cfg = TaskConfig(
-            org=self.org_override or self.config.task.org,
+            org=self.org or self.config.task.org,
             project=project or self.config.task.project,
             domain=domain or self.config.task.domain,
         )
@@ -105,7 +105,7 @@ class CLIConfig:
         updated_config = self.config.with_params(platform_cfg, task_cfg)
 
         logger.debug(f"Initializing CLI with config: {updated_config}")
-        flyte.init_auto_from_config(updated_config)
+        flyte.init_from_config(updated_config)
 
 
 class InvokeBaseMixin:

flyte/cli/_create.py

@@ -70,7 +70,7 @@ def secret(
     "--org",
     type=str,
     required=False,
-    help="Organization to use, this will override the organization in the configusraion file.",
+    help="Organization to use. This will override the organization in the configuration file.",
 )
 @click.option(
     "-o",
@@ -86,11 +86,9 @@ def secret(
     default=False,
     help="Force overwrite of the configuration file if it already exists.",
     show_default=True,
-    prompt="Are you sure you want to overwrite the configuration file?",
-    confirmation_prompt=True,
 )
 def config(
-    output: Path,
+    output: str,
     endpoint: str | None = None,
     insecure: bool = False,
     org: str | None = None,
@@ -105,8 +103,13 @@ def config(
     """
     import yaml
 
-    if output.exists() and not force:
-        raise click.BadParameter(f"Output file {output} already exists. Use --force to overwrite.")
+    output_path = Path(output)
+
+    if output_path.exists() and not force:
+        force = click.confirm(f"Overwrite [{output_path}]?", default=False)
+        if not force:
+            click.echo(f"Will not overwrite the existing config file at {output_path}")
+            return
 
     admin: Dict[str, Any] = {}
     if endpoint:
@@ -114,6 +117,11 @@ def config(
     if insecure:
         admin["insecure"] = insecure
 
+    if not org and endpoint:
+        from flyte._utils.org_discovery import org_from_endpoint
+
+        org = org_from_endpoint(endpoint)
+
     task: Dict[str, str] = {}
     if org:
         task["org"] = org
@@ -125,7 +133,7 @@ def config(
     if not admin and not task:
         raise click.BadParameter("At least one of --endpoint or --org must be provided.")
 
-    with open(output, "w") as f:
+    with open(output_path, "w") as f:
         d: Dict[str, Any] = {}
         if admin:
             d["admin"] = admin
@@ -133,4 +141,4 @@ def config(
             d["task"] = task
         yaml.dump(d, f)
 
-    click.echo(f"Config file created at {output}")
+    click.echo(f"Config file written to {output_path}")

flyte/cli/_params.py

@@ -23,7 +23,7 @@ from mashumaro.codecs.json import JSONEncoder
 
 from flyte._logging import logger
 from flyte.io import Dir, File
-from flyte.io.pickle.transformer import FlytePickleTransformer
+from flyte.types._pickle import FlytePickleTransformer
 
 
 class StructuredDataset:
@@ -110,7 +110,7 @@ class FileParamType(click.ParamType):
             p = pathlib.Path(value)
             if not p.exists() or not p.is_file():
                 raise click.BadParameter(f"parameter should be a valid file path, {value}")
-        return File(path=value)
+        return File.from_existing_remote(value)
 
 
 class PickleParamType(click.ParamType):

flyte/cli/_run.py

@@ -97,7 +97,7 @@ class RunTaskCommand(click.Command):
         if obj is None:
             import flyte.config
 
-            obj = CLIConfig(flyte.config.auto())
+            obj = CLIConfig(flyte.config.auto(), ctx)
 
         if not self.run_args.local:
             assert obj.endpoint, "CLI Config should have an endpoint"

flyte/cli/main.py

@@ -90,7 +90,7 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "config_file",
     required=False,
     type=click.Path(exists=True),
-    help="Path to the configuration file to use. If not specified, the default configuration file is used."
+    help="Path to the configuration file to use. If not specified, the default configuration file is used.",
 )
 @click.rich_config(help_config=help_config)
 @click.pass_context
@@ -146,15 +146,11 @@ def main(
     cfg = config.auto(config_file=config_file)
     logger.debug(f"Using config file discovered at location {cfg.source}")
 
-    final_insecure = cfg.platform.insecure
-    if insecure is not None:
-        final_insecure = insecure
-
     ctx.obj = CLIConfig(
         log_level=log_level,
-        endpoint=endpoint or cfg.platform.endpoint,
-        insecure=final_insecure,
-        org_override=org or cfg.task.org,
+        endpoint=endpoint,
+        insecure=insecure,
+        org=org,
         config=cfg,
         ctx=ctx,
     )

flyte/errors.py

@@ -150,3 +150,14 @@ class LogsNotYetAvailableError(BaseRuntimeError):
 
     def __init__(self, message: str):
         super().__init__("LogsNotYetAvailable", "system", message, None)
+
+
+class RuntimeDataValidationError(RuntimeUserError):
+    """
+    This error is raised when the user tries to access a resource that does not exist or is invalid.
+    """
+
+    def __init__(self, var: str, e: Exception, task_name: str = ""):
+        super().__init__(
+            "DataValiationError", f"In task {task_name} variable {var}, failed to serialize/deserialize because {e}"
+        )

flyte/extras/_container.py

@@ -9,20 +9,15 @@ from flyte._logging import logger
 from flyte._task import TaskTemplate
 from flyte.models import NativeInterface, SerializationContext
 
-_PRIMARY_CONTAINER_NAME_FIELD = "primary_container_name"
 
-
-def _extract_command_key(cmd: str, **kwargs) -> Any:
+def _extract_command_key(cmd: str, **kwargs) -> List[Any] | None:
     """
     Extract the key from the command using regex.
     """
     import re
 
-    input_regex = r"^\{\{\s*\.inputs\.(.*?)\s*\}\}$"
-    match = re.match(input_regex, cmd)
-    if match:
-        return match.group(1)
-    return None
+    input_regex = r"\{\{\.inputs\.([a-zA-Z0-9_]+)\}\}"
+    return re.findall(input_regex, cmd)
 
 
 def _extract_path_command_key(cmd: str, input_data_dir: Optional[str]) -> Optional[str]:
@@ -70,7 +65,7 @@ class ContainerTask(TaskTemplate):
         input_data_dir: str | pathlib.Path = "/var/inputs",
         output_data_dir: str | pathlib.Path = "/var/outputs",
         metadata_format: MetadataFormat = "JSON",
-        local_logs: bool = False,
+        local_logs: bool = True,
         **kwargs,
     ):
         super().__init__(
@@ -106,34 +101,33 @@ class ContainerTask(TaskTemplate):
         For FlyteFile and FlyteDirectory commands, e.g., "/var/inputs/inputs", we extract the key from strings that
          begin with the specified `input_data_dir`.
         """
-        # from flytekit.types.directory import FlyteDirectory
-        # from flytekit.types.file import FlyteFile
+        from flyte.io import Dir, File
 
         volume_binding: Dict[str, Dict[str, str]] = {}
         path_k = _extract_path_command_key(cmd, str(self._input_data_dir))
-        k = path_k if path_k else _extract_command_key(cmd)
-
-        if k:
-            input_val = kwargs.get(k)
-            # TODO: Add support file and directory transformer first
-            # if type(input_val) in [FlyteFile, FlyteDirectory]:
-            #     if not path_k:
-            #         raise AssertionError(
-            #             "FlyteFile and FlyteDirectory commands should not use the template syntax like this:
-            #             {{.inputs.infile}}\n"
-            #             "Please use a path-like syntax, such as: /var/inputs/infile.\n"
-            #             "This requirement is due to how Flyte Propeller processes template syntax inputs."
-            #         )
-            #     local_flyte_file_or_dir_path = str(input_val)
-            #     remote_flyte_file_or_dir_path = os.path.join(self._input_data_dir, k)  # type: ignore
-            #     volume_binding[local_flyte_file_or_dir_path] = {
-            #         "bind": remote_flyte_file_or_dir_path,
-            #         "mode": "rw",
-            #     }
-            #     command = remote_flyte_file_or_dir_path
-            command = str(input_val)
-        else:
-            command = cmd
+        keys = path_k if path_k else _extract_command_key(cmd)
+
+        if keys:
+            for k in keys:
+                input_val = kwargs.get(k)
+                # TODO: Add support file and directory transformer first
+                if type(input_val) in [File, Dir]:
+                    if not path_k:
+                        raise AssertionError(
+                            "File and Directory commands should not use the template syntax "
+                            "like this: {{.inputs.infile}}\n"
+                            "Please use a path-like syntax, such as: /var/inputs/infile.\n"
+                            "This requirement is due to how Flyte Propeller processes template syntax inputs."
+                        )
+                    local_flyte_file_or_dir_path = str(input_val)
+                    remote_flyte_file_or_dir_path = os.path.join(self._input_data_dir, k)  # type: ignore
+                    volume_binding[local_flyte_file_or_dir_path] = {
+                        "bind": remote_flyte_file_or_dir_path,
+                        "mode": "rw",
+                    }
+                    command = remote_flyte_file_or_dir_path
+                else:
+                    command = cmd
 
         return command, volume_binding
 
@@ -235,6 +229,7 @@ class ContainerTask(TaskTemplate):
             raise AssertionError(f"Only Image objects are supported, not strings. Got {self._image} instead.")
         uri = self._image.uri
         self._pull_image_if_not_exists(client, uri)
+        print(f"Command: {commands!r}")
 
         container = client.containers.run(uri, command=commands, remove=True, volumes=volume_bindings, detach=True)
 
@@ -266,8 +261,3 @@ class ContainerTask(TaskTemplate):
 
     def container_args(self, sctx: SerializationContext) -> List[str]:
         return self._cmd + (self._args if self._args else [])
-
-    def config(self, sctx: SerializationContext) -> Dict[str, str]:
-        if self.pod_template is None:
-            return {}
-        return {_PRIMARY_CONTAINER_NAME_FIELD: self.primary_container_name}

flyte/io/__init__.py

@@ -3,9 +3,25 @@
 
 This package contains additional data types beyond the primitive data types in python to abstract data flow
 of large datasets in Union.
+
 """
 
-__all__ = ["Dir", "File"]
+__all__ = [
+    "Dir",
+    "File",
+    "StructuredDataset",
+    "StructuredDatasetDecoder",
+    "StructuredDatasetEncoder",
+    "StructuredDatasetTransformerEngine",
+    "lazy_import_structured_dataset_handler",
+]
 
 from ._dir import Dir
 from ._file import File
+from ._structured_dataset import (
+    StructuredDataset,
+    StructuredDatasetDecoder,
+    StructuredDatasetEncoder,
+    StructuredDatasetTransformerEngine,
+    lazy_import_structured_dataset_handler,
+)

flyte/io/_file.py

@@ -232,6 +232,8 @@ class File(BaseModel, Generic[T], SerializableType):
             # This code is broadly similar to what storage.get_stream does, but without actually reading from the stream
             file_handle = None
             try:
+                if "b" not in mode:
+                    raise ValueError("Mode must include 'b' for binary access, when using remote files.")
                 if isinstance(fs, AsyncFileSystem):
                     file_handle = await fs.open_async(self.path, mode)
                     yield file_handle

flyte/io/{structured_dataset → _structured_dataset}/basic_dfs.py

@@ -9,7 +9,7 @@ from fsspec.core import split_protocol, strip_protocol
 import flyte.storage as storage
 from flyte._logging import logger
 from flyte._utils import lazy_module
-from flyte.io.structured_dataset.structured_dataset import (
+from flyte.io._structured_dataset.structured_dataset import (
     CSV,
     PARQUET,
     StructuredDataset,

flyte/io/{structured_dataset → _structured_dataset}/structured_dataset.py

@@ -168,7 +168,7 @@ class StructuredDataset(SerializableType, DataClassJSONMixin):
         return self._literal_sd
 
     def open(self, dataframe_type: Type[DF]):
-        from flyte.io.structured_dataset import lazy_import_structured_dataset_handler
+        from flyte.io._structured_dataset import lazy_import_structured_dataset_handler
 
         """
         Load the handler if needed. For the use case like:

flyte/models.py

@@ -95,6 +95,7 @@ class RawDataPath:
                 # Create a temporary directory for data storage
                 p = tempfile.mkdtemp()
                 logger.debug(f"Creating temporary directory for data storage: {p}")
+                pathlib.Path(p).mkdir(parents=True, exist_ok=True)
                 return RawDataPath(path=p)
             case str():
                 return RawDataPath(path=local_folder)

flyte/remote/_data.py

@@ -100,7 +100,8 @@ async def _upload_single_file(
             if put_resp.status_code != 200:
                 raise RuntimeSystemError(
                     "UploadFailed",
-                    f"Failed to upload {fp} to {resp.signed_url}, status code: {put_resp.status_code}",
+                    f"Failed to upload {fp} to {resp.signed_url}, status code: {put_resp.status_code}, "
+                    f"response: {put_resp.text}",
                 )
         # TODO in old code we did this
         #             if self._config.platform.insecure_skip_verify is True

flyte/types/__init__.py

@@ -1,9 +1,32 @@
+"""
+# Flyte Type System
+
+The Flyte type system provides a way to define, transform, and manipulate types in Flyte workflows.
+Since the data flowing through Flyte has to often cross process, container and langauge boundaries, the type system
+is designed to be serializable to a universal format that can be understood across different environments. This
+universal format is based on Protocol Buffers. The types are called LiteralTypes and the runtime
+representation of data is called Literals.
+
+The type system includes:
+- **TypeEngine**: The core engine that manages type transformations and serialization. This is the main entry point for
+  for all the internal type transformations and serialization logic.
+- **TypeTransformer**: A class that defines how to transform one type to another. This is extensible
+    allowing users to define custom types and transformations.
+- **Renderable**: An interface for types that can be rendered as HTML, that can be outputted to a flyte.report.
+
+It is always possible to bypass the type system and use the `FlytePickle` type to serialize any python object
+ into a pickle format. The pickle format is not human-readable, but can be passed between flyte tasks that are
+ written in python. The Pickled objects cannot be represented in the UI, and may be in-efficient for large datasets.
+"""
+
 from ._interface import guess_interface
+from ._pickle import FlytePickle
 from ._renderer import Renderable
 from ._string_literals import literal_string_repr
 from ._type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError
 
 __all__ = [
+    "FlytePickle",
     "Renderable",
     "TypeEngine",
     "TypeTransformer",

flyte/{io/pickle/transformer.py → types/_pickle.py}

@@ -8,7 +8,8 @@ import cloudpickle
 from flyteidl.core import literals_pb2, types_pb2
 
 import flyte.storage as storage
-from flyte.types import TypeEngine, TypeTransformer
+
+from ._type_engine import TypeEngine, TypeTransformer
 
 T = typing.TypeVar("T")
 

flyte/types/_type_engine.py

@@ -1008,7 +1008,7 @@ class TypeEngine(typing.Generic[T]):
             return cls._DATACLASS_TRANSFORMER
 
         display_pickle_warning(str(python_type))
-        from flyte.io.pickle.transformer import FlytePickleTransformer
+        from flyte.types._pickle import FlytePickleTransformer
 
         return FlytePickleTransformer()
 
@@ -1021,7 +1021,7 @@ class TypeEngine(typing.Generic[T]):
             # Avoid a race condition where concurrent threads may exit lazy_import_transformers before the transformers
             # have been imported. This could be implemented without a lock if you assume python assignments are atomic
             # and re-registering transformers is acceptable, but I decided to play it safe.
-            from flyte.io.structured_dataset import lazy_import_structured_dataset_handler
+            from flyte.io import lazy_import_structured_dataset_handler
 
             # todo: bring in extras transformers (pytorch, etc.)
             lazy_import_structured_dataset_handler()
@@ -1207,7 +1207,9 @@ class TypeEngine(typing.Generic[T]):
                 python_type = type_hints.get(k, type(d[k]))
                 e: BaseException = literal_map[k].exception()  # type: ignore
                 if isinstance(e, TypeError):
-                    raise TypeError(f"Error converting: {type(v)}, {python_type}, received_value {v}")
+                    raise TypeError(
+                        f"Error converting: Var:{k}, type:{type(v)}, into:{python_type}, received_value {v}"
+                    )
                 else:
                     raise e
             literal_map[k] = v.result()
@@ -1657,7 +1659,7 @@ class DictTransformer(TypeTransformer[dict]):
         Converts a Python dictionary to a Flyte-specific ``Literal`` using MessagePack encoding.
         Falls back to Pickle if encoding fails and `allow_pickle` is True.
         """
-        from flyte.io.pickle.transformer import FlytePickle
+        from flyte.types._pickle import FlytePickle
 
         try:
             # Handle dictionaries with non-string keys (e.g., Dict[int, Type])
@@ -1763,7 +1765,7 @@ class DictTransformer(TypeTransformer[dict]):
         # pr: han-ru is this part still necessary?
         if lv and lv.HasField("scalar") and lv.scalar.HasField("generic"):
             if lv.metadata and lv.metadata.get("format", None) == "pickle":
-                from flyte.io.pickle.transformer import FlytePickle
+                from flyte.types._pickle import FlytePickle
 
                 uri = json.loads(_json_format.MessageToJson(lv.scalar.generic)).get("pickle_file")
                 return await FlytePickle.from_pickle(uri)

{flyte-0.2.0b9.dist-info → flyte-0.2.0b11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 0.2.0b9
+Version: 0.2.0b11
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10
@@ -61,7 +61,6 @@ Note that the `--follow` command is optional. Use this to stream updates to the
 3. look at examples/... for various examples. 
 4. For a single script recommend using uv run scripts with metadata headers.
 
-
 ```python
 import flyte
 
@@ -80,14 +79,14 @@ async def say_hello_nested(data: str) -> str:
 
 if __name__ == "__main__":
     import asyncio
-    
+
     # to run pure python - the SDK is not invoked at all
     asyncio.run(say_hello_nested("test"))
-    
+
     # To run locally, but run through type system etc
     flyte.init()
     flyte.run(say_hello_nested, "World")
-    
+
     # To run remote
     flyte.init(endpoint="dns:///localhost:8090", insecure=True)
     flyte.run(say_hello_nested, "World")
@@ -95,7 +94,7 @@ if __name__ == "__main__":
     flyte.with_runcontext(mode="local").run(...)  # this will run locally only
 
     # To run remote with a config
-    flyte.init_auto_from_config("config.yaml")
+    flyte.init_from_config("config.yaml")
 ```
 
 # CLI