flyte 0.2.0b27__py3-none-any.whl → 0.2.0b30__py3-none-any.whl

flyte/_image.py

@@ -195,6 +195,15 @@ class WorkDir(Layer):
         hasher.update(self.workdir.encode("utf-8"))
 
 
+@rich.repr.auto
+@dataclass(frozen=True, repr=True)
+class DockerIgnore(Layer):
+    path: str
+
+    def update_hash(self, hasher: hashlib._Hash):
+        hasher.update(self.path.encode("utf-8"))
+
+
 @rich.repr.auto
 @dataclass(frozen=True, repr=True)
 class CopyConfig(Layer):
@@ -575,6 +584,13 @@ class Image:
 
         :return:
         """
+        if addl_layer and self.dockerfile:
+            # We don't know how to inspect dockerfiles to know what kind it is (OS, python version, uv vs poetry, etc)
+            # so there's no guarantee any of the layering logic will work.
+            raise ValueError(
+                "Flyte current cannot add additional layers to a Dockerfile-based Image."
+                " Please amend the dockerfile directly."
+            )
         registry = registry if registry else self.registry
         name = name if name else self.name
         if addl_layer and (not name):
@@ -595,17 +611,35 @@ class Image:
         return img
 
     @classmethod
-    def from_dockerfile(cls, file: Path, registry: str, name: str) -> Image:
+    def from_dockerfile(
+        cls, file: Path, registry: str, name: str, platform: Union[Architecture, Tuple[Architecture, ...], None] = None
+    ) -> Image:
         """
-        Use this method to create a new image with the specified dockerfile
+        Use this method to create a new image with the specified dockerfile. Note you cannot use additional layers
+        after this, as the system doesn't attempt to parse/understand the Dockerfile, and what kind of setup it has
+        (python version, uv vs poetry, etc), so please put all logic into the dockerfile itself.
+
+        Also since Python sees paths as from the calling directory, please use Path objects with absolute paths. The
+        context for the builder will be the directory where the dockerfile is located.
 
         :param file: path to the dockerfile
         :param name: name of the image
         :param registry: registry to use for the image
+        :param platform: architecture to use for the image, default is linux/amd64, use tuple for multiple values
+            Example: ("linux/amd64", "linux/arm64")
 
         :return:
         """
-        img = cls(dockerfile=file, registry=registry, name=name)
+        platform = _ensure_tuple(platform) if platform else None
+        file = file.absolute()  # for clarity when debugging
+        kwargs = {
+            "dockerfile": file,
+            "registry": registry,
+            "name": name,
+        }
+        if platform:
+            kwargs["platform"] = platform
+        img = cls._new(**kwargs)
 
         return img
 
@@ -753,6 +787,10 @@ class Image:
         new_image = self.clone(addl_layer=CopyConfig(path_type=0, src=src, dst=dst, _compute_identifier=lambda x: dst))
         return new_image
 
+    def with_dockerignore(self, path: Path) -> Image:
+        new_image = self.clone(addl_layer=DockerIgnore(path=str(path)))
+        return new_image
+
     def with_uv_project(
         self,
         pyproject_file: Path,

flyte/_internal/imagebuild/docker_builder.py

@@ -15,6 +15,7 @@ from flyte._image import (
     AptPackages,
     Commands,
     CopyConfig,
+    DockerIgnore,
     Env,
     Image,
     Layer,
@@ -205,12 +206,18 @@ class UVProjectHandler:
         return dockerfile
 
 
+class DockerIgnoreHandler:
+    @staticmethod
+    async def handle(layer: DockerIgnore, context_path: Path, _: str):
+        shutil.copy(layer.path, context_path)
+
+
 class CopyConfigHandler:
     @staticmethod
     async def handle(layer: CopyConfig, context_path: Path, dockerfile: str) -> str:
         # Copy the source config file or directory to the context path
         if layer.src.is_absolute() or ".." in str(layer.src):
-            dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/")
+            dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/", 1)
         else:
             dst_path = context_path / layer.src
 
@@ -276,6 +283,10 @@ async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> s
             # Handle commands
             dockerfile = await CommandsHandler.handle(layer, context_path, dockerfile)
 
+        case DockerIgnore():
+            # Handle dockerignore
+            await DockerIgnoreHandler.handle(layer, context_path, dockerfile)
+
         case WorkDir():
             # Handle workdir
             dockerfile = await WorkDirHandler.handle(layer, context_path, dockerfile)
@@ -305,14 +316,14 @@ class DockerImageBuilder(ImageBuilder):
         return [LocalDockerCommandImageChecker, LocalPodmanCommandImageChecker, DockerAPIImageChecker]
 
     async def build_image(self, image: Image, dry_run: bool = False) -> str:
-        if len(image._layers) == 0:
-            logger.warning("No layers to build, returning the image URI as is.")
-            return image.uri
-
         if image.dockerfile:
             # If a dockerfile is provided, use it directly
             return await self._build_from_dockerfile(image, push=True)
 
+        if len(image._layers) == 0:
+            logger.warning("No layers to build, returning the image URI as is.")
+            return image.uri
+
         return await self._build_image(
             image,
             push=True,
@@ -323,18 +334,27 @@ class DockerImageBuilder(ImageBuilder):
         """
         Build the image from a provided Dockerfile.
         """
+        assert image.dockerfile  # for mypy
+
         command = [
             "docker",
+            "buildx",
             "build",
+            "--builder",
+            DockerImageBuilder._builder_name,
+            "-f",
+            str(image.dockerfile),
             "--tag",
             f"{image.uri}",
             "--platform",
             ",".join(image.platform),
-            ".",
+            str(image.dockerfile.parent.absolute()),  # Use the parent directory of the Dockerfile as the context
         ]
 
         if image.registry and push:
             command.append("--push")
+        else:
+            command.append("--load")
 
         concat_command = " ".join(command)
         logger.debug(f"Build command: {concat_command}")

flyte/_internal/imagebuild/remote_builder.py

@@ -16,6 +16,7 @@ from flyte._image import (
     Architecture,
     Commands,
     CopyConfig,
+    DockerIgnore,
     Env,
     PipPackages,
     PythonWheels,
@@ -225,6 +226,8 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
                 commands=image_definition_pb2.Commands(cmd=list(layer.commands))
             )
             layers.append(commands_layer)
+        elif isinstance(layer, DockerIgnore):
+            shutil.copy(layer.path, context_path)
         elif isinstance(layer, CopyConfig):
             dst_path = _copy_files_to_context(layer.src, context_path)
 

flyte/_logging.py

@@ -83,7 +83,7 @@ def get_default_handler(log_level: int) -> logging.Handler:
     return handler
 
 
-def initialize_logger(log_level: int = DEFAULT_LOG_LEVEL, enable_rich: bool = False):
+def initialize_logger(log_level: int = get_env_log_level(), enable_rich: bool = False):
     """
     Initializes the global loggers to the default configuration.
     """

flyte/_task.py

@@ -20,8 +20,6 @@ from typing import (
     Union,
 )
 
-from flyteidl.core.tasks_pb2 import DataLoadingConfig
-
 from flyte._pod import PodTemplate
 from flyte.errors import RuntimeSystemError, RuntimeUserError
 
@@ -37,6 +35,8 @@ from ._timeout import TimeoutType
 from .models import NativeInterface, SerializationContext
 
 if TYPE_CHECKING:
+    from flyteidl.core.tasks_pb2 import DataLoadingConfig
+
     from ._task_environment import TaskEnvironment
 
 P = ParamSpec("P")  # capture the function's parameters

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b27'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b27')
+__version__ = version = '0.2.0b30'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b30')

flyte/models.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import inspect
 import os
 import pathlib
-import tempfile
 from dataclasses import dataclass, field, replace
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Literal, Optional, Tuple, Type
 
@@ -12,7 +11,6 @@ import rich.repr
 from flyte._docstring import Docstring
 from flyte._interface import extract_return_annotation
 from flyte._logging import logger
-from flyte._utils.helpers import base36_encode
 
 if TYPE_CHECKING:
     from flyteidl.core import literals_pb2
@@ -65,6 +63,8 @@ class ActionID:
         """Make a deterministic name"""
         import hashlib
 
+        from flyte._utils.helpers import base36_encode
+
         components = f"{self.name}-{input_hash}-{task_hash}-{task_call_seq}" + (f"-{group}" if group else "")
         logger.debug(f"----- Generating sub-action ID from components: {components}")
         # has the components into something deterministic
@@ -89,6 +89,8 @@ class RawDataPath:
         Create a new context attribute object, with local path given. Will be created if it doesn't exist.
         :return: Path to the temporary directory
         """
+        import tempfile
+
         match local_folder:
             case pathlib.Path():
                 local_folder.mkdir(parents=True, exist_ok=True)

flyte/remote/_data.py

@@ -79,16 +79,16 @@ async def _upload_single_file(
     except grpc.aio.AioRpcError as e:
         if e.code() == grpc.StatusCode.NOT_FOUND:
             raise RuntimeSystemError(
-                "NotFound", f"Failed to get signed url for {fp}, please check your project and domain."
+                "NotFound", f"Failed to get signed url for {fp}, please check your project and domain: {e.details()}"
             )
         elif e.code() == grpc.StatusCode.PERMISSION_DENIED:
             raise RuntimeSystemError(
-                "PermissionDenied", f"Failed to get signed url for {fp}, please check your permissions."
+                "PermissionDenied", f"Failed to get signed url for {fp}, please check your permissions: {e.details()}"
             )
         elif e.code() == grpc.StatusCode.UNAVAILABLE:
             raise InitializationError("EndpointUnavailable", "user", "Service is unavailable.")
         else:
-            raise RuntimeSystemError(e.code().value, f"Failed to get signed url for {fp}.")
+            raise RuntimeSystemError(e.code().value, f"Failed to get signed url for {fp}: {e.details()}")
     except Exception as e:
         raise RuntimeSystemError(type(e).__name__, f"Failed to get signed url for {fp}.") from e
     logger.debug(f"Uploading to {make_hyperlink('signed url', resp.signed_url)} for {fp}")

flyte/types/_type_engine.py

@@ -1594,6 +1594,18 @@ class UnionTransformer(TypeTransformer[T]):
         self, python_val: T, python_type: Type[T], expected: types_pb2.LiteralType
     ) -> literals_pb2.Literal:
         python_type = get_underlying_type(python_type)
+        inferred_type = type(python_val)
+        subtypes = get_args(python_type)
+
+        if inferred_type in subtypes:
+            # If the Python value's type matches one of the types in the Union,
+            # always use the transformer associated with that specific type.
+            transformer = TypeEngine.get_transformer(inferred_type)
+            res = await transformer.to_literal(
+                python_val, inferred_type, expected.union_type.variants[subtypes.index(inferred_type)]
+            )
+            res_type = _add_tag_to_type(transformer.get_literal_type(inferred_type), transformer.name)
+            return Literal(scalar=Scalar(union=Union(value=res, type=res_type)))
 
         potential_types = []
         found_res = False
@@ -1601,14 +1613,14 @@ class UnionTransformer(TypeTransformer[T]):
         res = None
         res_type = None
         t = None
-        for i in range(len(get_args(python_type))):
+        for i in range(len(subtypes)):
             try:
-                t = get_args(python_type)[i]
+                t = subtypes[i]
                 trans: TypeTransformer[T] = TypeEngine.get_transformer(t)
                 attempt = trans.to_literal(python_val, t, expected.union_type.variants[i])
                 res = await attempt
                 if found_res:
-                    logger.debug(f"Current type {get_args(python_type)[i]} old res {res_type}")
+                    logger.debug(f"Current type {subtypes[i]} old res {res_type}")
                     is_ambiguous = True
                 res_type = _add_tag_to_type(trans.get_literal_type(t), trans.name)
                 found_res = True

{flyte-0.2.0b27.dist-info → flyte-0.2.0b30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 0.2.0b27
+Version: 0.2.0b30
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10

{flyte-0.2.0b27.dist-info → flyte-0.2.0b30.dist-info}/RECORD

@@ -8,10 +8,10 @@ flyte/_environment.py,sha256=oKVXLBX0ky2eE_wjBdzvQGI_2LiT2Nbx58ur7GMt50c,3231
 flyte/_excepthook.py,sha256=nXts84rzEg6-7RtFarbKzOsRZTQR4plnbWVIFMAEprs,1310
 flyte/_group.py,sha256=7o1j16sZyUmYB50mOiq1ui4TBAKhRpDqLakV8Ya1kw4,803
 flyte/_hash.py,sha256=Of_Zl_DzzzF2jp4ZsLm-3o-xJFCCJ8_GubmLI1htx78,504
-flyte/_image.py,sha256=P7eGdGHOHenxZlgT8LkHWjv35pPqrhrBqA__nJ_QNOI,31845
+flyte/_image.py,sha256=NbHR_4pCpYVwd7gYW1CHaXm6cxVhAR40NFDy3QaiLbU,33604
 flyte/_initialize.py,sha256=xKl_LYMluRt21wWqa6RTKuLo0_DCbSaTfUk27_brtNk,18232
 flyte/_interface.py,sha256=1B9zIwFDjiVp_3l_mk8EpA4g3Re-6DUBEBi9z9vDvPs,3504
-flyte/_logging.py,sha256=_yNo-Nx2yzh0MLoZGbnIYHGKei4wtQmSGM0lE30Ev7w,3662
+flyte/_logging.py,sha256=QrT4Z30C2tsZ-yIojisQODTuq6Y6zSJYuTrLgF58UYc,3664
 flyte/_map.py,sha256=efPd8O-JKUg1OY3_MzU3KGbhsGYDVRNBwWr0ceNIXhQ,7444
 flyte/_pod.py,sha256=--72b0c6IkOEbBwZPLmgl-ll-j7ECfG-kh75LzBnNN8,1068
 flyte/_resources.py,sha256=L2JuvQDlMo1JLJeUmJPRwtWbunhR2xJEhFgQW5yc72c,9690
@@ -19,16 +19,16 @@ flyte/_retry.py,sha256=rfLv0MvWxzPByKESTglEmjPsytEAKiIvvmzlJxXwsfE,941
 flyte/_reusable_environment.py,sha256=f8Y1GilUwGcXH4n2Fckrnx0SrZmhk3nCfoe-TqUKivI,3740
 flyte/_run.py,sha256=htq4zyltmaWL-B_IkLX5joSzfB-pLTHQ5DtI49c220I,24247
 flyte/_secret.py,sha256=ogXmCNfYYIphV6p-2iiWmwr2cNUES5Cq01PPjY6uQNA,3217
-flyte/_task.py,sha256=hYId6MoJiLsGv3otAfXAnjIc1u39LRSSapHt0GrD-_Q,18014
+flyte/_task.py,sha256=aZpnLLMZLNmPPS6-_pafBEgOtvg7Z-kwI1fhexfI8JM,18018
 flyte/_task_environment.py,sha256=qYcTG9452a_POueQCHqkTafN4HG8Xo7KkBPwSMkKRRU,8536
 flyte/_task_plugins.py,sha256=9MH3nFPOH_e8_92BT4sFk4oyAnj6GJFvaPYWaraX7yE,1037
 flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
 flyte/_tools.py,sha256=tWb0sx3t3mm4jbaQVjCTc9y39oR_Ibo3z_KHToP3Lto,966
 flyte/_trace.py,sha256=C788bgoSc3st8kE8Cae2xegnLx2CT6uuRKKfaDrDUys,5122
-flyte/_version.py,sha256=1OZxtIinrzel7addSNq8I2MVEKwGJOtZ9VCRKrmBqxE,521
+flyte/_version.py,sha256=V4ZfeDEeYjX-hkTAGvUV-fZDolyr1Qdq3Nn5m5C-mng,521
 flyte/errors.py,sha256=IUXHuEgszis9ZyhBUOSSlZx7VPQk0BF4N875yj1FBKo,4963
 flyte/extend.py,sha256=GB4ZedGzKa30vYWRVPOdxEeK62xnUVFY4z2tD6H9eEw,376
-flyte/models.py,sha256=yqLwln7G30tpV5U8Zji4z8K4u7DbSCDdFo1q-F77WuM,15207
+flyte/models.py,sha256=g8QBIF9JjrNxDR_R_esAFG72hrq3fFV42bb7Nm0E0Zo,15225
 flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/_bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/_bin/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
@@ -53,9 +53,9 @@ flyte/_internal/controllers/remote/_core.py,sha256=FJe1ZAWu_0w5SQUFgAHY4Hjjf_Aju
 flyte/_internal/controllers/remote/_informer.py,sha256=StiPcQLLW0h36uEBhKsupMY79EeFCKA3QQzvv2IyvRo,14188
 flyte/_internal/controllers/remote/_service_protocol.py,sha256=B9qbIg6DiGeac-iSccLmX_AL2xUgX4ezNUOiAbSy4V0,1357
 flyte/_internal/imagebuild/__init__.py,sha256=dwXdJ1jMhw9RF8itF7jkPLanvX1yCviSns7hE5eoIts,102
-flyte/_internal/imagebuild/docker_builder.py,sha256=rQXbWG9eKcxBzjbd2pbkEJ1OqtDPeBxDGLqoluQvm5U,16129
+flyte/_internal/imagebuild/docker_builder.py,sha256=SZdQOaNwnXwHeCzcg7YdnmveypeLXGRiIEN3VzhqR-Q,16795
 flyte/_internal/imagebuild/image_builder.py,sha256=dXBXl62qcPabus6dR3eP8P9mBGNhpZHZ2Xm12AymKkk,11150
-flyte/_internal/imagebuild/remote_builder.py,sha256=JBeek4B4a6tUeyAD8dgQKVv4myMdeERqmxMStABVWN0,10452
+flyte/_internal/imagebuild/remote_builder.py,sha256=NndHL4dHKMIb8s4RS1fllr7oCtwbzZI0OufGNSLr14M,10566
 flyte/_internal/resolvers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flyte/_internal/resolvers/_task_module.py,sha256=jwy1QYygUK7xmpCZLt1SPTfJCkfox3Ck3mTlTsm66UI,1973
 flyte/_internal/resolvers/common.py,sha256=ADQLRoyGsJ4vuUkitffMGrMKKjy0vpk6X53g4FuKDLc,993
@@ -182,7 +182,7 @@ flyte/io/_structured_dataset/structured_dataset.py,sha256=ddRjz36RhNxIy0gakzdLSt
 flyte/remote/__init__.py,sha256=y9eke9JzEJkygk8eKZjSj44fJGlyepuW4i-j6lbCAPY,617
 flyte/remote/_action.py,sha256=r2rLmJoPUIOCUoBUy4RG_1mMYmC4uiI1tKXuZZ9iC9k,23564
 flyte/remote/_console.py,sha256=avmELJPx8nQMAVPrHlh6jEIRPjrMwFpdZjJsWOOa9rE,660
-flyte/remote/_data.py,sha256=h3oR2ZmwOEg6HpWyYVbVcHBs8_LX2RmRolHxgwBjlQE,6121
+flyte/remote/_data.py,sha256=zYXXlqEvPdsC44Gm7rP_hQjRgVe3EFfhZNEWKF0p4MQ,6163
 flyte/remote/_logs.py,sha256=xBx4ozfY-NBMw3uD5o5YLsI4no10zGroNMcx1Oj0ef0,6689
 flyte/remote/_project.py,sha256=CFNTGpgXU3X599tkJ_oxijs9zPzzCWOB6mAWn6WeDEU,2828
 flyte/remote/_run.py,sha256=HzpoDthSw50vuT-gmzyzdkf28H4SyBgMr0d6bFSmyNU,9747
@@ -224,11 +224,11 @@ flyte/types/_interface.py,sha256=5y9EC5r897xz03Hh0ltF8QVGKMfMfAznws-hKSEO4Go,167
 flyte/types/_pickle.py,sha256=PjdR66OTDMZ3OYq6GvM_Ua0cIo5t2XQaIjmpJ9xo4Ys,4050
 flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
 flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
-flyte/types/_type_engine.py,sha256=QBH-XNwyBHvKP8PjI_BaEPneonfoeNS86aMld87YLMA,97278
+flyte/types/_type_engine.py,sha256=9cgkE_gz4gFfuXc2Qy7cLEmf2p1ZJb2DhygK_PaiqOs,97934
 flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
-flyte-0.2.0b27.data/scripts/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
-flyte-0.2.0b27.dist-info/METADATA,sha256=vMxmOvEOeHKQwuOpB0A5vFPtvf7mlBbJhq8RiZ2MsTg,5857
-flyte-0.2.0b27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-flyte-0.2.0b27.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
-flyte-0.2.0b27.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
-flyte-0.2.0b27.dist-info/RECORD,,
+flyte-0.2.0b30.data/scripts/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
+flyte-0.2.0b30.dist-info/METADATA,sha256=KEy3-spKhC5A0GLF3a40UZQaT4DgCXso2fI2u9fwBrI,5857
+flyte-0.2.0b30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+flyte-0.2.0b30.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
+flyte-0.2.0b30.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
+flyte-0.2.0b30.dist-info/RECORD,,