flyte 2.0.0b19__py3-none-any.whl → 2.0.0b21__py3-none-any.whl

flyte/_internal/imagebuild/docker_builder.py

@@ -6,7 +6,7 @@ import tempfile
 import typing
 from pathlib import Path
 from string import Template
-from typing import ClassVar, Optional, Protocol, cast
+from typing import ClassVar, List, Optional, Protocol, cast
 
 import aiofiles
 import click
@@ -263,7 +263,37 @@ class DockerIgnoreHandler:
 
 class CopyConfigHandler:
     @staticmethod
-    async def handle(layer: CopyConfig, context_path: Path, dockerfile: str) -> str:
+    def list_dockerignore(root_path: Optional[Path], docker_ignore_file_path: Optional[Path]) -> List[str]:
+        patterns: List[str] = []
+        dockerignore_path: Optional[Path] = None
+        if root_path:
+            dockerignore_path = root_path / ".dockerignore"
+        # DockerIgnore layer should be first priority
+        if docker_ignore_file_path:
+            dockerignore_path = docker_ignore_file_path
+
+        # Return empty list if no .dockerignore file found
+        if not dockerignore_path or not dockerignore_path.exists() or not dockerignore_path.is_file():
+            logger.info(f".dockerignore file not found at path: {dockerignore_path}")
+            return patterns
+
+        try:
+            with open(dockerignore_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    stripped_line = line.strip()
+                    # Skip empty lines, whitespace-only lines, and comments
+                    if not stripped_line or stripped_line.startswith("#"):
+                        continue
+                    patterns.append(stripped_line)
+        except Exception as e:
+            logger.error(f"Failed to read .dockerignore file at {dockerignore_path}: {e}")
+            return []
+        return patterns
+
+    @staticmethod
+    async def handle(
+        layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_file_path: Optional[Path]
+    ) -> str:
         # Copy the source config file or directory to the context path
         if layer.src.is_absolute() or ".." in str(layer.src):
             dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/", 1)
@@ -272,18 +302,26 @@ class CopyConfigHandler:
 
         dst_path.parent.mkdir(parents=True, exist_ok=True)
         abs_path = layer.src.absolute()
+
         if layer.src.is_file():
             # Copy the file
             shutil.copy(abs_path, dst_path)
         elif layer.src.is_dir():
             # Copy the entire directory
-            shutil.copytree(abs_path, dst_path, dirs_exist_ok=True)
+            from flyte._initialize import _get_init_config
+
+            init_config = _get_init_config()
+            root_path = init_config.root_dir if init_config else None
+            docker_ignore_patterns = CopyConfigHandler.list_dockerignore(root_path, docker_ignore_file_path)
+            shutil.copytree(
+                abs_path, dst_path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*docker_ignore_patterns)
+            )
         else:
-            raise ValueError(f"Source path is neither file nor directory: {layer.src}")
+            logger.error(f"Source path not exists: {layer.src}")
+            return dockerfile
 
         # Add a copy command to the dockerfile
         dockerfile += f"\nCOPY {dst_path.relative_to(context_path)} {layer.dst}\n"
-
         return dockerfile
 
 
@@ -349,7 +387,9 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
     return secret_mounts_layer
 
 
-async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> str:
+async def _process_layer(
+    layer: Layer, context_path: Path, dockerfile: str, docker_ignore_file_path: Optional[Path]
+) -> str:
     match layer:
         case PythonWheels():
             # Handle Python wheels
@@ -385,7 +425,7 @@ async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> s
 
         case CopyConfig():
             # Handle local files and folders
-            dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile)
+            dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_file_path)
 
         case Commands():
             # Handle commands
@@ -512,6 +552,15 @@ class DockerImageBuilder(ImageBuilder):
         else:
             logger.info("Buildx builder already exists.")
 
+    def get_docker_ignore(self, image: Image) -> Optional[Path]:
+        """Get the .dockerignore file path from the image layers."""
+        # Look for DockerIgnore layer in the image layers
+        for layer in image._layers:
+            if isinstance(layer, DockerIgnore) and layer.path.strip():
+                return Path(layer.path)
+
+        return None
+
     async def _build_image(self, image: Image, *, push: bool = True, dry_run: bool = False) -> str:
         """
         if default image (only base image and locked), raise an error, don't have a dockerfile
@@ -541,8 +590,9 @@ class DockerImageBuilder(ImageBuilder):
                 PYTHON_VERSION=f"{image.python_version[0]}.{image.python_version[1]}",
             )
 
+            docker_ignore_file_path = self.get_docker_ignore(image)
             for layer in image._layers:
-                dockerfile = await _process_layer(layer, tmp_path, dockerfile)
+                dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_file_path)
 
             dockerfile += DOCKER_FILE_BASE_FOOTER.substitute(F_IMG_ID=image.uri)
 

flyte/_internal/imagebuild/remote_builder.py

@@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Optional, Tuple, cast
 from uuid import uuid4
 
 import aiofiles
-import click
 
 import flyte
 import flyte.errors
@@ -42,7 +41,6 @@ if TYPE_CHECKING:
     from flyte._protos.imagebuilder import definition_pb2 as image_definition_pb2
 
 IMAGE_TASK_NAME = "build-image"
-OPTIMIZE_TASK_NAME = "optimize-task"
 IMAGE_TASK_PROJECT = "system"
 IMAGE_TASK_DOMAIN = "production"
 
@@ -85,10 +83,10 @@ class RemoteImageChecker(ImageChecker):
                     raise ValueError("remote client should not be None")
                 cls._images_client = image_service_pb2_grpc.ImageServiceStub(cfg.client._channel)
             resp = await cls._images_client.GetImage(req)
-            logger.warning(click.style(f"Image {resp.image.fqin} found. Skip building.", fg="blue"))
+            logger.warning(f"[blue]Image {resp.image.fqin} found. Skip building.[/blue]")
             return resp.image.fqin
         except Exception:
-            logger.warning(click.style(f"Image {image_name} was not found or has expired.", fg="blue"))
+            logger.warning(f"[blue]Image {image_name} was not found or has expired.[/blue]", extra={"highlight": False})
             return None
 
 
@@ -110,37 +108,25 @@ class RemoteImageBuilder(ImageBuilder):
             domain=IMAGE_TASK_DOMAIN,
             auto_version="latest",
         ).override.aio(secrets=_get_build_secrets_from_image(image))
+
+        logger.warning("[bold blue]🐳 Submitting a new build...[/bold blue]")
         run = cast(
             Run,
             await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
                 entity, spec=spec, context=context, target_image=image_name
             ),
         )
-        logger.warning(click.style("🐳 Submitting a new build...", fg="blue", bold=True))
+        logger.warning(f"⏳ Waiting for build to finish at: [bold cyan link={run.url}]{run.url}[/bold cyan link]")
 
-        logger.warning(click.style("⏳ Waiting for build to finish at: " + click.style(run.url, fg="cyan"), bold=True))
         await run.wait.aio(quiet=True)
         run_details = await run.details.aio()
 
         elapsed = str(datetime.now(timezone.utc) - start).split(".")[0]
 
         if run_details.action_details.raw_phase == run_definition_pb2.PHASE_SUCCEEDED:
-            logger.warning(click.style(f"✅ Build completed in {elapsed}!", bold=True, fg="green"))
-            try:
-                entity = remote.Task.get(
-                    name=OPTIMIZE_TASK_NAME,
-                    project=IMAGE_TASK_PROJECT,
-                    domain=IMAGE_TASK_DOMAIN,
-                    auto_version="latest",
-                )
-                await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
-                    entity, target_image=image_name
-                )
-            except Exception as e:
-                # Ignore the error if optimize is not enabled in the backend.
-                logger.warning(f"Failed to run optimize task with error: {e}")
+            logger.warning(f"[bold green]✅ Build completed in {elapsed}![/bold green]")
         else:
-            raise flyte.errors.ImageBuildError(f"❌ Build failed in {elapsed} at {click.style(run.url, fg='cyan')}")
+            raise flyte.errors.ImageBuildError(f"❌ Build failed in {elapsed} at [cyan]{run.url}[/cyan]")
 
         outputs = await run_details.outputs()
         return _get_fully_qualified_image_name(outputs)
@@ -183,11 +169,8 @@ async def _validate_configuration(image: Image) -> Tuple[str, Optional[str]]:
         context_size = tar_path.stat().st_size
         if context_size > 5 * 1024 * 1024:
             logger.warning(
-                click.style(
-                    f"Context size is {context_size / (1024 * 1024):.2f} MB, which is larger than 5 MB. "
-                    "Upload and build speed will be impacted.",
-                    fg="yellow",
-                )
+                f"[yellow]Context size is {context_size / (1024 * 1024):.2f} MB, which is larger than 5 MB. "
+                "Upload and build speed will be impacted.[/yellow]",
             )
         _, context_url = await remote.upload_file.aio(context_dst)
     else:

flyte/_keyring/file.py

@@ -0,0 +1,85 @@
+from base64 import decodebytes, encodebytes
+from configparser import ConfigParser, NoOptionError, NoSectionError
+from pathlib import Path
+from typing import Optional
+
+from keyring.backend import KeyringBackend
+from keyring.errors import PasswordDeleteError
+
+_FLYTE_KEYRING_PATH: Path = Path.home() / ".flyte" / "keyring.cfg"
+
+
+class SimplePlainTextKeyring(KeyringBackend):
+    """Simple plain text keyring"""
+
+    priority = 0.5
+
+    def get_password(self, service: str, username: str) -> Optional[str]:
+        """Get password."""
+        if not self.file_path.exists():
+            return None
+
+        config = ConfigParser(interpolation=None)
+        config.read(self.file_path, encoding="utf-8")
+
+        try:
+            password_base64 = config.get(service, username).encode("utf-8")
+            return decodebytes(password_base64).decode("utf-8")
+        except (NoOptionError, NoSectionError):
+            return None
+
+    def delete_password(self, service: str, username: str) -> None:
+        """Delete password."""
+        if not self.file_path.exists():
+            raise PasswordDeleteError("Config file does not exist")
+
+        config = ConfigParser(interpolation=None)
+        config.read(self.file_path, encoding="utf-8")
+
+        try:
+            if not config.remove_option(service, username):
+                raise PasswordDeleteError("Password not found")
+        except NoSectionError:
+            raise PasswordDeleteError("Password not found")
+
+        with self.file_path.open("w", encoding="utf-8") as config_file:
+            config.write(config_file)
+
+    def set_password(self, service: str, username: str, password: str) -> None:
+        """Set password."""
+        if not username:
+            raise ValueError("Username must be provided")
+
+        file_path = self._ensure_file_path()
+        value = encodebytes(password.encode("utf-8")).decode("utf-8")
+
+        config = ConfigParser(interpolation=None)
+        config.read(file_path, encoding="utf-8")
+
+        if not config.has_section(service):
+            config.add_section(service)
+
+        config.set(service, username, value)
+
+        with file_path.open("w", encoding="utf-8") as config_file:
+            config.write(config_file)
+
+    def _ensure_file_path(self):
+        self.file_path.parent.mkdir(exist_ok=True, parents=True)
+        if not self.file_path.is_file():
+            self.file_path.touch(0o600)
+        return self.file_path
+
+    @property
+    def file_path(self) -> Path:
+        from flyte.config._reader import resolve_config_path
+
+        config_path = resolve_config_path()
+        if config_path and str(config_path.parent) == ".flyte":
+            # if the config is in a .flyte directory, use that as the path
+            return config_path.parent / "keyring.cfg"
+        # otherwise use the default path
+        return _FLYTE_KEYRING_PATH
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__}> at {self.file_path}>"

flyte/_logging.py

@@ -40,6 +40,21 @@ def log_format_from_env() -> str:
     return os.environ.get("LOG_FORMAT", "json")
 
 
+def _get_console():
+    """
+    Get the console.
+    """
+    from rich.console import Console
+
+    try:
+        width = os.get_terminal_size().columns
+    except Exception as e:
+        logger.debug(f"Failed to get terminal size: {e}")
+        width = 160
+
+    return Console(width=width)
+
+
 def get_rich_handler(log_level: int) -> Optional[logging.Handler]:
     """
     Upgrades the global loggers to use Rich logging.
@@ -51,23 +66,19 @@ def get_rich_handler(log_level: int) -> Optional[logging.Handler]:
         return None
 
     import click
-    from rich.console import Console
+    from rich.highlighter import NullHighlighter
     from rich.logging import RichHandler
 
-    try:
-        width = os.get_terminal_size().columns
-    except Exception as e:
-        logger.debug(f"Failed to get terminal size: {e}")
-        width = 160
-
     handler = RichHandler(
         tracebacks_suppress=[click],
         rich_tracebacks=True,
         omit_repeated_times=False,
         show_path=False,
         log_time_format="%H:%M:%S.%f",
-        console=Console(width=width),
+        console=_get_console(),
         level=log_level,
+        highlighter=NullHighlighter(),
+        markup=True,
     )
 
     formatter = logging.Formatter(fmt="%(filename)s:%(lineno)d - %(message)s")

flyte/_utils/coro_management.py

@@ -11,7 +11,8 @@ async def run_coros(*coros: typing.Coroutine, return_when: str = asyncio.FIRST_C
     :param return_when:
     :return:
     """
-    tasks: typing.List[asyncio.Task[typing.Never]] = [asyncio.create_task(c) for c in coros]
+    # tasks: typing.List[asyncio.Task[typing.Never]] = [asyncio.create_task(c) for c in coros] # Python 3.11+
+    tasks: typing.List[asyncio.Task] = [asyncio.create_task(c) for c in coros]
     done, pending = await asyncio.wait(tasks, return_when=return_when)
     # TODO we might want to handle asyncio.CancelledError here, for cases when the `action` is cancelled
     # and we want to propagate it to all tasks. Though the backend will handle it anyway,

flyte/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.0b19'
-__version_tuple__ = version_tuple = (2, 0, 0, 'b19')
+__version__ = version = '2.0.0b21'
+__version_tuple__ = version_tuple = (2, 0, 0, 'b21')
 
-__commit_id__ = commit_id = 'g172a8c6b7'
+__commit_id__ = commit_id = 'g7f82f6ddd'

flyte/config/_config.py

@@ -231,10 +231,12 @@ def auto(config_file: typing.Union[str, pathlib.Path, ConfigFile, None] = None)
       1. If specified, read the config from the provided file path.
       2. If not specified, the config file is searched in the default locations.
             a. ./config.yaml if it exists  (current working directory)
-            b. `UCTL_CONFIG` environment variable
-            c. `FLYTECTL_CONFIG` environment variable
-            d. ~/.union/config.yaml if it exists
-            e. ~/.flyte/config.yaml if it exists
+            b. ./.flyte/config.yaml if it exists (current working directory)
+            c. <git_root>/.flyte/config.yaml if it exists
+            d. `UCTL_CONFIG` environment variable
+            e. `FLYTECTL_CONFIG` environment variable
+            f. ~/.union/config.yaml if it exists
+            g. ~/.flyte/config.yaml if it exists
     3. If any value is not found in the config file, the default value is used.
     4. For any value there are environment variables that match the config variable names, those will override
 

flyte/config/_reader.py

@@ -135,15 +135,25 @@ class ConfigFile(object):
         return self._yaml_config
 
 
+def _config_path_from_git_root() -> pathlib.Path | None:
+    from flyte.git import config_from_root
+
+    try:
+        return config_from_root().source
+    except RuntimeError:
+        return None
+
+
 def resolve_config_path() -> pathlib.Path | None:
     """
     Config is read from the following locations in order of precedence:
     1. ./config.yaml if it exists
     2. ./.flyte/config.yaml if it exists
-    3. `UCTL_CONFIG` environment variable
-    4. `FLYTECTL_CONFIG` environment variable
-    5. ~/.union/config.yaml if it exists
-    6. ~/.flyte/config.yaml if it exists
+    3. <git_root>/.flyte/config.yaml if it exists
+    4. `UCTL_CONFIG` environment variable
+    5. `FLYTECTL_CONFIG` environment variable
+    6. ~/.union/config.yaml if it exists
+    7. ~/.flyte/config.yaml if it exists
     """
     current_location_config = Path("config.yaml")
     if current_location_config.exists():
@@ -155,6 +165,11 @@ def resolve_config_path() -> pathlib.Path | None:
         return dot_flyte_config
     logger.debug("No ./.flyte/config.yaml found")
 
+    git_root_config = _config_path_from_git_root()
+    if git_root_config:
+        return git_root_config
+    logger.debug("No .flyte/config.yaml found in git repo root")
+
     uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
     if uctl_path_from_env:
         return pathlib.Path(uctl_path_from_env)

flyte/errors.py

@@ -223,3 +223,12 @@ class RunAbortedError(RuntimeUserError):
 
     def __init__(self, message: str):
         super().__init__("RunAbortedError", message, "user")
+
+
+class SlowDownError(RuntimeUserError):
+    """
+    This error is raised when the user tries to access a resource that does not exist or is invalid.
+    """
+
+    def __init__(self, message: str):
+        super().__init__("SlowDownError", message, "user")

flyte/git/_config.py

@@ -14,4 +14,6 @@ def config_from_root(path: pathlib.Path | str = ".flyte/config.yaml") -> flyte.c
     if result.returncode != 0:
         raise RuntimeError(f"Failed to get git root directory: {result.stderr}")
     root = pathlib.Path(result.stdout.strip())
+    if not (root / path).exists():
+        raise RuntimeError(f"Config file {root / path} does not exist")
     return flyte.config.auto(root / path)

flyte/io/_dataframe/dataframe.py

@@ -904,7 +904,8 @@ class DataFrameTransformerEngine(TypeTransformer[DataFrame]):
         #   t1(input_a: DataFrame)  # or
         #   t1(input_a: Annotated[DataFrame, my_cols])
         if issubclass(expected_python_type, DataFrame):
-            fdf = DataFrame(format=metad.structured_dataset_type.format)
+            fdf = DataFrame(format=metad.structured_dataset_type.format, uri=lv.scalar.structured_dataset.uri)
+            fdf._already_uploaded = True
             fdf._literal_sd = lv.scalar.structured_dataset
             fdf._metadata = metad
             return fdf
@@ -1012,7 +1013,7 @@ class DataFrameTransformerEngine(TypeTransformer[DataFrame]):
     def guess_python_type(self, literal_type: types_pb2.LiteralType) -> Type[DataFrame]:
         # todo: technically we should return the dataframe type specified in the constructor, but to do that,
         #   we'd have to store that, which we don't do today. See possibly #1363
-        if literal_type.HasField("dataframe_type"):
+        if literal_type.HasField("structured_dataset_type"):
             return DataFrame
         raise ValueError(f"DataFrameTransformerEngine cannot reverse {literal_type}")