flyte 2.0.0b25__py3-none-any.whl → 2.0.0b28__py3-none-any.whl

flyte/_utils/__init__.py

@@ -9,12 +9,13 @@ from .coro_management import run_coros
 from .file_handling import filehash_update, update_hasher_for_source
 from .helpers import get_cwd_editable_install
 from .lazy_module import lazy_module
-from .module_loader import load_python_modules
+from .module_loader import adjust_sys_path, load_python_modules
 from .org_discovery import hostname_from_url, org_from_endpoint, sanitize_endpoint
 from .uv_script_parser import parse_uv_script_file
 
 __all__ = [
     "AsyncLRUCache",
+    "adjust_sys_path",
     "filehash_update",
     "get_cwd_editable_install",
     "hostname_from_url",

flyte/_utils/asyn.py

@@ -9,6 +9,8 @@ async def async_add(a: int, b: int) -> int:
 result = run_sync(async_add, a=10, b=12)
 """
 
+from __future__ import annotations
+
 import asyncio
 import atexit
 import functools
@@ -88,7 +90,7 @@ class _TaskRunner:
 
 
 class _AsyncLoopManager:
-    def __init__(self):
+    def __init__(self: _AsyncLoopManager):
         self._runner_map: dict[str, _TaskRunner] = {}
 
     def run_sync(self, coro_func: Callable[..., Awaitable[T]], *args, **kwargs) -> T:

flyte/_utils/docker_credentials.py

@@ -0,0 +1,173 @@
+"""Helper functions for creating Docker registry credentials for image pull secrets."""
+
+import base64
+import json
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_CONFIG_JSON = "config.json"
+_DEFAULT_CONFIG_PATH = f"~/.docker/{_CONFIG_JSON}"
+_CRED_HELPERS = "credHelpers"
+_CREDS_STORE = "credsStore"
+
+
+def _load_docker_config(config_path: str | Path | None = None) -> dict[str, Any]:
+    """
+    Load Docker config from specified path.
+
+    Args:
+        config_path: Path to Docker config file. If None, uses DOCKER_CONFIG env var
+                    or defaults to ~/.docker/config.json
+
+    Returns:
+        Dictionary containing Docker config
+
+    Raises:
+        FileNotFoundError: If the config file does not exist
+        json.JSONDecodeError: If the config file is not valid JSON
+    """
+    if not config_path:
+        docker_config_env = os.environ.get("DOCKER_CONFIG")
+        if docker_config_env:
+            config_path = Path(docker_config_env) / _CONFIG_JSON
+        else:
+            config_path = Path(_DEFAULT_CONFIG_PATH).expanduser()
+    else:
+        config_path = Path(config_path).expanduser()
+
+    with open(config_path) as f:
+        return json.load(f)
+
+
+def _get_credential_helper(config: dict[str, Any], registry: str | None = None) -> str | None:
+    """Get credential helper for registry or global default."""
+    if registry and _CRED_HELPERS in config and registry in config[_CRED_HELPERS]:
+        return config[_CRED_HELPERS].get(registry)
+    return config.get(_CREDS_STORE)
+
+
+def _get_credentials_from_helper(helper: str, registry: str) -> tuple[str, str] | None:
+    """
+    Get credentials from system credential helper.
+
+    Args:
+        helper: Name of the credential helper (e.g., "osxkeychain", "wincred")
+        registry: Registry hostname to get credentials for
+
+    Returns:
+        Tuple of (username, password) or None if credentials cannot be retrieved
+    """
+    helper_cmd = f"docker-credential-{helper}"
+
+    try:
+        process = subprocess.Popen(
+            [helper_cmd, "get"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        output, error = process.communicate(input=registry)
+
+        if process.returncode != 0:
+            logger.error(f"Credential helper error: {error}")
+            return None
+
+        creds = json.loads(output)
+        return creds.get("Username"), creds.get("Secret")
+    except FileNotFoundError:
+        logger.error(f"Credential helper {helper_cmd} not found in PATH")
+        return None
+    except Exception as e:
+        logger.error(f"Error getting credentials: {e!s}")
+        return None
+
+
+def create_dockerconfigjson_from_config(
+    registries: list[str] | None = None,
+    docker_config_path: str | Path | None = None,
+) -> str:
+    """
+    Create a dockerconfigjson string from existing Docker config.
+
+    This function extracts Docker registry credentials from the user's Docker config file
+    and creates a JSON string containing only the credentials for the specified registries.
+    It handles credentials stored directly in the config file as well as those managed by
+    credential helpers.
+
+    Args:
+        registries: List of registries to extract credentials for. If None, all registries
+                   from the config will be used.
+        docker_config_path: Path to the Docker config file. If None, the function will look
+                          for the config file in the standard locations.
+
+    Returns:
+        JSON string in dockerconfigjson format: {"auths": {"registry": {"auth": "..."}}}
+
+    Raises:
+        FileNotFoundError: If Docker config file cannot be found
+        ValueError: If no credentials can be extracted
+    """
+    config = _load_docker_config(docker_config_path)
+
+    # Create new config structure with empty auths
+    new_config: dict[str, Any] = {"auths": {}}
+
+    # Use specified registries or all from config
+    target_registries = registries or list(config.get("auths", {}).keys())
+
+    if not target_registries:
+        raise ValueError("No registries found in Docker config and none specified")
+
+    for registry in target_registries:
+        registry_config = config.get("auths", {}).get(registry, {})
+        if registry_config.get("auth"):
+            # Direct auth token exists
+            new_config["auths"][registry] = {"auth": registry_config["auth"]}
+        else:
+            # Try to get credentials from helper
+            helper = _get_credential_helper(config, registry)
+            if helper:
+                creds = _get_credentials_from_helper(helper, registry)
+                if creds:
+                    username, password = creds
+                    auth_string = f"{username}:{password}"
+                    new_config["auths"][registry] = {"auth": base64.b64encode(auth_string.encode()).decode()}
+                else:
+                    logger.warning(f"Could not retrieve credentials for {registry} from credential helper")
+            else:
+                logger.warning(f"No credentials found for {registry}")
+
+    if not new_config["auths"]:
+        raise ValueError(f"No credentials could be extracted for registries: {', '.join(target_registries)}")
+
+    return json.dumps(new_config)
+
+
+def create_dockerconfigjson_from_credentials(
+    registry: str,
+    username: str,
+    password: str,
+) -> str:
+    """
+    Create a dockerconfigjson string from explicit credentials.
+
+    Args:
+        registry: Registry hostname (e.g., "ghcr.io", "docker.io")
+        username: Username or token name for the registry
+        password: Password or access token for the registry
+
+    Returns:
+        JSON string in dockerconfigjson format: {"auths": {"registry": {"auth": "..."}}}
+    """
+    auth_string = f"{username}:{password}"
+    auth_token = base64.b64encode(auth_string.encode()).decode()
+
+    config = {"auths": {registry: {"auth": auth_token}}}
+
+    return json.dumps(config)

flyte/_utils/module_loader.py

@@ -6,6 +6,8 @@ from pathlib import Path
 from typing import List, Tuple
 
 import flyte.errors
+from flyte._constants import FLYTE_SYS_PATH
+from flyte._logging import logger
 
 
 def load_python_modules(path: Path, recursive: bool = False) -> Tuple[List[str], List[Tuple[Path, str]]]:
@@ -87,3 +89,16 @@ def _load_module_from_file(file_path: Path) -> str | None:
 
     except Exception as e:
         raise flyte.errors.ModuleLoadError(f"Failed to load module from {file_path}: {e}") from e
+
+
+def adjust_sys_path():
+    """
+    Adjust sys.path to include local sys.path entries under the root directory.
+    """
+    if "." not in sys.path or os.getcwd() not in sys.path:
+        sys.path.insert(0, ".")
+        logger.info(f"Added {os.getcwd()} to sys.path")
+    for p in os.environ.get(FLYTE_SYS_PATH, "").split(":"):
+        if p and p not in sys.path:
+            sys.path.insert(0, p)
+            logger.info(f"Added {p} to sys.path")

flyte/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.0b25'
-__version_tuple__ = version_tuple = (2, 0, 0, 'b25')
+__version__ = version = '2.0.0b28'
+__version_tuple__ = version_tuple = (2, 0, 0, 'b28')
 
-__commit_id__ = commit_id = 'gf704e4b75'
+__commit_id__ = commit_id = 'gb62a542c8'

flyte/cli/_common.py

@@ -120,6 +120,7 @@ class CLIConfig:
         domain: str | None = None,
         root_dir: str | None = None,
         images: tuple[str, ...] | None = None,
+        sync_local_sys_paths: bool = True,
     ):
         from flyte.config._config import TaskConfig
 
@@ -140,7 +141,13 @@ class CLIConfig:
 
         updated_config = self.config.with_params(platform_cfg, task_cfg)
 
-        flyte.init_from_config(updated_config, log_level=self.log_level, root_dir=root_dir, images=images)
+        flyte.init_from_config(
+            updated_config,
+            log_level=self.log_level,
+            root_dir=root_dir,
+            images=images,
+            sync_local_sys_paths=sync_local_sys_paths,
+        )
 
 
 class InvokeBaseMixin:
@@ -440,7 +447,12 @@ def parse_images(cfg: Config, values: tuple[str, ...] | None) -> None:
 
 @lru_cache()
 def initialize_config(
-    ctx: click.Context, project: str, domain: str, root_dir: str | None = None, images: tuple[str, ...] | None = None
+    ctx: click.Context,
+    project: str,
+    domain: str,
+    root_dir: str | None = None,
+    images: tuple[str, ...] | None = None,
+    sync_local_sys_paths: bool = True,
 ):
     obj: CLIConfig | None = ctx.obj
     if obj is None:
@@ -448,5 +460,5 @@ def initialize_config(
 
         obj = CLIConfig(flyte.config.auto(), ctx)
 
-    obj.init(project, domain, root_dir, images)
+    obj.init(project, domain, root_dir, images, sync_local_sys_paths)
     return obj

flyte/cli/_create.py

@@ -24,18 +24,49 @@ def create():
     prompt="Enter secret value",
     hide_input=True,
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["from_file"],
+    mutually_exclusive=["from_file", "from_docker_config", "registry"],
 )
 @click.option(
     "--from-file",
     type=click.Path(exists=True),
     help="Path to the file with the binary secret.",
     cls=MutuallyExclusiveOption,
-    mutually_exclusive=["value"],
+    mutually_exclusive=["value", "from_docker_config", "registry"],
 )
 @click.option(
     "--type", type=click.Choice(get_args(SecretTypes)), default="regular", help="Type of the secret.", show_default=True
 )
+@click.option(
+    "--from-docker-config",
+    is_flag=True,
+    help="Create image pull secret from Docker config file (only for --type image_pull).",
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["value", "from_file", "registry", "username", "password"],
+)
+@click.option(
+    "--docker-config-path",
+    type=click.Path(exists=True),
+    help="Path to Docker config file (defaults to ~/.docker/config.json or $DOCKER_CONFIG).",
+)
+@click.option(
+    "--registries",
+    help="Comma-separated list of registries to include (only with --from-docker-config).",
+)
+@click.option(
+    "--registry",
+    help="Registry hostname (e.g., ghcr.io, docker.io) for explicit credentials (only for --type image_pull).",
+    cls=MutuallyExclusiveOption,
+    mutually_exclusive=["value", "from_file", "from_docker_config"],
+)
+@click.option(
+    "--username",
+    help="Username for the registry (only with --registry).",
+)
+@click.option(
+    "--password",
+    help="Password for the registry (only with --registry). If not provided, will prompt.",
+    hide_input=True,
+)
 @click.pass_obj
 def secret(
     cfg: common.CLIConfig,
@@ -43,6 +74,12 @@ def secret(
     value: str | bytes | None = None,
     from_file: str | None = None,
     type: SecretTypes = "regular",
+    from_docker_config: bool = False,
+    docker_config_path: str | None = None,
+    registries: str | None = None,
+    registry: str | None = None,
+    username: str | None = None,
+    password: str | None = None,
     project: str | None = None,
     domain: str | None = None,
 ):
@@ -73,9 +110,24 @@ def secret(
     Other secrets should be specified as `regular`.
     If no type is specified, `regular` is assumed.
 
+    For image pull secrets, you have several options:
+
+    1. Interactive mode (prompts for registry, username, password):
     ```bash
     $ flyte create secret my_secret --type image_pull
     ```
+
+    2. With explicit credentials:
+    ```bash
+    $ flyte create secret my_secret --type image_pull --registry ghcr.io --username myuser
+    ```
+
+    3. Lastly, you can create a secret from your existing Docker installation (i.e., you've run `docker login` in
+    the past) and you just want to pull from those credentials. Since you may have logged in to multiple registries,
+    you can specify which registries to include. If no registries are specified, all registries are added.
+    ```bash
+    $ flyte create secret my_secret --type image_pull --from-docker-config --registries ghcr.io,docker.io
+    ```
     """
     from flyte.remote import Secret
 
@@ -84,9 +136,54 @@ def secret(
     project = "" if project is None else project
     domain = "" if domain is None else domain
     cfg.init(project, domain)
-    if from_file:
+
+    # Handle image pull secret creation
+    if type == "image_pull":
+        if project != "" or domain != "":
+            raise click.ClickException("Project and domain must not be set when creating an image pull secret.")
+
+        if from_docker_config:
+            # Mode 3: From Docker config
+            from flyte._utils.docker_credentials import create_dockerconfigjson_from_config
+
+            registry_list = [r.strip() for r in registries.split(",")] if registries else None
+            try:
+                value = create_dockerconfigjson_from_config(
+                    registries=registry_list,
+                    docker_config_path=docker_config_path,
+                )
+            except Exception as e:
+                raise click.ClickException(f"Failed to create dockerconfigjson from Docker config: {e}") from e
+
+        elif registry:
+            # Mode 2: Explicit credentials
+            from flyte._utils.docker_credentials import create_dockerconfigjson_from_credentials
+
+            if not username:
+                username = click.prompt("Username")
+            if not password:
+                password = click.prompt("Password", hide_input=True)
+
+            value = create_dockerconfigjson_from_credentials(registry, username, password)
+
+        else:
+            # Mode 1: Interactive prompts
+            from flyte._utils.docker_credentials import create_dockerconfigjson_from_credentials
+
+            registry = click.prompt("Registry (e.g., ghcr.io, docker.io)")
+            username = click.prompt("Username")
+            password = click.prompt("Password", hide_input=True)
+
+            value = create_dockerconfigjson_from_credentials(registry, username, password)
+
+    elif from_file:
         with open(from_file, "rb") as f:
             value = f.read()
+
+    # Encode string values to bytes
+    if isinstance(value, str):
+        value = value.encode("utf-8")
+
     Secret.create(name=name, value=value, type=type)
 
 

flyte/cli/_deploy.py

@@ -83,6 +83,19 @@ class DeployArguments:
             )
         },
     )
+    no_sync_local_sys_paths: bool = field(
+        default=True,
+        metadata={
+            "click.option": click.Option(
+                ["--no-sync-local-sys-paths"],
+                is_flag=True,
+                flag_value=True,
+                default=False,
+                help="Disable synchronization of local sys.path entries under the root directory "
+                "to the remote container.",
+            )
+        },
+    )
 
     @classmethod
     def from_dict(cls, d: Dict[str, Any]) -> "DeployArguments":
@@ -107,7 +120,12 @@ class DeployEnvCommand(click.RichCommand):
         console = common.get_console()
         console.print(f"Deploying root - environment: {self.env_name}")
         obj: CLIConfig = ctx.obj
-        obj.init(self.deploy_args.project, self.deploy_args.domain, root_dir=self.deploy_args.root_dir)
+        obj.init(
+            self.deploy_args.project,
+            self.deploy_args.domain,
+            root_dir=self.deploy_args.root_dir,
+            sync_local_sys_paths=not self.deploy_args.no_sync_local_sys_paths,
+        )
         with console.status("Deploying...", spinner="dots"):
             deployment = flyte.deploy(
                 self.env,
@@ -161,7 +179,11 @@ class DeployEnvRecursiveCommand(click.Command):
                 f"Failed to load {len(failed_paths)} files. Use --ignore-load-errors to ignore these errors."
             )
         # Now start connection and deploy all environments
-        obj.init(self.deploy_args.project, self.deploy_args.domain)
+        obj.init(
+            self.deploy_args.project,
+            self.deploy_args.domain,
+            sync_local_sys_paths=not self.deploy_args.no_sync_local_sys_paths,
+        )
         with console.status("Deploying...", spinner="dots"):
             deployments = flyte.deploy(
                 *all_envs,
@@ -190,11 +212,23 @@ class EnvPerFileGroup(common.ObjectsPerFileGroup):
         return {k: v for k, v in module.__dict__.items() if isinstance(v, flyte.Environment)}
 
     def list_commands(self, ctx):
-        common.initialize_config(ctx, self.deploy_args.project, self.deploy_args.domain, self.deploy_args.root_dir)
+        common.initialize_config(
+            ctx,
+            self.deploy_args.project,
+            self.deploy_args.domain,
+            self.deploy_args.root_dir,
+            sync_local_sys_paths=not self.deploy_args.no_sync_local_sys_paths,
+        )
         return super().list_commands(ctx)
 
     def get_command(self, ctx, obj_name):
-        common.initialize_config(ctx, self.deploy_args.project, self.deploy_args.domain, self.deploy_args.root_dir)
+        common.initialize_config(
+            ctx,
+            self.deploy_args.project,
+            self.deploy_args.domain,
+            self.deploy_args.root_dir,
+            sync_local_sys_paths=not self.deploy_args.no_sync_local_sys_paths,
+        )
         return super().get_command(ctx, obj_name)
 
     def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command: