flyte 0.2.0b5__py3-none-any.whl → 0.2.0b7__py3-none-any.whl

flyte/__init__.py

@@ -38,6 +38,7 @@ __all__ = [
     "deploy",
     "group",
     "init",
+    "init_auto_from_config",
     "run",
     "trace",
     "with_runcontext",
@@ -49,7 +50,7 @@ from ._deploy import deploy
 from ._environment import Environment
 from ._group import group
 from ._image import Image
-from ._initialize import init
+from ._initialize import init, init_auto_from_config
 from ._resources import GPU, TPU, Device, Resources
 from ._retry import RetryStrategy
 from ._reusable_environment import ReusePolicy

flyte/_code_bundle/_utils.py

@@ -14,7 +14,6 @@ import tempfile
 import typing
 from datetime import datetime, timezone
 from functools import lru_cache
-from pathlib import Path
 from types import ModuleType
 from typing import List, Literal, Optional, Tuple, Union
 
@@ -322,18 +321,3 @@ def hash_file(file_path: typing.Union[os.PathLike, str]) -> Tuple[bytes, str, in
             size += len(chunk)
 
     return h.digest(), h.hexdigest(), size
-
-
-def _find_project_root(source_path) -> str:
-    """
-    Find the root of the project.
-    The root of the project is considered to be the first ancestor from source_path that does
-    not contain a __init__.py file.
-
-    N.B.: This assumption only holds for regular packages (as opposed to namespace packages)
-    """
-    # Start from the directory right above source_path
-    path = Path(source_path).parent.resolve()
-    while os.path.exists(os.path.join(path, "__init__.py")):
-        path = path.parent
-    return str(path)

flyte/_code_bundle/bundle.py

@@ -146,7 +146,7 @@ async def build_code_bundle(
         logger.info(f"Code bundle created at {bundle_path}, size: {tar_size} MB, archive size: {archive_size} MB")
         if not dryrun:
             hash_digest, remote_path = await upload_file(bundle_path)
-            logger.info(f"Code bundle uploaded to {remote_path}")
+            logger.debug(f"Code bundle uploaded to {remote_path}")
         else:
             remote_path = "na"
             if copy_bundle_to:

flyte/_initialize.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Callable, List, Literal, Optional, TypeVar
 from flyte.errors import InitializationError
 from flyte.syncify import syncify
 
-from ._logging import initialize_logger
+from ._logging import initialize_logger, logger
 from ._tools import ipython_check
 
 if TYPE_CHECKING:
@@ -130,7 +130,6 @@ async def init(
     rpc_retries: int = 3,
     http_proxy_url: str | None = None,
     storage: Storage | None = None,
-    config: Config | None = None,
 ) -> None:
     """
     Initialize the Flyte system with the given configuration. This method should be called before any other Flyte
@@ -179,42 +178,72 @@ async def init(
     global _init_config  # noqa: PLW0603
 
     with _init_lock:
-        if config is None:
-            import flyte.config as _f_cfg
-
-            config = _f_cfg.Config()
-        platform_cfg = config.platform
-        task_cfg = config.task
         client = None
-        if endpoint or platform_cfg.endpoint or api_key:
+        if endpoint or api_key:
             client = await _initialize_client(
                 api_key=api_key,
-                auth_type=auth_type or platform_cfg.auth_mode,
-                endpoint=endpoint or platform_cfg.endpoint,
+                auth_type=auth_type,
+                endpoint=endpoint,
                 headless=headless,
-                insecure=insecure or platform_cfg.insecure,
-                insecure_skip_verify=insecure_skip_verify or platform_cfg.insecure_skip_verify,
-                ca_cert_file_path=ca_cert_file_path or platform_cfg.ca_cert_file_path,
-                command=command or platform_cfg.command,
-                proxy_command=proxy_command or platform_cfg.proxy_command,
-                client_id=client_id or platform_cfg.client_id,
-                client_credentials_secret=client_credentials_secret or platform_cfg.client_credentials_secret,
+                insecure=insecure,
+                insecure_skip_verify=insecure_skip_verify,
+                ca_cert_file_path=ca_cert_file_path,
+                command=command,
+                proxy_command=proxy_command,
+                client_id=client_id,
+                client_credentials_secret=client_credentials_secret,
                 client_config=auth_client_config,
-                rpc_retries=rpc_retries or platform_cfg.rpc_retries,
-                http_proxy_url=http_proxy_url or platform_cfg.http_proxy_url,
+                rpc_retries=rpc_retries,
+                http_proxy_url=http_proxy_url,
             )
 
         root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
         _init_config = _InitConfig(
             root_dir=root_dir,
-            project=project or task_cfg.project,
-            domain=domain or task_cfg.domain,
+            project=project,
+            domain=domain,
             client=client,
             storage=storage,
-            org=org or task_cfg.org,
+            org=org,
         )
 
 
+@syncify
+async def init_auto_from_config(path_or_config: str | Config | None = None) -> None:
+    """
+    Initialize the Flyte system using a configuration file or Config object. This method should be called before any
+    other Flyte remote API methods are called. Thread-safe implementation.
+
+    :param path_or_config: Path to the configuration file or Config object
+    :return: None
+    """
+    import flyte.config as config
+
+    cfg: config.Config
+    if path_or_config is None or isinstance(path_or_config, str):
+        # If a string is passed, treat it as a path to the config file
+        cfg = config.auto(path_or_config)
+    else:
+        # If a Config object is passed, use it directly
+        cfg = path_or_config
+
+    logger.debug(f"Flyte config initialized as {cfg}")
+    await init.aio(
+        org=cfg.task.org,
+        project=cfg.task.project,
+        domain=cfg.task.domain,
+        endpoint=cfg.platform.endpoint,
+        insecure=cfg.platform.insecure,
+        insecure_skip_verify=cfg.platform.insecure_skip_verify,
+        ca_cert_file_path=cfg.platform.ca_cert_file_path,
+        auth_type=cfg.platform.auth_mode,
+        command=cfg.platform.command,
+        proxy_command=cfg.platform.proxy_command,
+        client_id=cfg.platform.client_id,
+        client_credentials_secret=cfg.platform.client_credentials_secret,
+    )
+
+
 def _get_init_config() -> Optional[_InitConfig]:
     """
     Get the current initialization configuration. Thread-safe implementation.

flyte/_internal/controllers/remote/_core.py

@@ -32,7 +32,7 @@ class Controller:
         max_system_retries: int = 5,
         resource_log_interval_sec: float = 10.0,
         min_backoff_on_err_sec: float = 0.1,
-        thread_wait_timeout_sec: float = 0.5,
+        thread_wait_timeout_sec: float = 5.0,
         enqueue_timeout_sec: float = 5.0,
     ):
         """

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b5'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b5')
+__version__ = version = '0.2.0b7'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b7')

flyte/cli/_common.py

@@ -19,11 +19,13 @@ from rich.table import Table
 from rich.traceback import Traceback
 
 import flyte.errors
+from flyte._logging import logger
 from flyte.config import Config
 
 PREFERRED_BORDER_COLOR = "dim cyan"
 PREFERRED_ACCENT_COLOR = "bold #FFD700"
 HEADER_STYLE = f"{PREFERRED_ACCENT_COLOR} on black"
+PANELS = False
 
 PROJECT_OPTION = click.Option(
     param_decls=["-p", "--project"],
@@ -71,11 +73,11 @@ class CLIConfig:
     This is the global state for the CLI. It is manipulated by the main command.
     """
 
+    config: Config
     log_level: int | None = logging.ERROR
     endpoint: str | None = None
     insecure: bool = False
     org_override: str | None = None
-    config: Config | None = None
 
     def replace(self, **kwargs) -> CLIConfig:
         """
@@ -84,22 +86,26 @@ class CLIConfig:
         return replace(self, **kwargs)
 
     def init(self, project: str | None = None, domain: str | None = None):
-        import flyte
-
-        if self.config and self.config.task:
-            # If the config is already set, we can use it to initialize flyte
-            project = project or self.config.task.project
-            domain = domain or self.config.task.domain
-
-        flyte.init(
-            endpoint=self.endpoint,
-            insecure=self.insecure,
-            org=self.org_override,
-            project=project,
-            domain=domain,
-            config=self.config,
+        from flyte.config._config import TaskConfig
+
+        task_cfg = TaskConfig(
+            org=self.org_override or self.config.task.org,
+            project=project or self.config.task.project,
+            domain=domain or self.config.task.domain,
         )
 
+        kwargs: Dict[str, Any] = {}
+        if self.endpoint:
+            kwargs["endpoint"] = self.endpoint
+        if self.insecure is not None:
+            kwargs["insecure"] = self.insecure
+        platform_cfg = self.config.platform.replace(**kwargs)
+
+        updated_config = self.config.with_params(platform_cfg, task_cfg)
+
+        logger.debug(f"Initializing CLI with config: {updated_config}")
+        flyte.init_auto_from_config(updated_config)
+
 
 class InvokeBaseMixin:
     """
@@ -316,6 +322,8 @@ def get_panel(title: str, renderable: Any) -> Panel:
     """
     Get a panel from a list of values.
     """
+    if not PANELS:
+        return renderable
     return Panel.fit(
         renderable,
         title=f"[{PREFERRED_ACCENT_COLOR}]{title}[/{PREFERRED_ACCENT_COLOR}]",

flyte/cli/_run.py

@@ -12,8 +12,6 @@ from click import Context, Parameter
 from rich.console import Console
 from typing_extensions import get_args
 
-import flyte
-
 from .._code_bundle._utils import CopyFiles
 from .._task import TaskTemplate
 from ..remote import Run
@@ -95,11 +93,19 @@ class RunTaskCommand(click.Command):
         super().__init__(obj_name, *args, **kwargs)
 
     def invoke(self, ctx: Context):
-        obj: CLIConfig = ctx.obj or CLIConfig()
-        assert obj.endpoint, "CLI Config should have an endpoint"
+        obj: CLIConfig = ctx.obj
+        if obj is None:
+            import flyte.config
+
+            obj = CLIConfig(flyte.config.auto())
+
+        if not self.run_args.local:
+            assert obj.endpoint, "CLI Config should have an endpoint"
         obj.init(self.run_args.project, self.run_args.domain)
 
         async def _run():
+            import flyte
+
             r = flyte.with_runcontext(
                 copy_style=self.run_args.copy_style,
                 version=self.run_args.copy_style,
@@ -112,8 +118,8 @@ class RunTaskCommand(click.Command):
                     common.get_panel(
                         "Run",
                         f"[green bold]Created Run: {r.name} [/green bold] "
-                        f"(Project: {r.action.action_id.run.project}, Domain: {r.action.action_id.run.domain})\n\n"
-                        f"[blue bold]{r.url}[/blue bold]",
+                        f"(Project: {r.action.action_id.run.project}, Domain: {r.action.action_id.run.domain})\n"
+                        f"➡️  [blue bold]{r.url}[/blue bold]",
                     )
                 )
                 if self.run_args.follow:

flyte/cli/main.py

@@ -2,7 +2,6 @@ import rich_click as click
 
 from flyte._logging import initialize_logger, logger
 
-from ..config import Config
 from ._abort import abort
 from ._common import CLIConfig
 from ._create import create
@@ -39,12 +38,13 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     help="The endpoint to connect to, this will override any config and simply used pkce to connect.",
 )
 @click.option(
-    "--insecure",
+    "--insecure/--secure",
     is_flag=True,
     required=False,
-    help="insecure",
+    help="Use insecure connection to the endpoint. If secure is specified, the CLI will use TLS",
     type=bool,
     default=None,
+    show_default=True,
 )
 @click.option(
     "-v",
@@ -89,19 +89,25 @@ def main(
     The flyte cli follows a simple verb based structure, where the top-level commands are verbs that describe the action
 to be taken, and the subcommands are nouns that describe the object of the action.
     """
+    import flyte.config as config
+
     log_level = _verbosity_to_loglevel(verbose)
     if log_level is not None:
         initialize_logger(log_level)
 
-    config = Config.auto(config_file=config_file)
-    logger.debug(f"Using config file discovered at location {config.source}")
+    cfg = config.auto(config_file=config_file)
+    logger.debug(f"Using config file discovered at location {cfg.source}")
+
+    final_insecure = cfg.platform.insecure
+    if insecure is not None:
+        final_insecure = insecure
 
     ctx.obj = CLIConfig(
         log_level=log_level,
-        endpoint=endpoint or config.platform.endpoint,
-        insecure=insecure or config.platform.insecure,
-        org_override=org or config.task.org,
-        config=config,
+        endpoint=endpoint or cfg.platform.endpoint,
+        insecure=final_insecure,
+        org_override=org or cfg.task.org,
+        config=cfg,
     )
     logger.debug(f"Final materialized Cli config: {ctx.obj}")
 

flyte/config/__init__.py

@@ -1,190 +1,3 @@
-from __future__ import annotations
+from flyte.config._config import Config, auto, get_config_file, set_if_exists
 
-import os
-import pathlib
-import typing
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
-
-import rich.repr
-
-from flyte._logging import logger
-from flyte.config import _internal
-from flyte.config._config import ConfigFile, get_config_file, read_file_if_exists
-
-_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
-
-if TYPE_CHECKING:
-    from flyte.remote._client.auth import AuthType
-
-
-@rich.repr.auto
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class PlatformConfig(object):
-    """
-    This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
-
-    :param endpoint: DNS for Flyte backend
-    :param insecure: Whether or not to use SSL
-    :param insecure_skip_verify: Whether to skip SSL certificate verification
-    :param console_endpoint: endpoint for console if different from Flyte backend
-    :param command: This command is executed to return a token using an external process
-    :param proxy_command: This command is executed to return a token for proxy authorization using an external process
-    :param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
-      More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
-    :param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
-      allow the Flyte engine to read the password directly from the environment variable. Note that this is
-      less secure! Please only use this if mounting the secret as a file is impossible
-    :param scopes: List of scopes to request. This is only applicable to the client credentials flow
-    :param auth_mode: The OAuth mode to use. Defaults to pkce flow
-    :param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
-    :param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
-    """
-
-    endpoint: str | None = None
-    insecure: bool = False
-    insecure_skip_verify: bool = False
-    ca_cert_file_path: typing.Optional[str] = None
-    console_endpoint: typing.Optional[str] = None
-    command: typing.Optional[typing.List[str]] = None
-    proxy_command: typing.Optional[typing.List[str]] = None
-    client_id: typing.Optional[str] = None
-    client_credentials_secret: typing.Optional[str] = None
-    scopes: typing.List[str] = field(default_factory=list)
-    auth_mode: "AuthType" = "Pkce"
-    audience: typing.Optional[str] = None
-    rpc_retries: int = 3
-    http_proxy_url: typing.Optional[str] = None
-
-    @classmethod
-    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
-        """
-        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
-        :param config_file:
-        :return:
-        """
-
-        config_file = get_config_file(config_file)
-        kwargs: typing.Dict[str, typing.Any] = {}
-        kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
-        kwargs = set_if_exists(
-            kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
-        )
-        kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
-        kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
-        kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
-        kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
-
-        is_client_secret = False
-        client_credentials_secret = read_file_if_exists(
-            _internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
-        )
-        if client_credentials_secret:
-            is_client_secret = True
-            if client_credentials_secret.endswith("\n"):
-                logger.info("Newline stripped from client secret")
-                client_credentials_secret = client_credentials_secret.strip()
-        kwargs = set_if_exists(
-            kwargs,
-            "client_credentials_secret",
-            client_credentials_secret,
-        )
-
-        client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
-        if client_credentials_secret_env_var:
-            client_credentials_secret = os.getenv(client_credentials_secret_env_var)
-            if client_credentials_secret:
-                is_client_secret = True
-        kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
-        kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
-        kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
-        if is_client_secret:
-            kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
-        kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
-        kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
-
-        kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
-        return PlatformConfig(**kwargs)
-
-    @classmethod
-    def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
-        return PlatformConfig(endpoint=endpoint, insecure=insecure)
-
-
-@rich.repr.auto
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class TaskConfig(object):
-    org: str | None = None
-    project: str | None = None
-    domain: str | None = None
-
-    @classmethod
-    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
-        """
-        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
-        :param config_file:
-        :return:
-        """
-        config_file = get_config_file(config_file)
-        kwargs: typing.Dict[str, typing.Any] = {}
-        kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
-        kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
-        kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
-        return TaskConfig(**kwargs)
-
-
-@rich.repr.auto
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class Config(object):
-    """
-    This the parent configuration object and holds all the underlying configuration object types. An instance of
-    this object holds all the config necessary to
-
-    1. Interactive session with Flyte backend
-    2. Some parts are required for Serialization, for example Platform Config is not required
-    3. Runtime of a task
-    """
-
-    platform: PlatformConfig = field(default=PlatformConfig())
-    task: TaskConfig = field(default=TaskConfig())
-    source: pathlib.Path | None = None
-
-    def with_params(
-        self,
-        platform: PlatformConfig | None = None,
-        task: TaskConfig | None = None,
-    ) -> "Config":
-        return Config(
-            platform=platform or self.platform,
-            task=task or self.task,
-        )
-
-    @classmethod
-    def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
-        """
-        Automatically constructs the Config Object. The order of precedence is as follows
-          1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
-          2. If not found in environment then values ar read from the config file
-          3. If not found in the file, then the default values are used.
-
-        :param config_file: file path to read the config from, if not specified default locations are searched
-        :return: Config
-        """
-        config_file = get_config_file(config_file)
-        if config_file is None:
-            logger.debug("No config file found, using default values")
-            return Config()
-        return Config(
-            platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file), source=config_file.path
-        )
-
-
-def set_if_exists(d: dict, k: str, val: typing.Any) -> dict:
-    """
-    Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set
-    and return the updated dictionary.
-    """
-    exists = isinstance(val, bool) or bool(val is not None and val)
-    if exists:
-        d[k] = val
-    return d
+__all__ = ["Config", "auto", "get_config_file", "set_if_exists"]