flyte 0.0.1b3__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

flyte/_initialize.py

@@ -17,6 +17,7 @@ from ._logging import initialize_logger
 from ._tools import ipython_check
 
 if TYPE_CHECKING:
+    from flyte.config import Config
     from flyte.remote._client.auth import AuthType, ClientConfig
     from flyte.remote._client.controlplane import ClientSet
 
@@ -345,6 +346,7 @@ async def init(
     rpc_retries: int = 3,
     http_proxy_url: str | None = None,
     storage: Storage | None = None,
+    config: Config | None = None,
 ) -> None:
     """
     Initialize the Flyte system with the given configuration. This method should be called before any other Flyte
@@ -378,6 +380,7 @@ async def init(
     :param insecure: insecure flag for the client
     :param storage: Optional blob store (S3, GCS, Azure) configuration if needed to access (i.e. using Minio)
     :param org: Optional organization override for the client. Should be set by auth instead.
+    :param config: Optional config to override the init parameters
 
     :return: None
     """
@@ -392,33 +395,39 @@ async def init(
     global _init_config  # noqa: PLW0603
 
     with _init_lock:
+        if config is None:
+            from flyte.config import Config
+
+            config = Config.auto()
+        platform_cfg = config.platform
+        task_cfg = config.task
         client = None
-        if endpoint or api_key:
+        if endpoint or platform_cfg.endpoint or api_key:
             client = await _initialize_client(
                 api_key=api_key,
-                auth_type=auth_type,
-                endpoint=endpoint,
+                auth_type=auth_type or platform_cfg.auth_mode,
+                endpoint=endpoint or platform_cfg.endpoint,
                 headless=headless,
-                insecure=insecure,
-                insecure_skip_verify=insecure_skip_verify,
-                ca_cert_file_path=ca_cert_file_path,
-                command=command,
-                proxy_command=proxy_command,
-                client_id=client_id,
-                client_credentials_secret=client_credentials_secret,
+                insecure=insecure or platform_cfg.insecure,
+                insecure_skip_verify=insecure_skip_verify or platform_cfg.insecure_skip_verify,
+                ca_cert_file_path=ca_cert_file_path or platform_cfg.ca_cert_file_path,
+                command=command or platform_cfg.command,
+                proxy_command=proxy_command or platform_cfg.proxy_command,
+                client_id=client_id or platform_cfg.client_id,
+                client_credentials_secret=client_credentials_secret or platform_cfg.client_credentials_secret,
                 client_config=auth_client_config,
-                rpc_retries=rpc_retries,
-                http_proxy_url=http_proxy_url,
+                rpc_retries=rpc_retries or platform_cfg.rpc_retries,
+                http_proxy_url=http_proxy_url or platform_cfg.http_proxy_url,
             )
 
         root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
         _init_config = _InitConfig(
             root_dir=root_dir,
-            project=project,
-            domain=domain,
+            project=project or task_cfg.project,
+            domain=domain or task_cfg.domain,
             client=client,
             storage=storage,
-            org=org,
+            org=org or task_cfg.org,
         )
 
 

flyte/_internal/runtime/convert.py

@@ -60,6 +60,12 @@ async def convert_from_native_to_inputs(interface: NativeInterface, *args, **kwa
     kwargs = interface.convert_to_kwargs(*args, **kwargs)
     if len(kwargs) == 0:
         return Inputs.empty()
+    # fill in defaults if missing
+    for input_name, (input_type, default_value) in interface.inputs.items():
+        if input_name not in kwargs:
+            if default_value is not None:
+                kwargs[input_name] = default_value
+            # todo: fill in Nones for optional inputs
     if len(kwargs) < len(interface.inputs):
         raise ValueError(
             f"Received {len(kwargs)} inputs but interface has {len(interface.inputs)}. "

flyte/_run.py

@@ -277,7 +277,6 @@ class _Runner:
         from flyte._internal.runtime.entrypoints import direct_dispatch
 
         controller = create_controller(ct="local")
-
         inputs = await convert_from_native_to_inputs(obj.native_interface, *args, **kwargs)
         if self._name is None:
             action = ActionID.create_random()

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.1b3'
-__version_tuple__ = version_tuple = (0, 0, 1, 'b3')
+__version__ = version = '0.2.0b0'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b0')

flyte/config/__init__.py

@@ -0,0 +1,168 @@
+import os
+import typing
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from flyte._logging import logger
+from flyte.config import _internal
+from flyte.config._config import ConfigFile, get_config_file, read_file_if_exists
+
+_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
+
+if TYPE_CHECKING:
+    from flyte.remote._client.auth import AuthType
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class PlatformConfig(object):
+    """
+    This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
+
+    :param endpoint: DNS for Flyte backend
+    :param insecure: Whether or not to use SSL
+    :param insecure_skip_verify: Whether to skip SSL certificate verification
+    :param console_endpoint: endpoint for console if different from Flyte backend
+    :param command: This command is executed to return a token using an external process
+    :param proxy_command: This command is executed to return a token for proxy authorization using an external process
+    :param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
+      More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
+    :param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
+      allow the Flyte engine to read the password directly from the environment variable. Note that this is
+      less secure! Please only use this if mounting the secret as a file is impossible
+    :param scopes: List of scopes to request. This is only applicable to the client credentials flow
+    :param auth_mode: The OAuth mode to use. Defaults to pkce flow
+    :param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
+    :param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
+    """
+
+    endpoint: str | None = None
+    insecure: bool = False
+    insecure_skip_verify: bool = False
+    ca_cert_file_path: typing.Optional[str] = None
+    console_endpoint: typing.Optional[str] = None
+    command: typing.Optional[typing.List[str]] = None
+    proxy_command: typing.Optional[typing.List[str]] = None
+    client_id: typing.Optional[str] = None
+    client_credentials_secret: typing.Optional[str] = None
+    scopes: typing.List[str] = field(default_factory=list)
+    auth_mode: "AuthType" = "Pkce"
+    audience: typing.Optional[str] = None
+    rpc_retries: int = 3
+    http_proxy_url: typing.Optional[str] = None
+
+    @classmethod
+    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
+        """
+        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
+        :param config_file:
+        :return:
+        """
+        from .._initialize import set_if_exists
+
+        config_file = get_config_file(config_file)
+        kwargs: typing.Dict[str, typing.Any] = {}
+        kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
+        kwargs = set_if_exists(
+            kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
+        )
+        kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
+        kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
+        kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
+        kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
+
+        is_client_secret = False
+        client_credentials_secret = read_file_if_exists(
+            _internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
+        )
+        if client_credentials_secret:
+            is_client_secret = True
+            if client_credentials_secret.endswith("\n"):
+                logger.info("Newline stripped from client secret")
+                client_credentials_secret = client_credentials_secret.strip()
+        kwargs = set_if_exists(
+            kwargs,
+            "client_credentials_secret",
+            client_credentials_secret,
+        )
+
+        client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
+        if client_credentials_secret_env_var:
+            client_credentials_secret = os.getenv(client_credentials_secret_env_var)
+            if client_credentials_secret:
+                is_client_secret = True
+        kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
+        kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
+        kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
+        if is_client_secret:
+            kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
+        kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
+        kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
+
+        kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
+        return PlatformConfig(**kwargs)
+
+    @classmethod
+    def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
+        return PlatformConfig(endpoint=endpoint, insecure=insecure)
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class TaskConfig(object):
+    org: str | None = None
+    project: str | None = None
+    domain: str | None = None
+
+    @classmethod
+    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
+        """
+        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
+        :param config_file:
+        :return:
+        """
+        from flyte._initialize import set_if_exists
+
+        config_file = get_config_file(config_file)
+        kwargs: typing.Dict[str, typing.Any] = {}
+        kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
+        kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
+        kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
+        return TaskConfig(**kwargs)
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class Config(object):
+    """
+    This the parent configuration object and holds all the underlying configuration object types. An instance of
+    this object holds all the config necessary to
+
+    1. Interactive session with Flyte backend
+    2. Some parts are required for Serialization, for example Platform Config is not required
+    3. Runtime of a task
+    """
+
+    platform: PlatformConfig = field(default=PlatformConfig())
+    task: TaskConfig = field(default=TaskConfig())
+
+    def with_params(
+        self,
+        platform: PlatformConfig | None = None,
+        task: TaskConfig | None = None,
+    ) -> "Config":
+        return Config(
+            platform=platform or self.platform,
+            task=task or self.task,
+        )
+
+    @classmethod
+    def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
+        """
+        Automatically constructs the Config Object. The order of precedence is as follows
+          1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
+          2. If not found in environment then values ar read from the config file
+          3. If not found in the file, then the default values are used.
+
+        :param config_file: file path to read the config from, if not specified default locations are searched
+        :return: Config
+        """
+        config_file = get_config_file(config_file)
+        return Config(platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file))

flyte/config/_config.py

@@ -0,0 +1,196 @@
+import os
+import pathlib
+import typing
+from dataclasses import dataclass
+from functools import lru_cache
+from os import getenv
+from pathlib import Path
+
+import yaml
+
+from flyte._logging import logger
+
+# This is the default config file name for flyte
+FLYTECTL_CONFIG_ENV_VAR = "FLYTECTL_CONFIG"
+UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
+
+
+@dataclass
+class YamlConfigEntry(object):
+    """
+    Creates a record for the config entry.
+    Args:
+        switch: dot-delimited string that should match flytectl args. Leaving it as dot-delimited instead of a list
+          of strings because it's easier to maintain alignment with flytectl.
+        config_value_type: Expected type of the value
+    """
+
+    switch: str
+    config_value_type: typing.Type = str
+
+    def get_env_name(self) -> str:
+        var_name = self.switch.upper().replace(".", "_")
+        return f"FLYTE_{var_name}"
+
+    def read_from_env(self, transform: typing.Optional[typing.Callable] = None) -> typing.Optional[typing.Any]:
+        """
+        Reads the config entry from environment variable, the structure of the env var is current
+        ``FLYTE_{SECTION}_{OPTION}`` all upper cased. We will change this in the future.
+        :return:
+        """
+        env = self.get_env_name()
+        v = os.environ.get(env, None)
+        if v is None:
+            return None
+        return transform(v) if transform else v
+
+    def read_from_file(
+        self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
+    ) -> typing.Optional[typing.Any]:
+        if not cfg:
+            return None
+        try:
+            v = cfg.get(self)
+            if isinstance(v, bool) or bool(v is not None and v):
+                return transform(v) if transform else v
+        except Exception:
+            ...
+
+        return None
+
+
+@dataclass
+class ConfigEntry(object):
+    """
+    A top level Config entry holder, that holds multiple different representations of the config.
+    Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
+    when flytectl starts a sandbox
+    """
+
+    yaml_entry: YamlConfigEntry
+    transform: typing.Optional[typing.Callable[[str], typing.Any]] = None
+
+    def read(self, cfg: typing.Optional["ConfigFile"] = None) -> typing.Optional[typing.Any]:
+        """
+        Reads the config Entry from the various sources in the following order,
+        #. First try to read from the relevant environment variable,
+        #. If missing, then try to read from the legacy config file, if one was parsed.
+        #. If missing, then try to read from the yaml file.
+
+        The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
+
+        :param cfg:
+        :return:
+        """
+        from_env = self.yaml_entry.read_from_env(self.transform)
+        if from_env is not None:
+            return from_env
+        if cfg and cfg.yaml_config and self.yaml_entry:
+            return self.yaml_entry.read_from_file(cfg, self.transform)
+
+        return None
+
+
+class ConfigFile(object):
+    def __init__(self, location: str):
+        """
+        Load the config from this location
+        """
+        self._location = location
+        self._yaml_config = self._read_yaml_config(location)
+
+    @staticmethod
+    def _read_yaml_config(location: str) -> typing.Optional[typing.Dict[str, typing.Any]]:
+        with open(location, "r") as fh:
+            try:
+                yaml_contents = yaml.safe_load(fh)
+                return yaml_contents
+            except yaml.YAMLError as exc:
+                logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
+                return None
+
+    def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
+        keys = c.switch.split(".")  # flytectl switches are dot delimited
+        d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
+        try:
+            for k in keys:
+                d = d[k]
+            return d
+        except KeyError:
+            return None
+
+    def get(self, c: YamlConfigEntry) -> typing.Any:
+        return self._get_from_yaml(c)
+
+    @property
+    def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
+        return self._yaml_config
+
+
+def resolve_config_path() -> pathlib.Path | None:
+    """
+    Config is read from the following locations in order of precedence:
+    1. `UCTL_CONFIG` environment variable
+    2. `FLYTECTL_CONFIG` environment variable
+    3. ~/.union/config.yaml if it exists
+    4. ~/.flyte/config.yaml if it exists
+    5. ./config.yaml if it exists
+    """
+    uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
+    if uctl_path_from_env:
+        return pathlib.Path(uctl_path_from_env)
+    logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
+
+    flytectl_path_from_env = getenv(FLYTECTL_CONFIG_ENV_VAR, None)
+    if flytectl_path_from_env:
+        return pathlib.Path(flytectl_path_from_env)
+    logger.debug("No FLYTECTL_CONFIG environment variable found, checking default locations")
+
+    home_dir_union_config = Path(Path.home(), ".union", "config.yaml")
+    if home_dir_union_config.exists():
+        return home_dir_union_config
+    logger.debug("No ~/.union/config.yaml found, checking current directory")
+
+    home_dir_flytectl_config = Path(Path.home(), ".flyte", "config.yaml")
+    if home_dir_flytectl_config.exists():
+        return home_dir_flytectl_config
+    logger.debug("No ~/.flyte/config.yaml found, checking current directory")
+
+    current_location_config = Path("config.yaml")
+    if current_location_config.exists():
+        return current_location_config
+    logger.debug("No ./config.yaml found, returning None")
+    return None
+
+
+@lru_cache
+def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
+    """
+    Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
+    """
+    if "PYTEST_VERSION" in os.environ:
+        # Use default local config in the pytest environment
+        return None
+    if isinstance(c, str):
+        logger.debug(f"Using specified config file at {c}")
+        return ConfigFile(c)
+    config_path = resolve_config_path()
+    if config_path:
+        return ConfigFile(str(config_path))
+    return None
+
+
+def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
+    """
+    Reads the contents of the file if passed a path. Otherwise, returns None.
+
+    :param filename: The file path to load
+    :param encoding: The encoding to use when reading the file.
+    :return: The contents of the file as a string or None.
+    """
+    if not filename:
+        return None
+
+    file = pathlib.Path(filename)
+    logger.debug(f"Reading file contents from [{file}] with current directory [{os.getcwd()}].")
+    return file.read_text(encoding=encoding)

flyte/config/_internal.py

@@ -0,0 +1,64 @@
+from flyte.config._config import ConfigEntry, YamlConfigEntry
+
+
+class Platform(object):
+    URL = ConfigEntry(YamlConfigEntry("admin.endpoint"))
+    INSECURE = ConfigEntry(YamlConfigEntry("admin.insecure", bool))
+    INSECURE_SKIP_VERIFY = ConfigEntry(YamlConfigEntry("admin.insecureSkipVerify", bool))
+    CONSOLE_ENDPOINT = ConfigEntry(YamlConfigEntry("console.endpoint"))
+    CA_CERT_FILE_PATH = ConfigEntry(YamlConfigEntry("admin.caCertFilePath"))
+    HTTP_PROXY_URL = ConfigEntry(YamlConfigEntry("admin.httpProxyURL"))
+
+
+class Credentials(object):
+    SECTION = "credentials"
+    COMMAND = ConfigEntry(YamlConfigEntry("admin.command", list))
+    """
+    This command is executed to return a token using an external process.
+    """
+
+    PROXY_COMMAND = ConfigEntry(YamlConfigEntry("admin.proxyCommand", list))
+    """
+    This command is executed to return a token for authorization with a proxy
+     in front of Flyte using an external process.
+    """
+
+    CLIENT_ID = ConfigEntry(YamlConfigEntry("admin.clientId"))
+    """
+    This is the public identifier for the app which handles authorization for a Flyte deployment.
+    More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
+    """
+
+    CLIENT_CREDENTIALS_SECRET_LOCATION = ConfigEntry(YamlConfigEntry("admin.clientSecretLocation"))
+    """
+    Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
+    password from a mounted file.
+    """
+
+    CLIENT_CREDENTIALS_SECRET_ENV_VAR = ConfigEntry(YamlConfigEntry("admin.clientSecretEnvVar"))
+    """
+    Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
+    password from a mounted environment variable.
+    """
+
+    SCOPES = ConfigEntry(YamlConfigEntry("admin.scopes", list))
+    """
+    This setting can be used to manually pass in scopes into authenticator flows - eg.) for Auth0 compatibility
+    """
+
+    AUTH_MODE = ConfigEntry(YamlConfigEntry("admin.authType"))
+    """
+    The auth mode defines the behavior used to request and refresh credentials. The currently supported modes include:
+    - 'standard' or 'Pkce': This uses the pkce-enhanced authorization code flow by opening a browser window to initiate
+            credentials access.
+    - "DeviceFlow": This uses the Device Authorization Flow
+    - 'basic', 'client_credentials' or 'clientSecret': This uses symmetric key auth in which the end user enters a
+            client id and a client secret and public key encryption is used to facilitate authentication.
+    - None: No auth will be attempted.
+    """
+
+
+class Task(object):
+    ORG = ConfigEntry(YamlConfigEntry("task.org"))
+    PROJECT = ConfigEntry(YamlConfigEntry("task.project"))
+    DOMAIN = ConfigEntry(YamlConfigEntry("task.domain"))

flyte/remote/_console.py

@@ -7,7 +7,7 @@ def _get_http_domain(endpoint: str, insecure: bool) -> str:
     if parsed.scheme == "dns":
         domain = parsed.path.lstrip("/")
     else:
-        domain = parsed.netloc
+        domain = parsed.netloc or parsed.path
     # TODO: make console url configurable
     if domain.split(":")[0] == "localhost":
         domain = "localhost:8080"

flyte/types/_type_engine.py

@@ -450,7 +450,6 @@ class DataclassTransformer(TypeTransformer[object]):
         # However, FooSchema is created by flytekit and it's not equal to the user-defined dataclass (Foo).
         # Therefore, we should iterate all attributes in the dataclass and check the type of value in dataclass
         #   matches the expected_type.
-
         expected_fields_dict = {}
 
         for f in dataclasses.fields(expected_type):
@@ -1992,7 +1991,9 @@ DatetimeTransformer = SimpleTransformer(
     datetime.datetime,
     types_pb2.LiteralType(simple=types_pb2.SimpleType.DATETIME),
     lambda x: Literal(scalar=Scalar(primitive=Primitive(datetime=x))),
-    lambda x: x.scalar.primitive.datetime if x.scalar.primitive.HasField("datetime") else None,
+    lambda x: x.scalar.primitive.datetime.ToDatetime().replace(tzinfo=datetime.timezone.utc)
+    if x.scalar.primitive.HasField("datetime")
+    else None,
 )
 
 TimedeltaTransformer = SimpleTransformer(
@@ -2000,7 +2001,7 @@ TimedeltaTransformer = SimpleTransformer(
     datetime.timedelta,
     types_pb2.LiteralType(simple=types_pb2.SimpleType.DURATION),
     lambda x: Literal(scalar=Scalar(primitive=Primitive(duration=x))),
-    lambda x: x.scalar.primitive.duration if x.scalar.primitive.HasField("duration") else None,
+    lambda x: x.scalar.primitive.duration.ToTimedelta() if x.scalar.primitive.HasField("duration") else None,
 )
 
 DateTransformer = SimpleTransformer(

{flyte-0.0.1b3.dist-info → flyte-0.2.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 0.0.1b3
+Version: 0.2.0b0
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10