flyte 0.0.1b1__py3-none-any.whl → 0.0.1b3__py3-none-any.whl

union/types/_utils.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import importlib
+import typing
+
+from flyteidl.core.types_pb2 import EnumType, LiteralType, UnionType
+
+T = typing.TypeVar("T")
+
+
+def literal_types_match(downstream: LiteralType, upstream: LiteralType) -> bool:
+    """
+    Returns if two LiteralTypes are the same.
+    Takes into account arbitrary ordering of enums and unions, otherwise just an equivalence check.
+    """
+
+    # If the types are exactly the same, return True
+    if downstream == upstream:
+        return True
+
+    if downstream.collection_type:
+        if not upstream.collection_type:
+            return False
+        return literal_types_match(downstream.collection_type, upstream.collection_type)
+
+    if downstream.map_value_type:
+        if not upstream.map_value_type:
+            return False
+        return literal_types_match(downstream.map_value_type, upstream.map_value_type)
+
+    # Handle enum types
+    if downstream.enum_type and upstream.enum_type:
+        return _enum_types_match(downstream.enum_type, upstream.enum_type)
+
+    # Handle union types
+    if downstream.union_type and upstream.union_type:
+        return _union_types_match(downstream.union_type, upstream.union_type)
+
+    # If none of the above conditions are met, the types are not castable
+    return False
+
+
+def _enum_types_match(downstream: EnumType, upstream: EnumType) -> bool:
+    return set(upstream.values) == set(downstream.values)
+
+
+def _union_types_match(downstream: UnionType, upstream: UnionType) -> bool:
+    if len(downstream.variants) != len(upstream.variants):
+        return False
+
+    down_sorted = sorted(downstream.variants, key=lambda x: str(x))
+    up_sorted = sorted(upstream.variants, key=lambda x: str(x))
+
+    for downstream_variant, upstream_variant in zip(down_sorted, up_sorted):
+        if not literal_types_match(downstream_variant, upstream_variant):
+            return False
+
+    return True
+
+
+def load_type_from_tag(tag: str) -> typing.Type[T]:
+    """
+    Helper function for proto buf compatibility only. Used in type transformer
+    """
+
+    if "." not in tag:
+        raise ValueError(
+            f"Protobuf tag must include at least one '.' to delineate package and object name got {tag}",
+        )
+
+    module, name = tag.rsplit(".", 1)
+    try:
+        pb_module = importlib.import_module(module)
+    except ImportError:
+        raise ValueError(f"Could not resolve the protobuf definition @ {module}.  Is the protobuf library installed?")
+
+    if not hasattr(pb_module, name):
+        raise ValueError(f"Could not find the protobuf named: {name} @ {module}.")
+
+    return getattr(pb_module, name)

flyte/config/__init__.py

@@ -1,168 +0,0 @@
-import os
-import typing
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
-
-from flyte._logging import logger
-from flyte.config import _internal
-from flyte.config._config import ConfigFile, get_config_file, read_file_if_exists
-
-_all__ = ["ConfigFile", "PlatformConfig", "TaskConfig"]
-
-if TYPE_CHECKING:
-    from flyte.remote._client.auth import AuthType
-
-
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class PlatformConfig(object):
-    """
-    This object contains the settings to talk to a Flyte backend (the DNS location of your Admin server basically).
-
-    :param endpoint: DNS for Flyte backend
-    :param insecure: Whether or not to use SSL
-    :param insecure_skip_verify: Whether to skip SSL certificate verification
-    :param console_endpoint: endpoint for console if different from Flyte backend
-    :param command: This command is executed to return a token using an external process
-    :param proxy_command: This command is executed to return a token for proxy authorization using an external process
-    :param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
-      More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
-    :param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
-      allow the Flyte engine to read the password directly from the environment variable. Note that this is
-      less secure! Please only use this if mounting the secret as a file is impossible
-    :param scopes: List of scopes to request. This is only applicable to the client credentials flow
-    :param auth_mode: The OAuth mode to use. Defaults to pkce flow
-    :param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
-    :param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
-    """
-
-    endpoint: str | None = None
-    insecure: bool = False
-    insecure_skip_verify: bool = False
-    ca_cert_file_path: typing.Optional[str] = None
-    console_endpoint: typing.Optional[str] = None
-    command: typing.Optional[typing.List[str]] = None
-    proxy_command: typing.Optional[typing.List[str]] = None
-    client_id: typing.Optional[str] = None
-    client_credentials_secret: typing.Optional[str] = None
-    scopes: typing.List[str] = field(default_factory=list)
-    auth_mode: "AuthType" = "Pkce"
-    audience: typing.Optional[str] = None
-    rpc_retries: int = 3
-    http_proxy_url: typing.Optional[str] = None
-
-    @classmethod
-    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "PlatformConfig":
-        """
-        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
-        :param config_file:
-        :return:
-        """
-        from .._initialize import set_if_exists
-
-        config_file = get_config_file(config_file)
-        kwargs: typing.Dict[str, typing.Any] = {}
-        kwargs = set_if_exists(kwargs, "insecure", _internal.Platform.INSECURE.read(config_file))
-        kwargs = set_if_exists(
-            kwargs, "insecure_skip_verify", _internal.Platform.INSECURE_SKIP_VERIFY.read(config_file)
-        )
-        kwargs = set_if_exists(kwargs, "ca_cert_file_path", _internal.Platform.CA_CERT_FILE_PATH.read(config_file))
-        kwargs = set_if_exists(kwargs, "command", _internal.Credentials.COMMAND.read(config_file))
-        kwargs = set_if_exists(kwargs, "proxy_command", _internal.Credentials.PROXY_COMMAND.read(config_file))
-        kwargs = set_if_exists(kwargs, "client_id", _internal.Credentials.CLIENT_ID.read(config_file))
-
-        is_client_secret = False
-        client_credentials_secret = read_file_if_exists(
-            _internal.Credentials.CLIENT_CREDENTIALS_SECRET_LOCATION.read(config_file)
-        )
-        if client_credentials_secret:
-            is_client_secret = True
-            if client_credentials_secret.endswith("\n"):
-                logger.info("Newline stripped from client secret")
-                client_credentials_secret = client_credentials_secret.strip()
-        kwargs = set_if_exists(
-            kwargs,
-            "client_credentials_secret",
-            client_credentials_secret,
-        )
-
-        client_credentials_secret_env_var = _internal.Credentials.CLIENT_CREDENTIALS_SECRET_ENV_VAR.read(config_file)
-        if client_credentials_secret_env_var:
-            client_credentials_secret = os.getenv(client_credentials_secret_env_var)
-            if client_credentials_secret:
-                is_client_secret = True
-        kwargs = set_if_exists(kwargs, "client_credentials_secret", client_credentials_secret)
-        kwargs = set_if_exists(kwargs, "scopes", _internal.Credentials.SCOPES.read(config_file))
-        kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file))
-        if is_client_secret:
-            kwargs = set_if_exists(kwargs, "auth_mode", "ClientSecret")
-        kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file))
-        kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file))
-
-        kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file))
-        return PlatformConfig(**kwargs)
-
-    @classmethod
-    def for_endpoint(cls, endpoint: str, insecure: bool = False) -> "PlatformConfig":
-        return PlatformConfig(endpoint=endpoint, insecure=insecure)
-
-
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class TaskConfig(object):
-    org: str | None = None
-    project: str | None = None
-    domain: str | None = None
-
-    @classmethod
-    def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "TaskConfig":
-        """
-        Reads from a config file, and overrides from Environment variables. Refer to ConfigEntry for details
-        :param config_file:
-        :return:
-        """
-        from flyte._initialize import set_if_exists
-
-        config_file = get_config_file(config_file)
-        kwargs: typing.Dict[str, typing.Any] = {}
-        kwargs = set_if_exists(kwargs, "org", _internal.Task.ORG.read(config_file))
-        kwargs = set_if_exists(kwargs, "project", _internal.Task.PROJECT.read(config_file))
-        kwargs = set_if_exists(kwargs, "domain", _internal.Task.DOMAIN.read(config_file))
-        return TaskConfig(**kwargs)
-
-
-@dataclass(init=True, repr=True, eq=True, frozen=True)
-class Config(object):
-    """
-    This the parent configuration object and holds all the underlying configuration object types. An instance of
-    this object holds all the config necessary to
-
-    1. Interactive session with Flyte backend
-    2. Some parts are required for Serialization, for example Platform Config is not required
-    3. Runtime of a task
-    """
-
-    platform: PlatformConfig = field(default=PlatformConfig())
-    task: TaskConfig = field(default=TaskConfig())
-
-    def with_params(
-        self,
-        platform: PlatformConfig | None = None,
-        task: TaskConfig | None = None,
-    ) -> "Config":
-        return Config(
-            platform=platform or self.platform,
-            task=task or self.task,
-        )
-
-    @classmethod
-    def auto(cls, config_file: typing.Union[str, ConfigFile, None] = None) -> "Config":
-        """
-        Automatically constructs the Config Object. The order of precedence is as follows
-          1. first try to find any env vars that match the config vars specified in the FLYTE_CONFIG format.
-          2. If not found in environment then values ar read from the config file
-          3. If not found in the file, then the default values are used.
-
-        :param config_file: file path to read the config from, if not specified default locations are searched
-        :return: Config
-        """
-        config_file = get_config_file(config_file)
-        return Config(platform=PlatformConfig.auto(config_file), task=TaskConfig.auto(config_file))

flyte/config/_config.py

@@ -1,196 +0,0 @@
-import os
-import pathlib
-import typing
-from dataclasses import dataclass
-from functools import lru_cache
-from os import getenv
-from pathlib import Path
-
-import yaml
-
-from flyte._logging import logger
-
-# This is the default config file name for flyte
-FLYTECTL_CONFIG_ENV_VAR = "FLYTECTL_CONFIG"
-UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
-
-
-@dataclass
-class YamlConfigEntry(object):
-    """
-    Creates a record for the config entry.
-    Args:
-        switch: dot-delimited string that should match flytectl args. Leaving it as dot-delimited instead of a list
-          of strings because it's easier to maintain alignment with flytectl.
-        config_value_type: Expected type of the value
-    """
-
-    switch: str
-    config_value_type: typing.Type = str
-
-    def get_env_name(self) -> str:
-        var_name = self.switch.upper().replace(".", "_")
-        return f"FLYTE_{var_name}"
-
-    def read_from_env(self, transform: typing.Optional[typing.Callable] = None) -> typing.Optional[typing.Any]:
-        """
-        Reads the config entry from environment variable, the structure of the env var is current
-        ``FLYTE_{SECTION}_{OPTION}`` all upper cased. We will change this in the future.
-        :return:
-        """
-        env = self.get_env_name()
-        v = os.environ.get(env, None)
-        if v is None:
-            return None
-        return transform(v) if transform else v
-
-    def read_from_file(
-        self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
-    ) -> typing.Optional[typing.Any]:
-        if not cfg:
-            return None
-        try:
-            v = cfg.get(self)
-            if isinstance(v, bool) or bool(v is not None and v):
-                return transform(v) if transform else v
-        except Exception:
-            ...
-
-        return None
-
-
-@dataclass
-class ConfigEntry(object):
-    """
-    A top level Config entry holder, that holds multiple different representations of the config.
-    Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
-    when flytectl starts a sandbox
-    """
-
-    yaml_entry: YamlConfigEntry
-    transform: typing.Optional[typing.Callable[[str], typing.Any]] = None
-
-    def read(self, cfg: typing.Optional["ConfigFile"] = None) -> typing.Optional[typing.Any]:
-        """
-        Reads the config Entry from the various sources in the following order,
-        #. First try to read from the relevant environment variable,
-        #. If missing, then try to read from the legacy config file, if one was parsed.
-        #. If missing, then try to read from the yaml file.
-
-        The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
-
-        :param cfg:
-        :return:
-        """
-        from_env = self.yaml_entry.read_from_env(self.transform)
-        if from_env is not None:
-            return from_env
-        if cfg and cfg.yaml_config and self.yaml_entry:
-            return self.yaml_entry.read_from_file(cfg, self.transform)
-
-        return None
-
-
-class ConfigFile(object):
-    def __init__(self, location: str):
-        """
-        Load the config from this location
-        """
-        self._location = location
-        self._yaml_config = self._read_yaml_config(location)
-
-    @staticmethod
-    def _read_yaml_config(location: str) -> typing.Optional[typing.Dict[str, typing.Any]]:
-        with open(location, "r") as fh:
-            try:
-                yaml_contents = yaml.safe_load(fh)
-                return yaml_contents
-            except yaml.YAMLError as exc:
-                logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
-                return None
-
-    def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
-        keys = c.switch.split(".")  # flytectl switches are dot delimited
-        d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
-        try:
-            for k in keys:
-                d = d[k]
-            return d
-        except KeyError:
-            return None
-
-    def get(self, c: YamlConfigEntry) -> typing.Any:
-        return self._get_from_yaml(c)
-
-    @property
-    def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
-        return self._yaml_config
-
-
-def resolve_config_path() -> pathlib.Path | None:
-    """
-    Config is read from the following locations in order of precedence:
-    1. `UCTL_CONFIG` environment variable
-    2. `FLYTECTL_CONFIG` environment variable
-    3. ~/.union/config.yaml if it exists
-    4. ~/.flyte/config.yaml if it exists
-    5. ./config.yaml if it exists
-    """
-    uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
-    if uctl_path_from_env:
-        return pathlib.Path(uctl_path_from_env)
-    logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
-
-    flytectl_path_from_env = getenv(FLYTECTL_CONFIG_ENV_VAR, None)
-    if flytectl_path_from_env:
-        return pathlib.Path(flytectl_path_from_env)
-    logger.debug("No FLYTECTL_CONFIG environment variable found, checking default locations")
-
-    home_dir_union_config = Path(Path.home(), ".union", "config.yaml")
-    if home_dir_union_config.exists():
-        return home_dir_union_config
-    logger.debug("No ~/.union/config.yaml found, checking current directory")
-
-    home_dir_flytectl_config = Path(Path.home(), ".flyte", "config.yaml")
-    if home_dir_flytectl_config.exists():
-        return home_dir_flytectl_config
-    logger.debug("No ~/.flyte/config.yaml found, checking current directory")
-
-    current_location_config = Path("config.yaml")
-    if current_location_config.exists():
-        return current_location_config
-    logger.debug("No ./config.yaml found, returning None")
-    return None
-
-
-@lru_cache
-def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
-    """
-    Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
-    """
-    if "PYTEST_VERSION" in os.environ:
-        # Use default local config in the pytest environment
-        return None
-    if isinstance(c, str):
-        logger.debug(f"Using specified config file at {c}")
-        return ConfigFile(c)
-    config_path = resolve_config_path()
-    if config_path:
-        return ConfigFile(str(config_path))
-    return None
-
-
-def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
-    """
-    Reads the contents of the file if passed a path. Otherwise, returns None.
-
-    :param filename: The file path to load
-    :param encoding: The encoding to use when reading the file.
-    :return: The contents of the file as a string or None.
-    """
-    if not filename:
-        return None
-
-    file = pathlib.Path(filename)
-    logger.debug(f"Reading file contents from [{file}] with current directory [{os.getcwd()}].")
-    return file.read_text(encoding=encoding)

flyte/config/_internal.py

@@ -1,64 +0,0 @@
-from flyte.config._config import ConfigEntry, YamlConfigEntry
-
-
-class Platform(object):
-    URL = ConfigEntry(YamlConfigEntry("admin.endpoint"))
-    INSECURE = ConfigEntry(YamlConfigEntry("admin.insecure", bool))
-    INSECURE_SKIP_VERIFY = ConfigEntry(YamlConfigEntry("admin.insecureSkipVerify", bool))
-    CONSOLE_ENDPOINT = ConfigEntry(YamlConfigEntry("console.endpoint"))
-    CA_CERT_FILE_PATH = ConfigEntry(YamlConfigEntry("admin.caCertFilePath"))
-    HTTP_PROXY_URL = ConfigEntry(YamlConfigEntry("admin.httpProxyURL"))
-
-
-class Credentials(object):
-    SECTION = "credentials"
-    COMMAND = ConfigEntry(YamlConfigEntry("admin.command", list))
-    """
-    This command is executed to return a token using an external process.
-    """
-
-    PROXY_COMMAND = ConfigEntry(YamlConfigEntry("admin.proxyCommand", list))
-    """
-    This command is executed to return a token for authorization with a proxy
-     in front of Flyte using an external process.
-    """
-
-    CLIENT_ID = ConfigEntry(YamlConfigEntry("admin.clientId"))
-    """
-    This is the public identifier for the app which handles authorization for a Flyte deployment.
-    More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
-    """
-
-    CLIENT_CREDENTIALS_SECRET_LOCATION = ConfigEntry(YamlConfigEntry("admin.clientSecretLocation"))
-    """
-    Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
-    password from a mounted file.
-    """
-
-    CLIENT_CREDENTIALS_SECRET_ENV_VAR = ConfigEntry(YamlConfigEntry("admin.clientSecretEnvVar"))
-    """
-    Used for basic auth, which is automatically called during pyflyte. This will allow the Flyte engine to read the
-    password from a mounted environment variable.
-    """
-
-    SCOPES = ConfigEntry(YamlConfigEntry("admin.scopes", list))
-    """
-    This setting can be used to manually pass in scopes into authenticator flows - eg.) for Auth0 compatibility
-    """
-
-    AUTH_MODE = ConfigEntry(YamlConfigEntry("admin.authType"))
-    """
-    The auth mode defines the behavior used to request and refresh credentials. The currently supported modes include:
-    - 'standard' or 'Pkce': This uses the pkce-enhanced authorization code flow by opening a browser window to initiate
-            credentials access.
-    - "DeviceFlow": This uses the Device Authorization Flow
-    - 'basic', 'client_credentials' or 'clientSecret': This uses symmetric key auth in which the end user enters a
-            client id and a client secret and public key encryption is used to facilitate authentication.
-    - None: No auth will be attempted.
-    """
-
-
-class Task(object):
-    ORG = ConfigEntry(YamlConfigEntry("task.org"))
-    PROJECT = ConfigEntry(YamlConfigEntry("task.project"))
-    DOMAIN = ConfigEntry(YamlConfigEntry("task.domain"))