flyte 0.0.1b0__py3-none-any.whl

union/_initialize.py

@@ -0,0 +1,585 @@
+from __future__ import annotations
+
+import datetime
+import functools
+import os
+import threading
+import typing
+from abc import abstractmethod
+from dataclasses import dataclass, replace
+from datetime import timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Literal, Optional, TypeVar
+
+from union.errors import InitializationError
+
+from ._api_commons import syncer
+from ._logging import initialize_logger
+from ._tools import ipython_check
+
+if TYPE_CHECKING:
+    from union.remote._client.auth import AuthType, ClientConfig
+    from union.remote._client.controlplane import ClientSet
+
+Mode = Literal["local", "remote"]
+
+
+def set_if_exists(d: dict, k: str, val: typing.Any) -> dict:
+    """
+    Given a dict ``d`` sets the key ``k`` with value of config ``v``, if the config value ``v`` is set
+    and return the updated dictionary.
+    """
+    exists = isinstance(val, bool) or bool(val is not None and val)
+    if exists:
+        d[k] = val
+    return d
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class Storage(object):
+    """
+    Data storage configuration that applies across any provider.
+    """
+
+    retries: int = 3
+    backoff: datetime.timedelta = datetime.timedelta(seconds=5)
+    enable_debug: bool = False
+    attach_execution_metadata: bool = True
+
+    _KEY_ENV_VAR_MAPPING: ClassVar[typing.Dict[str, str]] = {
+        "enable_debug": "UNION_STORAGE_DEBUG",
+        "retries": "UNION_STORAGE_RETRIES",
+        "backoff": "UNION_STORAGE_BACKOFF_SECONDS",
+    }
+
+    @abstractmethod
+    def get_fsspec_kwargs(self, anonymous: bool = False, /, **kwargs) -> Dict[str, Any]:
+        """
+        Returns the configuration as kwargs for constructing an fsspec filesystem.
+        """
+        # TODO: Get attach_execution_metadata from config
+
+    @classmethod
+    def _auto_as_kwargs(cls) -> Dict[str, Any]:
+        retries = os.getenv(cls._KEY_ENV_VAR_MAPPING["retries"])
+        backoff = os.getenv(cls._KEY_ENV_VAR_MAPPING["backoff"])
+        enable_debug = os.getenv(cls._KEY_ENV_VAR_MAPPING["enable_debug"])
+
+        kwargs = {}
+        kwargs = set_if_exists(kwargs, "enable_debug", enable_debug)
+        kwargs = set_if_exists(kwargs, "retries", retries)
+        kwargs = set_if_exists(kwargs, "backoff", backoff)
+        return kwargs
+
+    @classmethod
+    @abstractmethod
+    def auto(cls) -> Storage:
+        """
+        Construct the config object automatically from environment variables.
+        """
+        ...
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class S3(Storage):
+    """
+    S3 specific configuration
+    """
+
+    endpoint: typing.Optional[str] = None
+    access_key_id: typing.Optional[str] = None
+    secret_access_key: typing.Optional[str] = None
+
+    _KEY_ENV_VAR_MAPPING: ClassVar[typing.Dict[str, str]] = {
+        "endpoint": "FLYTE_AWS_ENDPOINT",
+        "access_key_id": "FLYTE_AWS_ACCESS_KEY_ID",
+        "secret_access_key": "FLYTE_AWS_SECRET_ACCESS_KEY",
+    } | Storage._KEY_ENV_VAR_MAPPING
+
+    # Refer to https://github.com/developmentseed/obstore/blob/33654fc37f19a657689eb93327b621e9f9e01494/obstore/python/obstore/store/_aws.pyi#L11
+    # for key and secret
+    _CONFIG_KEY_FSSPEC_S3_KEY_ID: ClassVar = "access_key_id"
+    _CONFIG_KEY_FSSPEC_S3_SECRET: ClassVar = "secret_access_key"
+    _CONFIG_KEY_ENDPOINT: ClassVar = "endpoint_url"
+    _KEY_SKIP_SIGNATURE: ClassVar = "skip_signature"
+
+    @classmethod
+    def auto(cls) -> S3:
+        """
+        :return: Config
+        """
+        endpoint = os.getenv(cls._KEY_ENV_VAR_MAPPING["endpoint"], None)
+        access_key_id = os.getenv(cls._KEY_ENV_VAR_MAPPING["access_key_id"], None)
+        secret_access_key = os.getenv(cls._KEY_ENV_VAR_MAPPING["secret_access_key"], None)
+
+        kwargs = super()._auto_as_kwargs()
+        kwargs = set_if_exists(kwargs, "endpoint", endpoint)
+        kwargs = set_if_exists(kwargs, "access_key_id", access_key_id)
+        kwargs = set_if_exists(kwargs, "secret_access_key", secret_access_key)
+
+        return S3(**kwargs)
+
+    @classmethod
+    def for_sandbox(cls) -> S3:
+        """
+        :return:
+        """
+        kwargs = {
+            "endpoint": "http://localhost:4566",
+            "access_key_id": "minio",
+            "secret_access_key": "miniostorage",
+        }
+        return S3(**kwargs)
+
+    def get_fsspec_kwargs(self, anonymous: bool = False, /, **kwargs) -> Dict[str, Any]:
+        # Construct the config object
+        config: Dict[str, Any] = {}
+        if self._CONFIG_KEY_FSSPEC_S3_KEY_ID in kwargs or self.access_key_id:
+            config[self._CONFIG_KEY_FSSPEC_S3_KEY_ID] = kwargs.pop(
+                self._CONFIG_KEY_FSSPEC_S3_KEY_ID, self.access_key_id
+            )
+        if self._CONFIG_KEY_FSSPEC_S3_SECRET in kwargs or self.secret_access_key:
+            config[self._CONFIG_KEY_FSSPEC_S3_SECRET] = kwargs.pop(
+                self._CONFIG_KEY_FSSPEC_S3_SECRET, self.secret_access_key
+            )
+        if self._CONFIG_KEY_ENDPOINT in kwargs or self.endpoint:
+            config["endpoint_url"] = kwargs.pop(self._CONFIG_KEY_ENDPOINT, self.endpoint)
+
+        retries = kwargs.pop("retries", self.retries)
+        backoff = kwargs.pop("backoff", self.backoff)
+
+        if anonymous:
+            config[self._KEY_SKIP_SIGNATURE] = True
+
+        retry_config = {
+            "max_retries": retries,
+            "backoff": {
+                "base": 2,
+                "init_backoff": backoff,
+                "max_backoff": timedelta(seconds=16),
+            },
+            "retry_timeout": timedelta(minutes=3),
+        }
+
+        client_options = {"timeout": "99999s", "allow_http": True}
+
+        if config:
+            kwargs["config"] = config
+        kwargs["client_options"] = client_options or None
+        kwargs["retry_config"] = retry_config or None
+
+        return kwargs
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class GCS(Storage):
+    """
+    Any GCS specific configuration.
+    """
+
+    gsutil_parallelism: bool = False
+
+    _KEY_ENV_VAR_MAPPING: ClassVar[dict[str, str]] = {
+        "gsutil_parallelism": "GCP_GSUTIL_PARALLELISM",
+    }
+
+    @classmethod
+    def auto(cls) -> GCS:
+        gsutil_parallelism = os.getenv(cls._KEY_ENV_VAR_MAPPING["gsutil_parallelism"], None)
+
+        kwargs = {}
+        kwargs = set_if_exists(kwargs, "gsutil_parallelism", gsutil_parallelism)
+        return GCS(**kwargs)
+
+    def get_fsspec_kwargs(self, anonymous: bool = False, /, **kwargs) -> Dict[str, Any]:
+        return kwargs
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True)
+class ABFS(Storage):
+    """
+    Any Azure Blob Storage specific configuration.
+    """
+
+    account_name: typing.Optional[str] = None
+    account_key: typing.Optional[str] = None
+    tenant_id: typing.Optional[str] = None
+    client_id: typing.Optional[str] = None
+    client_secret: typing.Optional[str] = None
+
+    _KEY_ENV_VAR_MAPPING: ClassVar[dict[str, str]] = {
+        "account_name": "AZURE_STORAGE_ACCOUNT_NAME",
+        "account_key": "AZURE_STORAGE_ACCOUNT_KEY",
+        "tenant_id": "AZURE_TENANT_ID",
+        "client_id": "AZURE_CLIENT_ID",
+        "client_secret": "AZURE_CLIENT_SECRET",
+    }
+    _KEY_SKIP_SIGNATURE: ClassVar = "skip_signature"
+
+    @classmethod
+    def auto(cls) -> ABFS:
+        account_name = os.getenv(cls._KEY_ENV_VAR_MAPPING["account_name"], None)
+        account_key = os.getenv(cls._KEY_ENV_VAR_MAPPING["account_key"], None)
+        tenant_id = os.getenv(cls._KEY_ENV_VAR_MAPPING["tenant_id"], None)
+        client_id = os.getenv(cls._KEY_ENV_VAR_MAPPING["client_id"], None)
+        client_secret = os.getenv(cls._KEY_ENV_VAR_MAPPING["client_secret"], None)
+
+        kwargs = {}
+        kwargs = set_if_exists(kwargs, "account_name", account_name)
+        kwargs = set_if_exists(kwargs, "account_key", account_key)
+        kwargs = set_if_exists(kwargs, "tenant_id", tenant_id)
+        kwargs = set_if_exists(kwargs, "client_id", client_id)
+        kwargs = set_if_exists(kwargs, "client_secret", client_secret)
+        return ABFS(**kwargs)
+
+    def get_fsspec_kwargs(self, anonymous: bool = False, /, **kwargs) -> Dict[str, Any]:
+        config: Dict[str, Any] = {}
+        if "account_name" in kwargs or self.account_name:
+            config["account_name"] = kwargs.get("account_name", self.account_name)
+        if "account_key" in kwargs or self.account_key:
+            config["account_key"] = kwargs.get("account_key", self.account_key)
+        if "client_id" in kwargs or self.client_id:
+            config["client_id"] = kwargs.get("client_id", self.client_id)
+        if "client_secret" in kwargs or self.client_secret:
+            config["client_secret"] = kwargs.get("client_secret", self.client_secret)
+        if "tenant_id" in kwargs or self.tenant_id:
+            config["tenant_id"] = kwargs.get("tenant_id", self.tenant_id)
+
+        if anonymous:
+            config[self._KEY_SKIP_SIGNATURE] = True
+
+        client_options = {"timeout": "99999s", "allow_http": "true"}
+
+        if config:
+            kwargs["config"] = config
+        kwargs["client_options"] = client_options
+
+        return kwargs
+
+
+@dataclass(init=True, repr=True, eq=True, frozen=True, kw_only=True)
+class CommonInit:
+    """
+    Common initialization configuration for Union.
+    """
+
+    root_dir: Path
+    org: str | None = None
+    project: str | None = None
+    domain: str | None = None
+
+
+@dataclass(init=True, kw_only=True, repr=True, eq=True, frozen=True)
+class _InitConfig(CommonInit):
+    client: Optional[ClientSet] = None
+    storage: Optional[Storage] = None
+
+    def replace(self, **kwargs) -> _InitConfig:
+        return replace(self, **kwargs)
+
+
+# Global singleton to store initialization configuration
+_init_config: _InitConfig | None = None
+_init_lock = threading.RLock()  # Reentrant lock for thread safety
+
+
+async def _initialize_client(
+    api_key: str | None = None,
+    auth_type: AuthType = "Pkce",
+    endpoint: str | None = None,
+    client_config: ClientConfig | None = None,
+    headless: bool = False,
+    insecure: bool = False,
+    insecure_skip_verify: bool = False,
+    ca_cert_file_path: str | None = None,
+    command: List[str] | None = None,
+    proxy_command: List[str] | None = None,
+    client_id: str | None = None,
+    client_credentials_secret: str | None = None,
+    rpc_retries: int = 3,
+    http_proxy_url: str | None = None,
+) -> ClientSet:
+    """
+    Initialize the client based on the execution mode.
+    :return: The initialized client
+    """
+    from union.remote._client.controlplane import ClientSet
+
+    if endpoint is not None:
+        return await ClientSet.for_endpoint(
+            endpoint,
+            insecure=insecure,
+            api_key=api_key,
+            insecure_skip_verify=insecure_skip_verify,
+            auth_type=auth_type,
+            headless=headless,
+            ca_cert_file_path=ca_cert_file_path,
+            command=command,
+            proxy_command=proxy_command,
+            client_id=client_id,
+            client_credentials_secret=client_credentials_secret,
+            client_config=client_config,
+            rpc_retries=rpc_retries,
+            http_proxy_url=http_proxy_url,
+        )
+    raise NotImplementedError("Currently only endpoints are supported.")
+
+
+@syncer.wrap
+async def init(
+    org: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    root_dir: Path | None = None,
+    log_level: int | None = None,
+    endpoint: str | None = None,
+    headless: bool = False,
+    insecure: bool = False,
+    insecure_skip_verify: bool = False,
+    ca_cert_file_path: str | None = None,
+    auth_type: AuthType = "Pkce",
+    command: List[str] | None = None,
+    proxy_command: List[str] | None = None,
+    api_key: str | None = None,
+    client_id: str | None = None,
+    client_credentials_secret: str | None = None,
+    auth_client_config: ClientConfig | None = None,
+    rpc_retries: int = 3,
+    http_proxy_url: str | None = None,
+    storage: Storage | None = None,
+) -> None:
+    """
+    Initialize the Union system with the given configuration. This method should be called before any other Union
+    remote API methods are called. Thread-safe implementation.
+
+    :param project: Optional project name (not used in this implementation)
+    :param domain: Optional domain name (not used in this implementation)
+    :param root_dir: Optional root directory for the Union system
+    :param log_level: Optional logging level for the logger, default is set using the default initialization policies
+    :param api_key: Optional API key for authentication
+    :param endpoint: Optional API endpoint URL
+    :param headless: Optional Whether to run in headless mode
+    :param mode: Optional execution model (local, remote). Default is local. When local is used,
+           the execution will be done locally. When remote is used, the execution will be sent to a remote server,
+           In the remote case, the endpoint or api_key must be set.
+    :param insecure_skip_verify: Whether to skip SSL certificate verification
+    :param auth_client_config: Optional client configuration for authentication
+    :param auth_type: The authentication type to use (Pkce, ClientSecret, ExternalCommand, DeviceFlow)
+    :param command: This command is executed to return a token using an external process
+    :param proxy_command: This command is executed to return a token for proxy authorization using an external process
+    :param client_id: This is the public identifier for the app which handles authorization for a Flyte deployment.
+      More details here: https://www.oauth.com/oauth2-servers/client-registration/client-id-secret/.
+    :param client_credentials_secret: Used for service auth, which is automatically called during pyflyte. This will
+      allow the Flyte engine to read the password directly from the environment variable. Note that this is
+      less secure! Please only use this if mounting the secret as a file is impossible
+    :param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
+    :param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
+    :param rpc_retries: [optional] int Number of times to retry the platform calls
+    :param audience: oauth2 audience for the token request. This is used to validate the token
+    :param insecure: insecure flag for the client
+    :param storage: Optional blob store (S3, GCS, Azure) configuration if needed to access (i.e. using Minio)
+    :param org: Optional organization override for the client. Should be set by auth instead.
+
+    :return: None
+    """
+    interactive_mode = ipython_check()
+
+    initialize_logger(enable_rich=interactive_mode)
+    if log_level:
+        initialize_logger(log_level=log_level, enable_rich=interactive_mode)
+
+    global _init_config  # noqa: PLW0603
+
+    with _init_lock:
+        client = None
+        if endpoint or api_key:
+            client = await _initialize_client(
+                api_key=api_key,
+                auth_type=auth_type,
+                endpoint=endpoint,
+                headless=headless,
+                insecure=insecure,
+                insecure_skip_verify=insecure_skip_verify,
+                ca_cert_file_path=ca_cert_file_path,
+                command=command,
+                proxy_command=proxy_command,
+                client_id=client_id,
+                client_credentials_secret=client_credentials_secret,
+                client_config=auth_client_config,
+                rpc_retries=rpc_retries,
+                http_proxy_url=http_proxy_url,
+            )
+
+        _init_config = _InitConfig(
+            root_dir=root_dir or Path.cwd(),
+            project=project,
+            domain=domain,
+            client=client,
+            storage=storage,
+            org=org,
+        )
+
+
+def _get_init_config() -> _InitConfig | None:
+    """
+    Get the current initialization configuration. Thread-safe implementation.
+
+    :return: The current InitData if initialized, None otherwise
+    """
+    with _init_lock:
+        return _init_config
+
+
+def get_common_config() -> Optional[CommonInit]:
+    """
+    Get the current initialization configuration. Thread-safe implementation.
+
+    :return: The current InitData if initialized, None otherwise
+    """
+    with _init_lock:
+        return _init_config
+
+
+def get_storage() -> Optional[Storage]:
+    """
+    Get the current storage configuration. Thread-safe implementation.
+
+    :return: The current storage configuration
+    """
+    cfg = _get_init_config()
+    if cfg is None:
+        return None
+    return cfg.storage
+
+
+def get_client() -> Optional[ClientSet]:
+    """
+    Get the current client. Thread-safe implementation.
+
+    :return: The current client
+    """
+    cfg = _get_init_config()
+    if cfg is None:
+        return None
+    return cfg.client
+
+
+def is_initialized() -> bool:
+    """
+    Check if the system has been initialized.
+
+    :return: True if initialized, False otherwise
+    """
+    return _get_init_config() is not None
+
+
+def initialize_in_cluster(storage: Storage | None = None) -> None:
+    """
+    Initialize the system for in-cluster execution. This is a placeholder function and does not perform any actions.
+
+    :return: None
+    """
+    init(storage=storage)
+
+
+# Define a generic type variable for the decorated function
+T = TypeVar("T", bound=Callable)
+
+
+def requires_client(func: T) -> T:
+    """
+    Decorator that checks if the client has been initialized before executing the function.
+    Raises InitializationError if the client is not initialized.
+
+    :param func: Function to decorate
+    :return: Decorated function that checks for initialization
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if _get_init_config() is None or _get_init_config().client is None:
+            raise InitializationError(
+                "ClientNotInitializedError",
+                "user",
+                f"Function '{func.__name__}' requires client to be initialized. "
+                f"Call union.init() with a valid endpoint or api-key before using this function.",
+            )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def requires_upload_location(func: T) -> T:
+    """
+    Decorator that checks if the storage has been initialized before executing the function.
+    Raises InitializationError if the storage is not initialized.
+
+    :param func: Function to decorate
+    :return: Decorated function that checks for initialization
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        from ._context import internal_ctx
+
+        ctx = internal_ctx()
+        if not ctx.raw_data:
+            raise InitializationError(
+                "No upload path configured",
+                "user",
+                f"Function '{func.__name__}' requires client to be initialized. "
+                f"Call union.init() with storage configuration before using this function.",
+            )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def requires_initialization(func: T) -> T:
+    """
+    Decorator that checks if the system has been initialized before executing the function.
+    Raises InitializationError if the system is not initialized.
+
+    :param func: Function to decorate
+    :return: Decorated function that checks for initialization
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if not is_initialized():
+            raise InitializationError(
+                "NotInitConfiguredError",
+                "user",
+                f"Function '{func.__name__}' requires initialization. Call union.init() before using this function.",
+            )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+async def _init_for_testing(
+    project: str | None = None,
+    domain: str | None = None,
+    root_dir: Path | None = None,
+    log_level: int | None = None,
+    client: ClientSet | None = None,
+):
+    global _init_config  # noqa: PLW0603
+
+    if log_level:
+        initialize_logger(log_level=log_level)
+
+    with _init_lock:
+        _init_config = _InitConfig(
+            root_dir=root_dir or Path.cwd(),
+            project=project,
+            domain=domain,
+            client=client,
+        )
+
+
+def replace_client(client):
+    global _init_config  # noqa: PLW0603
+
+    with _init_lock:
+        _init_config = _init_config.replace(client=client)

union/_interface.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import inspect
+from typing import Dict, Generator, Tuple, Type, TypeVar, Union, cast, get_args, get_type_hints
+
+from union._logging import logger
+
+
+def default_output_name(index: int = 0) -> str:
+    return f"o{index}"
+
+
+def output_name_generator(length: int) -> Generator[str, None, None]:
+    for x in range(length):
+        yield default_output_name(x)
+
+
+def extract_return_annotation(return_annotation: Union[Type, Tuple, None]) -> Dict[str, Type]:
+    """
+    The input to this function should be sig.return_annotation where sig = inspect.signature(some_func)
+    The purpose of this function is to sort out whether a function is returning one thing, or multiple things, and to
+    name the outputs accordingly, either by using our default name function, or from a typing.NamedTuple.
+
+        # Option 1
+        nt1 = typing.NamedTuple("NT1", x_str=str, y_int=int)
+        def t(a: int, b: str) -> nt1: ...
+
+        # Option 2
+        def t(a: int, b: str) -> typing.NamedTuple("NT1", x_str=str, y_int=int): ...
+
+        # Option 3
+        def t(a: int, b: str) -> typing.Tuple[int, str]: ...
+
+        # Option 4
+        def t(a: int, b: str) -> (int, str): ...
+
+        # Option 5
+        def t(a: int, b: str) -> str: ...
+
+        # Option 6
+        def t(a: int, b: str) -> None: ...
+
+        # Options 7/8
+        def t(a: int, b: str) -> List[int]: ...
+        def t(a: int, b: str) -> Dict[str, int]: ...
+
+    Note that Options 1 and 2 are identical, just syntactic sugar. In the NamedTuple case, we'll use the names in the
+    definition. In all other cases, we'll automatically generate output names, indexed starting at 0.
+    """
+
+    # Handle Option 6
+    # We can think about whether we should add a default output name with type None in the future.
+    if return_annotation in (None, type(None), inspect.Signature.empty):
+        return {}
+
+    # This statement results in true for typing.Namedtuple, single and void return types, so this
+    # handles Options 1, 2. Even though NamedTuple for us is multi-valued, it's a single value for Python
+    if hasattr(return_annotation, "__bases__") and (
+        isinstance(return_annotation, type) or isinstance(return_annotation, TypeVar)
+    ):
+        # isinstance / issubclass does not work for Namedtuple.
+        # Options 1 and 2
+        bases = return_annotation.__bases__  # type: ignore
+        if len(bases) == 1 and bases[0] is tuple and hasattr(return_annotation, "_fields"):
+            logger.debug(f"Task returns named tuple {return_annotation}")
+            return dict(get_type_hints(cast(Type, return_annotation), include_extras=True))
+
+    if hasattr(return_annotation, "__origin__") and return_annotation.__origin__ is tuple:  # type: ignore
+        # Handle option 3
+        logger.debug(f"Task returns unnamed typing.Tuple {return_annotation}")
+        if len(return_annotation.__args__) == 1:  # type: ignore
+            raise TypeError("Tuples should be used to indicate multiple return values, found only one return variable.")
+        ra = get_args(return_annotation)
+        return dict(zip(list(output_name_generator(len(ra))), ra))
+
+    elif isinstance(return_annotation, tuple):
+        if len(return_annotation) == 1:
+            raise TypeError("Please don't use a tuple if you're just returning one thing.")
+        return dict(zip(list(output_name_generator(len(return_annotation))), return_annotation))
+
+    else:
+        # Handle all other single return types
+        logger.debug(f"Task returns unnamed native tuple {return_annotation}")
+        return {default_output_name(): cast(Type, return_annotation)}

union/_internal/__init__.py

@@ -0,0 +1,3 @@
+from .controllers import Controller, ControllerType, create_controller
+
+__all__ = ["Controller", "ControllerType", "create_controller"]