flyte 0.0.1b3__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

union/__init__.py

@@ -1,54 +0,0 @@
-"""
-Union SDK for authoring Compound AI applications, services and workflows.
-
-## Environments
-
-TaskEnvironment class to define a new environment for a set of tasks.
-
-Example usage:
-
-```python
-env = union.TaskEnvironment(name="my_env", image="my_image", resources=Resources(cpu="1", memory="1Gi"))
-
-@env.task
-async def my_task():
-    pass
-```
-"""
-
-__all__ = [
-    "ABFS",
-    "GCS",
-    "GPU",
-    "S3",
-    "TPU",
-    "Cache",
-    "CachePolicy",
-    "CacheRequest",
-    "Device",
-    "Image",
-    "Resources",
-    "ReusePolicy",
-    "Secret",
-    "TaskEnvironment",
-    "__version__",
-    "ctx",
-    "deploy",
-    "group",
-    "init",
-    "run",
-    "with_runcontext",
-]
-
-from ._cache import Cache, CachePolicy, CacheRequest
-from ._context import ctx
-from ._deploy import deploy
-from ._group import group
-from ._image import Image
-from ._initialize import ABFS, GCS, S3, init
-from ._resources import GPU, TPU, Device, Resources
-from ._reusable_environment import ReusePolicy
-from ._run import run, with_runcontext
-from ._secret import Secret
-from ._task_environment import TaskEnvironment
-from ._version import __version__

union/_api_commons.py

@@ -1,3 +0,0 @@
-from synchronicity import Synchronizer
-
-syncer = Synchronizer()

union/_bin/runtime.py

@@ -1,113 +0,0 @@
-"""
-Union runtime module, this is the entrypoint script for the Union runtime.
-
-Caution: Startup time for this module is very important, as it is the entrypoint for the Union runtime.
-Refrain from importing any modules here. If you need to import any modules, do it inside the main function.
-"""
-
-import asyncio
-import os
-import sys
-from typing import List
-
-import click
-
-# Todo: work with pvditt to make these the names
-# ACTION_NAME = "_U_ACTION_NAME"
-# RUN_NAME = "_U_RUN_NAME"
-# PROJECT_NAME = "_U_PROJECT_NAME"
-# DOMAIN_NAME = "_U_DOMAIN_NAME"
-# ORG_NAME = "_U_ORG_NAME"
-
-ACTION_NAME = "ACTION_NAME"
-RUN_NAME = "RUN_NAME"
-PROJECT_NAME = "FLYTE_INTERNAL_TASK_PROJECT"
-DOMAIN_NAME = "FLYTE_INTERNAL_TASK_DOMAIN"
-ORG_NAME = "_U_ORG_NAME"
-
-
-@click.command("urun")
-@click.option("--inputs", "-i", required=True)
-@click.option("--outputs-path", "-o", required=True)
-@click.option("--version", "-v", required=True)
-@click.option("--raw-data-path", "-r", required=False)
-@click.option("--checkpoint-path", "-c", required=False)
-@click.option("--prev-checkpoint", "-p", required=False)
-@click.option("--name", envvar=ACTION_NAME, required=False)
-@click.option("--run-name", envvar=RUN_NAME, required=False)
-@click.option("--project", envvar=PROJECT_NAME, required=False)
-@click.option("--domain", envvar=DOMAIN_NAME, required=False)
-@click.option("--org", envvar=ORG_NAME, required=False)
-@click.option("--image-cache", required=False)
-@click.option("--tgz", required=False)
-@click.option("--pkl", required=False)
-@click.option("--dest", required=False)
-@click.option("--resolver", required=False)
-@click.argument(
-    "resolver-args",
-    type=click.UNPROCESSED,
-    nargs=-1,
-)
-def main(
-    run_name: str,
-    name: str,
-    project: str,
-    domain: str,
-    org: str,
-    image_cache: str,
-    version: str,
-    inputs: str,
-    outputs_path: str,
-    raw_data_path: str,
-    checkpoint_path: str,
-    prev_checkpoint: str,
-    tgz: str,
-    pkl: str,
-    dest: str,
-    resolver: str,
-    resolver_args: List[str],
-):
-    sys.path.insert(0, ".")
-
-    from union._datastructures import ActionID, Checkpoints, CodeBundle, RawDataPath
-    from union._initialize import S3, initialize_in_cluster
-    from union._internal.controllers import create_controller
-    from union._internal.imagebuild.image_builder import ImageCache
-    from union._internal.runtime.entrypoints import load_and_run_task
-
-    if not org:
-        org = "testorg"
-
-    assert org, "Org is required for now"
-    assert project, "Project is required"
-    assert domain, "Domain is required"
-    assert run_name, "Run name is required"
-    assert name, "Action name is required"
-
-    if run_name.startswith("{{"):
-        run_name = os.environ.get("RUN_NAME")
-    if name.startswith("{{"):
-        name = os.environ.get("ACTION_NAME")
-
-    bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
-    # TODO configure storage correctly for cluster
-    initialize_in_cluster(storage=S3.auto())
-    controller = create_controller(ct="remote", endpoint="host.docker.internal:8090", insecure=True)
-
-    ic = ImageCache.from_transport(image_cache) if image_cache else None
-
-    asyncio.run(
-        load_and_run_task(
-            resolver=resolver,
-            resolver_args=resolver_args,
-            action=ActionID(name=name, run_name=run_name, project=project, domain=domain, org=org),
-            raw_data_path=RawDataPath(path=raw_data_path),
-            checkpoints=Checkpoints(checkpoint_path, prev_checkpoint),
-            code_bundle=bundle,
-            input_path=inputs,
-            output_path=outputs_path,
-            version=version,
-            controller=controller,
-            image_cache=ic,
-        )
-    )

union/_build.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-
-from ._api_commons import syncer
-from ._image import Image
-
-
-@syncer.wrap
-async def build(image: Image) -> str:
-    """
-    Build an image. The existing async context will be used.
-
-    Example:
-    ```
-    import union
-    image = union.Image("example_image")
-    if __name__ == "__main__":
-        asyncio.run(union.build.aio(image))
-    ```
-
-    :param image: The image(s) to build.
-    :return: The image URI.
-    """
-    from union._internal.imagebuild.image_builder import ImageBuildEngine
-
-    return await ImageBuildEngine.build(image)

union/_cache/__init__.py

@@ -1,12 +0,0 @@
-from .cache import Cache, CacheBehavior, CachePolicy, CacheRequest
-from .defaults import get_default_policies
-from .policy_function_body import FunctionBodyPolicy
-
-__all__ = [
-    "Cache",
-    "CacheBehavior",
-    "CachePolicy",
-    "CacheRequest",
-    "FunctionBodyPolicy",
-    "get_default_policies",
-]

union/_cache/cache.py

@@ -1,141 +0,0 @@
-import hashlib
-from dataclasses import dataclass, field
-from typing import (
-    Callable,
-    Generic,
-    List,
-    Optional,
-    Protocol,
-    Tuple,
-    Union,
-    runtime_checkable,
-)
-
-from typing_extensions import Literal, ParamSpec, TypeVar
-
-from union._datastructures import CodeBundle
-
-# if TYPE_CHECKING:
-from union._image import Image
-
-P = ParamSpec("P")
-FuncOut = TypeVar("FuncOut")
-
-CacheBehavior = Literal["auto", "override", "disable"]
-
-
-@dataclass
-class VersionParameters(Generic[P, FuncOut]):
-    """
-    Parameters used for cache version hash generation.
-
-    :param func: The function to generate a version for. This is a required parameter but can be any callable
-    :type func: Callable[P, FuncOut]
-    :param image: The container image to generate a version for. This can be a string representing the
-        image name or an Image object.
-    :type image: Optional[Union[str, Image]]
-    """
-
-    func: Callable[P, FuncOut]
-    image: Optional[Union[str, Image]] = None
-    code_bundle: Optional[CodeBundle] = None
-
-
-@runtime_checkable
-class CachePolicy(Protocol):
-    def get_version(self, salt: str, params: VersionParameters) -> str: ...
-
-
-@dataclass
-class Cache:
-    """
-    Cache configuration for a task.
-    :param behavior: The behavior of the cache. Can be "auto", "override" or "disable".
-    :param version_override: The version of the cache. If not provided, the version will be
-     generated based on the cache policies
-    :type version_override: Optional[str]
-    :param serialize: Boolean that indicates if identical (ie. same inputs) instances of this task should be executed in
-          serial when caching is enabled. This means that given multiple concurrent executions over identical inputs,
-          only a single instance executes and the rest wait to reuse the cached results.
-    :type serialize: bool
-    :param ignored_inputs: A tuple of input names to ignore when generating the version hash.
-    :type ignored_inputs: Union[Tuple[str, ...], str]
-    :param salt: A salt used in the hash generation.
-    :type salt: str
-    :param policies: A list of cache policies to generate the version hash.
-    :type policies: Optional[Union[List[CachePolicy], CachePolicy]]
-    """
-
-    behavior: CacheBehavior
-    version_override: Optional[str] = None
-    serialize: bool = False
-    ignored_inputs: Union[Tuple[str, ...], str] = field(default_factory=tuple)
-    salt: str = ""
-    policies: Optional[Union[List[CachePolicy], CachePolicy]] = None
-
-    def __post_init__(self):
-        if self.behavior not in ["auto", "override", "disable"]:
-            raise ValueError(f"Invalid cache behavior: {self.behavior}. Must be one of ['auto', 'override', 'disable']")
-        if self.behavior == "disable":
-            return
-
-        if isinstance(self.ignored_inputs, str):
-            self._ignored_inputs = (self.ignored_inputs,)
-        else:
-            self._ignored_inputs = self.ignored_inputs
-
-        # Normalize policies so that self._policies is always a list
-        if self.policies is None:
-            from union._cache.defaults import get_default_policies
-
-            self.policies = get_default_policies()
-        elif isinstance(self.policies, CachePolicy):
-            self.policies = [self.policies]
-
-        if self.version_override is None and not self.policies:
-            raise ValueError("If version is not defined then at least one cache policy needs to be set")
-
-    def is_enabled(self) -> bool:
-        """
-        Check if the cache policy is enabled.
-        """
-        return self.behavior in ["auto", "override"]
-
-    def get_ignored_inputs(self) -> Tuple[str, ...]:
-        return self._ignored_inputs
-
-    def get_version(self, params: Optional[VersionParameters] = None) -> str:
-        if not self.is_enabled():
-            return ""
-
-        if self.version_override is not None:
-            return self.version_override
-
-        if params is None:
-            raise ValueError("Version parameters must be provided when version_override is not set.")
-
-        if params.code_bundle is not None:
-            if params.code_bundle.pkl is not None:
-                return params.code_bundle.computed_version
-
-        task_hash = ""
-        for policy in self.policies:
-            try:
-                task_hash += policy.get_version(self.salt, params)
-            except Exception as e:
-                raise ValueError(f"Failed to generate version for cache policy {policy}.") from e
-
-        hash_obj = hashlib.sha256(task_hash.encode())
-        return hash_obj.hexdigest()
-
-
-CacheRequest = CacheBehavior | Cache
-
-
-def cache_from_request(cache: CacheRequest) -> Cache:
-    """
-    Coerce user input into a cache object.
-    """
-    if isinstance(cache, Cache):
-        return cache
-    return Cache(behavior=cache)

union/_cache/defaults.py

@@ -1,9 +0,0 @@
-from .cache import CachePolicy
-from .policy_function_body import FunctionBodyPolicy
-
-
-def get_default_policies() -> list[CachePolicy]:
-    """
-    Get default cache policies.
-    """
-    return [FunctionBodyPolicy()]

union/_cache/policy_function_body.py

@@ -1,42 +0,0 @@
-import ast
-import hashlib
-import inspect
-import textwrap
-
-from .cache import CachePolicy, VersionParameters
-
-
-class FunctionBodyPolicy(CachePolicy):
-    """
-    A class that implements a versioning mechanism for functions by generating
-    a SHA-256 hash of the function's source code combined with a salt.
-    """
-
-    def get_version(self, salt: str, params: VersionParameters) -> str:
-        """
-        This method generates a version string for a function by hashing the function's source code
-        combined with a salt.
-
-        :param salt: A string that is used to salt the hash.
-        :param params: VersionParameters object that contains the parameters (e.g. function, ImageSpec, etc.) that are
-                       used to generate the version.
-
-        :return: A string that represents the version of the function.
-        """
-        if params.func is None:
-            return ""
-
-        source = inspect.getsource(params.func)
-        dedented_source = textwrap.dedent(source)
-
-        # Parse the source code into an Abstract Syntax Tree (AST)
-        parsed_ast = ast.parse(dedented_source)
-
-        # Convert the AST into a string representation
-        ast_bytes = ast.dump(parsed_ast, include_attributes=False).encode("utf-8")
-
-        # Combine the AST bytes with the salt (encoded into bytes)
-        combined_data = ast_bytes + salt.encode("utf-8")
-
-        # Return the SHA-256 hash of the combined data (AST + salt)
-        return hashlib.sha256(combined_data).hexdigest()

union/_cli/_common.py

@@ -1,263 +0,0 @@
-from __future__ import annotations
-
-import importlib.util
-import logging
-import os
-import sys
-from abc import abstractmethod
-from dataclasses import Field, dataclass, field, replace
-from pathlib import Path
-from types import MappingProxyType, ModuleType
-from typing import Any, Dict, Iterable, List, Optional
-
-import rich_click as click
-from rich.table import Table
-
-import union.errors
-
-PROJECT_OPTION = click.Option(
-    param_decls=["-p", "--project"],
-    required=False,
-    type=str,
-    default="default",
-    help="Project to operate on",
-    show_default=True,
-)
-
-DOMAIN_OPTION = click.Option(
-    param_decls=["-d", "--domain"],
-    required=False,
-    type=str,
-    default="development",
-    help="Domain to operate on",
-    show_default=True,
-)
-
-DRY_RUN_OPTION = click.Option(
-    param_decls=["--dry-run", "--dryrun"],
-    required=False,
-    type=bool,
-    is_flag=True,
-    default=False,
-    help="Dry run, do not actually call the backend service.",
-    show_default=True,
-)
-
-
-def _common_options() -> List[click.Option]:
-    """
-    Common options for that will be added to all commands and groups that inherit from CommandBase or GroupBase.
-    """
-    return [PROJECT_OPTION, DOMAIN_OPTION]
-
-
-# This is global state for the CLI, it is manipulated by the main command
-
-
-@dataclass(frozen=True)
-class CLIConfig:
-    """
-    This is the global state for the CLI. It is manipulated by the main command.
-    """
-
-    log_level: int = logging.ERROR
-    endpoint: str = "dns:///dogfood.cloud-staging.union.ai"
-    insecure: bool = False
-    org_override: str | None = None
-
-    def replace(self, **kwargs) -> CLIConfig:
-        """
-        Replace the global state with a new one.
-        """
-        return replace(self, **kwargs)
-
-    def init(self, project: str | None = None, domain: str | None = None):
-        import union
-
-        union.init(
-            endpoint=self.endpoint,
-            project=project,
-            domain=domain,
-            log_level=self.log_level,
-            insecure=self.insecure,
-            org=self.org_override,
-        )
-
-
-class InvokeBaseMixin:
-    """
-    Mixin to catch grpc.RpcError, union.RpcError, other errors and other exceptions and raise them as
-     gclick.ClickException.
-    """
-
-    def invoke(self, ctx):
-        import grpc
-
-        try:
-            return super().invoke(ctx)  # type: ignore
-        except grpc.aio.AioRpcError as e:
-            if e.code() == grpc.StatusCode.UNAUTHENTICATED:
-                raise click.ClickException(f"Authentication failed. Please check your credentials. {e!s}")
-            if e.code() == grpc.StatusCode.NOT_FOUND:
-                raise click.ClickException(f"Requested object NOT FOUND. Please check your input. Error: {e!s}")
-            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
-                raise click.ClickException("Resource already exists.")
-            raise click.ClickException(f"Error invoking command: {e!s}") from e
-        except union.errors.InitializationError:
-            raise click.ClickException("Initialize the CLI with a remote configuration. For example, pass --endpoint")
-        except Exception as e:
-            raise click.ClickException(f"Error invoking command: {e}") from e
-
-
-class CommandBase(InvokeBaseMixin, click.RichCommand):
-    """
-    Base class for all commands, that adds common options to all commands if enabled.
-    """
-
-    common_options_enabled = True
-
-    def __init__(self, *args, **kwargs):
-        if "params" not in kwargs:
-            kwargs["params"] = []
-        if self.common_options_enabled:
-            kwargs["params"].extend(_common_options())
-        super().__init__(*args, **kwargs)
-
-
-class GroupBase(InvokeBaseMixin, click.RichGroup):
-    """
-    Base class for all commands, that adds common options to all commands if enabled.
-    """
-
-    common_options_enabled = True
-
-    def __init__(self, *args, **kwargs):
-        if "params" not in kwargs:
-            kwargs["params"] = []
-        if self.common_options_enabled:
-            kwargs["params"].extend(_common_options())
-        super().__init__(*args, **kwargs)
-
-
-class GroupBaseNoOptions(GroupBase):
-    common_options_enabled = False
-
-
-def make_click_option_field(o: click.Option) -> Field:
-    if o.multiple:
-        o.help = click.style("Multiple values allowed.", bold=True) + f"{o.help}"
-        return field(default_factory=lambda: o.default, metadata={"click.option": o})
-    return field(default=o.default, metadata={"click.option": o})
-
-
-def get_option_from_metadata(metadata: MappingProxyType) -> click.Option:
-    return metadata["click.option"]
-
-
-def key_value_callback(_: Any, param: str, values: List[str]) -> Optional[Dict[str, str]]:
-    """
-    Callback for click to parse key-value pairs.
-    """
-    if not values:
-        return None
-    result = {}
-    for v in values:
-        if "=" not in v:
-            raise click.BadParameter(f"Expected key-value pair of the form key=value, got {v}")
-        k, v_ = v.split("=", 1)
-        result[k.strip()] = v_.strip()
-    return result
-
-
-class ObjectsPerFileGroup(GroupBase):
-    """
-    Group that creates a command for each object in a python file.
-    """
-
-    def __init__(self, filename: Path, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if not filename.exists():
-            raise click.ClickException(f"{filename} does not exists")
-        self.filename = filename
-        self._objs = None
-
-    @abstractmethod
-    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
-        raise NotImplementedError
-
-    @property
-    def objs(self) -> Dict[str, Any]:
-        if self._objs is not None:
-            return self._objs
-
-        module_name = os.path.splitext(os.path.basename(self.filename))[0]
-        module_path = os.path.dirname(os.path.abspath(self.filename))
-
-        spec = importlib.util.spec_from_file_location(module_name, self.filename)
-        module = importlib.util.module_from_spec(spec)
-        sys.modules[module_name] = module
-
-        sys.path.append(module_path)
-        spec.loader.exec_module(module)
-
-        self._objs = self._filter_objects(module)
-        return self._objs
-
-    def list_commands(self, ctx):
-        m = list(self.objs.keys())
-        return sorted(m)
-
-    @abstractmethod
-    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command: ...
-
-    def get_command(self, ctx, obj_name):
-        obj = self.objs[obj_name]
-        return self._get_command_for_obj(ctx, obj_name, obj)
-
-
-class FileGroup(GroupBase):
-    """
-    Group that creates a command for each file in the current directory that is not __init__.py.
-    """
-
-    common_options_enabled = False
-
-    def __init__(
-        self,
-        *args,
-        directory: Path | None = None,
-        **kwargs,
-    ):
-        if "params" not in kwargs:
-            kwargs["params"] = []
-        super().__init__(*args, **kwargs)
-        self._files = None
-        self._dir = directory
-
-    @property
-    def files(self):
-        if self._files is None:
-            directory = self._dir or Path(".").absolute()
-            self._files = [os.fspath(p) for p in directory.glob("*.py") if p.name != "__init__.py"]
-        return self._files
-
-    def list_commands(self, ctx):
-        return self.files
-
-    def get_command(self, ctx, filename):
-        raise NotImplementedError
-
-
-def get_table(title: str, vals: Iterable[Any]) -> Table:
-    """
-    Get a table from a list of values.
-    """
-    table = Table(title=title)
-    headers = None
-    for p in vals:
-        if headers is None:
-            headers = [k for k, _ in p.__rich_repr__()]
-            for h in headers:
-                table.add_column(h.capitalize())
-        table.add_row(*[str(v) for _, v in p.__rich_repr__()])
-    return table