flyte 0.0.1b3__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

union/_cli/_create.py

@@ -1,40 +0,0 @@
-from typing import get_args
-
-import rich_click as click
-
-import union._cli._common as common
-from union.remote._secret import SecretTypes
-
-
-@click.group(name="create")
-def create():
-    """
-    Create a new task or environment.
-    """
-
-
-@create.command(cls=common.CommandBase)
-@click.argument("name", type=str, required=True)
-@click.argument("value", type=str, required=False)
-@click.option("--from-file", type=click.Path(exists=True), help="Path to the file with the binary secret.")
-@click.option("--type", type=click.Choice(get_args(SecretTypes)), default="regular", help="Type of the secret.")
-@click.pass_obj
-def secret(
-    cfg: common.CLIConfig,
-    name: str,
-    value: str | None = None,
-    from_file: str | None = None,
-    type: SecretTypes = "regular",
-    project: str | None = None,
-    domain: str | None = None,
-):
-    """
-    Create a new secret.
-    """
-    from union.remote import Secret
-
-    cfg.init(project, domain)
-    if from_file:
-        with open(from_file, "rb") as f:
-            value = f.read()
-    Secret.create(name=name, value=value, type=type)

union/_cli/_delete.py

@@ -1,23 +0,0 @@
-import rich_click as click
-
-import union._cli._common as common
-
-
-@click.group(name="delete")
-def delete():
-    """
-    Delete a task or environment.
-    """
-
-
-@click.command(cls=common.CommandBase)
-@click.argument("name", type=str, required=True)
-@click.pass_obj
-def secret(cfg: common.CLIConfig, name: str, project: str | None = None, domain: str | None = None):
-    """
-    Delete a secret.
-    """
-    from union.remote import Secret
-
-    cfg.init(project, domain)
-    Secret.delete(name=name)

union/_cli/_deploy.py

@@ -1,120 +0,0 @@
-from dataclasses import dataclass, fields
-from pathlib import Path
-from types import ModuleType
-from typing import Any, Dict, List, cast
-
-import click
-
-import union
-
-from .._code_bundle._utils import CopyFiles
-from . import _common as common
-from ._common import CLIConfig
-
-
-@dataclass
-class DeployArguments:
-    project: str = common.make_click_option_field(common.PROJECT_OPTION)
-    domain: str = common.make_click_option_field(common.DOMAIN_OPTION)
-    dry_run: bool = common.make_click_option_field(common.DRY_RUN_OPTION)
-    copy_style: CopyFiles = common.make_click_option_field(
-        click.Option(
-            ["--copy-style"],
-            type=click.Choice(["loaded_modules", "all_files", "none"]),
-            default="loaded_modules",
-            help="Copy style to use when running the task",
-        )
-    )
-    version: CopyFiles = common.make_click_option_field(
-        click.Option(
-            ["--version"],
-            type=str,
-            required=False,
-            help="If no files are copied, user must specify a version to register tasks with",
-        )
-    )
-
-    @classmethod
-    def from_dict(cls, d: Dict[str, Any]) -> "DeployArguments":
-        return cls(**d)
-
-    @classmethod
-    def options(cls) -> List[click.Option]:
-        """
-        Return the set of base parameters added to every pyflyte run workflow subcommand.
-        """
-        return [common.get_option_from_metadata(f.metadata) for f in fields(cls) if f.metadata]
-
-
-class DeployEnvCommand(click.Command):
-    def __init__(self, obj_name: str, obj: Any, deploy_args: DeployArguments, *args, **kwargs):
-        self.obj_name = obj_name
-        self.obj = obj
-        self.deploy_args = deploy_args
-        super().__init__(name=obj_name, *args, **kwargs)
-
-    def invoke(self, ctx):
-        print(f"Deploying environment: {self.obj_name}")
-        obj: CLIConfig = ctx.obj
-        obj.remote_init(self.deploy_args.project, self.deploy_args.domain)
-        return union.deploy(
-            self.obj,
-            dryrun=self.deploy_args.dry_run,
-            copy_style=self.deploy_args.copy_style,
-            version=self.deploy_args.version,
-        )
-
-
-class EnvPerFileGroup(common.ObjectsPerFileGroup):
-    """
-    Group that creates a command for each task in the current directory that is not __init__.py.
-    """
-
-    def __init__(self, filename: Path, deploy_args: DeployArguments, *args, **kwargs):
-        super().__init__(*args, filename=filename, **kwargs)
-        self.deploy_args = deploy_args
-
-    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
-        return {k: v for k, v in module.__dict__.items() if isinstance(v, union.TaskEnvironment)}
-
-    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
-        obj = cast(union.TaskEnvironment, obj)
-        return DeployEnvCommand(
-            obj_name=obj_name,
-            obj=obj,
-            help=obj.description,
-            deploy_args=self.deploy_args,
-        )
-
-
-class EnvFiles(common.FileGroup):
-    """
-    Group that creates a command for each file in the current directory that is not __init__.py.
-    """
-
-    common_options_enabled = False
-
-    def __init__(
-        self,
-        *args,
-        **kwargs,
-    ):
-        if "params" not in kwargs:
-            kwargs["params"] = []
-        kwargs["params"].extend(DeployArguments.options())
-        super().__init__(*args, **kwargs)
-
-    def get_command(self, ctx, filename):
-        deploy_args = DeployArguments.from_dict(ctx.params)
-        return EnvPerFileGroup(
-            filename=Path(filename),
-            deploy_args=deploy_args,
-            name=filename,
-            help=f"Run, functions decorated `env.task` or instances of Tasks in {filename}",
-        )
-
-
-deploy = EnvFiles(
-    name="deploy",
-    help="deploy one or more environments from a python file.",
-)

union/_cli/_get.py

@@ -1,162 +0,0 @@
-import asyncio
-
-import rich_click as click
-from rich.console import Console
-
-from . import _common as common
-
-
-@click.group(name="get")
-def get():
-    """
-    Get the value of a task or environment.
-    """
-
-
-@get.command()
-@click.argument("name", type=str, required=False)
-@click.pass_obj
-def project(cfg: common.CLIConfig, name: str | None = None):
-    """
-    Get the current project.
-    """
-    from union.remote import Project
-
-    print(cfg)
-    cfg.init()
-
-    console = Console()
-    if name:
-        console.print(Project.get(name))
-    else:
-        console.print(common.get_table("Projects", Project.listall()))
-
-
-@get.command(cls=common.CommandBase)
-@click.argument("name", type=str, required=False)
-@click.pass_obj
-def run(cfg: common.CLIConfig, name: str | None = None, project: str | None = None, domain: str | None = None):
-    """
-    Get the current run.
-    """
-    from union.remote import Run, RunDetails
-
-    cfg.init(project=project, domain=domain)
-
-    console = Console()
-    if name:
-        details = RunDetails.get(name=name)
-        console.print(details)
-    else:
-        console.print(common.get_table("Runs", Run.listall()))
-
-
-@get.command(cls=common.CommandBase)
-@click.argument("name", type=str, required=False)
-@click.argument("version", type=str, required=False)
-@click.pass_obj
-def task(
-    cfg: common.CLIConfig,
-    name: str | None = None,
-    version: str | None = None,
-    project: str | None = None,
-    domain: str | None = None,
-):
-    """
-    Get the current task.
-    """
-    from union.remote import Task
-
-    cfg.init(project=project, domain=domain)
-
-    console = Console()
-    if name:
-        v = Task.get(name=name, version=version)
-        if v is None:
-            raise click.BadParameter(f"Task {name} not found.")
-        t = v.fetch()
-        console.print(t)
-    else:
-        raise click.BadParameter("Task listing is not supported yet, please provide a name.")
-        # console.print(common.get_table("Tasks", Task.listall()))
-
-
-@get.command(cls=common.CommandBase)
-@click.argument("run_name", type=str, required=True)
-@click.argument("action_name", type=str, required=False)
-@click.pass_obj
-def action(
-    cfg: common.CLIConfig,
-    run_name: str,
-    action_name: str | None = None,
-    project: str | None = None,
-    domain: str | None = None,
-):
-    """
-    Get all actions for a run or details for a specific action.
-    """
-    import union.remote as remote
-
-    cfg.init(project=project, domain=domain)
-
-    console = Console()
-    if action_name:
-        console.print(remote.Action.get(run_name=run_name, name=action_name))
-    else:
-        # List all actions for the run
-        console.print(common.get_table(f"Actions for {run_name}", remote.Action.listall(for_run_name=run_name)))
-
-
-@get.command(cls=common.CommandBase)
-@click.argument("run_name", type=str, required=False)
-@click.argument("action_name", type=str, required=False)
-@click.pass_obj
-def logs(
-    cfg: common.CLIConfig,
-    run_name: str,
-    action_name: str | None = None,
-    project: str | None = None,
-    domain: str | None = None,
-):
-    """
-    Get the current run.
-    """
-    import union.remote as remote
-
-    cfg.init(project=project, domain=domain)
-
-    async def _run_log_view(_obj):
-        task = asyncio.create_task(_obj.show_logs())
-        try:
-            await task
-        except KeyboardInterrupt:
-            task.cancel()
-
-    if action_name:
-        obj = remote.Action.get(run_name=run_name, name=action_name)
-    else:
-        obj = remote.Run.get(run_name)
-    asyncio.run(_run_log_view(obj))
-
-
-@get.command(cls=common.CommandBase)
-@click.argument("name", type=str, required=False)
-@click.pass_obj
-def secret(
-    cfg: common.CLIConfig,
-    name: str | None = None,
-    project: str | None = None,
-    domain: str | None = None,
-):
-    """
-    Get the current secret.
-    """
-    import union.remote as remote
-
-    cfg.init(project=project, domain=domain)
-
-    console = Console()
-    if name:
-        console.print(remote.Secret.get(name))
-    else:
-        console.print(common.get_table("Secrets", remote.Secret.listall()))

union/_cli/_run.py

@@ -1,150 +0,0 @@
-from __future__ import annotations
-
-from union._logging import logger
-from dataclasses import dataclass, fields
-from pathlib import Path
-from types import ModuleType
-from typing import Any, Dict, List, cast
-
-import click
-
-import union
-
-from .._code_bundle._utils import CopyFiles
-from .._initialize import initialize_in_cluster, S3
-from .._task import TaskTemplate
-from . import _common as common
-from ._common import CLIConfig
-
-
-@dataclass
-class RunArguments:
-    project: str = common.make_click_option_field(common.PROJECT_OPTION)
-    domain: str = common.make_click_option_field(common.DOMAIN_OPTION)
-    local: bool = common.make_click_option_field(
-        click.Option(
-            ["--local"],
-            is_flag=True,
-            help="Run the task locally",
-        )
-    )
-    hybrid: bool = common.make_click_option_field(
-        click.Option(
-            ["--hybrid"],
-            is_flag=True,
-            help="hybrid mode",
-        )
-    )
-    copy_style: CopyFiles = common.make_click_option_field(
-        click.Option(
-            ["--copy-style"],
-            type=click.Choice(["loaded_modules", "all_files", "none"]),
-            default="loaded_modules",
-            help="Copy style to use when running the task",
-        )
-    )
-
-    @classmethod
-    def from_dict(cls, d: Dict[str, Any]) -> RunArguments:
-        return cls(**d)
-
-    @classmethod
-    def options(cls) -> List[click.Option]:
-        """
-        Return the set of base parameters added to run subcommand.
-        """
-        return [common.get_option_from_metadata(f.metadata) for f in fields(cls) if f.metadata]
-
-
-class RunTaskCommand(click.Command):
-    def __init__(self, obj_name: str, obj: Any, run_args: RunArguments, *args, **kwargs):
-        self.obj_name = obj_name
-        self.obj = cast(TaskTemplate, obj)
-        self.run_args = run_args
-        super().__init__(name=obj_name, *args, **kwargs)
-
-    def invoke(self, ctx):
-        obj: CLIConfig = ctx.obj
-        obj.init(self.run_args.project, self.run_args.domain)
-
-        # todo: remove this when backend supports
-        from union.storage import get_random_local_path
-
-        run_name = "random-run-" + str(get_random_local_path())[-8:]
-        if self.run_args.local:
-            mode = "local"
-        elif self.run_args.hybrid:
-            mode = "hybrid"
-            # TODO configure storage correctly for cluster
-            initialize_in_cluster(storage=S3.auto())
-        else:
-            mode = "remote"
-        logger.debug(f"Running {self.obj_name} in {mode} mode")
-        r = union.with_runcontext(
-            name=run_name,
-            copy_style=self.run_args.copy_style,
-            version=self.run_args.copy_style,
-            mode=mode,
-        ).run(self.obj)
-        # click.secho(f"Created Run: {r.action.id}", fg="green")
-
-
-class TaskPerFileGroup(common.ObjectsPerFileGroup):
-    """
-    Group that creates a command for each task in the current directory that is not __init__.py.
-    """
-
-    def __init__(self, filename: Path, run_args: RunArguments, *args, **kwargs):
-        super().__init__(*args, filename=filename, **kwargs)
-        self.run_args = run_args
-
-    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
-        return {k: v for k, v in module.__dict__.items() if isinstance(v, TaskTemplate)}
-
-    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
-        obj = cast(TaskTemplate, obj)
-        return RunTaskCommand(
-            obj_name=obj_name,
-            obj=obj,
-            help=obj.docs.__help__str__() if obj.docs else None,
-            run_args=self.run_args,
-        )
-
-
-class TaskFiles(common.FileGroup):
-    """
-    Group that creates a command for each file in the current directory that is not __init__.py.
-    """
-
-    common_options_enabled = False
-
-    def __init__(
-        self,
-        *args,
-        directory: Path | None = None,
-        **kwargs,
-    ):
-        if "params" not in kwargs:
-            kwargs["params"] = []
-        kwargs["params"].extend(RunArguments.options())
-        super().__init__(*args, directory=directory, **kwargs)
-
-    def get_command(self, ctx, filename):
-        run_args = RunArguments.from_dict(ctx.params)
-        fp = Path(filename)
-        if not fp.exists():
-            raise click.BadParameter(f"File {filename} does not exist")
-        if fp.is_dir():
-            return TaskFiles(directory=fp)
-        return TaskPerFileGroup(
-            filename=Path(filename),
-            run_args=run_args,
-            name=filename,
-            help=f"Run, functions decorated `env.task` or instances of Tasks in {filename}",
-        )
-
-
-run = TaskFiles(
-    name="run",
-    help="Run a task from a python file",
-)

union/_cli/main.py

@@ -1,72 +0,0 @@
-import rich_click as click
-
-from ._common import CLIConfig
-from ._create import create
-from ._deploy import deploy
-from ._get import get
-from ._run import run
-
-
-def _verbosity_to_loglevel(verbosity: int) -> int | None:
-    """
-    Converts a verbosity level from the CLI to a logging level.
-
-    :param verbosity: verbosity level from the CLI
-    :return: logging level
-    """
-    import logging
-
-    match verbosity:
-        case 0:
-            return None
-        case 1:
-            return logging.WARNING
-        case 2:
-            return logging.INFO
-        case _:
-            return logging.DEBUG
-
-
-@click.group(cls=click.RichGroup)
-@click.option(
-    "--endpoint",
-    type=str,
-    required=False,
-    help="The endpoint to connect to, this will override any config and simply used pkce to connect.",
-)
-@click.option(
-    "--insecure",
-    is_flag=True,
-    required=False,
-    help="insecure",
-    type=bool,
-    default=False,
-)
-@click.option(
-    "-v",
-    "--verbose",
-    required=False,
-    help="Show verbose messages and exception traces",
-    count=True,
-    default=0,
-    type=int,
-)
-@click.option(
-    "--org-override",
-    type=str,
-    required=False,
-    help="Override for org",
-)
-@click.pass_context
-def main(ctx: click.Context, endpoint: str | None, insecure: bool, verbose: int, org_override: str | None):
-    """
-    v2 cli. Root command, please use one of the subcommands.
-    """
-    log_level = _verbosity_to_loglevel(verbose)
-    ctx.obj = CLIConfig(log_level=log_level, endpoint=endpoint, insecure=insecure, org_override=org_override)
-
-
-main.add_command(run)
-main.add_command(deploy)
-main.add_command(get)  # type: ignore
-main.add_command(create)  # type: ignore

union/_code_bundle/__init__.py

@@ -1,8 +0,0 @@
-from ._ignore import GitIgnore, IgnoreGroup, StandardIgnore
-from ._utils import CopyFiles
-from .bundle import build_code_bundle, build_pkl_bundle, download_bundle
-
-__all__ = ["CopyFiles", "build_code_bundle", "build_pkl_bundle", "default_ignores", "download_bundle"]
-
-
-default_ignores = [GitIgnore, StandardIgnore, IgnoreGroup]

union/_code_bundle/_ignore.py

@@ -1,113 +0,0 @@
-import os
-import pathlib
-import subprocess
-import tarfile as _tarfile
-from abc import ABC, abstractmethod
-from fnmatch import fnmatch
-from pathlib import Path
-from shutil import which
-from typing import List, Optional, Type
-
-from union._logging import logger
-
-
-class Ignore(ABC):
-    """Base for Ignores, implements core logic. Children have to implement _is_ignored"""
-
-    def __init__(self, root: Path):
-        self.root = root
-
-    def is_ignored(self, path: pathlib.Path) -> bool:
-        return self._is_ignored(path)
-
-    def tar_filter(self, tarinfo: _tarfile.TarInfo) -> Optional[_tarfile.TarInfo]:
-        if self.is_ignored(pathlib.Path(tarinfo.name)):
-            return None
-        return tarinfo
-
-    @abstractmethod
-    def _is_ignored(self, path: pathlib.Path) -> bool:
-        pass
-
-
-class GitIgnore(Ignore):
-    """Uses git cli (if available) to list all ignored files and compare with those."""
-
-    def __init__(self, root: Path):
-        super().__init__(root)
-        self.has_git = which("git") is not None
-        self.ignored_files = self._list_ignored_files()
-        self.ignored_dirs = self._list_ignored_dirs()
-
-    def _git_wrapper(self, extra_args: List[str]) -> set[str]:
-        if self.has_git:
-            out = subprocess.run(
-                ["git", "ls-files", "-io", "--exclude-standard", *extra_args],
-                cwd=self.root,
-                capture_output=True,
-                check=False,
-            )
-            if out.returncode == 0:
-                return set(out.stdout.decode("utf-8").split("\n")[:-1])
-            logger.info(f"Could not determine ignored paths due to:\n{out.stderr}\nNot applying any filters")
-            return set()
-        logger.info("No git executable found, not applying any filters")
-        return set()
-
-    def _list_ignored_files(self) -> set[str]:
-        return self._git_wrapper([])
-
-    def _list_ignored_dirs(self) -> set[str]:
-        return self._git_wrapper(["--directory"])
-
-    def _is_ignored(self, path: str) -> bool:
-        if self.ignored_files:
-            # git-ls-files uses POSIX paths
-            if Path(path).as_posix() in self.ignored_files:
-                return True
-            # Ignore empty directories
-            if os.path.isdir(os.path.join(self.root, path)) and self.ignored_dirs:
-                return Path(path).as_posix() + "/" in self.ignored_dirs
-        return False
-
-
-STANDARD_IGNORE_PATTERNS = ["*.pyc", ".cache", ".cache/*", "__pycache__", "**/__pycache__"]
-
-
-class StandardIgnore(Ignore):
-    """Retains the standard ignore functionality that previously existed. Could in theory
-    by fed with custom ignore patterns from cli."""
-
-    def __init__(self, root: Path, patterns: Optional[List[str]] = None):
-        super().__init__(root)
-        self.patterns = patterns if patterns else STANDARD_IGNORE_PATTERNS
-
-    def _is_ignored(self, path: pathlib.Path) -> bool:
-        for pattern in self.patterns:
-            if fnmatch(str(path), pattern):
-                return True
-        return False
-
-
-class IgnoreGroup(Ignore):
-    """Groups multiple Ignores and checks a path against them. A file is ignored if any
-    Ignore considers it ignored."""
-
-    def __init__(self, root: Path, *ignores: Type[Ignore]):
-        super().__init__(root)
-        self.ignores = [ignore(root) for ignore in ignores]
-
-    def _is_ignored(self, path: pathlib.Path) -> bool:
-        for ignore in self.ignores:
-            if ignore.is_ignored(path):
-                return True
-        return False
-
-    def list_ignored(self) -> List[str]:
-        ignored = []
-        for dir, _, files in self.root.walk():
-            for file in files:
-                abs_path = dir / file
-                if self.is_ignored(abs_path):
-                    ignored.append(str(abs_path.relative_to(self.root)))
-        return ignored