flyte 0.0.1b0__py3-none-any.whl

flyte/_cli/_get.py

@@ -0,0 +1,235 @@
+import asyncio
+from typing import Tuple, Union
+
+import rich_click as click
+from rich.console import Console
+from rich.pretty import pretty_repr
+
+from . import _common as common
+
+
+@click.group(name="get")
+def get():
+    """
+    Get the value of a task or environment.
+    """
+
+
+@get.command()
+@click.argument("name", type=str, required=False)
+@click.pass_obj
+def project(cfg: common.CLIConfig, name: str | None = None):
+    """
+    Get the current project.
+    """
+    from flyte.remote import Project
+
+    print(cfg)
+    cfg.init()
+
+    console = Console()
+    if name:
+        console.print(pretty_repr(Project.get(name)))
+    else:
+        console.print(common.get_table("Projects", Project.listall()))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.pass_obj
+def run(cfg: common.CLIConfig, name: str | None = None, project: str | None = None, domain: str | None = None):
+    """
+    Get the current run.
+    """
+    from flyte.remote import Run, RunDetails
+
+    cfg.init(project=project, domain=domain)
+
+    console = Console()
+    if name:
+        details = RunDetails.get(name=name)
+        console.print(pretty_repr(details))
+    else:
+        console.print(common.get_table("Runs", Run.listall()))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.argument("version", type=str, required=False)
+@click.pass_obj
+def task(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    version: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get the current task.
+    """
+    from flyte.remote import Task
+
+    cfg.init(project=project, domain=domain)
+
+    console = Console()
+    if name:
+        if not version:
+            raise click.BadParameter("Version is required when getting a task by name.")
+        v = Task.get(name=name, version=version)
+        if v is None:
+            raise click.BadParameter(f"Task {name} not found.")
+        t = v.fetch(v)
+        console.print(pretty_repr(t))
+    else:
+        raise click.BadParameter("Task listing is not supported yet, please provide a name.")
+        # console.print(common.get_table("Tasks", Task.listall()))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=True)
+@click.argument("action_name", type=str, required=False)
+@click.pass_obj
+def action(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get all actions for a run or details for a specific action.
+    """
+    import flyte.remote as remote
+
+    cfg.init(project=project, domain=domain)
+
+    console = Console()
+    if action_name:
+        console.print(pretty_repr(remote.Action.get(run_name=run_name, name=action_name)))
+    else:
+        # List all actions for the run
+        console.print(common.get_table(f"Actions for {run_name}", remote.Action.listall(for_run_name=run_name)))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=False)
+@click.argument("action_name", type=str, required=False)
+@click.option("--lines", "-l", type=int, default=30, help="Number of lines to show")
+@click.option("--show-ts", is_flag=True, help="Show timestamps")
+@click.option(
+    "--follow",
+    "-f",
+    is_flag=True,
+    default=False,
+    help="Follow the logs (like tail -f), will open a panel with logs scrolling.",
+)
+@click.pass_obj
+def logs(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    lines: int = 30,
+    show_ts: bool = False,
+    follow: bool = True,
+):
+    """
+    Get the current run.
+    """
+    import flyte.remote as remote
+
+    cfg.init(project=project, domain=domain)
+
+    async def _run_log_view(_obj):
+        task = asyncio.create_task(_obj.show_logs(max_lines=lines, show_ts=show_ts, raw=not follow))
+        try:
+            await task
+        except KeyboardInterrupt:
+            task.cancel()
+
+    if action_name:
+        obj = remote.Action.get(run_name=run_name, name=action_name)
+    else:
+        obj = remote.Run.get(run_name)
+    asyncio.run(_run_log_view(obj))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.pass_obj
+def secret(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get the current secret.
+    """
+    import flyte.remote as remote
+
+    cfg.init(project=project, domain=domain)
+
+    console = Console()
+    if name:
+        console.print(pretty_repr(remote.Secret.get(name)))
+    else:
+        console.print(common.get_table("Secrets", remote.Secret.listall()))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=True)
+@click.argument("action_name", type=str, required=False)
+@click.option("--inputs-only", "-i", is_flag=True, help="Show only inputs")
+@click.option("--outputs-only", "-o", is_flag=True, help="Show only outputs")
+@click.pass_obj
+def io(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    inputs_only: bool = False,
+    outputs_only: bool = False,
+):
+    """
+    Get the inputs and outputs of a run or action.
+    """
+    if inputs_only and outputs_only:
+        raise click.BadParameter("Cannot use both --inputs-only and --outputs-only")
+
+    import flyte.remote as remote
+
+    cfg.init(project=project, domain=domain)
+    console = Console()
+    if action_name:
+        obj = remote.ActionDetails.get(run_name=run_name, name=action_name)
+    else:
+        obj = remote.RunDetails.get(run_name)
+
+    async def _get_io(
+        details: Union[remote.RunDetails, remote.ActionDetails],
+    ) -> Tuple[remote.ActionInputs | None, remote.ActionOutputs | None | str]:
+        if inputs_only or outputs_only:
+            if inputs_only:
+                return await details.inputs(), None
+            elif outputs_only:
+                return None, await details.outputs()
+        inputs = await details.inputs()
+        outputs: remote.ActionOutputs | None | str = None
+        try:
+            outputs = await details.outputs()
+        except Exception:
+            # If the outputs are not available, we can still show the inputs
+            outputs = "[red]not yet available[/red]"
+        return inputs, outputs
+
+    inputs, outputs = asyncio.run(_get_io(obj))
+    # Show inputs and outputs side by side
+    console.print(
+        common.get_panel(
+            "Inputs & Outputs",
+            f"[green bold]Inputs[/green bold]\n{inputs}\n\n[blue bold]Outputs[/blue bold]\n{outputs}",
+        )
+    )

flyte/_cli/_run.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Dict, List, cast
+
+import click
+from click import Context
+from rich.console import Console
+from typing_extensions import get_args
+
+import flyte
+
+from .._code_bundle._utils import CopyFiles
+from .._task import TaskTemplate
+from ..remote import Run
+from . import _common as common
+from ._common import CLIConfig
+
+
+@dataclass
+class RunArguments:
+    project: str = field(
+        default=cast(str, common.PROJECT_OPTION.default), metadata={"click.option": common.PROJECT_OPTION}
+    )
+    domain: str = field(
+        default=cast(str, common.DOMAIN_OPTION.default), metadata={"click.option": common.DOMAIN_OPTION}
+    )
+    local: bool = field(
+        default=False,
+        metadata={
+            "click.option": click.Option(
+                ["--local"],
+                is_flag=True,
+                help="Run the task locally",
+            )
+        },
+    )
+    copy_style: CopyFiles = field(
+        default="loaded_modules",
+        metadata={
+            "click.option": click.Option(
+                ["--copy-style"],
+                type=click.Choice(get_args(CopyFiles)),
+                default="loaded_modules",
+                help="Copy style to use when running the task",
+            )
+        },
+    )
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> RunArguments:
+        return cls(**d)
+
+    @classmethod
+    def options(cls) -> List[click.Option]:
+        """
+        Return the set of base parameters added to run subcommand.
+        """
+        return [common.get_option_from_metadata(f.metadata) for f in fields(cls) if f.metadata]
+
+
+class RunTaskCommand(click.Command):
+    def __init__(self, obj_name: str, obj: Any, run_args: RunArguments, *args, **kwargs):
+        self.obj_name = obj_name
+        self.obj = cast(TaskTemplate, obj)
+        self.run_args = run_args
+        kwargs.pop("name", None)
+        super().__init__(obj_name, *args, **kwargs)
+
+    def invoke(self, ctx: Context):
+        obj: CLIConfig = ctx.obj or CLIConfig()
+        obj.init(self.run_args.project, self.run_args.domain)
+
+        r = flyte.with_runcontext(
+            copy_style=self.run_args.copy_style,
+            version=self.run_args.copy_style,
+            mode="local" if self.run_args.local else "remote",
+        ).run(self.obj)
+        if isinstance(r, Run) and r.action is not None:
+            console = Console()
+            console.print(
+                common.get_panel(
+                    "Run",
+                    f"[green bold]Created Run: {r.name} [/green bold] "
+                    f"(Project: {r.action.action_id.run.project}, Domain: {r.action.action_id.run.domain})\n\n"
+                    f"[blue bold]{r.url}[/blue bold]",
+                )
+            )
+
+
+class TaskPerFileGroup(common.ObjectsPerFileGroup):
+    """
+    Group that creates a command for each task in the current directory that is not __init__.py.
+    """
+
+    def __init__(self, filename: Path, run_args: RunArguments, *args, **kwargs):
+        args = (filename, *args)
+        super().__init__(*args, **kwargs)
+        self.run_args = run_args
+
+    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
+        return {k: v for k, v in module.__dict__.items() if isinstance(v, TaskTemplate)}
+
+    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
+        obj = cast(TaskTemplate, obj)
+        return RunTaskCommand(
+            obj_name=obj_name,
+            obj=obj,
+            help=obj.docs.__help__str__() if obj.docs else None,
+            run_args=self.run_args,
+        )
+
+
+class TaskFiles(common.FileGroup):
+    """
+    Group that creates a command for each file in the current directory that is not __init__.py.
+    """
+
+    common_options_enabled = False
+
+    def __init__(
+        self,
+        *args,
+        directory: Path | None = None,
+        **kwargs,
+    ):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        kwargs["params"].extend(RunArguments.options())
+        super().__init__(*args, directory=directory, **kwargs)
+
+    def get_command(self, ctx, filename):
+        run_args = RunArguments.from_dict(ctx.params)
+        fp = Path(filename)
+        if not fp.exists():
+            raise click.BadParameter(f"File {filename} does not exist")
+        if fp.is_dir():
+            return TaskFiles(directory=fp)
+        return TaskPerFileGroup(
+            filename=Path(filename),
+            run_args=run_args,
+            name=filename,
+            help=f"Run, functions decorated `env.task` or instances of Tasks in {filename}",
+        )
+
+
+run = TaskFiles(
+    name="run",
+    help="Run a task from a python file",
+)

flyte/_cli/main.py

@@ -0,0 +1,72 @@
+import rich_click as click
+
+from ._common import CLIConfig
+from ._create import create
+from ._deploy import deploy
+from ._get import get
+from ._run import run
+
+
+def _verbosity_to_loglevel(verbosity: int) -> int | None:
+    """
+    Converts a verbosity level from the CLI to a logging level.
+
+    :param verbosity: verbosity level from the CLI
+    :return: logging level
+    """
+    import logging
+
+    match verbosity:
+        case 0:
+            return None
+        case 1:
+            return logging.WARNING
+        case 2:
+            return logging.INFO
+        case _:
+            return logging.DEBUG
+
+
+@click.group(cls=click.RichGroup)
+@click.option(
+    "--endpoint",
+    type=str,
+    required=False,
+    help="The endpoint to connect to, this will override any config and simply used pkce to connect.",
+)
+@click.option(
+    "--insecure",
+    is_flag=True,
+    required=False,
+    help="insecure",
+    type=bool,
+    default=False,
+)
+@click.option(
+    "-v",
+    "--verbose",
+    required=False,
+    help="Show verbose messages and exception traces",
+    count=True,
+    default=0,
+    type=int,
+)
+@click.option(
+    "--org-override",
+    type=str,
+    required=False,
+    help="Override for org",
+)
+@click.pass_context
+def main(ctx: click.Context, endpoint: str | None, insecure: bool, verbose: int, org_override: str | None):
+    """
+    v2 cli. Root command, please use one of the subcommands.
+    """
+    log_level = _verbosity_to_loglevel(verbose)
+    ctx.obj = CLIConfig(log_level=log_level, endpoint=endpoint, insecure=insecure, org_override=org_override)
+
+
+main.add_command(run)
+main.add_command(deploy)
+main.add_command(get)  # type: ignore
+main.add_command(create)  # type: ignore

flyte/_code_bundle/__init__.py

@@ -0,0 +1,8 @@
+from ._ignore import GitIgnore, IgnoreGroup, StandardIgnore
+from ._utils import CopyFiles
+from .bundle import build_code_bundle, build_pkl_bundle, download_bundle
+
+__all__ = ["CopyFiles", "build_code_bundle", "build_pkl_bundle", "default_ignores", "download_bundle"]
+
+
+default_ignores = [GitIgnore, StandardIgnore, IgnoreGroup]

flyte/_code_bundle/_ignore.py

@@ -0,0 +1,113 @@
+import os
+import pathlib
+import subprocess
+import tarfile as _tarfile
+from abc import ABC, abstractmethod
+from fnmatch import fnmatch
+from pathlib import Path
+from shutil import which
+from typing import List, Optional, Type
+
+from flyte._logging import logger
+
+
+class Ignore(ABC):
+    """Base for Ignores, implements core logic. Children have to implement _is_ignored"""
+
+    def __init__(self, root: Path):
+        self.root = root
+
+    def is_ignored(self, path: pathlib.Path) -> bool:
+        return self._is_ignored(path)
+
+    def tar_filter(self, tarinfo: _tarfile.TarInfo) -> Optional[_tarfile.TarInfo]:
+        if self.is_ignored(pathlib.Path(tarinfo.name)):
+            return None
+        return tarinfo
+
+    @abstractmethod
+    def _is_ignored(self, path: pathlib.Path) -> bool:
+        pass
+
+
+class GitIgnore(Ignore):
+    """Uses git cli (if available) to list all ignored files and compare with those."""
+
+    def __init__(self, root: Path):
+        super().__init__(root)
+        self.has_git = which("git") is not None
+        self.ignored_files = self._list_ignored_files()
+        self.ignored_dirs = self._list_ignored_dirs()
+
+    def _git_wrapper(self, extra_args: List[str]) -> set[str]:
+        if self.has_git:
+            out = subprocess.run(
+                ["git", "ls-files", "-io", "--exclude-standard", *extra_args],
+                cwd=self.root,
+                capture_output=True,
+                check=False,
+            )
+            if out.returncode == 0:
+                return set(out.stdout.decode("utf-8").split("\n")[:-1])
+            logger.info(f"Could not determine ignored paths due to:\n{out.stderr!r}\nNot applying any filters")
+            return set()
+        logger.info("No git executable found, not applying any filters")
+        return set()
+
+    def _list_ignored_files(self) -> set[str]:
+        return self._git_wrapper([])
+
+    def _list_ignored_dirs(self) -> set[str]:
+        return self._git_wrapper(["--directory"])
+
+    def _is_ignored(self, path: pathlib.Path) -> bool:
+        if self.ignored_files:
+            # git-ls-files uses POSIX paths
+            if Path(path).as_posix() in self.ignored_files:
+                return True
+            # Ignore empty directories
+            if os.path.isdir(os.path.join(self.root, path)) and self.ignored_dirs:
+                return Path(path).as_posix() + "/" in self.ignored_dirs
+        return False
+
+
+STANDARD_IGNORE_PATTERNS = ["*.pyc", ".cache", ".cache/*", "__pycache__", "**/__pycache__"]
+
+
+class StandardIgnore(Ignore):
+    """Retains the standard ignore functionality that previously existed. Could in theory
+    by fed with custom ignore patterns from cli."""
+
+    def __init__(self, root: Path, patterns: Optional[List[str]] = None):
+        super().__init__(root)
+        self.patterns = patterns if patterns else STANDARD_IGNORE_PATTERNS
+
+    def _is_ignored(self, path: pathlib.Path) -> bool:
+        for pattern in self.patterns:
+            if fnmatch(str(path), pattern):
+                return True
+        return False
+
+
+class IgnoreGroup(Ignore):
+    """Groups multiple Ignores and checks a path against them. A file is ignored if any
+    Ignore considers it ignored."""
+
+    def __init__(self, root: Path, *ignores: Type[Ignore]):
+        super().__init__(root)
+        self.ignores = [ignore(root) for ignore in ignores]
+
+    def _is_ignored(self, path: pathlib.Path) -> bool:
+        for ignore in self.ignores:
+            if ignore.is_ignored(path):
+                return True
+        return False
+
+    def list_ignored(self) -> List[str]:
+        ignored = []
+        for dir, _, files in self.root.walk():
+            for file in files:
+                abs_path = dir / file
+                if self.is_ignored(abs_path):
+                    ignored.append(str(abs_path.relative_to(self.root)))
+        return ignored