flyte 0.0.1b0__py3-none-any.whl

flyte/_cli/_common.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+import importlib.util
+import logging
+import os
+import sys
+from abc import abstractmethod
+from dataclasses import dataclass, replace
+from pathlib import Path
+from types import MappingProxyType, ModuleType
+from typing import Any, Dict, Iterable, List, Optional
+
+import rich.box
+import rich_click as click
+from rich.panel import Panel
+from rich.table import Table
+
+import flyte.errors
+
+PREFERRED_BORDER_COLOR = "dim cyan"
+PREFERRED_ACCENT_COLOR = "bold #FFD700"
+HEADER_STYLE = f"{PREFERRED_ACCENT_COLOR} on black"
+
+PROJECT_OPTION = click.Option(
+    param_decls=["-p", "--project"],
+    required=False,
+    type=str,
+    default="default",
+    help="Project to operate on",
+    show_default=True,
+)
+
+DOMAIN_OPTION = click.Option(
+    param_decls=["-d", "--domain"],
+    required=False,
+    type=str,
+    default="development",
+    help="Domain to operate on",
+    show_default=True,
+)
+
+DRY_RUN_OPTION = click.Option(
+    param_decls=["--dry-run", "--dryrun"],
+    required=False,
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Dry run, do not actually call the backend service.",
+    show_default=True,
+)
+
+
+def _common_options() -> List[click.Option]:
+    """
+    Common options for that will be added to all commands and groups that inherit from CommandBase or GroupBase.
+    """
+    return [PROJECT_OPTION, DOMAIN_OPTION]
+
+
+# This is global state for the CLI, it is manipulated by the main command
+
+
+@dataclass(frozen=True)
+class CLIConfig:
+    """
+    This is the global state for the CLI. It is manipulated by the main command.
+    """
+
+    log_level: int | None = logging.ERROR
+    endpoint: str | None = None
+    insecure: bool = False
+    org_override: str | None = None
+
+    def replace(self, **kwargs) -> CLIConfig:
+        """
+        Replace the global state with a new one.
+        """
+        return replace(self, **kwargs)
+
+    def init(self, project: str | None = None, domain: str | None = None):
+        import flyte
+
+        flyte.init(
+            endpoint=self.endpoint,
+            insecure=self.insecure,
+            org=self.org_override,
+            project=project,
+            domain=domain,
+            log_level=self.log_level,
+        )
+
+
+class InvokeBaseMixin:
+    """
+    Mixin to catch grpc.RpcError, flyte.RpcError, other errors and other exceptions and raise them as
+     gclick.ClickException.
+    """
+
+    def invoke(self, ctx):
+        import grpc
+
+        try:
+            return super().invoke(ctx)  # type: ignore
+        except grpc.aio.AioRpcError as e:
+            if e.code() == grpc.StatusCode.UNAUTHENTICATED:
+                raise click.ClickException(f"Authentication failed. Please check your credentials. {e.details()}")
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise click.ClickException(f"Requested object NOT FOUND. Please check your input. Error: {e.details()}")
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise click.ClickException("Resource already exists.")
+            raise click.ClickException(f"RPC error invoking command: {e!s}") from e
+        except flyte.errors.InitializationError:
+            raise click.ClickException("Initialize the CLI with a remote configuration. For example, pass --endpoint")
+        except Exception as e:
+            raise click.ClickException(f"Error invoking command: {e}") from e
+
+
+class CommandBase(InvokeBaseMixin, click.RichCommand):
+    """
+    Base class for all commands, that adds common options to all commands if enabled.
+    """
+
+    common_options_enabled = True
+
+    def __init__(self, *args, **kwargs):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        if self.common_options_enabled:
+            kwargs["params"].extend(_common_options())
+        super().__init__(*args, **kwargs)
+
+
+class GroupBase(InvokeBaseMixin, click.RichGroup):
+    """
+    Base class for all commands, that adds common options to all commands if enabled.
+    """
+
+    common_options_enabled = True
+
+    def __init__(self, *args, **kwargs):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        if self.common_options_enabled:
+            kwargs["params"].extend(_common_options())
+        super().__init__(*args, **kwargs)
+
+
+class GroupBaseNoOptions(GroupBase):
+    common_options_enabled = False
+
+
+def get_option_from_metadata(metadata: MappingProxyType) -> click.Option:
+    return metadata["click.option"]
+
+
+def key_value_callback(_: Any, param: str, values: List[str]) -> Optional[Dict[str, str]]:
+    """
+    Callback for click to parse key-value pairs.
+    """
+    if not values:
+        return None
+    result = {}
+    for v in values:
+        if "=" not in v:
+            raise click.BadParameter(f"Expected key-value pair of the form key=value, got {v}")
+        k, v_ = v.split("=", 1)
+        result[k.strip()] = v_.strip()
+    return result
+
+
+class ObjectsPerFileGroup(GroupBase):
+    """
+    Group that creates a command for each object in a python file.
+    """
+
+    def __init__(self, filename: Path, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not filename.exists():
+            raise click.ClickException(f"{filename} does not exists")
+        self.filename = filename
+        self._objs: Dict[str, Any] | None = None
+
+    @abstractmethod
+    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
+        """
+        Filter the objects in the module to only include the ones we want to expose.
+        """
+        raise NotImplementedError
+
+    @property
+    def objs(self) -> Dict[str, Any]:
+        if self._objs is not None:
+            return self._objs
+
+        module_name = os.path.splitext(os.path.basename(self.filename))[0]
+        module_path = os.path.dirname(os.path.abspath(self.filename))
+
+        spec = importlib.util.spec_from_file_location(module_name, self.filename)
+        if spec is None or spec.loader is None:
+            raise click.ClickException(f"Could not load module {module_name} from {self.filename}")
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+
+        sys.path.append(module_path)
+        spec.loader.exec_module(module)
+
+        self._objs = self._filter_objects(module)
+        if not self._objs:
+            raise click.ClickException(f"No objects found in {self.filename}")
+        return self._objs
+
+    def list_commands(self, ctx):
+        m = list(self.objs.keys())
+        return sorted(m)
+
+    @abstractmethod
+    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command: ...
+
+    def get_command(self, ctx, obj_name):
+        obj = self.objs[obj_name]
+        return self._get_command_for_obj(ctx, obj_name, obj)
+
+
+class FileGroup(GroupBase):
+    """
+    Group that creates a command for each file in the current directory that is not __init__.py.
+    """
+
+    common_options_enabled = False
+
+    def __init__(
+        self,
+        *args,
+        directory: Path | None = None,
+        **kwargs,
+    ):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        super().__init__(*args, **kwargs)
+        self._files = None
+        self._dir = directory
+
+    @property
+    def files(self):
+        if self._files is None:
+            directory = self._dir or Path(".").absolute()
+            self._files = [os.fspath(p) for p in directory.glob("*.py") if p.name != "__init__.py"]
+        return self._files
+
+    def list_commands(self, ctx):
+        return self.files
+
+    def get_command(self, ctx, filename):
+        raise NotImplementedError
+
+
+def get_table(title: str, vals: Iterable[Any]) -> Table:
+    """
+    Get a table from a list of values.
+    """
+    table = Table(
+        title=title,
+        box=rich.box.SQUARE_DOUBLE_HEAD,
+        header_style=HEADER_STYLE,
+        show_header=True,
+        border_style=PREFERRED_BORDER_COLOR,
+    )
+    headers = None
+    for p in vals:
+        if headers is None:
+            headers = [k for k, _ in p.__rich_repr__()]
+            for h in headers:
+                table.add_column(h.capitalize())
+        table.add_row(*[str(v) for _, v in p.__rich_repr__()])
+    return table
+
+
+def get_panel(title: str, renderable: Any) -> Panel:
+    """
+    Get a panel from a list of values.
+    """
+    return Panel.fit(
+        renderable,
+        title=f"[{PREFERRED_ACCENT_COLOR}]{title}[/{PREFERRED_ACCENT_COLOR}]",
+        border_style=PREFERRED_BORDER_COLOR,
+    )

flyte/_cli/_create.py

@@ -0,0 +1,42 @@
+from typing import get_args
+
+import rich_click as click
+
+import flyte._cli._common as common
+from flyte.remote._secret import SecretTypes
+
+
+@click.group(name="create")
+def create():
+    """
+    Create a new task or environment.
+    """
+
+
+@create.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=True)
+@click.argument("value", type=str, required=False)
+@click.option("--from-file", type=click.Path(exists=True), help="Path to the file with the binary secret.")
+@click.option(
+    "--type", type=click.Choice(get_args(SecretTypes)), default="regular", help="Type of the secret.", show_default=True
+)
+@click.pass_obj
+def secret(
+    cfg: common.CLIConfig,
+    name: str,
+    value: str | bytes | None = None,
+    from_file: str | None = None,
+    type: SecretTypes = "regular",
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Create a new secret.
+    """
+    from flyte.remote import Secret
+
+    cfg.init(project, domain)
+    if from_file:
+        with open(from_file, "rb") as f:
+            value = f.read()
+    Secret.create(name=name, value=value, type=type)

flyte/_cli/_delete.py

@@ -0,0 +1,23 @@
+import rich_click as click
+
+import flyte._cli._common as common
+
+
+@click.group(name="delete")
+def delete():
+    """
+    Delete a task or environment.
+    """
+
+
+@click.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=True)
+@click.pass_obj
+def secret(cfg: common.CLIConfig, name: str, project: str | None = None, domain: str | None = None):
+    """
+    Delete a secret.
+    """
+    from flyte.remote import Secret
+
+    cfg.init(project, domain)
+    Secret.delete(name=name)

flyte/_cli/_deploy.py

@@ -0,0 +1,140 @@
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Dict, List, cast, get_args
+
+import click
+from click import Context
+
+import flyte
+
+from .._code_bundle._utils import CopyFiles
+from . import _common as common
+from ._common import CLIConfig
+
+
+@dataclass
+class DeployArguments:
+    project: str = field(
+        default=cast(str, common.PROJECT_OPTION.default), metadata={"click.option": common.PROJECT_OPTION}
+    )
+    domain: str = field(
+        default=cast(str, common.DOMAIN_OPTION.default), metadata={"click.option": common.DOMAIN_OPTION}
+    )
+    version: str = field(
+        default="",
+        metadata={
+            "click.option": click.Option(
+                ["--version"],
+                type=str,
+                help="Version of the environment to deploy",
+            )
+        },
+    )
+    dry_run: bool = field(default=False, metadata={"click.option": common.DRY_RUN_OPTION})
+    local: bool = field(
+        default=False,
+        metadata={
+            "click.option": click.Option(
+                ["--local"],
+                is_flag=True,
+                help="Run the task locally",
+            )
+        },
+    )
+    copy_style: CopyFiles = field(
+        default="loaded_modules",
+        metadata={
+            "click.option": click.Option(
+                ["--copy-style"],
+                type=click.Choice(get_args(CopyFiles)),
+                default="loaded_modules",
+                help="Copy style to use when running the task",
+            )
+        },
+    )
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "DeployArguments":
+        return cls(**d)
+
+    @classmethod
+    def options(cls) -> List[click.Option]:
+        """
+        Return the set of base parameters added to every pyflyte run workflow subcommand.
+        """
+        return [common.get_option_from_metadata(f.metadata) for f in fields(cls) if f.metadata]
+
+
+class DeployEnvCommand(click.Command):
+    def __init__(self, obj_name: str, obj: Any, deploy_args: DeployArguments, *args, **kwargs):
+        self.obj_name = obj_name
+        self.obj = obj
+        self.deploy_args = deploy_args
+        super().__init__(*args, **kwargs)
+
+    def invoke(self, ctx: Context):
+        print(f"Deploying environment: {self.obj_name}")
+        obj: CLIConfig = ctx.obj
+        obj.init(self.deploy_args.project, self.deploy_args.domain)
+        return flyte.deploy(
+            self.obj,
+            dryrun=self.deploy_args.dry_run,
+            copy_style=self.deploy_args.copy_style,
+            version=self.deploy_args.version,
+        )
+
+
+class EnvPerFileGroup(common.ObjectsPerFileGroup):
+    """
+    Group that creates a command for each task in the current directory that is not __init__.py.
+    """
+
+    def __init__(self, filename: Path, deploy_args: DeployArguments, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.deploy_args = deploy_args
+
+    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
+        return {k: v for k, v in module.__dict__.items() if isinstance(v, flyte.TaskEnvironment)}
+
+    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
+        obj = cast(flyte.TaskEnvironment, obj)
+        return DeployEnvCommand(
+            obj_name=obj_name,
+            obj=obj,
+            help=obj.description,
+            deploy_args=self.deploy_args,
+        )
+
+
+class EnvFiles(common.FileGroup):
+    """
+    Group that creates a command for each file in the current directory that is not __init__.py.
+    """
+
+    common_options_enabled = False
+
+    def __init__(
+        self,
+        *args,
+        **kwargs,
+    ):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        kwargs["params"].extend(DeployArguments.options())
+        super().__init__(*args, **kwargs)
+
+    def get_command(self, ctx, filename):
+        deploy_args = DeployArguments.from_dict(ctx.params)
+        return EnvPerFileGroup(
+            filename=Path(filename),
+            deploy_args=deploy_args,
+            name=filename,
+            help=f"Run, functions decorated `env.task` or instances of Tasks in {filename}",
+        )
+
+
+deploy = EnvFiles(
+    name="deploy",
+    help="deploy one or more environments from a python file.",
+)