flyte 0.2.0b1__py3-none-any.whl → 2.0.0b46__py3-none-any.whl

flyte/cli/_get.py

@@ -0,0 +1,446 @@
+import asyncio
+import os
+from typing import Tuple, Union
+
+import rich_click as click
+from rich.pretty import pretty_repr
+
+import flyte.remote as remote
+from flyte.models import ActionPhase
+
+from . import _common as common
+
+
+@click.group(name="get")
+def get():
+    """
+    Retrieve resources from a Flyte deployment.
+
+    You can get information about projects, runs, tasks, actions, secrets, logs and input/output values.
+
+    Each command supports optional parameters to filter or specify the resource you want to retrieve.
+
+    Using a `get` subcommand without any arguments will retrieve a list of available resources to get.
+    For example:
+
+    * `get project` (without specifying a project), will list all projects.
+    * `get project my_project` will return the details of the project named `my_project`.
+
+    In some cases, a partially specified command will act as a filter and return available further parameters.
+    For example:
+
+    * `get action my_run` will return all actions for the run named `my_run`.
+    * `get action my_run my_action` will return the details of the action named `my_action` for the run `my_run`.
+    """
+
+
+@get.command()
+@click.argument("name", type=str, required=False)
+@click.pass_obj
+def project(cfg: common.CLIConfig, name: str | None = None):
+    """
+    Get a list of all projects, or details of a specific project by name.
+    """
+    cfg.init()
+
+    console = common.get_console()
+    if name:
+        console.print(pretty_repr(remote.Project.get(name)))
+    else:
+        console.print(common.format("Projects", remote.Project.listall(), cfg.output_format))
+    os._exit(0)
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.option("--limit", type=int, default=100, help="Limit the number of runs to fetch when listing.")
+@click.option(
+    "--in-phase",  # multiple=True, TODO support multiple phases once values in works
+    type=click.Choice([p.value for p in ActionPhase], case_sensitive=False),
+    help="Filter runs by their status.",
+)
+@click.option("--only-mine", is_flag=True, default=False, help="Show only runs created by the current user (you).")
+@click.option("--task-name", type=str, default=None, help="Filter runs by task name.")
+@click.option("--task-version", type=str, default=None, help="Filter runs by task version.")
+@click.pass_obj
+def run(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    limit: int = 100,
+    in_phase: str | Tuple[str, ...] | None = None,
+    only_mine: bool = False,
+    task_name: str | None = None,
+    task_version: str | None = None,
+):
+    """
+    Get a list of all runs, or details of a specific run by name.
+
+    The run details will include information about the run, its status, but only the root action will be shown.
+
+    If you want to see the actions for a run, use `get action <run_name>`.
+
+    You can filter runs by task name and optionally task version:
+
+    ```bash
+    $ flyte get run --task-name my_task
+    $ flyte get run --task-name my_task --task-version v1.0
+    ```
+    """
+
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if name:
+        details = remote.RunDetails.get(name=name)
+        console.print(common.format(f"Run {name}", [details], "json"))
+    else:
+        if in_phase and isinstance(in_phase, str):
+            in_phase = (ActionPhase(in_phase),)
+
+        subject = None
+        if only_mine:
+            usr = remote.User.get()
+            subject = usr.subject()
+
+        console.print(
+            common.format(
+                "Runs",
+                remote.Run.listall(
+                    limit=limit,
+                    in_phase=in_phase,
+                    created_by_subject=subject,
+                    task_name=task_name,
+                    task_version=task_version,
+                ),
+                cfg.output_format,
+            )
+        )
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.argument("version", type=str, required=False)
+@click.option("--limit", type=int, default=100, help="Limit the number of tasks to fetch.")
+@click.pass_obj
+def task(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    limit: int = 100,
+    version: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Retrieve a list of all tasks, or details of a specific task by name and version.
+
+    Currently, both `name` and `version` are required to get a specific task.
+    """
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if name:
+        if version:
+            v = remote.Task.get(name=name, version=version)
+            if v is None:
+                raise click.BadParameter(f"Task {name} not found.")
+            t = v.fetch()
+            console.print(common.format(f"Task {name}", [t], "json"))
+        else:
+            console.print(
+                common.format("Tasks", remote.Task.listall(by_task_name=name, limit=limit), cfg.output_format)
+            )
+    else:
+        console.print(common.format("Tasks", remote.Task.listall(limit=limit), cfg.output_format))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=True)
+@click.argument("action_name", type=str, required=False)
+@click.option(
+    "--in-phase",
+    type=click.Choice([p.value for p in ActionPhase], case_sensitive=False),
+    help="Filter actions by their phase.",
+)
+@click.pass_obj
+def action(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    in_phase: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get all actions for a run or details for a specific action.
+    """
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if action_name:
+        console.print(
+            common.format(
+                f"Action {run_name}.{action_name}", [remote.Action.get(run_name=run_name, name=action_name)], "json"
+            )
+        )
+    else:
+        # List all actions for the run
+        if in_phase:
+            in_phase_tuple = (ActionPhase(in_phase),)
+        else:
+            in_phase_tuple = None
+
+        console.print(
+            common.format(
+                f"Actions for {run_name}",
+                remote.Action.listall(for_run_name=run_name, in_phase=in_phase_tuple),
+                cfg.output_format,
+            )
+        )
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=True)
+@click.argument("action_name", type=str, required=False)
+@click.option("--lines", "-l", type=int, default=30, help="Number of lines to show, only useful for --pretty")
+@click.option("--show-ts", is_flag=True, help="Show timestamps")
+@click.option(
+    "--pretty",
+    is_flag=True,
+    default=False,
+    help="Show logs in an auto-scrolling box, where number of lines is limited to `--lines`",
+)
+@click.option(
+    "--attempt", "-a", type=int, default=None, help="Attempt number to show logs for, defaults to the latest attempt."
+)
+@click.option("--filter-system", is_flag=True, default=False, help="Filter all system logs from the output.")
+@click.pass_obj
+def logs(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    lines: int = 30,
+    show_ts: bool = False,
+    pretty: bool = True,
+    attempt: int | None = None,
+    filter_system: bool = False,
+):
+    """
+    Stream logs for the provided run or action.
+    If only the run is provided, only the logs for the parent action will be streamed:
+
+    ```bash
+    $ flyte get logs my_run
+    ```
+
+    If you want to see the logs for a specific action, you can provide the action name as well:
+
+    ```bash
+    $ flyte get logs my_run my_action
+    ```
+
+    By default, logs will be shown in the raw format and will scroll the terminal.
+    If automatic scrolling and only tailing `--lines` number of lines is desired, use the `--pretty` flag:
+
+    ```bash
+    $ flyte get logs my_run my_action --pretty --lines 50
+    ```
+    """
+    cfg.init(project=project, domain=domain)
+
+    async def _run_log_view(_obj):
+        task = asyncio.create_task(
+            _obj.show_logs.aio(
+                max_lines=lines, show_ts=show_ts, raw=not pretty, attempt=attempt, filter_system=filter_system
+            )
+        )
+        try:
+            await task
+        except KeyboardInterrupt:
+            task.cancel()
+
+    obj: Union[remote.Action, remote.Run]
+    if action_name:
+        obj = remote.Action.get(run_name=run_name, name=action_name)
+    else:
+        obj = remote.Run.get(name=run_name)
+    asyncio.run(_run_log_view(obj))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.pass_obj
+def secret(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get a list of all secrets, or details of a specific secret by name.
+    """
+    if project is None:
+        project = ""
+    if domain is None:
+        domain = ""
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if name:
+        console.print(common.format("Secret", [remote.Secret.get(name)], "json"))
+    else:
+        console.print(common.format("Secrets", remote.Secret.listall(), cfg.output_format))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("run_name", type=str, required=True)
+@click.argument("action_name", type=str, required=False)
+@click.option("--inputs-only", "-i", is_flag=True, help="Show only inputs")
+@click.option("--outputs-only", "-o", is_flag=True, help="Show only outputs")
+@click.pass_obj
+def io(
+    cfg: common.CLIConfig,
+    run_name: str,
+    action_name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    inputs_only: bool = False,
+    outputs_only: bool = False,
+):
+    """
+    Get the inputs and outputs of a run or action.
+    If only the run name is provided, it will show the inputs and outputs of the root action of that run.
+    If an action name is provided, it will show the inputs and outputs for that action.
+    If `--inputs-only` or `--outputs-only` is specified, it will only show the inputs or outputs respectively.
+
+    Examples:
+
+    ```bash
+    $ flyte get io my_run
+    ```
+
+    ```bash
+    $ flyte get io my_run my_action
+    ```
+    """
+    if inputs_only and outputs_only:
+        raise click.BadParameter("Cannot use both --inputs-only and --outputs-only")
+
+    cfg.init(project=project, domain=domain)
+    console = common.get_console()
+    obj: Union[remote.ActionDetails, remote.RunDetails]
+    if action_name:
+        obj = remote.ActionDetails.get(run_name=run_name, name=action_name)
+    else:
+        obj = remote.RunDetails.get(name=run_name)
+
+    async def _get_io(
+        details: Union[remote.RunDetails, remote.ActionDetails],
+    ) -> Tuple[remote.ActionInputs | None, remote.ActionOutputs | None | str]:
+        if inputs_only or outputs_only:
+            if inputs_only:
+                return await details.inputs(), None
+            elif outputs_only:
+                return None, await details.outputs()
+        inputs = await details.inputs()
+        outputs: remote.ActionOutputs | None | str = None
+        try:
+            outputs = await details.outputs()
+        except Exception:
+            # If the outputs are not available, we can still show the inputs
+            outputs = "[red]not yet available[/red]"
+        return inputs, outputs
+
+    inputs, outputs = asyncio.run(_get_io(obj))
+    # Show inputs and outputs side by side
+    console.print(
+        common.get_panel(
+            "Inputs & Outputs",
+            f"[green bold]Inputs[/green bold]\n{inputs}\n\n[blue bold]Outputs[/blue bold]\n{outputs}",
+            cfg.output_format,
+        )
+    )
+
+
+@get.command(cls=click.RichCommand)
+@click.pass_obj
+def config(cfg: common.CLIConfig):
+    """
+    Shows the automatically detected configuration to connect with the remote backend.
+
+    The configuration will include the endpoint, organization, and other settings that are used by the CLI.
+    """
+    console = common.get_console()
+    console.print(cfg)
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("task_name", type=str, required=False)
+@click.argument("name", type=str, required=False)
+@click.option("--limit", type=int, default=100, help="Limit the number of triggers to fetch.")
+@click.pass_obj
+def trigger(
+    cfg: common.CLIConfig,
+    task_name: str | None = None,
+    name: str | None = None,
+    limit: int = 100,
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Get a list of all triggers, or details of a specific trigger by name.
+    """
+    if name and not task_name:
+        raise click.BadParameter("If you provide a trigger name, you must also provide the task name.")
+
+    from flyte.remote import Trigger
+
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if name:
+        console.print(pretty_repr(Trigger.get(name=name, task_name=task_name)))
+    else:
+        console.print(common.format("Triggers", Trigger.listall(task_name=task_name, limit=limit), cfg.output_format))
+
+
+@get.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=False)
+@click.option("--limit", type=int, default=100, help="Limit the number of apps to fetch when listing.")
+@click.option("--only-mine", is_flag=True, default=False, help="Show only apps created by the current user (you).")
+@click.pass_obj
+def app(
+    cfg: common.CLIConfig,
+    name: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    limit: int = 100,
+    only_mine: bool = False,
+):
+    """
+    Get a list of all apps, or details of a specific app by name.
+
+    Apps are long-running services deployed on the Flyte platform.
+    """
+    cfg.init(project=project, domain=domain)
+
+    console = common.get_console()
+    if name:
+        app_details = remote.App.get(name=name)
+        console.print(common.format(f"App {name}", [app_details], "json"))
+    else:
+        subject = None
+        if only_mine:
+            usr = remote.User.get()
+            subject = usr.subject()
+
+        console.print(
+            common.format(
+                "Apps",
+                remote.App.listall(limit=limit, created_by_subject=subject),
+                cfg.output_format,
+            )
+        )

flyte/cli/_option.py

@@ -0,0 +1,33 @@
+from click import Option, UsageError
+
+
+class MutuallyExclusiveMixin:
+    def __init__(self, *args, **kwargs):
+        self.mutually_exclusive = set(kwargs.pop("mutually_exclusive", []))
+        self.error_format = kwargs.pop(
+            "error_msg", "Illegal usage: options '{name}' and '{invalid}' are mutually exclusive"
+        )
+        super().__init__(*args, **kwargs)
+
+    def handle_parse_result(self, ctx, opts, args):
+        self_present = self.name in opts and opts[self.name] is not None
+        others_intersect = self.mutually_exclusive.intersection(opts)
+        others_present = others_intersect and any(opts[value] is not None for value in others_intersect)
+
+        if others_present:
+            if self_present:
+                raise UsageError(self.error_format.format(name=self.name, invalid=", ".join(self.mutually_exclusive)))
+            else:
+                self.prompt = None
+
+        return super().handle_parse_result(ctx, opts, args)
+
+
+# See https://stackoverflow.com/a/37491504/499285 and https://stackoverflow.com/a/44349292/499285
+class MutuallyExclusiveOption(MutuallyExclusiveMixin, Option):
+    def __init__(self, *args, **kwargs):
+        mutually_exclusive = kwargs.get("mutually_exclusive", [])
+        help = kwargs.get("help", "")
+        if mutually_exclusive:
+            kwargs["help"] = help + f" Mutually exclusive with {', '.join(mutually_exclusive)}."
+        super().__init__(*args, **kwargs)

flyte/{_cli → cli}/_params.py

@@ -14,16 +14,16 @@ from typing import get_args
 
 import rich_click as click
 import yaml
-from click import Parameter
-from flyteidl.core.interface_pb2 import Variable
-from flyteidl.core.literals_pb2 import Literal
-from flyteidl.core.types_pb2 import BlobType, LiteralType, SimpleType
+from click import Context, Parameter
+from flyteidl2.core.interface_pb2 import Variable
+from flyteidl2.core.literals_pb2 import Literal
+from flyteidl2.core.types_pb2 import BlobType, LiteralType, SimpleType
 from google.protobuf.json_format import MessageToDict
 from mashumaro.codecs.json import JSONEncoder
 
 from flyte._logging import logger
 from flyte.io import Dir, File
-from flyte.io.pickle.transformer import FlytePickleTransformer
+from flyte.types._pickle import FlytePickleTransformer
 
 
 class StructuredDataset:
@@ -69,6 +69,9 @@ def labels_callback(_: typing.Any, param: str, values: typing.List[str]) -> typi
 class DirParamType(click.ParamType):
     name = "directory path"
 
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return "Remote Dir Path"
+
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
     ) -> typing.Any:
@@ -88,6 +91,9 @@ class StructuredDatasetParamType(click.ParamType):
 
     name = "structured dataset path (dir/file)"
 
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return "Remote parquet URI"
+
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
     ) -> typing.Any:
@@ -101,6 +107,9 @@ class StructuredDatasetParamType(click.ParamType):
 class FileParamType(click.ParamType):
     name = "file path"
 
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return "Remote File Path"
+
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
     ) -> typing.Any:
@@ -110,14 +119,15 @@ class FileParamType(click.ParamType):
             p = pathlib.Path(value)
             if not p.exists() or not p.is_file():
                 raise click.BadParameter(f"parameter should be a valid file path, {value}")
-        return File(path=value)
+            raise click.BadParameter(f"Only remote paths are supported currently, {value}")
+        return File.from_existing_remote(value)
 
 
 class PickleParamType(click.ParamType):
     name = "pickle"
 
-    def get_metavar(self, param: "Parameter", *args) -> t.Optional[str]:
-        return "Python Object <Module>:<Object>"
+    def get_metavar(self, param: "Parameter", ctx) -> t.Optional[str]:
+        return "Python Object Instance <Module>:<Object>"
 
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
@@ -142,6 +152,9 @@ class PickleParamType(click.ParamType):
 class JSONIteratorParamType(click.ParamType):
     name = "json iterator"
 
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return "JSON Value"
+
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
     ) -> typing.Any:
@@ -257,6 +270,9 @@ class DateTimeType(click.DateTime):
 class DurationParamType(click.ParamType):
     name = "[1:24 | :22 | 1 minute | 10 days | ...]"
 
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return "ISO8601 duration"
+
     def convert(
         self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context]
     ) -> typing.Any:
@@ -283,13 +299,20 @@ class UnionParamType(click.ParamType):
     A composite type that allows for multiple types to be specified. This is used for union types.
     """
 
-    def __init__(self, types: typing.List[click.ParamType]):
+    def __init__(self, types: typing.List[click.ParamType | None]):
         super().__init__()
         self._types = self._sort_precedence(types)
-        self.name = "|".join([t.name for t in self._types])
+        self.name = "|".join([t.name for t in self._types if t is not None])
+        self.optional = False
+        if None in types:
+            self.name = f"Optional[{self.name}]"
+            self.optional = True
+
+    def get_metavar(self, param: Parameter, ctx: typing.Optional[click.Context]) -> str | None:
+        return self.name
 
     @staticmethod
-    def _sort_precedence(tp: typing.List[click.ParamType]) -> typing.List[click.ParamType]:
+    def _sort_precedence(tp: typing.List[click.ParamType | None]) -> typing.List[click.ParamType]:
         unprocessed = []
         str_types = []
         others = []
@@ -311,6 +334,8 @@ class UnionParamType(click.ParamType):
         """
         for p in self._types:
             try:
+                if p is None and value is None:
+                    return None
                 return p.convert(value, param, ctx)
             except Exception as e:
                 logger.debug(f"Ignoring conversion error for type {p} trying other variants in Union. Error: {e}")
@@ -405,7 +430,7 @@ def literal_type_to_click_type(lt: LiteralType, python_type: typing.Type) -> cli
             return ct
         if lt.simple in SIMPLE_TYPE_CONVERTER:
             return SIMPLE_TYPE_CONVERTER[lt.simple]
-        raise NotImplementedError(f"Type {lt.simple} is not supported in pyflyte run")
+        raise NotImplementedError(f"Type {lt.simple} is not supported in `flyte run`")
 
     if lt.HasField("structured_dataset_type"):
         return StructuredDatasetParamType()
@@ -429,11 +454,17 @@ def literal_type_to_click_type(lt: LiteralType, python_type: typing.Type) -> cli
         return DirParamType()
 
     if lt.HasField("union_type"):
+        python_args = typing.get_args(python_type)
+        if len(python_args) == 0:
+            return PickleParamType()
         cts = []
         for i in range(len(lt.union_type.variants)):
             variant = lt.union_type.variants[i]
-            variant_python_type = typing.get_args(python_type)[i]
-            cts.append(literal_type_to_click_type(variant, variant_python_type))
+            variant_python_type = python_args[i]
+            if variant_python_type is type(None):
+                cts.append(None)
+            else:
+                cts.append(literal_type_to_click_type(variant, variant_python_type))
         return UnionParamType(cts)
 
     if lt.HasField("enum_type"):
@@ -461,6 +492,9 @@ class FlyteLiteralConverter(object):
     def is_bool(self) -> bool:
         return self.click_type == click.BOOL
 
+    def is_optional(self) -> bool:
+        return isinstance(self.click_type, UnionParamType) and self.click_type.optional
+
     def convert(
         self, ctx: click.Context, param: typing.Optional[click.Parameter], value: typing.Any
     ) -> typing.Union[Literal, typing.Any]:
@@ -470,6 +504,8 @@ class FlyteLiteralConverter(object):
         try:
             # If the expected Python type is datetime.date, adjust the value to date
             if self._python_type is datetime.date:
+                if value is None:
+                    return None
                 # Click produces datetime, so converting to date to avoid type mismatch error
                 value = value.date()
 
@@ -493,7 +529,7 @@ def to_click_option(
     This handles converting workflow input types to supported click parameters with callbacks to initialize
     the input values to their expected types.
     """
-    from flyteidl.core.types_pb2 import SimpleType
+    from flyteidl2.core.types_pb2 import SimpleType
 
     if input_name != input_name.lower():
         # Click does not support uppercase option names: https://github.com/pallets/click/issues/837
@@ -519,15 +555,19 @@ def to_click_option(
         if literal_var.type.metadata:
             description_extra = f": {MessageToDict(literal_var.type.metadata)}"
 
-    # If a query has been specified, the input is never strictly required at this layer
     required = False if default_val is not None else True
     is_flag: typing.Optional[bool] = None
+    param_decls = [f"--{input_name}"]
     if literal_converter.is_bool():
         required = False
         is_flag = True
+        if default_val is True:
+            param_decls = [f"--{input_name}/--no-{input_name}"]
+    if literal_converter.is_optional():
+        required = False
 
     return click.Option(
-        param_decls=[f"--{input_name}"],
+        param_decls=param_decls,
         type=literal_converter.click_type,
         is_flag=is_flag,
         default=default_val,