flyte 0.2.0b7__py3-none-any.whl → 0.2.0b9__py3-none-any.whl

flyte/cli/_gen.py

@@ -0,0 +1,163 @@
+import textwrap
+from os import getcwd
+from typing import Generator, Tuple
+
+import rich_click as click
+
+import flyte.cli._common as common
+
+
+@click.group(name="gen")
+def gen():
+    """
+    Generate documentation.
+    """
+
+
+@gen.command(cls=common.CommandBase)
+@click.option("--type", "doc_type", type=str, required=True, help="Type of documentation (valid: markdown)")
+@click.pass_obj
+def docs(cfg: common.CLIConfig, doc_type: str, project: str | None = None, domain: str | None = None):
+    """
+    Generate documentation.
+    """
+    if doc_type == "markdown":
+        markdown(cfg)
+    else:
+        raise click.ClickException("Invalid documentation type: {}".format(doc_type))
+
+
+def walk_commands(ctx: click.Context) -> Generator[Tuple[str, click.Command], None, None]:
+    """
+    Recursively walk a Click command tree, starting from the given context.
+
+    Yields:
+        (full_command_path, command_object)
+    """
+    command = ctx.command
+
+    if not isinstance(command, click.Group):
+        yield ctx.command_path, command
+    else:
+        for name in command.list_commands(ctx):
+            subcommand = command.get_command(ctx, name)
+            if subcommand is None:
+                continue
+
+            full_name = f"{ctx.command_path} {name}".strip()
+            yield full_name, subcommand
+
+            # Recurse if subcommand is a MultiCommand (i.e., has its own subcommands)
+            if isinstance(subcommand, click.Group):
+                sub_ctx = click.Context(subcommand, info_name=name, parent=ctx)
+                yield from walk_commands(sub_ctx)
+
+
+def markdown(cfg: common.CLIConfig):
+    """
+    Generate documentation in Markdown format
+    """
+    ctx = cfg.ctx
+
+    output = []
+    output_verb_groups: dict[str, list[str]] = {}
+    output_noun_groups: dict[str, list[str]] = {}
+
+    commands = [*[("flyte", ctx.command)], *walk_commands(ctx)]
+    for cmd_path, cmd in commands:
+        output.append("")
+
+        cmd_path_parts = cmd_path.split(" ")
+
+        if len(cmd_path_parts) > 1:
+            if cmd_path_parts[1] not in output_verb_groups:
+                output_verb_groups[cmd_path_parts[1]] = []
+            if len(cmd_path_parts) > 2:
+                output_verb_groups[cmd_path_parts[1]].append(cmd_path_parts[2])
+
+        if len(cmd_path_parts) == 3:
+            if cmd_path_parts[2] not in output_noun_groups:
+                output_noun_groups[cmd_path_parts[2]] = []
+            output_noun_groups[cmd_path_parts[2]].append(cmd_path_parts[1])
+
+        output.append(f"{'#' * (len(cmd_path_parts) + 1)} {cmd_path}")
+        if cmd.help:
+            output.append("")
+            output.append(f"{dedent(cmd.help)}")
+
+        if not cmd.params:
+            continue
+
+        params = cmd.get_params(click.Context(cmd))
+
+        # Collect all data first to calculate column widths
+        table_data = []
+        for param in params:
+            if isinstance(param, click.Option):
+                # Format each option with backticks before joining
+                all_opts = param.opts + param.secondary_opts
+                if len(all_opts) == 1:
+                    opts = f"`{all_opts[0]}`"
+                else:
+                    opts = "".join(
+                        [
+                            "{{< multiline >}}",
+                            "\n".join([f"`{opt}`" for opt in all_opts]),
+                            "{{< /multiline >}}",
+                        ]
+                    )
+                default_value = ""
+                if param.default is not None:
+                    default_value = f"`{param.default}`"
+                    default_value = default_value.replace(f"{getcwd()}/", "")
+                help_text = dedent(param.help) if param.help else ""
+                table_data.append([opts, f"`{param.type.name}`", default_value, help_text])
+
+        if not table_data:
+            continue
+
+        # Add table header with proper alignment
+        output.append("")
+        output.append("| Option | Type | Default | Description |")
+        output.append("|--------|------|---------|-------------|")
+
+        # Add table rows with proper alignment
+        for row in table_data:
+            output.append(f"| {row[0]} | {row[1]} | {row[2]} | {row[3]} |")
+
+    output_verb_index = []
+
+    if len(output_verb_groups) > 0:
+        output_verb_index.append("| Action | On |")
+        output_verb_index.append("| ------ | -- |")
+        for verb, nouns in output_verb_groups.items():
+            entries = [f"[`{noun}`](#flyte-{verb}-{noun})" for noun in nouns]
+            output_verb_index.append(f"| `{verb}` | {', '.join(entries)}  |")
+
+    output_noun_index = []
+
+    if len(output_noun_groups) > 0:
+        output_noun_index.append("| Object | Action |")
+        output_noun_index.append("| ------ | -- |")
+        for obj, actions in output_noun_groups.items():
+            entries = [f"[`{action}`](#flyte-{action}-{obj})" for action in actions]
+            output_noun_index.append(f"| `{obj}` | {', '.join(entries)}  |")
+
+    print()
+    print("{{< grid >}}")
+    print("{{< markdown >}}")
+    print("\n".join(output_noun_index))
+    print("{{< /markdown >}}")
+    print("{{< markdown >}}")
+    print("\n".join(output_verb_index))
+    print("{{< /markdown >}}")
+    print("{{< /grid >}}")
+    print()
+    print("\n".join(output))
+
+
+def dedent(text: str) -> str:
+    """
+    Remove leading whitespace from a string.
+    """
+    return textwrap.dedent(text).strip("\n")

flyte/cli/_get.py

@@ -11,7 +11,23 @@ from . import _common as common
 @click.group(name="get")
 def get():
     """
-    Get the value of a task or environment.
+    Retrieve resources from a Flyte deployment.
+
+    You can get information about projects, runs, tasks, actions, secrets, logs and input/output values.
+
+    Each command supports optional parameters to filter or specify the resource you want to retrieve.
+
+    Using a `get` subcommand without any arguments will retrieve a list of available resources to get.
+    For example:
+
+    * `get project` (without specifiying aproject), will list all projects.
+    * `get project my_project` will return the details of the project named `my_project`.
+
+    In some cases, a partially specified command will act as a filter and return available further parameters.
+    For example:
+
+    * `get action my_run` will return all actions for the run named `my_run`.
+    * `get action my_run my_action` will return the details of the action named `my_action` for the run `my_run`.
     """
 
 
@@ -20,7 +36,7 @@ def get():
 @click.pass_obj
 def project(cfg: common.CLIConfig, name: str | None = None):
     """
-    Get the current project.
+    Get a list of all projects, or details of a specific project by name.
     """
     from flyte.remote import Project
 
@@ -39,7 +55,11 @@ def project(cfg: common.CLIConfig, name: str | None = None):
 @click.pass_obj
 def run(cfg: common.CLIConfig, name: str | None = None, project: str | None = None, domain: str | None = None):
     """
-    Get the current run.
+    Get a list of all runs, or details of a specific run by name.
+
+    The run details will include information about the run, its status, but only the root action will be shown.
+
+    If you want to see the actions for a run, use `get action <run_name>`.
     """
     from flyte.remote import Run, RunDetails
 
@@ -65,7 +85,9 @@ def task(
     domain: str | None = None,
 ):
     """
-    Get the current task.
+    Retrieve a list of all tasks, or details of a specific task by name and version.
+
+    Currently, both `name` and `version` are required to get a specific task.
     """
     from flyte.remote import Task
 
@@ -140,8 +162,25 @@ def logs(
     filter_system: bool = False,
 ):
     """
-    Stream logs for the provided run or action. If the run is provided, only the logs for the parent action will be
-    streamed.
+    Stream logs for the provided run or action.
+    If only the run is provided, only the logs for the parent action will be streamed:
+
+    ```bash
+    $ flyte get logs my_run
+    ```
+
+    If you want to see the logs for a specific action, you can provide the action name as well:
+
+    ```bash
+    $ flyte get logs my_run my_action
+    ```
+
+    By default, logs will be shown in the raw format and will scroll the terminal.
+    If automatic scrolling and only tailing `--lines` number of lines is desired, use the `--pretty` flag:
+
+    ```bash
+    $ flyte get logs my_run my_action --pretty --lines 50
+    ```
     """
     import flyte.remote as remote
 
@@ -175,7 +214,7 @@ def secret(
     domain: str | None = None,
 ):
     """
-    Get the current secret.
+    Get a list of all secrets, or details of a specific secret by name.
     """
     import flyte.remote as remote
 
@@ -205,6 +244,19 @@ def io(
 ):
     """
     Get the inputs and outputs of a run or action.
+    If only the run name is provided, it will show the inputs and outputs of the root action of that run.
+    If an action name is provided, it will show the inputs and outputs for that action.
+    If `--inputs-only` or `--outputs-only` is specified, it will only show the inputs or outputs respectively.
+
+    Examples:
+
+    ```bash
+    $ flyte get io my_run
+    ```
+
+    ```bash
+    $ flyte get io my_run my_action
+    ```
     """
     if inputs_only and outputs_only:
         raise click.BadParameter("Cannot use both --inputs-only and --outputs-only")
@@ -249,7 +301,9 @@ def io(
 @click.pass_obj
 def config(cfg: common.CLIConfig):
     """
-    Shows the automatically detected configuration to connect with remote Flyte services.
+    Shows the automatically detected configuration to connect with the remote backend.
+
+    The configuration will include the endpoint, organization, and other settings that are used by the CLI.
     """
     console = Console()
     console.print(cfg)

flyte/cli/_run.py

@@ -204,11 +204,31 @@ class TaskFiles(common.FileGroup):
             filename=Path(filename),
             run_args=run_args,
             name=filename,
-            help=f"Run, functions decorated `env.task` {filename}",
+            help=f"Run, functions decorated with `env.task` in {filename}",
         )
 
 
 run = TaskFiles(
     name="run",
-    help="Run a task from a python file.",
+    help="""
+Run a task from a python file.
+
+Example usage:
+```bash
+flyte run --name examples/basics/hello.py my_task --arg1 value1 --arg2 value2
+```
+Note: all arguments for the run command are provided right after the `run` command and before the file name.
+
+You can also specify the project and domain using the `--project` and `--domain` options, respectively. These
+options can be set in the config file or passed as command line arguments.
+
+Note: The arguments for the task are provided after the task name and can be retrieved using `--help`
+Example:
+```bash
+flyte run --name examples/basics/hello.py my_task --help
+```
+
+To run a task locally, use the `--local` flag. This will run the task in the local environment instead of the remote
+ Flyte environment.
+""",
 )

flyte/cli/main.py

@@ -6,9 +6,32 @@ from ._abort import abort
 from ._common import CLIConfig
 from ._create import create
 from ._deploy import deploy
+from ._gen import gen
 from ._get import get
 from ._run import run
 
+click.rich_click.COMMAND_GROUPS = {
+    "flyte": [
+        {
+            "name": "Running workflows",
+            "commands": ["run", "abort"],
+        },
+        {
+            "name": "Management",
+            "commands": ["create", "deploy", "get"],
+        },
+        {
+            "name": "Documentation generation",
+            "commands": ["gen"],
+        },
+    ]
+}
+
+help_config = click.RichHelpConfiguration(
+    use_markdown=True,
+    use_markdown_emoji=True,
+)
+
 
 def _verbosity_to_loglevel(verbosity: int) -> int | None:
     """
@@ -35,13 +58,13 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "--endpoint",
     type=str,
     required=False,
-    help="The endpoint to connect to, this will override any config and simply used pkce to connect.",
+    help="The endpoint to connect to. This will override any configuration file and simply use `pkce` to connect.",
 )
 @click.option(
-    "--insecure/--secure",
+    "--insecure",
     is_flag=True,
     required=False,
-    help="Use insecure connection to the endpoint. If secure is specified, the CLI will use TLS",
+    help="Use an insecure connection to the endpoint. If not specified, the CLI will use TLS.",
     type=bool,
     default=None,
     show_default=True,
@@ -50,7 +73,7 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "-v",
     "--verbose",
     required=False,
-    help="Show verbose messages and exception traces",
+    help="Show verbose messages and exception traces. Repeating multiple times increases the verbosity (e.g., -vvv).",
     count=True,
     default=0,
     type=int,
@@ -59,7 +82,7 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "--org",
     type=str,
     required=False,
-    help="Organization to use",
+    help="The organization to which the command applies.",
 )
 @click.option(
     "-c",
@@ -67,9 +90,9 @@ def _verbosity_to_loglevel(verbosity: int) -> int | None:
     "config_file",
     required=False,
     type=click.Path(exists=True),
-    help="Path to config file (YAML format) to use for the CLI. If not specified,"
-    " the default config file will be used.",
+    help="Path to the configuration file to use. If not specified, the default configuration file is used."
 )
+@click.rich_config(help_config=help_config)
 @click.pass_context
 def main(
     ctx: click.Context,
@@ -80,14 +103,39 @@ def main(
     config_file: str | None,
 ):
     """
+    The Flyte CLI is the the command line interface for working with the Flyte SDK and backend.
+
+    It follows a simple verb/noun structure,
+    where the top-level commands are verbs that describe the action to be taken,
+    and the subcommands are nouns that describe the object of the action.
+
+    The root command can be used to configure the CLI for persistent settings,
+    such as the endpoint, organization, and verbosity level.
+
+    Set endpoint and organization:
+
+    ```bash
+    $ flyte --endpoint <endpoint> --org <org> get project <project_name>
+    ```
+
+    Increase verbosity level (This is useful for debugging,
+    this will show more logs and exception traces):
+
+    ```bash
+    $ flyte -vvv get logs <run-name>
+    ```
+
+    Override the default config file:
+
+    ```bash
+    $ flyte --config /path/to/config.yaml run ...
+    ```
 
- ____  __    _  _  ____  ____    _  _  ____     __
-(  __)(  )  ( \\/ )(_  _)(  __)  / )( \\(___ \\   /  \
- ) _) / (_/\\ )  /   )(   ) _)   \\ \\/ / / __/ _(  0 )
-(__)  \\____/(__/   (__) (____)   \\__/ (____)(_)\\__/
+    * [Documentation](https://www.union.ai/docs/flyte/user-guide/)
+    * [GitHub](https://github.com/flyteorg/flyte): Please leave a star if you like Flyte!
+    * [Slack](https://slack.flyte.org): Join the community and ask questions.
+    * [Issues](https://github.com/flyteorg/flyte/issues)
 
-    The flyte cli follows a simple verb based structure, where the top-level commands are verbs that describe the action
-to be taken, and the subcommands are nouns that describe the object of the action.
     """
     import flyte.config as config
 
@@ -108,6 +156,7 @@ to be taken, and the subcommands are nouns that describe the object of the actio
         insecure=final_insecure,
         org_override=org or cfg.task.org,
         config=cfg,
+        ctx=ctx,
     )
     logger.debug(f"Final materialized Cli config: {ctx.obj}")
 
@@ -117,3 +166,4 @@ main.add_command(deploy)
 main.add_command(get)  # type: ignore
 main.add_command(create)  # type: ignore
 main.add_command(abort)  # type: ignore
+main.add_command(gen)  # type: ignore

flyte/extras/_container.py

@@ -215,7 +215,7 @@ class ContainerTask(TaskTemplate):
                 output_dict[k] = self._convert_output_val_to_correct_type(output_val, output_type)
         return output_dict
 
-    def execute(self, **kwargs) -> Any:
+    async def execute(self, **kwargs) -> Any:
         try:
             import docker
         except ImportError:

flyte/models.py

@@ -5,7 +5,9 @@ import os
 import pathlib
 import tempfile
 from dataclasses import dataclass, field, replace
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type
+from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, Optional, Tuple, Type
+
+import rich.repr
 
 from flyte._docstring import Docstring
 from flyte._interface import extract_return_annotation
@@ -27,6 +29,7 @@ def generate_random_name() -> str:
     return str(uuid4())  # Placeholder for actual random name generation logic
 
 
+@rich.repr.auto
 @dataclass(frozen=True, kw_only=True)
 class ActionID:
     """
@@ -68,6 +71,7 @@ class ActionID:
         return self.new_sub_action(new_name)
 
 
+@rich.repr.auto
 @dataclass(frozen=True, kw_only=True)
 class RawDataPath:
     """
@@ -133,11 +137,13 @@ class RawDataPath:
         return remote_path
 
 
+@rich.repr.auto
 @dataclass(frozen=True)
 class GroupData:
     name: str
 
 
+@rich.repr.auto
 @dataclass(frozen=True, kw_only=True)
 class TaskContext:
     """
@@ -160,6 +166,7 @@ class TaskContext:
     code_bundle: CodeBundle | None = None
     compiled_image_cache: ImageCache | None = None
     data: Dict[str, Any] = field(default_factory=dict)
+    mode: Literal["local", "remote", "hybrid"] = "remote"
 
     def replace(self, **kwargs) -> TaskContext:
         if "data" in kwargs:
@@ -177,6 +184,7 @@ class TaskContext:
         return self.data.get(key)
 
 
+@rich.repr.auto
 @dataclass(frozen=True, kw_only=True)
 class CodeBundle:
     """
@@ -211,6 +219,7 @@ class CodeBundle:
         return replace(self, downloaded_path=path)
 
 
+@rich.repr.auto
 @dataclass(frozen=True)
 class Checkpoints:
     """

flyte/remote/_run.py

@@ -50,6 +50,7 @@ def _action_rich_repr(action: run_definition_pb2.Action) -> rich.repr.Result:
     """
     Rich representation of the action.
     """
+    yield "run", action.id.run.name
     if action.metadata.HasField("task"):
         yield "task", action.metadata.task.id.name
         yield "type", "task"

flyte/syncify/__init__.py

@@ -1,3 +1,54 @@
+"""
+# Syncify Module
+This module provides the `syncify` decorator and the `Syncify` class.
+The decorator can be used to convert asynchronous functions or methods into synchronous ones.
+This is useful for integrating async code into synchronous contexts.
+
+Every asynchronous function or method wrapped with `syncify` can be called synchronously using the
+parenthesis `()` operator, or asynchronously using the `.aio()` method.
+
+Example::
+
+```python
+from flyte.syncify import syncify
+
+@syncify
+async def async_function(x: str) -> str:
+    return f"Hello, Async World {x}!"
+
+
+# now you can call it synchronously
+result = async_function("Async World")   # Note: no .aio() needed for sync calls
+print(result)
+# Output: Hello, Async World Async World!
+
+# or call it asynchronously
+async def main():
+    result = await async_function.aio("World")    # Note the use of .aio() for async calls
+    print(result)
+```
+
+## Creating a Syncify Instance
+```python
+from flyte.syncify. import Syncify
+
+syncer = Syncify("my_syncer")
+
+# Now you can use `syncer` to decorate your async functions or methods
+
+```
+
+## How does it work?
+The Syncify class wraps asynchronous functions, classmethods, instance methods, and static methods to
+ provide a synchronous interface. The wrapped methods are always executed in the context of a background loop,
+ whether they are called synchronously or asynchronously. This allows for seamless integration of async code, as
+ certain async libraries capture the event loop. An example is grpc.aio, which captures the event loop.
+ In such a case, the Syncify class ensures that the async function is executed in the context of the background loop.
+
+To use it correctly with grpc.aio, you should wrap every grpc.aio channel creation, and client invocation
+with the same `Syncify` instance. This ensures that the async code runs in the correct event loop context.
+"""
+
 from flyte.syncify._api import Syncify
 
 syncify = Syncify()