runnable 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/__init__.py

@@ -1,6 +1,5 @@
 # ruff: noqa
 
-# TODO: Might need to add Rich to pyinstaller part
 import logging
 import os
 from logging.config import dictConfig
@@ -20,6 +19,7 @@ task_console = Console(record=True)
 from runnable.sdk import (  # noqa
     Catalog,
     Fail,
+    Job,
     Map,
     NotebookTask,
     Parallel,

runnable/catalog.py

@@ -10,6 +10,8 @@ from runnable.datastore import DataCatalog
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
+# TODO: Should ** be allowed as glob pattern as it can potentially copy everything to catalog
+
 
 def is_catalog_out_of_sync(
     catalog, synced_catalogs=Optional[List[DataCatalog]]

runnable/cli.py

@@ -1,137 +1,154 @@
-# A dummy to trigger the PR
 import logging
+from enum import Enum
+from typing import Annotated
 
-import click
-from click_plugins import with_plugins
-from pkg_resources import iter_entry_points
+import typer
 
 from runnable import defaults, entrypoints
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-@with_plugins(iter_entry_points("runnable.cli_plugins"))
-@click.group()
-@click.version_option()
-def cli():
-    """
-    Welcome to runnable. Please provide the command that you want to use.
-    All commands have options that you can see by runnable <command> --help
-    """
-    pass
+app = typer.Typer(
+    help=(
+        "Welcome to runnable. Please provide the command that you want to use."
+        "All commands have options that you can see by runnable <command> --help"
+    ),
+)
 
 
-@cli.command("execute", short_help="Execute/translate a pipeline")
-@click.argument("filename")
-@click.option(
-    "-c",
-    "--config-file",
-    default=None,
-    help="config file, in yaml, to be used for the run",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
-@click.option(
-    "--run-id", help="An optional run_id, one would be generated if not provided"
-)
+class LogLevel(str, Enum):
+    INFO = "INFO"
+    DEBUG = "DEBUG"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    FATAL = "FATAL"
+
+
+class ExecutionMode(str, Enum):
+    YAML = "yaml"
+    PYTHON = "python"
+
+
+class FanMode(str, Enum):
+    IN = "in"
+    OUT = "out"
+
+
+@app.command()
 def execute(
-    filename, config_file, parameters_file, log_level, tag, run_id
-):  # pragma: no cover
+    yaml_file: Annotated[str, typer.Argument(help="The pipeline definition file")],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    run_id: Annotated[
+        str,
+        typer.Option(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ] = "",
+):
     """
-    Execute a pipeline
+    Execute a pipeline defined by yaml file.
 
-    Usage: runnable execute [OPTIONS]
+    The executor is defined by executor block of the configuration file.
 
-    Options:
-    -f, --file TEXT               The pipeline definition file [default: pipeline.yaml]
-    -c, --config-file TEXT        config file, in yaml, to be used for the run [default: None]
-    -p, --parameters-file TEXT    Parameters, in yaml,  accessible by the application [default: None]
-    --log-level                     One of [INFO|DEBUG|WARNING|ERROR|FATAL]
-                                    The log level
-                                    [default: INFO]
-    --tag TEXT                   A tag attached to the run
-                                    [default: ]
-    --run-id TEXT                An optional run_id, one would be generated if not
-                                    provided
+    The behavior of this command depends on the executor type:
+
+    -- For local executors (local, local-container), the pipeline is executed in the current environment.
+
+    -- For remote executors (argo, airflow), the pipeline translated to the specification.
     """
-    logger.setLevel(log_level)
+    logger.setLevel(log_level.value)
 
-    entrypoints.execute(
+    entrypoints.execute_pipeline_yaml_spec(
         configuration_file=config_file,
-        pipeline_file=filename,
+        pipeline_file=yaml_file,
         tag=tag,
         run_id=run_id,
         parameters_file=parameters_file,
     )
 
 
-@cli.command(
-    "execute_single_node",
-    short_help="Internal entry point to execute a single node",
-    hidden=True,
-)
-@click.argument("run_id")
-@click.argument("step_name")
-@click.option(
-    "--map-variable",
-    default="",
-    help="The map variable dictionary in str",
-    show_default=True,
-)
-@click.option(
-    "-f", "--file", default="", help="The pipeline definition file", show_default=True
-)
-@click.option(
-    "-c",
-    "--config-file",
-    default=None,
-    help="config file, in yaml, to be used for the run",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
+@app.command(hidden=True)
 def execute_single_node(
-    run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag
+    run_id: Annotated[
+        str,
+        typer.Argument(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ],
+    yaml_or_python_file: Annotated[
+        str, typer.Argument(help="The pipeline definition file")
+    ],
+    step_name: Annotated[str, typer.Argument(help="The step name to execute")],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters-file",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.INFO,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    mode: Annotated[
+        ExecutionMode,
+        typer.Option(
+            "--mode",
+            "-m",
+            help="spec in yaml or python sdk",
+        ),
+    ] = ExecutionMode.YAML,
+    map_variable: Annotated[
+        str,
+        typer.Option(
+            "--map-variable",
+            help="The map variable dictionary in str",
+            show_default=True,
+        ),
+    ] = "",
 ):
-    """
-    Internal entrypoint for runnable to execute a single node.
-
-    Other than local executor, every other executor uses this entry point to execute a step in the context of runnable.
-    Only chained executions should use this method. Unchained executions should use execute_
-    """
-    logger.setLevel(log_level)
-
-    # Execute the node as part of the graph execution.
+    logger.setLevel(log_level.value)
     entrypoints.execute_single_node(
         configuration_file=config_file,
-        pipeline_file=file,
+        pipeline_file=yaml_or_python_file,
+        mode=mode,
         step_name=step_name,
         map_variable=map_variable,
         run_id=run_id,
@@ -140,228 +157,168 @@ def execute_single_node(
     )
 
 
-@cli.command("execute_notebook", short_help="Entry point to execute a notebook")
-@click.argument("filename")
-@click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
-@click.option(
-    "-c",
-    "--config-file",
-    default=None,
-    help="config file, in yaml, to be used for the run",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option(
-    "--put-in-catalog",
-    "-put",
-    default=None,
-    multiple=True,
-    help="The data to put from the catalog",
-)
-@click.option(
-    "--notebook-output-path", default="", help="The output path for the notebook"
-)
-@click.option("--tag", help="A tag attached to the run")
-@click.option(
-    "--run-id", help="An optional run_id, one would be generated if not provided"
-)
-def execute_notebook(
-    filename,
-    entrypoint,
-    config_file,
-    parameters_file,
-    log_level,
-    data_folder,
-    put_in_catalog,
-    notebook_output_path,
-    tag,
-    run_id,
+@app.command(hidden=True)
+def fan(
+    run_id: Annotated[str, typer.Argument(help="The run id")],
+    step_name: Annotated[str, typer.Argument(help="The step name")],
+    python_or_yaml_file: Annotated[
+        str, typer.Argument(help="The pipeline definition file")
+    ],
+    mode: Annotated[FanMode, typer.Option(help="fan in or fan out")],
+    map_variable: Annotated[
+        str,
+        typer.Option(
+            "--map-variable",
+            help="The map variable dictionary in str",
+            show_default=True,
+        ),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config-file", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters-file",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.INFO,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
 ):
-    """
-    External entry point to execute a Jupyter notebook in isolation.
-
-    The notebook would be executed in the environment defined by the config file or default if none.
-    The execution plan is unchained.
-    """
-    logger.setLevel(log_level)
-    catalog_config = {
-        "compute_data_folder": data_folder,
-        "put": list(put_in_catalog) if put_in_catalog else None,
-    }
-    if not filename.endswith(".ipynb"):
-        raise Exception("A notebook should always have ipynb as the extension")
+    logger.setLevel(log_level.value)
 
-    entrypoints.execute_notebook(
-        entrypoint=entrypoint,
-        notebook_file=filename,
-        catalog_config=catalog_config,
+    # Fan in or out
+    entrypoints.fan(
         configuration_file=config_file,
-        parameters_file=parameters_file,
-        notebook_output_path=notebook_output_path,
-        tag=tag,
+        pipeline_file=python_or_yaml_file,
+        step_name=step_name,
+        mode=mode,
+        map_variable=map_variable,
         run_id=run_id,
+        tag=tag,
+        parameters_file=parameters_file,
     )
 
 
-@cli.command("execute_function", short_help="Entry point to execute a python function")
-@click.argument("command")
-@click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
-@click.option(
-    "-c",
-    "--config-file",
-    default=None,
-    help="config file, in yaml, to be used for the run",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option(
-    "--put-in-catalog",
-    "-put",
-    default=None,
-    multiple=True,
-    help="The data to put from the catalog",
-)
-@click.option("--tag", help="A tag attached to the run")
-@click.option(
-    "--run-id", help="An optional run_id, one would be generated if not provided"
-)
-def execute_function(
-    command,
-    entrypoint,
-    config_file,
-    parameters_file,
-    log_level,
-    data_folder,
-    put_in_catalog,
-    tag,
-    run_id,
+@app.command()
+def submit_job(
+    job_definition_file: Annotated[
+        str,
+        typer.Argument(
+            help=("The yaml file containing the job definition"),
+        ),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    run_id: Annotated[
+        str,
+        typer.Option(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ] = "",
 ):
-    """
-    External entry point to execute a python function in isolation.
+    logger.setLevel(log_level.value)
 
-    The function would be executed in the environment defined by the config file or default if none.
-    The execution plan is unchained.
-    """
-    logger.setLevel(log_level)
-    catalog_config = {
-        "compute_data_folder": data_folder,
-        "put": list(put_in_catalog) if put_in_catalog else None,
-    }
-    entrypoints.execute_function(
-        entrypoint=entrypoint,
-        command=command,
-        catalog_config=catalog_config,
+    entrypoints.execute_job_yaml_spec(
         configuration_file=config_file,
-        parameters_file=parameters_file,
+        job_definition_file=job_definition_file,
         tag=tag,
         run_id=run_id,
+        parameters_file=parameters_file,
     )
 
 
-@cli.command(
-    "fan",
-    short_help="Internal entry point to fan in or out a composite node",
-    hidden=True,
-)
-@click.argument("run_id")
-@click.argument("step_name")
-@click.option(
-    "-m",
-    "--mode",
-    help="fan in or fan out",
-    required=True,
-    type=click.Choice(["in", "out"]),
-)
-@click.option(
-    "--map-variable",
-    default="",
-    help="The map variable dictionary in str",
-    show_default=True,
-)
-@click.option(
-    "-f", "--file", default="", help="The pipeline definition file", show_default=True
-)
-@click.option(
-    "-c",
-    "--config-file",
-    default=None,
-    help="config file, in yaml, to be used for the run",
-    show_default=True,
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
-def fan(
-    run_id,
-    step_name,
-    mode,
-    map_variable,
-    file,
-    config_file,
-    parameters_file,
-    log_level,
-    tag,
+@app.command(hidden=True)
+def execute_job(
+    job_definition_file: Annotated[
+        str,
+        typer.Argument(
+            help=("The yaml file containing the job definition"),
+        ),
+    ],
+    run_id: Annotated[
+        str,
+        typer.Argument(help="An run_id, one would be generated if its not provided"),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    mode: Annotated[
+        ExecutionMode,
+        typer.Option(
+            "--mode",
+            "-m",
+            help="spec in yaml or python sdk",
+        ),
+    ] = ExecutionMode.YAML,
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
 ):
-    """
-    Internal entrypoint for runnable to fan in or out a composite node.
-
-    Only 3rd party orchestrators should use this entry point.
-    """
-    logger.setLevel(log_level)
+    logger.setLevel(log_level.value)
 
-    # Fan in or out
-    entrypoints.fan(
+    entrypoints.execute_job_non_local(
         configuration_file=config_file,
-        pipeline_file=file,
-        step_name=step_name,
+        job_definition_file=job_definition_file,
         mode=mode,
-        map_variable=map_variable,
-        run_id=run_id,
         tag=tag,
+        run_id=run_id,
         parameters_file=parameters_file,
     )
 
 
-# Needed for the binary creation
 if __name__ == "__main__":
-    cli()
+    app()