runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/__init__.py

@@ -1,6 +1,5 @@
 # ruff: noqa
 
-# TODO: Might need to add Rich to pyinstaller part
 import logging
 import os
 from logging.config import dictConfig
@@ -20,6 +19,7 @@ task_console = Console(record=True)
 from runnable.sdk import (  # noqa
     Catalog,
     Fail,
+    Job,
     Map,
     NotebookTask,
     Parallel,
@@ -34,14 +34,3 @@ from runnable.sdk import (  # noqa
 
 # Needed to disable ploomber telemetry
 os.environ["PLOOMBER_STATS_ENABLED"] = "false"
-
-## TODO: Summary should be a bit better for catalog.
-## If the execution fails, hint them about the retry executor.
-# Make the retry executor loose!
-
-# TODO: Think of model registry as a central place to store models.
-# TODO: Implement Sagemaker pipelines as a executor.
-
-
-# TODO: Think of way of generating dag hash without executor configuration
-# Try to get a release

runnable/catalog.py

@@ -10,6 +10,28 @@ from runnable.datastore import DataCatalog
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
+# TODO: Should ** be allowed as glob pattern as it can potentially copy everything to catalog
+
+
+def is_catalog_out_of_sync(
+    catalog, synced_catalogs=Optional[List[DataCatalog]]
+) -> bool:
+    """
+    Check if the catalog items are out of sync from already cataloged objects.
+    If they are, return False.
+    If the object does not exist or synced catalog does not exist, return True
+    """
+    if not synced_catalogs:
+        return True  # If nothing has been synced in the past
+
+    for synced_catalog in synced_catalogs:
+        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
+            if synced_catalog.data_hash == catalog.data_hash:
+                return False
+            return True
+
+    return True  # The object does not exist, sync it
+
 
 # --8<-- [start:docs]
 
@@ -26,8 +48,7 @@ class BaseCatalog(ABC, BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     @abstractmethod
-    def get_summary(self) -> Dict[str, Any]:
-        ...
+    def get_summary(self) -> Dict[str, Any]: ...
 
     @property
     def _context(self):
@@ -38,7 +59,9 @@ class BaseCatalog(ABC, BaseModel):
         return defaults.COMPUTE_DATA_FOLDER
 
     @abstractmethod
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
+    def get(
+        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
+    ) -> List[DataCatalog]:
         """
         Get the catalog item by 'name' for the 'run id' and store it in compute data folder.
 
@@ -119,7 +142,9 @@ class DoNothingCatalog(BaseCatalog):
     def get_summary(self) -> Dict[str, Any]:
         return {}
 
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
+    def get(
+        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
+    ) -> List[DataCatalog]:
         """
         Does nothing
         """
@@ -145,4 +170,3 @@ class DoNothingCatalog(BaseCatalog):
         Does nothing
         """
         logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
-        ...

runnable/cli.py

@@ -1,112 +1,154 @@
-# A dummy to trigger the PR
 import logging
+from enum import Enum
+from typing import Annotated
 
-import click
-from click_plugins import with_plugins
-from pkg_resources import iter_entry_points
+import typer
 
 from runnable import defaults, entrypoints
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-@with_plugins(iter_entry_points("runnable.cli_plugins"))
-@click.group()
-@click.version_option()
-def cli():
-    """
-    Welcome to runnable. Please provide the command that you want to use.
-    All commands have options that you can see by runnable <command> --help
-    """
-    pass
+app = typer.Typer(
+    help=(
+        "Welcome to runnable. Please provide the command that you want to use."
+        "All commands have options that you can see by runnable <command> --help"
+    ),
+)
 
 
-@cli.command("execute", short_help="Execute/translate a pipeline")
-@click.option("-f", "--file", default="pipeline.yaml", help="The pipeline definition file", show_default=True)
-@click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
-def execute(file, config_file, parameters_file, log_level, tag, run_id):  # pragma: no cover
+class LogLevel(str, Enum):
+    INFO = "INFO"
+    DEBUG = "DEBUG"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    FATAL = "FATAL"
+
+
+class ExecutionMode(str, Enum):
+    YAML = "yaml"
+    PYTHON = "python"
+
+
+class FanMode(str, Enum):
+    IN = "in"
+    OUT = "out"
+
+
+@app.command()
+def execute(
+    yaml_file: Annotated[str, typer.Argument(help="The pipeline definition file")],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    run_id: Annotated[
+        str,
+        typer.Option(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ] = "",
+):
     """
-    Execute a pipeline
+    Execute a pipeline defined by yaml file.
+
+    The executor is defined by executor block of the configuration file.
+
+    The behavior of this command depends on the executor type:
 
-    Usage: runnable execute [OPTIONS]
+    -- For local executors (local, local-container), the pipeline is executed in the current environment.
 
-    Options:
-    -f, --file TEXT               The pipeline definition file [default: pipeline.yaml]
-    -c, --config-file TEXT        config file, in yaml, to be used for the run [default: None]
-    -p, --parameters-file TEXT    Parameters, in yaml,  accessible by the application [default: None]
-    --log-level                     One of [INFO|DEBUG|WARNING|ERROR|FATAL]
-                                    The log level
-                                    [default: INFO]
-    --tag TEXT                   A tag attached to the run
-                                    [default: ]
-    --run-id TEXT                An optional run_id, one would be generated if not
-                                    provided
+    -- For remote executors (argo, airflow), the pipeline translated to the specification.
     """
-    logger.setLevel(log_level)
+    logger.setLevel(log_level.value)
 
-    entrypoints.execute(
+    entrypoints.execute_pipeline_yaml_spec(
         configuration_file=config_file,
-        pipeline_file=file,
+        pipeline_file=yaml_file,
         tag=tag,
         run_id=run_id,
         parameters_file=parameters_file,
     )
 
 
-@cli.command("execute_single_node", short_help="Internal entry point to execute a single node", hidden=True)
-@click.argument("run_id")
-@click.argument("step_name")
-@click.option("--map-variable", default="", help="The map variable dictionary in str", show_default=True)
-@click.option("-f", "--file", default="", help="The pipeline definition file", show_default=True)
-@click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
-def execute_single_node(run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag):
-    """
-    Internal entrypoint for runnable to execute a single node.
-
-    Other than local executor, every other executor uses this entry point to execute a step in the context of runnable.
-    Only chained executions should use this method. Unchained executions should use execute_
-    """
-    logger.setLevel(log_level)
-
-    # Execute the node as part of the graph execution.
+@app.command(hidden=True)
+def execute_single_node(
+    run_id: Annotated[
+        str,
+        typer.Argument(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ],
+    yaml_or_python_file: Annotated[
+        str, typer.Argument(help="The pipeline definition file")
+    ],
+    step_name: Annotated[str, typer.Argument(help="The step name to execute")],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters-file",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.INFO,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    mode: Annotated[
+        ExecutionMode,
+        typer.Option(
+            "--mode",
+            "-m",
+            help="spec in yaml or python sdk",
+        ),
+    ] = ExecutionMode.YAML,
+    map_variable: Annotated[
+        str,
+        typer.Option(
+            "--map-variable",
+            help="The map variable dictionary in str",
+            show_default=True,
+        ),
+    ] = "",
+):
+    logger.setLevel(log_level.value)
     entrypoints.execute_single_node(
         configuration_file=config_file,
-        pipeline_file=file,
+        pipeline_file=yaml_or_python_file,
+        mode=mode,
         step_name=step_name,
         map_variable=map_variable,
         run_id=run_id,
@@ -115,157 +157,168 @@ def execute_single_node(run_id, step_name, map_variable, file, config_file, para
     )
 
 
-@cli.command("execute_notebook", short_help="Entry point to execute a notebook")
-@click.argument("filename")
-@click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
-@click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option("--put-in-catalog", "-put", default=None, multiple=True, help="The data to put from the catalog")
-@click.option("--notebook-output-path", default="", help="The output path for the notebook")
-@click.option("--tag", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
-def execute_notebook(
-    filename,
-    entrypoint,
-    config_file,
-    parameters_file,
-    log_level,
-    data_folder,
-    put_in_catalog,
-    notebook_output_path,
-    tag,
-    run_id,
+@app.command(hidden=True)
+def fan(
+    run_id: Annotated[str, typer.Argument(help="The run id")],
+    step_name: Annotated[str, typer.Argument(help="The step name")],
+    python_or_yaml_file: Annotated[
+        str, typer.Argument(help="The pipeline definition file")
+    ],
+    mode: Annotated[FanMode, typer.Option(help="fan in or fan out")],
+    map_variable: Annotated[
+        str,
+        typer.Option(
+            "--map-variable",
+            help="The map variable dictionary in str",
+            show_default=True,
+        ),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config-file", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters-file",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.INFO,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
 ):
-    """
-    External entry point to execute a Jupyter notebook in isolation.
-
-    The notebook would be executed in the environment defined by the config file or default if none.
-    The execution plan is unchained.
-    """
-    logger.setLevel(log_level)
-    catalog_config = {"compute_data_folder": data_folder, "put": list(put_in_catalog) if put_in_catalog else None}
-    if not filename.endswith(".ipynb"):
-        raise Exception("A notebook should always have ipynb as the extension")
+    logger.setLevel(log_level.value)
 
-    entrypoints.execute_notebook(
-        entrypoint=entrypoint,
-        notebook_file=filename,
-        catalog_config=catalog_config,
+    # Fan in or out
+    entrypoints.fan(
         configuration_file=config_file,
-        parameters_file=parameters_file,
-        notebook_output_path=notebook_output_path,
-        tag=tag,
+        pipeline_file=python_or_yaml_file,
+        step_name=step_name,
+        mode=mode,
+        map_variable=map_variable,
         run_id=run_id,
+        tag=tag,
+        parameters_file=parameters_file,
     )
 
 
-@cli.command("execute_function", short_help="Entry point to execute a python function")
-@click.argument("command")
-@click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
-@click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option("--put-in-catalog", "-put", default=None, multiple=True, help="The data to put from the catalog")
-@click.option("--tag", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
-def execute_function(
-    command, entrypoint, config_file, parameters_file, log_level, data_folder, put_in_catalog, tag, run_id
+@app.command()
+def submit_job(
+    job_definition_file: Annotated[
+        str,
+        typer.Argument(
+            help=("The yaml file containing the job definition"),
+        ),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+    run_id: Annotated[
+        str,
+        typer.Option(
+            help="An optional run_id, one would be generated if its not provided"
+        ),
+    ] = "",
 ):
-    """
-    External entry point to execute a python function in isolation.
+    logger.setLevel(log_level.value)
 
-    The function would be executed in the environment defined by the config file or default if none.
-    The execution plan is unchained.
-    """
-    logger.setLevel(log_level)
-    catalog_config = {"compute_data_folder": data_folder, "put": list(put_in_catalog) if put_in_catalog else None}
-    entrypoints.execute_function(
-        entrypoint=entrypoint,
-        command=command,
-        catalog_config=catalog_config,
+    entrypoints.execute_job_yaml_spec(
         configuration_file=config_file,
-        parameters_file=parameters_file,
+        job_definition_file=job_definition_file,
         tag=tag,
         run_id=run_id,
+        parameters_file=parameters_file,
     )
 
 
-@cli.command("fan", short_help="Internal entry point to fan in or out a composite node", hidden=True)
-@click.argument("run_id")
-@click.argument("step_name")
-@click.option("-m", "--mode", help="fan in or fan out", required=True, type=click.Choice(["in", "out"]))
-@click.option("--map-variable", default="", help="The map variable dictionary in str", show_default=True)
-@click.option("-f", "--file", default="", help="The pipeline definition file", show_default=True)
-@click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
-)
-@click.option(
-    "-p",
-    "--parameters-file",
-    default=None,
-    help="Parameters, in yaml,  accessible by the application",
-    show_default=True,
-)
-@click.option(
-    "--log-level",
-    default=defaults.LOG_LEVEL,
-    help="The log level",
-    show_default=True,
-    type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
-)
-@click.option("--tag", default="", help="A tag attached to the run")
-def fan(run_id, step_name, mode, map_variable, file, config_file, parameters_file, log_level, tag):
-    """
-    Internal entrypoint for runnable to fan in or out a composite node.
-
-    Only 3rd party orchestrators should use this entry point.
-    """
-    logger.setLevel(log_level)
+@app.command(hidden=True)
+def execute_job(
+    job_definition_file: Annotated[
+        str,
+        typer.Argument(
+            help=("The yaml file containing the job definition"),
+        ),
+    ],
+    run_id: Annotated[
+        str,
+        typer.Argument(help="An run_id, one would be generated if its not provided"),
+    ],
+    config_file: Annotated[
+        str,
+        typer.Option(
+            "--config", "-c", help="The configuration file specifying the services"
+        ),
+    ] = "",
+    parameters_file: Annotated[
+        str,
+        typer.Option(
+            "--parameters",
+            "-p",
+            help="Parameters, in yaml,  accessible by the application",
+        ),
+    ] = "",
+    mode: Annotated[
+        ExecutionMode,
+        typer.Option(
+            "--mode",
+            "-m",
+            help="spec in yaml or python sdk",
+        ),
+    ] = ExecutionMode.YAML,
+    log_level: Annotated[
+        LogLevel,
+        typer.Option(
+            "--log-level",
+            help="The log level",
+            show_default=True,
+            case_sensitive=False,
+        ),
+    ] = LogLevel.WARNING,
+    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
+):
+    logger.setLevel(log_level.value)
 
-    # Fan in or out
-    entrypoints.fan(
+    entrypoints.execute_job_non_local(
         configuration_file=config_file,
-        pipeline_file=file,
-        step_name=step_name,
+        job_definition_file=job_definition_file,
         mode=mode,
-        map_variable=map_variable,
-        run_id=run_id,
         tag=tag,
+        run_id=run_id,
         parameters_file=parameters_file,
     )
 
 
-# Needed for the binary creation
 if __name__ == "__main__":
-    cli()
+    app()

runnable/context.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny
 from rich.progress import Progress
@@ -9,6 +9,7 @@ from runnable.executor import BaseExecutor
 from runnable.graph import Graph
 from runnable.pickler import BasePickler
 from runnable.secrets import BaseSecrets
+from runnable.tasks import BaseTaskType
 
 
 class Context(BaseModel):
@@ -22,15 +23,21 @@ class Context(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     pipeline_file: Optional[str] = ""
+    job_definition_file: Optional[str] = ""
     parameters_file: Optional[str] = ""
     configuration_file: Optional[str] = ""
+    from_sdk: bool = False
 
-    tag: str = ""
     run_id: str = ""
+
+    tag: str = ""
     variables: Dict[str, str] = {}
+
     dag: Optional[Graph] = None
     dag_hash: str = ""
-    execution_plan: str = ""
+
+    job: Optional[BaseTaskType] = None
+    job_catalog_settings: Optional[List[str]] = []
 
 
 run_context = None  # type: Context # type: ignore