runnable 0.34.0a1__py3-none-any.whl → 1.0.0__py3-none-any.whl

extensions/pipeline_executor/local.py

@@ -4,7 +4,7 @@ from pydantic import Field, PrivateAttr
 
 from extensions.pipeline_executor import GenericPipelineExecutor
 from runnable import defaults
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -32,14 +32,14 @@ class LocalExecutor(GenericPipelineExecutor):
 
     _is_local: bool = PrivateAttr(default=True)
 
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         if not self.object_serialisation:
             self._context.object_serialisation = False
 
         super().execute_from_graph(node=node, map_variable=map_variable)
 
     def trigger_node_execution(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         In this mode of execution, we prepare for the node execution and execute the node
@@ -50,7 +50,7 @@ class LocalExecutor(GenericPipelineExecutor):
         """
         self.execute_node(node=node, map_variable=map_variable)
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         For local execution, we just execute the node.
 

extensions/pipeline_executor/local_container.py

@@ -2,12 +2,12 @@ import logging
 from pathlib import Path
 from typing import Dict
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from extensions.pipeline_executor import GenericPipelineExecutor
-from runnable import console, defaults, task_console, utils
+from runnable import defaults, utils
 from runnable.datastore import StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -70,7 +70,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
     auto_remove_container: bool = True
     environment: Dict[str, str] = Field(default_factory=dict)
 
-    _is_local: bool = False
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
@@ -104,7 +104,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
             code_id.code_identifier_url = "local docker host"
             step_log.code_identities.append(code_id)
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         We are already in the container, we just execute the node.
         The node is already prepared for execution.
@@ -112,69 +112,8 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         self._use_volumes()
         return self._execute_node(node, map_variable)
 
-    def execute_from_graph(
-        self,
-        node: BaseNode,
-        map_variable: TypeMapVariable = None,
-    ):
-        """
-        This is the entry point to from the graph execution.
-
-        While the self.execute_graph is responsible for traversing the graph, this function is responsible for
-        actual execution of the node.
-
-        If the node type is:
-            * task : We can delegate to _execute_node after checking the eligibility for re-run in cases of a re-run
-            * success: We can delegate to _execute_node
-            * fail: We can delegate to _execute_node
-
-        For nodes that are internally graphs:
-            * parallel: Delegate the responsibility of execution to the node.execute_as_graph()
-            * dag: Delegate the responsibility of execution to the node.execute_as_graph()
-            * map: Delegate the responsibility of execution to the node.execute_as_graph()
-
-        Transpilers will NEVER use this method and will NEVER call ths method.
-        This method should only be used by interactive executors.
-
-        Args:
-            node (Node): The node to execute
-            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of iterable.
-                    Defaults to None.
-        """
-        step_log = self._context.run_log_store.create_step_log(
-            node.name, node._get_step_log_name(map_variable)
-        )
-
-        self.add_code_identities(node=node, step_log=step_log)
-
-        step_log.step_type = node.node_type
-        step_log.status = defaults.PROCESSING
-
-        self._context.run_log_store.add_step_log(step_log, self._context.run_id)
-
-        logger.info(f"Executing node: {node.get_summary()}")
-
-        # Add the step log to the database as per the situation.
-        # If its a terminal node, complete it now
-        if node.node_type in ["success", "fail"]:
-            self._execute_node(node, map_variable=map_variable)
-            return
-
-        # We call an internal function to iterate the sub graphs and execute them
-        if node.is_composite:
-            node.execute_as_graph(map_variable=map_variable)
-            return
-
-        task_console.export_text(clear=True)
-
-        task_name = node._resolve_map_placeholders(node.internal_name, map_variable)
-        console.print(
-            f":runner: Executing the node {task_name} ... ", style="bold color(208)"
-        )
-        self.trigger_node_execution(node=node, map_variable=map_variable)
-
     def trigger_node_execution(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         We come into this step via execute from graph, use trigger job to spin up the container.
@@ -192,7 +131,9 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         logger.debug("Here is the resolved executor config")
         logger.debug(executor_config)
 
-        command = utils.get_node_execution_command(node, map_variable=map_variable)
+        command = self._context.get_node_callable_command(
+            node, map_variable=map_variable
+        )
 
         self._spin_container(
             node=node,
@@ -218,7 +159,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         self,
         node: BaseNode,
         command: str,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         auto_remove_container: bool = True,
     ):
         """
@@ -294,6 +235,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         """
         Mount the volumes for the container
         """
+        # TODO: There should be an abstraction on top of service providers
         match self._context.run_log_store.service_name:
             case "file-system":
                 write_to = self._context.run_log_store.log_folder
@@ -308,17 +250,17 @@ class LocalContainerExecutor(GenericPipelineExecutor):
                     "mode": "rw",
                 }
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                catalog_location = self._context.catalog_handler.catalog_location
+                catalog_location = self._context.catalog.catalog_location
                 self._volumes[str(Path(catalog_location).resolve())] = {
                     "bind": f"{self._container_catalog_location}",
                     "mode": "rw",
                 }
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                secrets_location = self._context.secrets_handler.location
+                secrets_location = self._context.secrets.location
                 self._volumes[str(Path(secrets_location).resolve())] = {
                     "bind": f"{self._container_secrets_location}",
                     "mode": "ro",
@@ -331,14 +273,12 @@ class LocalContainerExecutor(GenericPipelineExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                self._context.secrets_handler.location = (
-                    self._container_secrets_location
-                )
+                self._context.secrets.location = self._container_secrets_location

extensions/pipeline_executor/mocked.py

@@ -4,10 +4,10 @@ from typing import Any, Dict, Type, cast
 
 from pydantic import ConfigDict, Field
 
-from extensions.nodes.nodes import TaskNode
+from extensions.nodes.task import TaskNode
 from extensions.pipeline_executor import GenericPipelineExecutor
-from runnable import context, defaults
-from runnable.defaults import TypeMapVariable
+from runnable import defaults
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 from runnable.tasks import BaseTaskType
 
@@ -32,11 +32,7 @@ class MockedExecutor(GenericPipelineExecutor):
 
     patches: Dict[str, Any] = Field(default_factory=dict)
 
-    @property
-    def _context(self):
-        return context.run_context
-
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -140,7 +136,7 @@ class MockedExecutor(GenericPipelineExecutor):
 
         return effective_node_config
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         The entry point for all executors apart from local.
         We have already prepared for node execution.

extensions/pipeline_executor/retry.py

@@ -3,9 +3,9 @@ from functools import cached_property
 from typing import Any, Dict, Optional
 
 from extensions.pipeline_executor import GenericPipelineExecutor
-from runnable import context, defaults, exceptions
+from runnable import defaults, exceptions
 from runnable.datastore import RunLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -33,10 +33,6 @@ class RetryExecutor(GenericPipelineExecutor):
     _original_run_log: Optional[RunLog] = None
     _restart_initiated: bool = False
 
-    @property
-    def _context(self):
-        return context.run_context
-
     @cached_property
     def original_run_log(self):
         return self._context.run_log_store.get_run_log_by_id(
@@ -46,7 +42,7 @@ class RetryExecutor(GenericPipelineExecutor):
 
     def _set_up_for_re_run(self, params: Dict[str, Any]) -> None:
         # Sync the previous run log catalog to this one.
-        self._context.catalog_handler.sync_between_runs(
+        self._context.catalog.sync_between_runs(
             previous_run_id=self.run_id, run_id=self._context.run_id
         )
 
@@ -63,7 +59,7 @@ class RetryExecutor(GenericPipelineExecutor):
         # Should the parameters be copied from previous execution
         # self._set_up_for_re_run(params=params)
 
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -124,7 +120,7 @@ class RetryExecutor(GenericPipelineExecutor):
         self.execute_node(node=node, map_variable=map_variable)
 
     def _is_step_eligible_for_rerun(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is
@@ -172,5 +168,5 @@ class RetryExecutor(GenericPipelineExecutor):
         self._restart_initiated = True
         return True
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         self._execute_node(node, map_variable=map_variable)

runnable/__init__.py

@@ -1,24 +1,17 @@
 # ruff: noqa
 
-
-import logging
 import os
-from logging.config import dictConfig
 
 from rich.console import Console
 
-from runnable import defaults
-
-dictConfig(defaults.LOGGING_CONFIG)
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
 console = Console(record=True)
 console.print(":runner: Lets go!!")
 
 task_console = Console(record=True)
 
-from runnable.sdk import (  # noqa
+from runnable.sdk import (  # noqa;
     Catalog,
+    Conditional,
     Fail,
     Map,
     NotebookJob,
@@ -31,8 +24,6 @@ from runnable.sdk import (  # noqa
     ShellTask,
     Stub,
     Success,
-    TorchJob,
-    TorchTask,
     metric,
     pickled,
 )

runnable/catalog.py

@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -11,26 +11,6 @@ from runnable.datastore import DataCatalog
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-def is_catalog_out_of_sync(
-    catalog, synced_catalogs=Optional[List[DataCatalog]]
-) -> bool:
-    """
-    Check if the catalog items are out of sync from already cataloged objects.
-    If they are, return False.
-    If the object does not exist or synced catalog does not exist, return True
-    """
-    if not synced_catalogs:
-        return True  # If nothing has been synced in the past
-
-    for synced_catalog in synced_catalogs:
-        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
-            if synced_catalog.data_hash == catalog.data_hash:
-                return False
-            return True
-
-    return True  # The object does not exist, sync it
-
-
 # --8<-- [start:docs]
 
 
@@ -77,7 +57,7 @@ class BaseCatalog(ABC, BaseModel):
 
     @abstractmethod
     def put(
-        self, name: str, allow_file_not_found_exc: bool = False
+        self, name: str, allow_file_not_found_exc: bool = False, store_copy: bool = True
     ) -> List[DataCatalog]:
         """
         Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
@@ -140,7 +120,10 @@ class DoNothingCatalog(BaseCatalog):
         return []
 
     def put(
-        self, name: str, allow_file_not_found_exc: bool = False
+        self,
+        name: str,
+        allow_file_not_found_exc: bool = False,
+        store_copy: bool = True,
     ) -> List[DataCatalog]:
         """
         Does nothing

runnable/cli.py

@@ -1,10 +1,13 @@
 import logging
+import os
 from enum import Enum
+from pathlib import Path
 from typing import Annotated
 
 import typer
 
 from runnable import defaults, entrypoints
+from runnable.gantt import SimpleVisualizer, generate_html_timeline, visualize_simple
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -223,14 +226,21 @@ def fan(
     )
 
 
-@app.command()
-def submit_job(
+@app.command(hidden=True)
+def execute_job(
     job_definition_file: Annotated[
         str,
         typer.Argument(
             help=("The yaml file containing the job definition"),
         ),
     ],
+    run_id: Annotated[
+        str,
+        typer.Argument(
+            envvar="RUNNABLE_RUN_ID",
+            help="An optional run_id, one would be generated if its not provided",
+        ),
+    ] = "",
     config_file: Annotated[
         str,
         typer.Option(
@@ -255,16 +265,10 @@ def submit_job(
         ),
     ] = LogLevel.WARNING,
     tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
-    run_id: Annotated[
-        str,
-        typer.Option(
-            help="An optional run_id, one would be generated if its not provided"
-        ),
-    ] = "",
 ):
     logger.setLevel(log_level.value)
 
-    entrypoints.execute_job_yaml_spec(
+    entrypoints.execute_job_non_local(
         configuration_file=config_file,
         job_definition_file=job_definition_file,
         tag=tag,
@@ -273,43 +277,29 @@ def submit_job(
     )
 
 
-@app.command(hidden=True)
-def execute_job(
-    job_definition_file: Annotated[
-        str,
-        typer.Argument(
-            help=("The yaml file containing the job definition"),
-        ),
+@app.command()
+def timeline(
+    run_id_or_path: Annotated[
+        str, typer.Argument(help="Run ID to visualize, or path to JSON run log file")
     ],
-    run_id: Annotated[
-        str,
-        typer.Argument(
-            envvar="RUNNABLE_RUN_ID",
-            help="An optional run_id, one would be generated if its not provided",
-        ),
-    ] = "",
-    config_file: Annotated[
+    output: Annotated[
         str,
-        typer.Option(
-            "--config", "-c", help="The configuration file specifying the services"
-        ),
+        typer.Option("--output", "-o", help="Output HTML file path"),
     ] = "",
-    parameters_file: Annotated[
-        str,
+    console: Annotated[
+        bool,
         typer.Option(
-            "--parameters",
-            "-p",
-            help="Parameters, in yaml,  accessible by the application",
+            "--console/--no-console",
+            help="Show console timeline output (default: true)",
         ),
-    ] = "",
-    mode: Annotated[
-        ExecutionMode,
+    ] = True,
+    open_browser: Annotated[
+        bool,
         typer.Option(
-            "--mode",
-            "-m",
-            help="spec in yaml or python sdk",
+            "--open/--no-open",
+            help="Automatically open the generated file in default browser",
         ),
-    ] = ExecutionMode.YAML,
+    ] = True,
     log_level: Annotated[
         LogLevel,
         typer.Option(
@@ -319,18 +309,125 @@ def execute_job(
             case_sensitive=False,
         ),
     ] = LogLevel.WARNING,
-    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
 ):
+    """
+    Visualize pipeline execution as an interactive timeline.
+
+    This command creates lightweight timeline visualizations that effectively
+    show composite nodes (parallel, map, conditional) with hierarchical structure,
+    timing information, and execution metadata.
+
+    The new visualization system provides:
+    - Clean console output with hierarchical display
+    - Interactive HTML with hover tooltips and expandable sections
+    - Proper support for all composite pipeline types
+    - Rich metadata including commands, parameters, and catalog operations
+
+    By default, shows console output AND generates HTML file with browser opening.
+
+    Input Options:
+    - Run ID: Looks up JSON file in .run_log_store/ directory
+    - JSON Path: Direct path to run log JSON file (flexible for any config)
+
+    Examples:
+        # Using Run ID (looks in .run_log_store/)
+        runnable timeline forgiving-joliot-0645                    # Console + HTML + browser
+        runnable timeline parallel-run --output custom.html       # Console + custom HTML + browser
+
+        # Using JSON file path (any location)
+        runnable timeline /path/to/my-run.json                    # Console + HTML + browser
+        runnable timeline ../logs/pipeline-run.json --no-open     # Console + HTML, no browser
+        runnable timeline ~/experiments/run.json --no-console     # HTML + browser only
+
+        # Other options
+        runnable timeline complex-pipeline --no-open              # Console + HTML, no browser
+        runnable timeline simple-run --no-console --no-open       # HTML only, no browser
+    """
     logger.setLevel(log_level.value)
 
-    entrypoints.execute_job_non_local(
-        configuration_file=config_file,
-        job_definition_file=job_definition_file,
-        mode=mode,
-        tag=tag,
-        run_id=run_id,
-        parameters_file=parameters_file,
-    )
+    # Determine if input is a file path or run ID
+    if os.path.exists(run_id_or_path) or run_id_or_path.endswith(".json"):
+        # Input is a file path
+        json_file_path = Path(run_id_or_path)
+        if not json_file_path.exists():
+            print(f"❌ JSON file not found: {json_file_path}")
+            return
+
+        # Extract run ID from the file for default naming
+        run_id = json_file_path.stem
+        mode = "file"
+    else:
+        # Input is a run ID - use existing behavior
+        run_id = run_id_or_path
+        json_file_path = None
+        mode = "run_id"
+
+    # Default console behavior: always show console output
+    show_console = console if console is not None else True
+
+    if output:
+        # Generate HTML file with console output
+        output_file = output
+        print(f"🌐 Generating timeline: {output_file}")
+
+        if show_console:
+            # Show console output first, then generate HTML
+            if mode == "file":
+                _visualize_simple_from_file(json_file_path, show_summary=False)
+            else:
+                visualize_simple(run_id, show_summary=False)
+            print(f"\n🌐 Generating HTML timeline: {output_file}")
+
+        if mode == "file":
+            _generate_html_timeline_from_file(json_file_path, output_file, open_browser)
+        else:
+            generate_html_timeline(run_id, output_file, open_browser)
+    else:
+        # Default behavior: show console + generate HTML with browser
+        if show_console:
+            if mode == "file":
+                _visualize_simple_from_file(json_file_path, show_summary=False)
+            else:
+                visualize_simple(run_id, show_summary=False)
+
+        # Always generate HTML file and open browser by default
+        output_file = f"{run_id}_timeline.html"
+        print(f"\n🌐 Generating HTML timeline: {output_file}")
+        if mode == "file":
+            _generate_html_timeline_from_file(json_file_path, output_file, open_browser)
+        else:
+            generate_html_timeline(run_id, output_file, open_browser)
+
+
+def _visualize_simple_from_file(json_file_path, show_summary: bool = False) -> None:
+    """Visualize timeline from JSON file path."""
+
+    try:
+        viz = SimpleVisualizer(json_file_path)
+        viz.print_simple_timeline()
+        if show_summary:
+            viz.print_execution_summary()
+    except Exception as e:
+        print(f"❌ Error reading JSON file: {e}")
+
+
+def _generate_html_timeline_from_file(
+    json_file_path, output_file: str, open_browser: bool = True
+) -> None:
+    """Generate HTML timeline from JSON file path."""
+
+    try:
+        viz = SimpleVisualizer(json_file_path)
+        viz.generate_html_timeline(output_file)
+
+        if open_browser:
+            import webbrowser
+
+            file_path = Path(output_file).absolute()
+            print(f"🌐 Opening timeline in browser: {file_path.name}")
+            webbrowser.open(file_path.as_uri())
+    except Exception as e:
+        print(f"❌ Error generating HTML: {e}")
 
 
 if __name__ == "__main__":