runnable 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

runnable/__init__.py

@@ -24,7 +24,18 @@ from runnable.interaction import (
     set_parameter,
     track_this,
 )  # noqa
-from runnable.sdk import Stub, Pipeline, Task, Parallel, Map, Catalog, Success, Fail  # noqa
+from runnable.sdk import (
+    Stub,
+    Pipeline,
+    Parallel,
+    Map,
+    Catalog,
+    Success,
+    Fail,
+    PythonTask,
+    NotebookTask,
+    ShellTask,
+)  # noqa
 
 
 # TODO: Think of model registry as a central place to store models.

runnable/catalog.py

@@ -43,7 +43,7 @@ class BaseCatalog(ABC, BaseModel):
         Args:
             name (str): The name of the catalog item
             run_id (str): The run_id of the run.
-            compute_data_folder (str, optional): The compute data folder. Defaults to magnus default (data/)
+            compute_data_folder (str, optional): The compute data folder. Defaults to runnable default (data/)
 
         Raises:
             NotImplementedError: Base class, hence not implemented
@@ -70,7 +70,7 @@ class BaseCatalog(ABC, BaseModel):
         Args:
             name (str): The name of the catalog item.
             run_id (str): The run_id of the run.
-            compute_data_folder (str, optional): The compute data folder. Defaults to magnus default (data/)
+            compute_data_folder (str, optional): The compute data folder. Defaults to runnable default (data/)
             synced_catalogs (dict, optional): Any previously synced catalogs. Defaults to None.
 
         Raises:

runnable/cli.py

@@ -9,7 +9,7 @@ from runnable import defaults, entrypoints
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-@with_plugins(iter_entry_points("magnus.cli_plugins"))
+@with_plugins(iter_entry_points("runnable.cli_plugins"))
 @click.group()
 @click.version_option()
 def cli():
@@ -46,7 +46,7 @@ def execute(file, config_file, parameters_file, log_level, tag, run_id, use_cach
     """
     Execute a pipeline
 
-    Usage: magnus execute [OPTIONS]
+    Usage: runnable execute [OPTIONS]
 
     Options:
     -f, --file TEXT               The pipeline definition file [default: pipeline.yaml]
@@ -97,9 +97,9 @@ def execute(file, config_file, parameters_file, log_level, tag, run_id, use_cach
 @click.option("--tag", default="", help="A tag attached to the run")
 def execute_single_node(run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag):
     """
-    Internal entrypoint for magnus to execute a single node.
+    Internal entrypoint for runnable to execute a single node.
 
-    Other than local executor, every other executor uses this entry point to execute a step in the context of magnus.
+    Other than local executor, every other executor uses this entry point to execute a step in the context of runnable.
     Only chained executions should use this method. Unchained executions should use execute_
     """
     logger.setLevel(log_level)
@@ -248,7 +248,7 @@ def execute_function(
 @click.option("--tag", default="", help="A tag attached to the run")
 def fan(run_id, step_name, mode, map_variable, file, config_file, parameters_file, log_level, tag):
     """
-    Internal entrypoint for magnus to fan in or out a composite node.
+    Internal entrypoint for runnable to fan in or out a composite node.
 
     Only 3rd party orchestrators should use this entry point.
     """

runnable/datastore.py

@@ -12,7 +12,7 @@ from runnable import defaults, exceptions
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 # Once defined these classes are sealed to any additions unless a default is provided
-# Breaking this rule might make magnus backwardly incompatible
+# Breaking this rule might make runnable backwardly incompatible
 
 
 class DataCatalog(BaseModel, extra="allow"):
@@ -53,7 +53,8 @@ class StepAttempt(BaseModel):
     duration: str = ""  #  end_time - start_time
     status: str = "FAIL"
     message: str = ""
-    parameters: Dict[str, Any] = Field(default_factory=dict)
+    input_parameters: Dict[str, Any] = Field(default_factory=dict)
+    output_parameters: Dict[str, Any] = Field(default_factory=dict)
 
 
 class CodeIdentity(BaseModel, extra="allow"):

runnable/defaults.py

@@ -12,8 +12,8 @@ except ImportError:  # pragma: no cover
     from typing_extensions import TypedDict  # type: ignore[unused-ignore]
 
 
-NAME = "magnus"
-LOGGER_NAME = "magnus"
+NAME = "runnable"
+LOGGER_NAME = "runnable"
 
 # CLI settings
 LOG_LEVEL = "WARNING"
@@ -21,7 +21,7 @@ LOG_LEVEL = "WARNING"
 
 class EXECUTION_PLAN(Enum):
     """
-    The possible execution plans for a magnus job.
+    The possible execution plans for a runnable job.
     """
 
     CHAINED = "chained"  #  121 relationship between run log and the dag.
@@ -35,7 +35,7 @@ class ServiceConfig(TypedDict):
     config: Mapping[str, Any]
 
 
-class MagnusConfig(TypedDict, total=False):
+class runnableConfig(TypedDict, total=False):
     run_log_store: Optional[ServiceConfig]
     secrets: Optional[ServiceConfig]
     catalog: Optional[ServiceConfig]
@@ -47,17 +47,20 @@ TypeMapVariable: TypeAlias = Optional[Dict[str, Union[str, int, float]]]
 
 
 # Config file environment variable
-MAGNUS_CONFIG_FILE = "MAGNUS_CONFIG_FILE"
-MAGNUS_RUN_TAG = "MAGNUS_RUN_TAG"
+RUNNABLE_CONFIG_FILE = "RUNNABLE_CONFIG_FILE"
+RUNNABLE_RUN_TAG = "RUNNABLE_RUN_TAG"
 
 # Interaction settings
-TRACK_PREFIX = "MAGNUS_TRACK_"
+TRACK_PREFIX = "RUNNABLE_TRACK_"
 STEP_INDICATOR = "_STEP_"
-PARAMETER_PREFIX = "MAGNUS_PRM_"
-MAP_VARIABLE = "MAGNUS_MAP_VARIABLE"
-VARIABLE_PREFIX = "MAGNUS_VAR_"
-ENV_RUN_ID = "MAGNUS_RUN_ID"
-ATTEMPT_NUMBER = "MAGNUS_STEP_ATTEMPT"
+PARAMETER_PREFIX = "RUNNABLE_PRM_"
+MAP_VARIABLE = "RUNNABLE_MAP_VARIABLE"
+VARIABLE_PREFIX = "RUNNABLE_VAR_"
+ENV_RUN_ID = "RUNNABLE_RUN_ID"
+ATTEMPT_NUMBER = "RUNNABLE_STEP_ATTEMPT"
+
+## Generated pipeline file
+GENERATED_PIPELINE_FILE = "generated_pipeline.yaml"
 
 # STATUS progression
 # For Branch, CREATED -> PROCESSING -> SUCCESS OR FAIL
@@ -72,7 +75,7 @@ TRIGGERED = "TRIGGERED"
 COMMAND_TYPE = "python"
 NODE_SPEC_FILE = "node_spec.yaml"
 COMMAND_FRIENDLY_CHARACTER = "%"
-DEFAULT_CONTAINER_CONTEXT_PATH = "/opt/magnus/"
+DEFAULT_CONTAINER_CONTEXT_PATH = "/opt/runnable/"
 DEFAULT_CONTAINER_DATA_PATH = "data/"
 DEFAULT_CONTAINER_OUTPUT_PARAMETERS = "parameters.json"
 
@@ -94,7 +97,7 @@ RANDOM_RUN_ID_LEN = 6
 MAX_TIME = 86400  # 1 day in seconds
 
 # User extensions
-USER_CONFIG_FILE = "magnus-config.yaml"
+USER_CONFIG_FILE = "runnable-config.yaml"
 
 # Executor settings
 ENABLE_PARALLEL = False
@@ -155,7 +158,7 @@ LOGGING_CONFIG = {
     "disable_existing_loggers": True,
     "formatters": {
         "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"},
-        "magnus_formatter": {"format": "%(message)s", "datefmt": "[%X]"},
+        "runnable_formatter": {"format": "%(message)s", "datefmt": "[%X]"},
     },
     "handlers": {
         "default": {
@@ -163,8 +166,8 @@ LOGGING_CONFIG = {
             "class": "logging.StreamHandler",
             "stream": "ext://sys.stdout",  # Default is stderr
         },
-        "magnus_handler": {
-            "formatter": "magnus_formatter",
+        "runnable_handler": {
+            "formatter": "runnable_formatter",
             "class": "rich.logging.RichHandler",
             "rich_tracebacks": True,
         },
@@ -174,6 +177,6 @@ LOGGING_CONFIG = {
             "handlers": ["default"],
             "propagate": True,
         },  # Root logger
-        LOGGER_NAME: {"handlers": ["magnus_handler"], "propagate": False},
+        LOGGER_NAME: {"handlers": ["runnable_handler"], "propagate": False},
     },
 }

runnable/entrypoints.py

@@ -1,19 +1,22 @@
+import importlib
 import json
 import logging
+import os
+import sys
 from typing import Optional, cast
 
 from rich import print
 
 import runnable.context as context
 from runnable import defaults, graph, utils
-from runnable.defaults import MagnusConfig, ServiceConfig
+from runnable.defaults import ServiceConfig, runnableConfig
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-def get_default_configs() -> MagnusConfig:
+def get_default_configs() -> runnableConfig:
     """
-    User can provide extensions as part of their code base, magnus-config.yaml provides the place to put them.
+    User can provide extensions as part of their code base, runnable-config.yaml provides the place to put them.
     """
     user_configs = {}
     if utils.does_file_exist(defaults.USER_CONFIG_FILE):
@@ -53,7 +56,7 @@ def prepare_configurations(
     Returns:
         executor.BaseExecutor : A prepared executor as per the dag/config
     """
-    magnus_defaults = get_default_configs()
+    runnable_defaults = get_default_configs()
 
     variables = utils.gather_variables()
 
@@ -61,31 +64,31 @@ def prepare_configurations(
     if configuration_file:
         templated_configuration = utils.load_yaml(configuration_file) or {}
 
-    configuration: MagnusConfig = cast(MagnusConfig, templated_configuration)
+    configuration: runnableConfig = cast(runnableConfig, templated_configuration)
 
     # Run log settings, configuration over-rides everything
     run_log_config: Optional[ServiceConfig] = configuration.get("run_log_store", None)
     if not run_log_config:
-        run_log_config = cast(ServiceConfig, magnus_defaults.get("run_log_store", defaults.DEFAULT_RUN_LOG_STORE))
+        run_log_config = cast(ServiceConfig, runnable_defaults.get("run_log_store", defaults.DEFAULT_RUN_LOG_STORE))
     run_log_store = utils.get_provider_by_name_and_type("run_log_store", run_log_config)
 
     # Catalog handler settings, configuration over-rides everything
     catalog_config: Optional[ServiceConfig] = configuration.get("catalog", None)
     if not catalog_config:
-        catalog_config = cast(ServiceConfig, magnus_defaults.get("catalog", defaults.DEFAULT_CATALOG))
+        catalog_config = cast(ServiceConfig, runnable_defaults.get("catalog", defaults.DEFAULT_CATALOG))
     catalog_handler = utils.get_provider_by_name_and_type("catalog", catalog_config)
 
     # Secret handler settings, configuration over-rides everything
     secrets_config: Optional[ServiceConfig] = configuration.get("secrets", None)
     if not secrets_config:
-        secrets_config = cast(ServiceConfig, magnus_defaults.get("secrets", defaults.DEFAULT_SECRETS))
+        secrets_config = cast(ServiceConfig, runnable_defaults.get("secrets", defaults.DEFAULT_SECRETS))
     secrets_handler = utils.get_provider_by_name_and_type("secrets", secrets_config)
 
     # experiment tracker settings, configuration over-rides everything
     tracker_config: Optional[ServiceConfig] = configuration.get("experiment_tracker", None)
     if not tracker_config:
         tracker_config = cast(
-            ServiceConfig, magnus_defaults.get("experiment_tracker", defaults.DEFAULT_EXPERIMENT_TRACKER)
+            ServiceConfig, runnable_defaults.get("experiment_tracker", defaults.DEFAULT_EXPERIMENT_TRACKER)
         )
     tracker_handler = utils.get_provider_by_name_and_type("experiment_tracker", tracker_config)
 
@@ -95,7 +98,7 @@ def prepare_configurations(
         executor_config = ServiceConfig(type="local", config={})
 
     if not executor_config:
-        executor_config = cast(ServiceConfig, magnus_defaults.get("executor", defaults.DEFAULT_EXECUTOR))
+        executor_config = cast(ServiceConfig, runnable_defaults.get("executor", defaults.DEFAULT_EXECUTOR))
     configured_executor = utils.get_provider_by_name_and_type("executor", executor_config)
 
     # Construct the context
@@ -113,20 +116,30 @@ def prepare_configurations(
     )
 
     if pipeline_file:
-        # There are use cases where we are only preparing the executor
-        pipeline_config = utils.load_yaml(pipeline_file)
+        if pipeline_file.endswith(".py"):
+            # converting a pipeline defined in python to a dag in yaml
+            module_file = pipeline_file.strip(".py")
+            module, func = utils.get_module_and_attr_names(module_file)
+            sys.path.insert(0, os.getcwd())  # Need to add the current directory to path
+            imported_module = importlib.import_module(module)
 
-        logger.info("The input pipeline:")
-        logger.info(json.dumps(pipeline_config, indent=4))
+            os.environ["RUNNABLE_PY_TO_YAML"] = "true"
+            dag = getattr(imported_module, func)().return_dag()
 
-        # Create the graph
-        dag_config = pipeline_config["dag"]
-        dag_hash = utils.get_dag_hash(dag_config)
-        dag = graph.create_graph(dag_config)
+        else:
+            pipeline_config = utils.load_yaml(pipeline_file)
+
+            logger.info("The input pipeline:")
+            logger.info(json.dumps(pipeline_config, indent=4))
+
+            dag_config = pipeline_config["dag"]
+
+            dag_hash = utils.get_dag_hash(dag_config)
+            dag = graph.create_graph(dag_config)
+            run_context.dag_hash = dag_hash
 
         run_context.pipeline_file = pipeline_file
         run_context.dag = dag
-        run_context.dag_hash = dag_hash
 
     run_context.use_cached = False
     if use_cached:
@@ -148,7 +161,7 @@ def execute(
 ):
     # pylint: disable=R0914,R0913
     """
-    The entry point to magnus execution. This method would prepare the configurations and delegates traversal to the
+    The entry point to runnable execution. This method would prepare the configurations and delegates traversal to the
     executor
 
     Args:
@@ -176,7 +189,7 @@ def execute(
 
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
 
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
     # Prepare for graph execution
     executor.prepare_for_graph_execution()
@@ -197,7 +210,7 @@ def execute_single_node(
     parameters_file: str = "",
 ):
     """
-    The entry point into executing a single node of magnus. Orchestration modes should extensively use this
+    The entry point into executing a single node of runnable. Orchestration modes should extensively use this
     entry point.
 
     It should have similar set up of configurations to execute because orchestrator modes can initiate the execution.
@@ -226,7 +239,7 @@ def execute_single_node(
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
     executor.prepare_for_node_execution()
 
@@ -247,55 +260,6 @@ def execute_single_node(
     executor.send_return_code(stage="execution")
 
 
-def execute_single_brach(
-    configuration_file: str,
-    pipeline_file: str,
-    branch_name: str,
-    map_variable: str,
-    run_id: str,
-    tag: str,
-):
-    """
-    The entry point into executing a branch of the graph. Interactive modes in parallel runs use this to execute
-    branches in parallel.
-
-    This entry point is never used by its own but rather from a node. So the arguments sent into this are fewer.
-
-    Args:
-        variables_file (str): The variables file, if used or None
-        branch_name : The name of the branch to execute, in dot.path.convention
-        pipeline_file (str): The config/dag file
-        run_id (str): The run id of the run.
-        tag (str): If a tag is provided at the run time
-    """
-    from runnable import nodes
-
-    run_context = prepare_configurations(
-        configuration_file=configuration_file,
-        pipeline_file=pipeline_file,
-        run_id=run_id,
-        tag=tag,
-        use_cached="",
-    )
-    print("Working with context:")
-    print(run_context)
-
-    executor = run_context.executor
-    run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
-
-    branch_internal_name = nodes.BaseNode._get_internal_name_from_command_name(branch_name)
-
-    map_variable_dict = utils.json_to_ordered_dict(map_variable)
-
-    branch_to_execute = graph.search_branch_by_internal_name(run_context.dag, branch_internal_name)  # type: ignore
-
-    logger.info("Executing the single branch of %s", branch_to_execute)
-    executor.execute_graph(dag=branch_to_execute, map_variable=map_variable_dict)
-
-    executor.send_return_code()
-
-
 def execute_notebook(
     entrypoint: str,
     notebook_file: str,
@@ -307,7 +271,7 @@ def execute_notebook(
     parameters_file: str = "",
 ):
     """
-    The entry point to magnus execution of a notebook. This method would prepare the configurations and
+    The entry point to runnable execution of a notebook. This method would prepare the configurations and
     delegates traversal to the executor
     """
     run_id = utils.generate_run_id(run_id=run_id)
@@ -321,7 +285,7 @@ def execute_notebook(
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.UNCHAINED.value
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
     print("Working with context:")
     print(run_context)
@@ -368,7 +332,7 @@ def execute_function(
     parameters_file: str = "",
 ):
     """
-    The entry point to magnus execution of a function. This method would prepare the configurations and
+    The entry point to runnable execution of a function. This method would prepare the configurations and
     delegates traversal to the executor
     """
     run_id = utils.generate_run_id(run_id=run_id)
@@ -383,7 +347,7 @@ def execute_function(
     executor = run_context.executor
 
     run_context.execution_plan = defaults.EXECUTION_PLAN.UNCHAINED.value
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
     print("Working with context:")
     print(run_context)
@@ -460,7 +424,7 @@ def fan(
 
     executor = run_context.executor
     run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
     executor.prepare_for_node_execution()
 

runnable/executor.py

@@ -25,7 +25,7 @@ class BaseExecutor(ABC, BaseModel):
     The skeleton of an executor class.
     Any implementation of an executor should inherit this class and over-ride accordingly.
 
-    There is a extension available in magnus/extensions/executor/__init__.py
+    There is a extension available in runnable/extensions/executor/__init__.py
     which implements the most common functionality which is easier to
     extend/override in most scenarios.
 
@@ -34,11 +34,12 @@ class BaseExecutor(ABC, BaseModel):
     service_name: str = ""
     service_type: str = "executor"
 
-    enable_parallel: bool = defaults.ENABLE_PARALLEL
     overrides: dict = {}
 
+    # TODO: This needs to go away
     _previous_run_log: Optional[RunLog] = None
     _single_step: str = ""
+    _local: bool = False  # This is a flag to indicate whether the executor is local or not.
 
     _context_step_log = None  # type : StepLog
     _context_node = None  # type: BaseNode
@@ -48,19 +49,6 @@ class BaseExecutor(ABC, BaseModel):
     def _context(self):
         return context.run_context
 
-    def _is_parallel_execution(self) -> bool:
-        """
-        Controls the parallelization of branches in map and parallel state.
-        Defaults to False and left for the compute modes to decide.
-
-        Interactive executors like local and local-container need decisions.
-        For most transpilers it is inconsequential as its always True and supported by platforms.
-
-        Returns:
-            bool: True if the execution allows parallel execution of branches.
-        """
-        return self.enable_parallel
-
     @abstractmethod
     def _get_parameters(self) -> Dict[str, Any]:
         """
@@ -72,6 +60,7 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
+    # TODO: This needs to go away
     @abstractmethod
     def _set_up_for_re_run(self, parameters: Dict[str, Any]) -> None:
         """
@@ -251,7 +240,7 @@ class BaseExecutor(ABC, BaseModel):
     @abstractmethod
     def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
         """
-        Executor specific way of triggering jobs when magnus does both traversal and execution
+        Executor specific way of triggering jobs when runnable does both traversal and execution
 
         Transpilers will NEVER use this method and will NEVER call them.
         Only interactive executors who need execute_from_graph will ever implement it.
@@ -304,6 +293,7 @@ class BaseExecutor(ABC, BaseModel):
         """
         ...
 
+    # TODO: This needs to go away
     @abstractmethod
     def _is_step_eligible_for_rerun(self, node: BaseNode, map_variable: TypeMapVariable = None):
         """

runnable/extensions/catalog/file_system/implementation.py

@@ -144,7 +144,8 @@ class FileSystemCatalog(BaseCatalog):
         if not utils.does_dir_exist(copy_from):
             msg = (
                 f"Expected compute data folder to be present at: {compute_data_folder} but not found. \n"
-                "Note: Magnus does not create the compute data folder for you. Please ensure that the folder exists.\n"
+                "Note: runnable does not create the compute data folder for you. Please ensure that the "
+                "folder exists.\n"
             )
             raise Exception(msg)
 

runnable/extensions/executor/__init__.py

@@ -47,12 +47,12 @@ class GenericExecutor(BaseExecutor):
 
         This function is used by the decorator function.
         The design idea is we can over-ride this method in different implementations to retrieve the run_id.
-        But is it really intrusive to ask to set the environmental variable MAGNUS_RUN_ID?
+        But is it really intrusive to ask to set the environmental variable runnable_RUN_ID?
 
         Returns:
             _type_: _description_
         """
-        return os.environ.get("MAGNUS_RUN_ID", None)
+        return os.environ.get("runnable_RUN_ID", None)
 
     def _get_parameters(self) -> Dict[str, Any]:
         """
@@ -117,6 +117,7 @@ class GenericExecutor(BaseExecutor):
         # Consolidate and get the parameters
         parameters = self._get_parameters()
 
+        # TODO: This needs to go away
         if self._context.use_cached:
             self._set_up_for_re_run(parameters=parameters)
 
@@ -300,6 +301,8 @@ class GenericExecutor(BaseExecutor):
         We set them as environment variables, serialized as json strings.
         """
         params = self._context.run_log_store.get_parameters(run_id=self._context.run_id)
+        params_copy = copy.deepcopy(params)
+        # This is only for the API to work.
         parameters.set_user_defined_params_as_environment_variables(params)
 
         attempt = self.step_attempt_number
@@ -311,22 +314,26 @@ class GenericExecutor(BaseExecutor):
 
         data_catalogs_get: Optional[List[DataCatalog]] = self._sync_catalog(step_log, stage="get")
         try:
-            attempt_log = node.execute(executor=self, mock=step_log.mock, map_variable=map_variable, **kwargs)
+            attempt_log = node.execute(
+                executor=self,
+                mock=step_log.mock,
+                map_variable=map_variable,
+                params=params,
+                **kwargs,
+            )
         except Exception as e:
-            # Any exception here is a magnus exception as node suppresses exceptions.
-            msg = "This is clearly magnus fault, please report a bug and the logs"
+            # Any exception here is a runnable exception as node suppresses exceptions.
+            msg = "This is clearly runnable fault, please report a bug and the logs"
             logger.exception(msg)
             raise Exception(msg) from e
         finally:
             attempt_log.attempt_number = attempt
-            attempt_log.parameters = params.copy()
             step_log.attempts.append(attempt_log)
 
             tracked_data = get_tracked_data()
 
             self._context.experiment_tracker.publish_data(tracked_data)
-            # By this point, the updated parameters are deserialized as json strings.
-            parameters_out = parameters.get_user_set_parameters(remove=True)
+            parameters_out = attempt_log.output_parameters
 
             if attempt_log.status == defaults.FAIL:
                 logger.exception(f"Node: {node} failed")
@@ -339,10 +346,12 @@ class GenericExecutor(BaseExecutor):
                 step_log.status = defaults.SUCCESS
                 self._sync_catalog(step_log, stage="put", synced_catalogs=data_catalogs_get)
                 step_log.user_defined_metrics = tracked_data
-                diff_parameters = utils.diff_dict(params, parameters_out)
+
+                diff_parameters = utils.diff_dict(params_copy, parameters_out)
                 self._context.run_log_store.set_parameters(self._context.run_id, diff_parameters)
 
             # Remove the step context
+            parameters.get_user_set_parameters(remove=True)
             self._context_step_log = None
             self._context_node = None  # type: ignore
             self._context_metrics = {}  # type: ignore
@@ -400,6 +409,7 @@ class GenericExecutor(BaseExecutor):
             self._execute_node(node, map_variable=map_variable, **kwargs)
             return
 
+        # TODO: This needs to go away
         # In single step
         if (self._single_step and not node.name == self._single_step) or not self._is_step_eligible_for_rerun(
             node, map_variable=map_variable
@@ -533,6 +543,7 @@ class GenericExecutor(BaseExecutor):
             run_log = self._context.run_log_store.get_run_log_by_id(run_id=self._context.run_id, full=True)
         print(json.dumps(run_log.model_dump(), indent=4))
 
+    # TODO: This needs to go away
     def _is_step_eligible_for_rerun(self, node: BaseNode, map_variable: TypeMapVariable = None):
         """
         In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is