runnable 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

runnable/extensions/executor/k8s_job/implementation_FF.py

@@ -4,10 +4,10 @@
 
 # from pydantic import BaseModel
 
-# from magnus import defaults, integration, utils
-# from magnus.executor import BaseExecutor
-# from magnus.graph import Graph
-# from magnus.nodes import BaseNode
+# from runnable import defaults, integration, utils
+# from runnable.executor import BaseExecutor
+# from runnable.graph import Graph
+# from runnable.nodes import BaseNode
 
 # logger = logging.getLogger(defaults.NAME)
 

runnable/extensions/executor/local/implementation.py

@@ -25,6 +25,7 @@ class LocalExecutor(GenericExecutor):
     """
 
     service_name: str = "local"
+    _local: bool = True
 
     def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
         """

runnable/extensions/executor/local_container/implementation.py

@@ -55,6 +55,8 @@ class LocalContainerExecutor(GenericExecutor):
     run_in_local: bool = False
     environment: Dict[str, str] = Field(default_factory=dict)
 
+    _local: bool = False
+
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
     _container_secrets_location = "/tmp/dotenv"
@@ -131,7 +133,7 @@ class LocalContainerExecutor(GenericExecutor):
 
 
         If the config has "run_in_local: True", we compute it on local system instead of container.
-        In local container execution, we just spin the container to execute magnus execute_single_node.
+        In local container execution, we just spin the container to execute runnable execute_single_node.
 
         Args:
             node (BaseNode): The node we are currently executing
@@ -198,6 +200,7 @@ class LocalContainerExecutor(GenericExecutor):
 
         try:
             logger.info(f"Running the command {command}")
+            print(command)
             #  Overrides global config with local
             executor_config = self._resolve_executor_config(node)
 
@@ -256,15 +259,6 @@ class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
     service_type = "run_log_store"  # One of secret, catalog, datastore
     service_provider = "file-system"  # The actual implementation of the service
 
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = (
-                "Run log generated by file-system run log store are not thread safe. "
-                "Inconsistent results are possible because of race conditions to write to the same file.\n"
-                "Consider using partitioned run log store like database for consistent results."
-            )
-            logger.warning(msg)
-
     def configure_for_traversal(self, **kwargs):
         from runnable.extensions.run_log_store.file_system.implementation import FileSystemRunLogstore
 

runnable/extensions/executor/mocked/implementation.py

@@ -8,7 +8,6 @@ from runnable import context, defaults
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
 from runnable.extensions.nodes import TaskNode
-from runnable.integration import BaseIntegration
 from runnable.nodes import BaseNode
 from runnable.tasks import BaseTaskType
 
@@ -25,8 +24,7 @@ def create_executable(params: Dict[str, Any], model: Type[BaseTaskType], node_na
 
 class MockedExecutor(GenericExecutor):
     service_name: str = "mocked"
-
-    enable_parallel: bool = defaults.ENABLE_PARALLEL
+    _local_executor: bool = True
 
     patches: Dict[str, Any] = Field(default_factory=dict)
 
@@ -119,6 +117,7 @@ class MockedExecutor(GenericExecutor):
         self.prepare_for_node_execution()
         self.execute_node(node=node, map_variable=map_variable, **kwargs)
 
+    # TODO: This needs to go away
     def _is_step_eligible_for_rerun(self, node: BaseNode, map_variable: TypeMapVariable = None):
         """
         In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is
@@ -188,33 +187,3 @@ class MockedExecutor(GenericExecutor):
             map_variable (dict[str, str], optional): _description_. Defaults to None.
         """
         self._execute_node(node=node, map_variable=map_variable, **kwargs)
-
-
-class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
-    """
-    Integration between local container and file system run log store
-    """
-
-    executor_type = "local-container"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "file-system"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = "Mocked executor does not support parallel execution. "
-            logger.warning(msg)
-
-
-class LocalContainerComputeChunkedFSRunLogstore(BaseIntegration):
-    """
-    Integration between local container and file system run log store
-    """
-
-    executor_type = "local-container"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "chunked-fs"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = "Mocked executor does not support parallel execution. "
-            logger.warning(msg)

runnable/extensions/nodes.py

@@ -1,10 +1,9 @@
-import json
+import copy
 import logging
-import multiprocessing
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Dict, cast
+from typing import Any, Dict, Optional, cast
 
 from pydantic import ConfigDict, Field, ValidationInfo, field_serializer, field_validator
 from typing_extensions import Annotated
@@ -44,9 +43,15 @@ class TaskNode(ExecutableNode):
         executable = create_task(task_config)
         return cls(executable=executable, **node_config, **task_config)
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
-        All that we do in magnus is to come to this point where we actually execute the command.
+        All that we do in runnable is to come to this point where we actually execute the command.
 
         Args:
             executor (_type_): The executor class
@@ -62,9 +67,11 @@ class TaskNode(ExecutableNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = copy.deepcopy(params)
             if not mock:
                 # Do not run if we are mocking the execution, could be useful for caching and dry runs
-                self.executable.execute_command(map_variable=map_variable)
+                output_parameters = self.executable.execute_command(map_variable=map_variable, params=params)
+                attempt_log.output_parameters = output_parameters
         except Exception as _e:  # pylint: disable=W0703
             logger.exception("Task failed")
             attempt_log.status = defaults.FAIL
@@ -88,7 +95,13 @@ class FailNode(TerminalNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "FailNode":
         return cast("FailNode", super().parse_from_config(config))
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Execute the failure node.
         Set the run or branch log status to failure.
@@ -105,6 +118,7 @@ class FailNode(TerminalNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = params
             #  could be a branch or run log
             run_or_branch_log = self._context.run_log_store.get_branch_log(
                 self._get_branch_log_name(map_variable), self._context.run_id
@@ -133,7 +147,13 @@ class SuccessNode(TerminalNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "SuccessNode":
         return cast("SuccessNode", super().parse_from_config(config))
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Execute the success node.
         Set the run or branch log status to success.
@@ -150,6 +170,7 @@ class SuccessNode(TerminalNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = params
             #  could be a branch or run log
             run_or_branch_log = self._context.run_log_store.get_branch_log(
                 self._get_branch_log_name(map_variable), self._context.run_id
@@ -257,35 +278,11 @@ class ParallelNode(CompositeNode):
             executor (Executor): The Executor as per the use config
             **kwargs: Optional kwargs passed around
         """
-        from runnable import entrypoints
 
         self.fan_out(map_variable=map_variable, **kwargs)
 
-        jobs = []
-        # Given that we can have nesting and complex graphs, controlling the number of processes is hard.
-        # A better way is to actually submit the job to some process scheduler which does resource management
-        for internal_branch_name, branch in self.branches.items():
-            if self._context.executor._is_parallel_execution():
-                # Trigger parallel jobs
-                action = entrypoints.execute_single_brach
-                kwargs = {
-                    "configuration_file": self._context.configuration_file,
-                    "pipeline_file": self._context.pipeline_file,
-                    "branch_name": internal_branch_name.replace(" ", defaults.COMMAND_FRIENDLY_CHARACTER),
-                    "run_id": self._context.run_id,
-                    "map_variable": json.dumps(map_variable),
-                    "tag": self._context.tag,
-                }
-                process = multiprocessing.Process(target=action, kwargs=kwargs)
-                jobs.append(process)
-                process.start()
-
-            else:
-                # If parallel is not enabled, execute them sequentially
-                self._context.executor.execute_graph(branch, map_variable=map_variable, **kwargs)
-
-        for job in jobs:
-            job.join()  # Find status of the branches
+        for _, branch in self.branches.items():
+            self._context.executor.execute_graph(branch, map_variable=map_variable, **kwargs)
 
         self.fan_in(map_variable=map_variable, **kwargs)
 
@@ -418,7 +415,6 @@ class MapNode(CompositeNode):
             map_variable (dict): The map variables the graph belongs to
             **kwargs: Optional kwargs passed around
         """
-        from runnable import entrypoints
 
         iterate_on = None
         try:
@@ -433,34 +429,11 @@ class MapNode(CompositeNode):
 
         self.fan_out(map_variable=map_variable, **kwargs)
 
-        jobs = []
-        # Given that we can have nesting and complex graphs, controlling the number of processess is hard.
-        # A better way is to actually submit the job to some process scheduler which does resource management
         for iter_variable in iterate_on:
             effective_map_variable = map_variable or OrderedDict()
             effective_map_variable[self.iterate_as] = iter_variable
 
-            if self._context.executor._is_parallel_execution():
-                # Trigger parallel jobs
-                action = entrypoints.execute_single_brach
-                kwargs = {
-                    "configuration_file": self._context.configuration_file,
-                    "pipeline_file": self._context.pipeline_file,
-                    "branch_name": self.branch.internal_branch_name.replace(" ", defaults.COMMAND_FRIENDLY_CHARACTER),
-                    "run_id": self._context.run_id,
-                    "map_variable": json.dumps(effective_map_variable),
-                    "tag": self._context.tag,
-                }
-                process = multiprocessing.Process(target=action, kwargs=kwargs)
-                jobs.append(process)
-                process.start()
-
-            else:
-                # If parallel is not enabled, execute them sequentially
-                self._context.executor.execute_graph(self.branch, map_variable=effective_map_variable, **kwargs)
-
-        for job in jobs:
-            job.join()
+            self._context.executor.execute_graph(self.branch, map_variable=effective_map_variable, **kwargs)
 
         self.fan_in(map_variable=map_variable, **kwargs)
 
@@ -652,7 +625,13 @@ class StubNode(ExecutableNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "StubNode":
         return cls(**config)
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Do Nothing node.
         We just send an success attempt log back to the caller
@@ -666,6 +645,7 @@ class StubNode(ExecutableNode):
             [type]: [description]
         """
         attempt_log = self._context.run_log_store.create_attempt_log()
+        attempt_log.input_parameters = params
 
         attempt_log.start_time = str(datetime.now())
         attempt_log.status = defaults.SUCCESS  # This is a dummy node and always will be success

runnable/integration.py

@@ -84,7 +84,7 @@ def get_integration_handler(executor: "BaseExecutor", service: object) -> BaseIn
             logger.info(f"Identified an integration pattern {kls.obj}")
             integrations.append(kls.obj)
 
-    # Get all the implementations defined by the magnus package
+    # Get all the implementations defined by the runnable package
     for kls in BaseIntegration.__subclasses__():
         # Match the exact service type
         if kls.service_type == service_type and kls.service_provider == service_name:
@@ -95,7 +95,7 @@ def get_integration_handler(executor: "BaseExecutor", service: object) -> BaseIn
     if len(integrations) > 1:
         msg = (
             f"Multiple integrations between {executor.service_name} and {service_name} of type {service_type} found. "
-            "If you defined an integration pattern, please ensure it is specific and does not conflict with magnus "
+            "If you defined an integration pattern, please ensure it is specific and does not conflict with runnable "
             " implementations."
         )
         logger.exception(msg)

runnable/interaction.py

@@ -58,6 +58,11 @@ def track_this(step: int = 0, **kwargs):
         os.environ[prefix + key + f"{defaults.STEP_INDICATOR}{step}"] = json.dumps(value)
 
 
+# TODO: Do we need the API for parameters?
+# If we still want them, what takes precedence? API or returns?
+# Once we decide that, collect the parameters and update them in tasks
+
+
 @check_context
 def set_parameter(**kwargs) -> None:
     """
@@ -279,7 +284,7 @@ def get_run_id() -> str:
     """
     Returns the run_id of the current run.
 
-    You can also access this from the environment variable `MAGNUS_RUN_ID`.
+    You can also access this from the environment variable `runnable_RUN_ID`.
     """
     return context.run_context.run_id
 
@@ -321,14 +326,14 @@ def get_experiment_tracker_context() -> ContextManager:
 
 def start_interactive_session(run_id: str = "", config_file: str = "", tag: str = "", parameters_file: str = ""):
     """
-    During interactive python coding, either via notebooks or ipython, you can start a magnus session by calling
+    During interactive python coding, either via notebooks or ipython, you can start a runnable session by calling
     this function. The executor would always be local executor as its interactive.
 
     If this was called during a pipeline/function/notebook execution, it will be ignored.
 
     Args:
         run_id (str, optional): The run id to use. Defaults to "" and would be created if not provided.
-        config_file (str, optional): The configuration file to use. Defaults to "" and magnus defaults.
+        config_file (str, optional): The configuration file to use. Defaults to "" and runnable defaults.
         tag (str, optional): The tag to attach to the run. Defaults to "".
         parameters_file (str, optional): The parameters file to use. Defaults to "".
     """
@@ -350,7 +355,7 @@ def start_interactive_session(run_id: str = "", config_file: str = "", tag: str
 
     executor = context.run_context.executor
 
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=config_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=config_file, tag=tag)
 
     context.run_context.execution_plan = defaults.EXECUTION_PLAN.INTERACTIVE.value
     executor.prepare_for_graph_execution()

runnable/nodes.py

@@ -64,7 +64,7 @@ class BaseNode(ABC, BaseModel):
     @classmethod
     def _get_internal_name_from_command_name(cls, command_name: str) -> str:
         """
-        Replace Magnus specific character (%) with whitespace.
+        Replace runnable specific character (%) with whitespace.
         The opposite of _command_friendly_name.
 
         Args:
@@ -274,7 +274,13 @@ class BaseNode(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         The actual function that does the execution of the command in the config.
 
@@ -282,7 +288,7 @@ class BaseNode(ABC, BaseModel):
         composite nodes.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor class
+            executor (runnable.executor.BaseExecutor): The executor class
             mock (bool, optional): Don't run, just pretend. Defaults to False.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
                 Defaults to ''.
@@ -301,7 +307,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
 
         Raises:
             NotImplementedError: Base class, hence not implemented.
@@ -317,7 +323,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
 
         Raises:
@@ -334,7 +340,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
 
         Raises:
@@ -449,7 +455,13 @@ class CompositeNode(TraversalNode):
     def _get_max_attempts(self) -> int:
         raise Exception("This is a composite node and does not have a max_attempts")
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         raise Exception("This is a composite node and does not have an execute function")
 
 

runnable/parameters.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 
 def get_user_set_parameters(remove: bool = False) -> Dict[str, Any]:
     """
-    Scans the environment variables for any user returned parameters that have a prefix MAGNUS_PRM_.
+    Scans the environment variables for any user returned parameters that have a prefix runnable_PRM_.
 
     This function does not deal with any type conversion of the parameters.
     It just deserializes the parameters and returns them as a dictionary.

runnable/sdk.py

@@ -7,7 +7,6 @@ from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, computed_field, field_validator, model_validator
 from rich import print
-from ruamel.yaml import YAML
 from typing_extensions import Self
 
 from runnable import defaults, entrypoints, graph, utils
@@ -33,7 +32,7 @@ class Catalog(BaseModel):
         put (List[str]): List of glob patterns to put into central catalog from the compute data folder.
 
     Examples:
-        >>> from magnus import Catalog, Task
+        >>> from runnable import Catalog, Task
         >>> catalog = Catalog(compute_data_folder="/path/to/data", get=["*.csv"], put=["*.csv"])
 
         >>> task = Task(name="task", catalog=catalog, command="echo 'hello'")
@@ -107,6 +106,9 @@ class BaseTraversal(ABC, BaseModel):
         ...
 
 
+## TODO: Add python task, shell task, and notebook task.
+
+
 class Task(BaseTraversal):
     """
     An execution node of the pipeline.
@@ -133,10 +135,10 @@ class Task(BaseTraversal):
             executor:
               type: local-container
               config:
-                docker_image: "magnus/magnus:latest"
+                docker_image: "runnable/runnable:latest"
                 overrides:
                   custom_docker_image:
-                    docker_image: "magnus/magnus:custom"
+                    docker_image: "runnable/runnable:custom"
             ```
             ### Task specific configuration
             ```python
@@ -148,7 +150,7 @@ class Task(BaseTraversal):
         optional_ploomber_args (Optional[Dict[str, Any]]): Any optional ploomber args.
             Only used when command_type is 'notebook', defaults to {}
         output_cell_tag (Optional[str]): The tag of the output cell.
-            Only used when command_type is 'notebook', defaults to "magnus_output"
+            Only used when command_type is 'notebook', defaults to "runnable_output"
         terminate_with_failure (bool): Whether to terminate the pipeline with a failure after this node.
         terminate_with_success (bool): Whether to terminate the pipeline with a success after this node.
         on_failure (str): The name of the node to execute if the step fails.
@@ -385,6 +387,9 @@ class Pipeline(BaseModel):
 
         self._dag.check_graph()
 
+    def return_dag(self) -> graph.Graph:
+        return self._dag
+
     def execute(
         self,
         configuration_file: str = "",
@@ -393,7 +398,6 @@ class Pipeline(BaseModel):
         parameters_file: str = "",
         use_cached: str = "",
         log_level: str = defaults.LOG_LEVEL,
-        output_pipeline_definition: str = "magnus-pipeline.yaml",
     ):
         """
         *Execute* the Pipeline.
@@ -408,7 +412,7 @@ class Pipeline(BaseModel):
 
         Args:
             configuration_file (str, optional): The path to the configuration file. Defaults to "".
-                The configuration file can be overridden by the environment variable MAGNUS_CONFIGURATION_FILE.
+                The configuration file can be overridden by the environment variable runnable_CONFIGURATION_FILE.
 
             run_id (str, optional): The ID of the run. Defaults to "".
             tag (str, optional): The tag of the run. Defaults to "".
@@ -419,18 +423,18 @@ class Pipeline(BaseModel):
                 Provide the run_id of the older execution to recover.
 
             log_level (str, optional): The log level. Defaults to defaults.LOG_LEVEL.
-            output_pipeline_definition (str, optional): The path to the output pipeline definition file.
-                Defaults to "magnus-pipeline.yaml".
-
-                Only applicable for the execution via SDK for non ```local``` executors.
         """
-        from runnable.extensions.executor.local.implementation import LocalExecutor
-        from runnable.extensions.executor.mocked.implementation import MockedExecutor
+
+        # py_to_yaml is used by non local executors to generate the yaml representation of the pipeline.
+        py_to_yaml = os.environ.get("RUNNABLE_PY_TO_YAML", "false")
+
+        if py_to_yaml == "true":
+            return
 
         logger.setLevel(log_level)
 
         run_id = utils.generate_run_id(run_id=run_id)
-        configuration_file = os.environ.get("MAGNUS_CONFIGURATION_FILE", configuration_file)
+        configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
         run_context = entrypoints.prepare_configurations(
             configuration_file=configuration_file,
             run_id=run_id,
@@ -440,7 +444,7 @@ class Pipeline(BaseModel):
         )
 
         run_context.execution_plan = defaults.EXECUTION_PLAN.CHAINED.value
-        utils.set_magnus_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
+        utils.set_runnable_environment_variables(run_id=run_id, configuration_file=configuration_file, tag=tag)
 
         dag_definition = self._dag.model_dump(by_alias=True, exclude_none=True)
 
@@ -449,17 +453,14 @@ class Pipeline(BaseModel):
         print("Working with context:")
         print(run_context)
 
-        if not (isinstance(run_context.executor, LocalExecutor) or isinstance(run_context.executor, MockedExecutor)):
-            logger.debug(run_context.dag.model_dump(by_alias=True))
-            yaml = YAML()
+        if not run_context.executor._local:
+            # We are working with non local executor
+            import inspect
 
-            with open(output_pipeline_definition, "w", encoding="utf-8") as f:
-                yaml.dump(
-                    {"dag": run_context.dag.model_dump(by_alias=True, exclude_none=True)},
-                    f,
-                )
+            caller_stack = inspect.stack()[1]
+            module_to_call = f"{caller_stack.filename.replace('/', '.').replace('.py', '')}.{caller_stack.function}"
 
-            return
+            run_context.pipeline_file = f"{module_to_call}.py"
 
         # Prepare for graph execution
         run_context.executor.prepare_for_graph_execution()
@@ -467,4 +468,5 @@ class Pipeline(BaseModel):
         logger.info("Executing the graph")
         run_context.executor.execute_graph(dag=run_context.dag)
 
-        return run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)
+        if run_context.executor._local:
+            return run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)