runnable 0.11.2__py3-none-any.whl → 0.11.4__py3-none-any.whl

runnable/__init__.py

@@ -2,6 +2,7 @@
 
 # TODO: Might need to add Rich to pyinstaller part
 import logging
+import os
 from logging.config import dictConfig
 
 from rich.console import Console
@@ -29,6 +30,8 @@ from runnable.sdk import (  # noqa
     pickled,
 )
 
+os.environ["_PLOOMBER_TELEMETRY_DEBUG"] = "false"
+
 ## TODO: Summary should be a bit better for catalog.
 ## If the execution fails, hint them about the retry executor.
 # Make the retry executor loose!
@@ -38,4 +41,4 @@ from runnable.sdk import (  # noqa
 
 
 # TODO: Think of way of generating dag hash without executor configuration
-# TODO: Add examples of map parameters and task types
+# Try to get a release

runnable/datastore.py

@@ -402,7 +402,10 @@ class RunLog(BaseModel):
         """
         dot_path = i_name.split(".")
         if len(dot_path) == 1:
-            return self.steps[i_name], None
+            try:
+                return self.steps[i_name], None
+            except KeyError as e:
+                raise exceptions.StepLogNotFoundError(self.run_id, i_name) from e
 
         current_steps = self.steps
         current_step = None

runnable/entrypoints.py

@@ -60,6 +60,8 @@ def prepare_configurations(
     variables = utils.gather_variables()
 
     templated_configuration = {}
+    configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
+
     if configuration_file:
         templated_configuration = utils.load_yaml(configuration_file) or {}
 
@@ -144,8 +146,8 @@ def prepare_configurations(
 
 
 def execute(
-    configuration_file: str,
     pipeline_file: str,
+    configuration_file: str = "",
     tag: str = "",
     run_id: str = "",
     parameters_file: str = "",
@@ -196,6 +198,10 @@ def execute(
             run_context.progress = progress
             executor.execute_graph(dag=run_context.dag)  # type: ignore
 
+            if not executor._local:
+                executor.send_return_code(stage="traversal")
+                return
+
             run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id, full=False)
 
             if run_log.status == defaults.SUCCESS:
@@ -205,6 +211,10 @@ def execute(
         except Exception as e:  # noqa: E722
             console.print(e, style=defaults.error_style)
             progress.update(pipeline_execution_task, description="[red] Errored execution", completed=True)
+            run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id, full=False)
+            run_log.status = defaults.FAIL
+            run_context.run_log_store.add_branch_log(run_log, run_context.run_id)
+            raise e
 
     executor.send_return_code()
 
@@ -235,6 +245,8 @@ def execute_single_node(
     """
     from runnable import nodes
 
+    configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
+
     run_context = prepare_configurations(
         configuration_file=configuration_file,
         pipeline_file=pipeline_file,
@@ -422,6 +434,8 @@ def fan(
     """
     from runnable import nodes
 
+    configuration_file = os.environ.get("RUNNABLE_CONFIGURATION_FILE", configuration_file)
+
     run_context = prepare_configurations(
         configuration_file=configuration_file,
         pipeline_file=pipeline_file,

runnable/extensions/executor/__init__.py

@@ -185,14 +185,11 @@ class GenericExecutor(BaseExecutor):
         data_catalogs = []
         for name_pattern in node_catalog_settings.get(stage) or []:
             if stage == "get":
-                get_catalog_progress = self._context.progress.add_task(f"Getting from catalog {name_pattern}", total=1)
                 data_catalog = self._context.catalog_handler.get(
                     name=name_pattern, run_id=self._context.run_id, compute_data_folder=compute_data_folder
                 )
-                self._context.progress.update(get_catalog_progress, completed=True, visible=False, refresh=True)
 
             elif stage == "put":
-                put_catalog_progress = self._context.progress.add_task(f"Putting in catalog {name_pattern}", total=1)
                 data_catalog = self._context.catalog_handler.put(
                     name=name_pattern,
                     run_id=self._context.run_id,
@@ -200,8 +197,6 @@ class GenericExecutor(BaseExecutor):
                     synced_catalogs=synced_catalogs,
                 )
 
-                self._context.progress.update(put_catalog_progress, completed=True, visible=False)
-
             logger.debug(f"Added data catalog: {data_catalog} to step log")
             data_catalogs.extend(data_catalog)
 

runnable/extensions/executor/argo/implementation.py

@@ -1033,6 +1033,9 @@ class ArgoExecutor(GenericExecutor):
             if working_on.node_type not in ["success", "fail"] and working_on._get_on_failure_node():
                 failure_node = dag.get_node_by_name(working_on._get_on_failure_node())
 
+                render_obj = get_renderer(working_on)(executor=self, node=failure_node)
+                render_obj.render(list_of_iter_values=list_of_iter_values.copy())
+
                 failure_template_name = self.get_clean_name(failure_node)
                 # If a task template for clean name exists, retrieve it
                 failure_template = templates.get(
@@ -1040,7 +1043,6 @@ class ArgoExecutor(GenericExecutor):
                     DagTaskTemplate(name=failure_template_name, template=failure_template_name),
                 )
                 failure_template.depends.append(f"{clean_name}.Failed")
-
                 templates[failure_template_name] = failure_template
 
             # If we are in a map node, we need to add the values as arguments

runnable/extensions/executor/local_container/implementation.py

@@ -5,7 +5,7 @@ from typing import Dict, cast
 from pydantic import Field
 from rich import print
 
-from runnable import defaults, integration, utils
+from runnable import defaults, utils
 from runnable.datastore import StepLog
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
@@ -145,16 +145,6 @@ class LocalContainerExecutor(GenericExecutor):
         logger.debug("Here is the resolved executor config")
         logger.debug(executor_config)
 
-        if executor_config.get("run_in_local", False):
-            # Do not change config but only validate the configuration.
-            # Trigger the job on local system instead of a container
-            integration.validate(self, self._context.run_log_store)
-            integration.validate(self, self._context.catalog_handler)
-            integration.validate(self, self._context.secrets_handler)
-
-            self.execute_node(node=node, map_variable=map_variable, **kwargs)
-            return
-
         command = utils.get_node_execution_command(node, map_variable=map_variable)
 
         self._spin_container(
@@ -172,7 +162,7 @@ class LocalContainerExecutor(GenericExecutor):
                 "Note: If you do not see any docker issue from your side and the code works properly on local execution"
                 "please raise a bug report."
             )
-            logger.warning(msg)
+            logger.error(msg)
             step_log.status = defaults.FAIL
             self._context.run_log_store.add_step_log(step_log, self._context.run_id)
 
@@ -212,6 +202,7 @@ class LocalContainerExecutor(GenericExecutor):
                     f"Please provide a docker_image using executor_config of the step {node.name} or at global config"
                 )
 
+            print("container", self._volumes)
             # TODO: Should consider using getpass.getuser() when running the docker container? Volume permissions
             container = client.containers.create(
                 image=docker_image,
@@ -260,7 +251,9 @@ class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
     service_provider = "file-system"  # The actual implementation of the service
 
     def configure_for_traversal(self, **kwargs):
-        from runnable.extensions.run_log_store.file_system.implementation import FileSystemRunLogstore
+        from runnable.extensions.run_log_store.file_system.implementation import (
+            FileSystemRunLogstore,
+        )
 
         self.executor = cast(LocalContainerExecutor, self.executor)
         self.service = cast(FileSystemRunLogstore, self.service)
@@ -272,7 +265,9 @@ class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
         }
 
     def configure_for_execution(self, **kwargs):
-        from runnable.extensions.run_log_store.file_system.implementation import FileSystemRunLogstore
+        from runnable.extensions.run_log_store.file_system.implementation import (
+            FileSystemRunLogstore,
+        )
 
         self.executor = cast(LocalContainerExecutor, self.executor)
         self.service = cast(FileSystemRunLogstore, self.service)
@@ -280,6 +275,40 @@ class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
         self.service.log_folder = self.executor._container_log_location
 
 
+class LocalContainerComputeChunkedFS(BaseIntegration):
+    """
+    Integration pattern between Local container and File System catalog
+    """
+
+    executor_type = "local-container"
+    service_type = "run_log_store"  # One of secret, catalog, datastore
+    service_provider = "chunked-fs"  # The actual implementation of the service
+
+    def configure_for_traversal(self, **kwargs):
+        from runnable.extensions.run_log_store.chunked_file_system.implementation import (
+            ChunkedFileSystemRunLogStore,
+        )
+
+        self.executor = cast(LocalContainerExecutor, self.executor)
+        self.service = cast(ChunkedFileSystemRunLogStore, self.service)
+
+        write_to = self.service.log_folder
+        self.executor._volumes[str(Path(write_to).resolve())] = {
+            "bind": f"{self.executor._container_log_location}",
+            "mode": "rw",
+        }
+
+    def configure_for_execution(self, **kwargs):
+        from runnable.extensions.run_log_store.chunked_file_system.implementation import (
+            ChunkedFileSystemRunLogStore,
+        )
+
+        self.executor = cast(LocalContainerExecutor, self.executor)
+        self.service = cast(ChunkedFileSystemRunLogStore, self.service)
+
+        self.service.log_folder = self.executor._container_log_location
+
+
 class LocalContainerComputeFileSystemCatalog(BaseIntegration):
     """
     Integration pattern between Local container and File System catalog
@@ -290,7 +319,9 @@ class LocalContainerComputeFileSystemCatalog(BaseIntegration):
     service_provider = "file-system"  # The actual implementation of the service
 
     def configure_for_traversal(self, **kwargs):
-        from runnable.extensions.catalog.file_system.implementation import FileSystemCatalog
+        from runnable.extensions.catalog.file_system.implementation import (
+            FileSystemCatalog,
+        )
 
         self.executor = cast(LocalContainerExecutor, self.executor)
         self.service = cast(FileSystemCatalog, self.service)
@@ -302,7 +333,9 @@ class LocalContainerComputeFileSystemCatalog(BaseIntegration):
         }
 
     def configure_for_execution(self, **kwargs):
-        from runnable.extensions.catalog.file_system.implementation import FileSystemCatalog
+        from runnable.extensions.catalog.file_system.implementation import (
+            FileSystemCatalog,
+        )
 
         self.executor = cast(LocalContainerExecutor, self.executor)
         self.service = cast(FileSystemCatalog, self.service)

runnable/extensions/executor/mocked/implementation.py

@@ -18,7 +18,7 @@ def create_executable(params: Dict[str, Any], model: Type[BaseTaskType], node_na
     class EasyModel(model):  # type: ignore
         model_config = ConfigDict(extra="ignore")
 
-    swallow_all = EasyModel(**params, node_name=node_name)
+    swallow_all = EasyModel(node_name=node_name, **params)
     return swallow_all
 
 
@@ -26,6 +26,8 @@ class MockedExecutor(GenericExecutor):
     service_name: str = "mocked"
     _local_executor: bool = True
 
+    model_config = ConfigDict(extra="ignore")
+
     patches: Dict[str, Any] = Field(default_factory=dict)
 
     @property
@@ -64,6 +66,10 @@ class MockedExecutor(GenericExecutor):
         step_log.step_type = node.node_type
         step_log.status = defaults.PROCESSING
 
+        self._context.run_log_store.add_step_log(step_log, self._context.run_id)
+
+        logger.info(f"Executing node: {node.get_summary()}")
+
         # Add the step log to the database as per the situation.
         # If its a terminal node, complete it now
         if node.node_type in ["success", "fail"]:
@@ -132,3 +138,17 @@ class MockedExecutor(GenericExecutor):
 
     def execute_job(self, node: TaskNode):
         pass
+
+    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
+        """
+        The entry point for all executors apart from local.
+        We have already prepared for node execution.
+
+        Args:
+            node (BaseNode): The node to execute
+            map_variable (dict, optional): If the node is part of a map, send in the map dictionary. Defaults to None.
+
+        Raises:
+            NotImplementedError: _description_
+        """
+        ...

runnable/extensions/executor/retry/implementation.py

@@ -6,6 +6,7 @@ from runnable import context, defaults, exceptions
 from runnable.datastore import RunLog
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
+from runnable.extensions.nodes import TaskNode
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -31,6 +32,7 @@ class RetryExecutor(GenericExecutor):
 
     _local: bool = True
     _original_run_log: Optional[RunLog] = None
+    _restart_initiated: bool = False
 
     @property
     def _context(self):
@@ -38,7 +40,7 @@ class RetryExecutor(GenericExecutor):
 
     @cached_property
     def original_run_log(self):
-        self.original_run_log = self._context.run_log_store.get_run_log_by_id(
+        return self._context.run_log_store.get_run_log_by_id(
             run_id=self.run_id,
             full=True,
         )
@@ -140,10 +142,14 @@ class RetryExecutor(GenericExecutor):
         node_step_log_name = node._get_step_log_name(map_variable=map_variable)
         logger.info(f"Scanning previous run logs for node logs of: {node_step_log_name}")
 
+        if self._restart_initiated:
+            return True
+
         try:
             previous_attempt_log, _ = self.original_run_log.search_step_by_internal_name(node_step_log_name)
         except exceptions.StepLogNotFoundError:
             logger.warning(f"Did not find the node {node.name} in previous run log")
+            self._restart_initiated = True
             return True  # We should re-run the node.
 
         logger.info(f"The original step status: {previous_attempt_log.status}")
@@ -152,7 +158,11 @@ class RetryExecutor(GenericExecutor):
             return False  # We need not run the node
 
         logger.info(f"The new execution should start executing graph from this node {node.name}")
+        self._restart_initiated = True
         return True
 
     def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
         self._execute_node(node, map_variable=map_variable, **kwargs)
+
+    def execute_job(self, node: TaskNode):
+        pass

runnable/extensions/nodes.py

@@ -5,7 +5,7 @@ import sys
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Annotated, Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from pydantic import (
     ConfigDict,
@@ -14,10 +14,15 @@ from pydantic import (
     field_serializer,
     field_validator,
 )
-from typing_extensions import Annotated
 
 from runnable import datastore, defaults, utils
-from runnable.datastore import JsonParameter, MetricParameter, ObjectParameter, StepLog
+from runnable.datastore import (
+    JsonParameter,
+    MetricParameter,
+    ObjectParameter,
+    Parameter,
+    StepLog,
+)
 from runnable.defaults import TypeMapVariable
 from runnable.graph import Graph, create_graph
 from runnable.nodes import CompositeNode, ExecutableNode, TerminalNode
@@ -46,8 +51,6 @@ class TaskNode(ExecutableNode):
         task_config = {k: v for k, v in config.items() if k not in TaskNode.model_fields.keys()}
         node_config = {k: v for k, v in config.items() if k in TaskNode.model_fields.keys()}
 
-        task_config["node_name"] = config.get("name")
-
         executable = create_task(task_config)
         return cls(executable=executable, **node_config, **task_config)
 
@@ -543,10 +546,14 @@ class MapNode(CompositeNode):
         iterate_on = None
         try:
             iterate_on = self._context.run_log_store.get_parameters(self._context.run_id)[self.iterate_on].get_value()
-        except KeyError:
+        except KeyError as e:
             raise Exception(
-                f"Expected parameter {self.iterate_on} not present in Run Log parameters, was it ever set before?"
-            )
+                (
+                    f"Expected parameter {self.iterate_on}",
+                    "not present in Run Log parameters",
+                    "was it ever set before?",
+                )
+            ) from e
 
         if not isinstance(iterate_on, list):
             raise Exception("Only list is allowed as a valid iterator type")
@@ -599,29 +606,44 @@ class MapNode(CompositeNode):
         # The final value of the parameter is the result of the reduce function.
         reducer_f = self.get_reducer_function()
 
-        if map_variable:
-            # If we are in a map state already, the param should have an index of the map variable.
-            for _, v in map_variable.items():
-                for branch_return in self.branch_returns:
-                    param_name, _ = branch_return
-                    to_reduce = []
-                    for iter_variable in iterate_on:
-                        to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
+        def update_param(params: Dict[str, Parameter], reducer_f: Callable, map_prefix: str = ""):
+            from runnable.extensions.executor.mocked.implementation import (
+                MockedExecutor,
+            )
 
-                    param_name = f"{v}_{param_name}"
-                    params[param_name].value = reducer_f(to_reduce)
-                    params[param_name].reduced = True
-        else:
             for branch_return in self.branch_returns:
                 param_name, _ = branch_return
 
                 to_reduce = []
                 for iter_variable in iterate_on:
-                    to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
-
-                params[param_name].value = reducer_f(*to_reduce)
+                    try:
+                        to_reduce.append(params[f"{iter_variable}_{param_name}"].get_value())
+                    except KeyError as e:
+                        if isinstance(self._context.executor, MockedExecutor):
+                            pass
+                        else:
+                            raise Exception(
+                                (
+                                    f"Expected parameter {iter_variable}_{param_name}",
+                                    "not present in Run Log parameters",
+                                    "was it ever set before?",
+                                )
+                            ) from e
+
+                param_name = f"{map_prefix}{param_name}"
+                if to_reduce:
+                    params[param_name].value = reducer_f(*to_reduce)
+                else:
+                    params[param_name].value = ""
                 params[param_name].reduced = True
 
+        if map_variable:
+            # If we are in a map state already, the param should have an index of the map variable.
+            for _, v in map_variable.items():
+                update_param(params, reducer_f, map_prefix=f"{v}_")
+        else:
+            update_param(params, reducer_f)
+
         self._context.run_log_store.set_parameters(parameters=params, run_id=self._context.run_id)
 
 

runnable/graph.py

@@ -74,6 +74,7 @@ class Graph(BaseModel):
         for _, value in self.nodes.items():
             if value.internal_name == internal_name:
                 return value
+        print("graph", internal_name)
         raise exceptions.NodeNotFoundError(internal_name)
 
     def __str__(self):  # pragma: no cover

runnable/parameters.py

@@ -36,7 +36,7 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
             try:
                 parameters[key.lower()] = JsonParameter(kind="json", value=json.loads(value))
             except json.decoder.JSONDecodeError:
-                logger.error(f"Parameter {key} could not be JSON decoded, adding the literal value")
+                logger.warning(f"Parameter {key} could not be JSON decoded, adding the literal value")
                 parameters[key.lower()] = JsonParameter(kind="json", value=value)
 
             if remove:

runnable/sdk.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import os
+import re
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Union
@@ -25,7 +26,7 @@ from rich.progress import (
 from rich.table import Column
 from typing_extensions import Self
 
-from runnable import console, defaults, entrypoints, graph, utils
+from runnable import console, defaults, entrypoints, exceptions, graph, utils
 from runnable.extensions.nodes import (
     FailNode,
     MapNode,
@@ -310,8 +311,6 @@ class NotebookTask(BaseTask):
     """
 
     notebook: str = Field(serialization_alias="command")
-
-    notebook_output_path: Optional[str] = Field(default=None, alias="notebook_output_path", validate_default=True)
     optional_ploomber_args: Optional[Dict[str, Any]] = Field(default=None, alias="optional_ploomber_args")
 
     @computed_field
@@ -591,6 +590,7 @@ class Pipeline(BaseModel):
 
                 Any definition of pipeline should have one node that terminates with success.
         """
+        # TODO: Bug with repeat names
 
         success_path: List[StepType] = []
         on_failure_paths: List[List[StepType]] = []
@@ -637,7 +637,8 @@ class Pipeline(BaseModel):
         self._dag.check_graph()
 
     def return_dag(self) -> graph.Graph:
-        return self._dag
+        dag_definition = self._dag.model_dump(by_alias=True, exclude_none=True)
+        return graph.create_graph(dag_definition)
 
     def execute(
         self,
@@ -708,7 +709,8 @@ class Pipeline(BaseModel):
             caller_stack = inspect.stack()[1]
             relative_to_root = str(Path(caller_stack.filename).relative_to(Path.cwd()))
 
-            module_to_call = f"{relative_to_root.replace('/', '.').replace('.py', '')}.{caller_stack.function}"
+            module_name = re.sub(r"\b.py\b", "", relative_to_root.replace("/", "."))
+            module_to_call = f"{module_name}.{caller_stack.function}"
 
             run_context.pipeline_file = f"{module_to_call}.py"
 
@@ -728,15 +730,20 @@ class Pipeline(BaseModel):
                 pipeline_execution_task = progress.add_task("[dark_orange] Starting execution .. ", total=1)
                 run_context.executor.execute_graph(dag=run_context.dag)
 
+                if not run_context.executor._local:
+                    return {}
+
                 run_log = run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id, full=False)
 
                 if run_log.status == defaults.SUCCESS:
                     progress.update(pipeline_execution_task, description="[green] Success", completed=True)
                 else:
                     progress.update(pipeline_execution_task, description="[red] Failed", completed=True)
+                    raise exceptions.ExecutionFailedError(run_context.run_id)
             except Exception as e:  # noqa: E722
                 console.print(e, style=defaults.error_style)
                 progress.update(pipeline_execution_task, description="[red] Errored execution", completed=True)
+                raise
 
         if run_context.executor._local:
             return run_context.run_log_store.get_run_log_by_id(run_id=run_context.run_id)

runnable/tasks.py

@@ -8,11 +8,14 @@ import os
 import subprocess
 import sys
 from datetime import datetime
+from pathlib import Path
 from pickle import PicklingError
 from string import Template
-from typing import Any, Dict, List, Literal, Optional, Tuple
+from typing import Any, Dict, List, Literal, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+# from rich import print
 from rich.console import Console
 from stevedore import driver
 
@@ -34,9 +37,6 @@ logging.getLogger("stevedore").setLevel(logging.CRITICAL)
 # TODO: Can we add memory peak, cpu usage, etc. to the metrics?
 
 
-console = Console(file=io.StringIO())
-
-
 class TaskReturns(BaseModel):
     name: str
     kind: Literal["json", "object", "metric"] = Field(default="json")
@@ -46,7 +46,6 @@ class BaseTaskType(BaseModel):
     """A base task class which does the execution of command defined by the user."""
 
     task_type: str = Field(serialization_alias="command_type")
-    node_name: str = Field(exclude=True)
     secrets: List[str] = Field(default_factory=list)
     returns: List[TaskReturns] = Field(default_factory=list, alias="returns")
 
@@ -153,7 +152,7 @@ class BaseTaskType(BaseModel):
         if not allow_complex:
             params = {key: value for key, value in params.items() if isinstance(value, JsonParameter)}
 
-        log_file_name = self.node_name  # + ".execution.log"
+        log_file_name = self._context.executor._context_node.internal_name
         if map_variable:
             for _, value in map_variable.items():
                 log_file_name += "_" + str(value)
@@ -163,16 +162,17 @@ class BaseTaskType(BaseModel):
         log_file = open(log_file_name, "w")
 
         parameters_in = copy.deepcopy(params)
-
         f = io.StringIO()
+        task_console = Console(file=io.StringIO())
         try:
             with contextlib.redirect_stdout(f):
                 # with contextlib.nullcontext():
-                yield params
-                print(console.file.getvalue())  # type: ignore
+                yield params, task_console
+                print(task_console.file.getvalue())  # type: ignore
         except Exception as e:  # pylint: disable=broad-except
             logger.exception(e)
         finally:
+            task_console = None  # type: ignore
             print(f.getvalue())  # print to console
             log_file.write(f.getvalue())  # Print to file
 
@@ -233,7 +233,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
         """Execute the notebook as defined by the command."""
         attempt_log = StepAttempt(status=defaults.FAIL, start_time=str(datetime.now()))
 
-        with self.execution_context(map_variable=map_variable) as params, self.expose_secrets() as _:
+        with self.execution_context(map_variable=map_variable) as (params, task_console), self.expose_secrets() as _:
             module, func = utils.get_module_and_attr_names(self.command)
             sys.path.insert(0, os.getcwd())  # Need to add the current directory to path
             imported_module = importlib.import_module(module)
@@ -245,7 +245,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                     logger.info(f"Calling {func} from {module} with {filtered_parameters}")
                     user_set_parameters = f(**filtered_parameters)  # This is a tuple or single value
                 except Exception as e:
-                    console.log(e, style=defaults.error_style, markup=False)
+                    task_console.log(e, style=defaults.error_style, markup=False)
                     raise exceptions.CommandCallError(f"Function call: {self.command} did not succeed.\n") from e
 
                 attempt_log.input_parameters = params.copy()
@@ -289,8 +289,8 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
             except Exception as _e:
                 msg = f"Call to the function {self.command} did not succeed.\n"
                 attempt_log.message = msg
-                console.print_exception(show_locals=False)
-                console.log(_e, style=defaults.error_style)
+                task_console.print_exception(show_locals=False)
+                task_console.log(_e, style=defaults.error_style)
 
         attempt_log.end_time = str(datetime.now())
 
@@ -302,25 +302,25 @@ class NotebookTaskType(BaseTaskType):
 
     task_type: str = Field(default="notebook", serialization_alias="command_type")
     command: str
-    notebook_output_path: Optional[str] = Field(default=None, validate_default=True)
     optional_ploomber_args: dict = {}
 
     @field_validator("command")
     @classmethod
-    def notebook_should_end_with_ipynb(cls, command: str):
+    def notebook_should_end_with_ipynb(cls, command: str) -> str:
         if not command.endswith(".ipynb"):
             raise Exception("Notebook task should point to a ipynb file")
 
         return command
 
-    @field_validator("notebook_output_path")
-    @classmethod
-    def correct_notebook_output_path(cls, notebook_output_path: str, info: ValidationInfo):
-        if notebook_output_path:
-            return notebook_output_path
+    @property
+    def notebook_output_path(self) -> str:
+        node_name = self._context.executor._context_node.internal_name
+        sane_name = "".join(x for x in node_name if x.isalnum())
+
+        output_path = Path(".", self.command)
+        file_name = output_path.parent / (output_path.stem + f"{sane_name}_out.ipynb")
 
-        command = info.data["command"]
-        return "".join(command.split(".")[:-1]) + "_out.ipynb"
+        return str(file_name)
 
     def get_cli_options(self) -> Tuple[str, dict]:
         return "notebook", {"command": self.command, "notebook-output-path": self.notebook_output_path}
@@ -344,17 +344,26 @@ class NotebookTaskType(BaseTaskType):
             import ploomber_engine as pm
             from ploomber_engine.ipython import PloomberClient
 
-            notebook_output_path = self.notebook_output_path or ""
+            notebook_output_path = self.notebook_output_path
 
-            with self.execution_context(
-                map_variable=map_variable, allow_complex=False
-            ) as params, self.expose_secrets() as _:
+            with self.execution_context(map_variable=map_variable, allow_complex=False) as (
+                params,
+                _,
+            ), self.expose_secrets() as _:
                 if map_variable:
                     for key, value in map_variable.items():
                         notebook_output_path += "_" + str(value)
-                        params[key] = value
+                        params[key] = JsonParameter(kind="json", value=value)
+
+                # Remove any {v}_unreduced parameters from the parameters
+                copy_params = copy.deepcopy(params)
+                unprocessed_params = [k for k, v in copy_params.items() if not v.reduced]
 
-                notebook_params = {k: v.get_value() for k, v in params.items()}
+                for key in list(copy_params.keys()):
+                    if any(key.endswith(f"_{k}") for k in unprocessed_params):
+                        del copy_params[key]
+
+                notebook_params = {k: v.get_value() for k, v in copy_params.items()}
 
                 ploomber_optional_args = self.optional_ploomber_args
 
@@ -377,6 +386,11 @@ class NotebookTaskType(BaseTaskType):
                 try:
                     for task_return in self.returns:
                         param_name = Template(task_return.name).safe_substitute(map_variable)  # type: ignore
+
+                        if map_variable:
+                            for _, v in map_variable.items():
+                                param_name = f"{v}_{param_name}"
+
                         output_parameters[param_name] = task_return_to_parameter(
                             task_return=task_return,
                             value=namespace[task_return.name],
@@ -453,95 +467,98 @@ class ShellTaskType(BaseTaskType):
                 secret_value = context.run_context.secrets_handler.get(key)
                 subprocess_env[key] = secret_value
 
-        with self.execution_context(map_variable=map_variable, allow_complex=False) as params:
-            subprocess_env.update({k: v.get_value() for k, v in params.items()})
-
-            # Json dumps all runnable environment variables
-            for key, value in subprocess_env.items():
-                if isinstance(value, str):
-                    continue
-                subprocess_env[key] = json.dumps(value)
-
-            collect_delimiter = "=== COLLECT ==="
-
-            command = self.command.strip() + f" && echo '{collect_delimiter}'  && env"
-            logger.info(f"Executing shell command: {command}")
-
-            capture = False
-            return_keys = {x.name: x for x in self.returns}
-
-            proc = subprocess.Popen(
-                command,
-                shell=True,
-                env=subprocess_env,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-            )
-            result = proc.communicate()
-            logger.debug(result)
-            logger.info(proc.returncode)
-
-            if proc.returncode != 0:
-                msg = ",".join(result[1].split("\n"))
-                attempt_log.status = defaults.FAIL
-                attempt_log.end_time = str(datetime.now())
-                attempt_log.message = msg
-                console.print(msg, style=defaults.error_style)
-                return attempt_log
-
-            # for stderr
-            for line in result[1].split("\n"):
-                if line.strip() == "":
-                    continue
-                console.print(line, style=defaults.warning_style)
+        try:
+            with self.execution_context(map_variable=map_variable, allow_complex=False) as (params, task_console):
+                subprocess_env.update({k: v.get_value() for k, v in params.items()})
+
+                # Json dumps all runnable environment variables
+                for key, value in subprocess_env.items():
+                    if isinstance(value, str):
+                        continue
+                    subprocess_env[key] = json.dumps(value)
+
+                collect_delimiter = "=== COLLECT ==="
+
+                command = self.command.strip() + f" && echo '{collect_delimiter}'  && env"
+                logger.info(f"Executing shell command: {command}")
+
+                capture = False
+                return_keys = {x.name: x for x in self.returns}
+
+                proc = subprocess.Popen(
+                    command,
+                    shell=True,
+                    env=subprocess_env,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                )
+                result = proc.communicate()
+                logger.debug(result)
+                logger.info(proc.returncode)
+
+                if proc.returncode != 0:
+                    msg = ",".join(result[1].split("\n"))
+                    task_console.print(msg, style=defaults.error_style)
+                    raise exceptions.CommandCallError(msg)
+
+                # for stderr
+                for line in result[1].split("\n"):
+                    if line.strip() == "":
+                        continue
+                    task_console.print(line, style=defaults.warning_style)
 
-            output_parameters: Dict[str, Parameter] = {}
-            metrics: Dict[str, Parameter] = {}
+                output_parameters: Dict[str, Parameter] = {}
+                metrics: Dict[str, Parameter] = {}
 
-            # only from stdout
-            for line in result[0].split("\n"):
-                if line.strip() == "":
-                    continue
+                # only from stdout
+                for line in result[0].split("\n"):
+                    if line.strip() == "":
+                        continue
 
-                logger.info(line)
-                console.print(line)
+                    logger.info(line)
+                    task_console.print(line)
 
-                if line.strip() == collect_delimiter:
-                    # The lines from now on should be captured
-                    capture = True
-                    continue
+                    if line.strip() == collect_delimiter:
+                        # The lines from now on should be captured
+                        capture = True
+                        continue
 
-                if capture:
-                    key, value = line.strip().split("=", 1)
-                    if key in return_keys:
-                        task_return = return_keys[key]
+                    if capture:
+                        key, value = line.strip().split("=", 1)
+                        if key in return_keys:
+                            task_return = return_keys[key]
 
-                        try:
-                            value = json.loads(value)
-                        except json.JSONDecodeError:
-                            value = value
+                            try:
+                                value = json.loads(value)
+                            except json.JSONDecodeError:
+                                value = value
 
-                        output_parameter = task_return_to_parameter(
-                            task_return=task_return,
-                            value=value,
-                        )
+                            output_parameter = task_return_to_parameter(
+                                task_return=task_return,
+                                value=value,
+                            )
 
-                        if task_return.kind == "metric":
-                            metrics[task_return.name] = output_parameter
+                            if task_return.kind == "metric":
+                                metrics[task_return.name] = output_parameter
 
-                        param_name = task_return.name
-                        if map_variable:
-                            for _, v in map_variable.items():
-                                param_name = f"{param_name}_{v}"
+                            param_name = task_return.name
+                            if map_variable:
+                                for _, v in map_variable.items():
+                                    param_name = f"{v}_{param_name}"
 
-                        output_parameters[param_name] = output_parameter
+                            output_parameters[param_name] = output_parameter
 
-                attempt_log.output_parameters = output_parameters
-                attempt_log.user_defined_metrics = metrics
-                params.update(output_parameters)
+                    attempt_log.output_parameters = output_parameters
+                    attempt_log.user_defined_metrics = metrics
+                    params.update(output_parameters)
 
-            attempt_log.status = defaults.SUCCESS
+                attempt_log.status = defaults.SUCCESS
+        except exceptions.CommandCallError as e:
+            msg = f"Call to the command {self.command} did not succeed"
+            logger.exception(msg)
+            logger.exception(e)
+            attempt_log.status = defaults.FAIL
 
         attempt_log.end_time = str(datetime.now())
         return attempt_log

{runnable-0.11.2.dist-info → runnable-0.11.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.11.2
+Version: 0.11.4
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.11.2.dist-info → runnable-0.11.4.dist-info}/RECORD

@@ -1,10 +1,10 @@
-runnable/__init__.py,sha256=V3Ihmzbb56k0qCNiBxB8ELDhgffhr_qctIy8qL0o4QM,924
+runnable/__init__.py,sha256=Ov2G5QdLFuM4eqRDJDYKGjt8YHfizVhYEBUTl_hkOb4,954
 runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
 runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
 runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
-runnable/datastore.py,sha256=ViyAyjZQuJkRE1Q8CEEkVJXRKCmozQPe4_ZPl1X3wxo,27773
+runnable/datastore.py,sha256=8aQZ15KAMdre7a7G61bNRmcTeJFzOdnx_9O9UP4JQc8,27910
 runnable/defaults.py,sha256=MOX7I2S6yO4FphZaZREFQca94a20oO8uvzXLd6GLKQs,4703
-runnable/entrypoints.py,sha256=a8M7vb954as_ni7lM0t65czXQj2AHjB-KrQJ3zt3sWo,16397
+runnable/entrypoints.py,sha256=A76Fpa08RmEdXevZSRvhlWN4aQiIoLODa5NYSp8Kd00,17049
 runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
 runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
 runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -14,9 +14,9 @@ runnable/extensions/catalog/file_system/implementation.py,sha256=mFPsAwPMNGWbHcz
 runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
 runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
-runnable/extensions/executor/__init__.py,sha256=eV3q_dL2cRqYaJ8RWV6Xk1__KMWMM2hUnQFT7Z5pRso,26698
+runnable/extensions/executor/__init__.py,sha256=0385OpNSpjyA0GjXlLw7gZtqJFFOHGLmYHzWAGBzU98,26247
 runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/argo/implementation.py,sha256=_BfxCe742S6uV-7PuQ53KjzwY-8Rq-5y9txOXMYf20U,43670
+runnable/extensions/executor/argo/implementation.py,sha256=Wd__bOwLxhIrHKwk0iMgavD44F0nEzR7CcSJzAe7hKk,43840
 runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
 runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
@@ -24,12 +24,12 @@ runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAn
 runnable/extensions/executor/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/executor/local/implementation.py,sha256=e8Tzv-FgQmJeUXVut96jeNERTR83JVG_zkQZMEjCVAs,2469
 runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/local_container/implementation.py,sha256=6kYMgdgE5JxZkVAidxsBSpqkHvyKMfEctgZWSZQEpXA,13979
+runnable/extensions/executor/local_container/implementation.py,sha256=CmvQK77V98Gyzkyuh4WSDrgnoonN89I7xvNSPfeqMxA,14894
 runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/mocked/implementation.py,sha256=ChdUyUsiXXjG_v80d0uLp76Nz4jqqGEry36gs9gNn9k,5082
+runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
 runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/retry/implementation.py,sha256=ZBSYpxSiAIt-SXPD-qIPP-MMo8b7sQ6UKOTJemAjXlI,6625
-runnable/extensions/nodes.py,sha256=Z2LuAxeZpx1pKZmI7G2u90jAm0sdi4U2pqCIFmm0JB4,31965
+runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
+runnable/extensions/nodes.py,sha256=JUmovDBissri3oyTYS5K68gOx5sm3e-rThoTE5xKy0Y,32645
 runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
@@ -47,20 +47,18 @@ runnable/extensions/run_log_store/k8s_pvc/integration.py,sha256=lxQg327mwC0ykhNp
 runnable/extensions/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
-runnable/extensions/secrets/env_secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/secrets/env_secrets/implementation.py,sha256=5XiHdJvIr0-jkl4fGfEf26UsgE5Q2Z4oCc0RwjlJdJA,1236
-runnable/graph.py,sha256=18IpGYw5kgHP32m12WwXscx-kG5Kx-AuWS1LFbMfBLg,16202
+runnable/graph.py,sha256=lVSfODa61zCWZcEyzQbg9bwOYvqoW89-3i92YOkmWII,16240
 runnable/integration.py,sha256=eb9qJVZR7Ehg0N1UnGPuyjJvoA-xQ1-xP7AlZHUXHqM,6705
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
-runnable/parameters.py,sha256=KGGW8_uoIK2hd3EwzzBmoHBOrai3fh-SESNPpJRTfj4,5161
+runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
 runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
-runnable/sdk.py,sha256=JsM27GUc3c57ZepK996FHtfzXP6FGs8MP-s96RC-_fo,27648
+runnable/sdk.py,sha256=t6d1Q3BoovixqC29QuSjFEwsleVgM0E-pAQlfCfMz_o,27923
 runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
-runnable/tasks.py,sha256=CKmZoQAHAQgQLGEX3S0l6qvDL5hcqHoUyTXH_gHe61M,21261
+runnable/tasks.py,sha256=XiFQGTrUvoXub99915lYLvHair8sVWfcUzhY0OceUXo,22351
 runnable/utils.py,sha256=okZFGbJWqStl5Rq5vLhNUQZDv_vhcT58bq9MDrTVxhc,19449
-runnable-0.11.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.11.2.dist-info/METADATA,sha256=5FKWYUkN4EidqFwOckXPOY0DZFqJykmaJdZAf1w__Yo,17020
-runnable-0.11.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-runnable-0.11.2.dist-info/entry_points.txt,sha256=Wy-dimdD2REO2a36Ri84fqGqA5iwGy2RIbdgRNtCNdM,1540
-runnable-0.11.2.dist-info/RECORD,,
+runnable-0.11.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.11.4.dist-info/METADATA,sha256=YaiI3bdujfDjcv-QDsiKhBf1COOKxQROME_m23yy9qc,17020
+runnable-0.11.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+runnable-0.11.4.dist-info/entry_points.txt,sha256=amb6ISqKBSIz47um8_6LKnYgpoZ4d_p6-O1-7uUb1cU,1447
+runnable-0.11.4.dist-info/RECORD,,

{runnable-0.11.2.dist-info → runnable-0.11.4.dist-info}/entry_points.txt

@@ -32,7 +32,6 @@ file-system=runnable.extensions.run_log_store.file_system.implementation:FileSys
 [secrets]
 do-nothing=runnable.secrets:DoNothingSecretManager
 dotenv=runnable.extensions.secrets.dotenv.implementation:DotEnvSecrets
-env-secrets-manager=runnable.extensions.secrets.env_secrets.implementation:EnvSecretsManager
 
 [tasks]
 notebook=runnable.tasks:NotebookTaskType

runnable/extensions/secrets/env_secrets/implementation.py

@@ -1,42 +0,0 @@
-import logging
-import os
-
-from runnable import defaults, exceptions
-from runnable.secrets import BaseSecrets
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class EnvSecretsManager(BaseSecrets):
-    """
-    A secret manager via environment variables.
-
-    This secret manager returns nothing if the key does not match
-    """
-
-    service_name: str = "env-secrets-manager"
-    prefix: str = ""
-    suffix: str = ""
-
-    def get(self, name: str = "", **kwargs) -> str:
-        """
-        If a name is provided, we look for that in the environment.
-        If a environment variable by that name is not found, we raise an Exception.
-
-        If a name is not provided, we return an empty dictionary.
-
-        Args:
-            name (str): The name of the secret to retrieve
-
-        Raises:
-            Exception: If the secret by the name is not found.
-
-        Returns:
-            [type]: [description]
-        """
-
-        try:
-            return os.environ[f"{self.prefix}{name}{self.suffix}"]
-        except KeyError as _e:
-            logger.exception(f"Secret {self.prefix}{name}{self.suffix} not found in environment")
-            raise exceptions.SecretNotFoundError(secret_name=name, secret_setting="environment") from _e