runnable 0.23.0__tar.gz → 0.25.0__tar.gz

{runnable-0.23.0 → runnable-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.23.0
+Version: 0.25.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.23.0 → runnable-0.25.0}/extensions/nodes/nodes.py

@@ -661,6 +661,10 @@ class MapNode(CompositeNode):
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
 
+        # If we failed, we return without any collection
+        if not step_log.status == defaults.SUCCESS:
+            return
+
         # Apply the reduce function and reduce the returns of the task nodes.
         # The final value of the parameter is the result of the reduce function.
         reducer_f = self.get_reducer_function()

{runnable-0.23.0 → runnable-0.25.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.23.0"
+version = "0.25.0"
 description = "Add your description here"
 readme = "README.md"
 authors = [

{runnable-0.23.0 → runnable-0.25.0}/runnable/cli.py

@@ -64,8 +64,9 @@ def execute(
     tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
     run_id: Annotated[
         str,
-        typer.Option(
-            help="An optional run_id, one would be generated if its not provided"
+        typer.Argument(
+            envvar=defaults.ENV_RUN_ID,
+            help="An optional run_id, one would be generated if its not provided",
         ),
     ] = "",
 ):
@@ -282,8 +283,11 @@ def execute_job(
     ],
     run_id: Annotated[
         str,
-        typer.Argument(help="An run_id, one would be generated if its not provided"),
-    ],
+        typer.Argument(
+            envvar="RUNNABLE_RUN_ID",
+            help="An optional run_id, one would be generated if its not provided",
+        ),
+    ] = "",
     config_file: Annotated[
         str,
         typer.Option(

{runnable-0.23.0 → runnable-0.25.0}/runnable/graph.py

@@ -76,7 +76,6 @@ class Graph(BaseModel):
         for _, value in self.nodes.items():
             if value.internal_name == internal_name:
                 return value
-        print("graph", internal_name)
         raise exceptions.NodeNotFoundError(internal_name)
 
     def __str__(self):  # pragma: no cover

{runnable-0.23.0 → runnable-0.25.0}/runnable/tasks.py

@@ -31,7 +31,26 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 logging.getLogger("stevedore").setLevel(logging.CRITICAL)
 
 
-# TODO: This has to be an extension
+class TeeIO(io.StringIO):
+    """
+    A custom class to write to the buffer and the output stream at the same time.
+    """
+
+    def __init__(self, output_stream=sys.stdout):
+        super().__init__()
+        self.output_stream = output_stream
+
+    def write(self, s):
+        super().write(s)  # Write to the buffer
+        self.output_stream.write(s)  # Write to the output stream
+
+    def flush(self):
+        super().flush()
+        self.output_stream.flush()
+
+
+buffer = TeeIO()
+sys.stdout = buffer
 
 
 class TaskReturns(BaseModel):
@@ -152,6 +171,7 @@ class BaseTaskType(BaseModel):
                 key: value
                 for key, value in params.items()
                 if isinstance(value, JsonParameter)
+                or isinstance(value, MetricParameter)
             }
 
         parameters_in = copy.deepcopy(params)
@@ -274,7 +294,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         f"Calling {func} from {module} with {filtered_parameters}"
                     )
 
-                    out_file = io.StringIO()
+                    out_file = TeeIO()
                     with contextlib.redirect_stdout(out_file):
                         user_set_parameters = f(
                             **filtered_parameters
@@ -284,16 +304,15 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                     raise exceptions.CommandCallError(
                         f"Function call: {self.command} did not succeed.\n"
                     ) from e
-
-                attempt_log.input_parameters = params.copy()
-
-                if map_variable:
-                    attempt_log.input_parameters.update(
-                        {
-                            k: JsonParameter(value=v, kind="json")
-                            for k, v in map_variable.items()
-                        }
-                    )
+                finally:
+                    attempt_log.input_parameters = params.copy()
+                    if map_variable:
+                        attempt_log.input_parameters.update(
+                            {
+                                k: JsonParameter(value=v, kind="json")
+                                for k, v in map_variable.items()
+                            }
+                        )
 
                 if self.returns:
                     if not isinstance(user_set_parameters, tuple):  # make it a tuple
@@ -448,6 +467,7 @@ class NotebookTaskType(BaseTaskType):
                 ) as params,
                 self.expose_secrets() as _,
             ):
+                attempt_log.input_parameters = params.copy()
                 copy_params = copy.deepcopy(params)
 
                 if map_variable:
@@ -476,7 +496,7 @@ class NotebookTaskType(BaseTaskType):
                 }
                 kwds.update(ploomber_optional_args)
 
-                out_file = io.StringIO()
+                out_file = TeeIO()
                 with contextlib.redirect_stdout(out_file):
                     pm.execute_notebook(**kwds)
                 task_console.print(out_file.getvalue())
@@ -635,6 +655,7 @@ class ShellTaskType(BaseTaskType):
             ) as params:
                 subprocess_env.update({k: v.get_value() for k, v in params.items()})
 
+                attempt_log.input_parameters = params.copy()
                 # Json dumps all runnable environment variables
                 for key, value in subprocess_env.items():
                     if isinstance(value, str):

{runnable-0.23.0 → runnable-0.25.0}/runnable/utils.py

@@ -77,7 +77,11 @@ def generate_run_id(run_id: str = "") -> str:
     Returns:
         str: A generated run_id
     """
-    # If we are not provided with a run_id, generate one
+    # If we are not provided with a run_id, check env var
+    if not run_id:
+        run_id = os.environ.get(defaults.ENV_RUN_ID, "")
+
+    # If both are not given, generate one
     if not run_id:
         now = datetime.now()
         run_id = f"{names.get_random_name()}-{now.hour:02}{now.minute:02}"