runnable 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

extensions/nodes/nodes.py

@@ -661,6 +661,10 @@ class MapNode(CompositeNode):
 
         self._context.run_log_store.add_step_log(step_log, self._context.run_id)
 
+        # If we failed, we return without any collection
+        if not step_log.status == defaults.SUCCESS:
+            return
+
         # Apply the reduce function and reduce the returns of the task nodes.
         # The final value of the parameter is the result of the reduce function.
         reducer_f = self.get_reducer_function()

runnable/cli.py

@@ -64,8 +64,9 @@ def execute(
     tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
     run_id: Annotated[
         str,
-        typer.Option(
-            help="An optional run_id, one would be generated if its not provided"
+        typer.Argument(
+            envvar=defaults.ENV_RUN_ID,
+            help="An optional run_id, one would be generated if its not provided",
         ),
     ] = "",
 ):
@@ -282,8 +283,11 @@ def execute_job(
     ],
     run_id: Annotated[
         str,
-        typer.Argument(help="An run_id, one would be generated if its not provided"),
-    ],
+        typer.Argument(
+            envvar="RUNNABLE_RUN_ID",
+            help="An optional run_id, one would be generated if its not provided",
+        ),
+    ] = "",
     config_file: Annotated[
         str,
         typer.Option(

runnable/graph.py

@@ -76,7 +76,6 @@ class Graph(BaseModel):
         for _, value in self.nodes.items():
             if value.internal_name == internal_name:
                 return value
-        print("graph", internal_name)
         raise exceptions.NodeNotFoundError(internal_name)
 
     def __str__(self):  # pragma: no cover

runnable/tasks.py

@@ -31,7 +31,26 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 logging.getLogger("stevedore").setLevel(logging.CRITICAL)
 
 
-# TODO: This has to be an extension
+class TeeIO(io.StringIO):
+    """
+    A custom class to write to the buffer and the output stream at the same time.
+    """
+
+    def __init__(self, output_stream=sys.stdout):
+        super().__init__()
+        self.output_stream = output_stream
+
+    def write(self, s):
+        super().write(s)  # Write to the buffer
+        self.output_stream.write(s)  # Write to the output stream
+
+    def flush(self):
+        super().flush()
+        self.output_stream.flush()
+
+
+buffer = TeeIO()
+sys.stdout = buffer
 
 
 class TaskReturns(BaseModel):
@@ -152,6 +171,7 @@ class BaseTaskType(BaseModel):
                 key: value
                 for key, value in params.items()
                 if isinstance(value, JsonParameter)
+                or isinstance(value, MetricParameter)
             }
 
         parameters_in = copy.deepcopy(params)
@@ -274,7 +294,7 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                         f"Calling {func} from {module} with {filtered_parameters}"
                     )
 
-                    out_file = io.StringIO()
+                    out_file = TeeIO()
                     with contextlib.redirect_stdout(out_file):
                         user_set_parameters = f(
                             **filtered_parameters
@@ -284,16 +304,15 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
                     raise exceptions.CommandCallError(
                         f"Function call: {self.command} did not succeed.\n"
                     ) from e
-
-                attempt_log.input_parameters = params.copy()
-
-                if map_variable:
-                    attempt_log.input_parameters.update(
-                        {
-                            k: JsonParameter(value=v, kind="json")
-                            for k, v in map_variable.items()
-                        }
-                    )
+                finally:
+                    attempt_log.input_parameters = params.copy()
+                    if map_variable:
+                        attempt_log.input_parameters.update(
+                            {
+                                k: JsonParameter(value=v, kind="json")
+                                for k, v in map_variable.items()
+                            }
+                        )
 
                 if self.returns:
                     if not isinstance(user_set_parameters, tuple):  # make it a tuple
@@ -448,6 +467,7 @@ class NotebookTaskType(BaseTaskType):
                 ) as params,
                 self.expose_secrets() as _,
             ):
+                attempt_log.input_parameters = params.copy()
                 copy_params = copy.deepcopy(params)
 
                 if map_variable:
@@ -476,7 +496,7 @@ class NotebookTaskType(BaseTaskType):
                 }
                 kwds.update(ploomber_optional_args)
 
-                out_file = io.StringIO()
+                out_file = TeeIO()
                 with contextlib.redirect_stdout(out_file):
                     pm.execute_notebook(**kwds)
                 task_console.print(out_file.getvalue())
@@ -635,6 +655,7 @@ class ShellTaskType(BaseTaskType):
             ) as params:
                 subprocess_env.update({k: v.get_value() for k, v in params.items()})
 
+                attempt_log.input_parameters = params.copy()
                 # Json dumps all runnable environment variables
                 for key, value in subprocess_env.items():
                     if isinstance(value, str):

runnable/utils.py

@@ -77,7 +77,11 @@ def generate_run_id(run_id: str = "") -> str:
     Returns:
         str: A generated run_id
     """
-    # If we are not provided with a run_id, generate one
+    # If we are not provided with a run_id, check env var
+    if not run_id:
+        run_id = os.environ.get(defaults.ENV_RUN_ID, "")
+
+    # If both are not given, generate one
     if not run_id:
         now = datetime.now()
         run_id = f"{names.get_random_name()}-{now.hour:02}{now.minute:02}"

{runnable-0.23.0.dist-info → runnable-0.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.23.0
+Version: 0.25.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.23.0.dist-info → runnable-0.25.0.dist-info}/RECORD

@@ -11,7 +11,7 @@ extensions/job_executor/local.py,sha256=FvxTk0vyxdrbLOAyNkLyjvmmowypabWOSITQBK_f
 extensions/job_executor/local_container.py,sha256=hyFnpicCp3_87mZsW64P6KSVbz7XMLjwJUWVjeCJ0_I,6627
 extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
 extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,34645
+extensions/nodes/nodes.py,sha256=WdOmep4uxmY2mTOtsuVZ5QhYl96jqJprkG6jkIg7BVg,34774
 extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
 extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/pipeline_executor/__init__.py,sha256=bobyC4BWmDKCnMQsuyj9buQX7tZOFxuwU3Coq9-QgR0,25568
@@ -34,24 +34,24 @@ extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
 runnable/__init__.py,sha256=n14AnTUUEYxXlTJ6-YLT0tMmeFb7Co_3kNldV6pgKSs,662
 runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
-runnable/cli.py,sha256=xHb2VSC16z9IT45SnhUYHuXUrzPqF_pK-jyWiTnz5sM,8670
+runnable/cli.py,sha256=3BiKSj95h2Drn__YlchMPZ5rBMafuRb2OGIsVpbsO5Y,8788
 runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
 runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
 runnable/entrypoints.py,sha256=xkUa568-7x9xALz13qW14DxS1nnLDKwLwdIBJZG-vM0,18982
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
 runnable/executor.py,sha256=ne-iRQqGuEmmuApnkBDz1_hokVcjFrbe7BvWqXCG1Ys,15684
-runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
+runnable/graph.py,sha256=poQz5zcvq89ju_u5sYlunQLPbHnXTaUmjcvstPwvT4U,16536
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
 runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
 runnable/sdk.py,sha256=T1nqDpLN9fULvvU9L-oY0EHqYdKUI9qk7oekLynm02Y,33568
 runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
-runnable/tasks.py,sha256=YVKpKxSpsRZWcU3MOqoBoqxeo1XSqv5crkOu6kyu63o,28520
-runnable/utils.py,sha256=6yIqPO8TrUMwOj9pid37I2qg6LD6ILDz9acdO3rQ1zo,19702
-runnable-0.23.0.dist-info/METADATA,sha256=8wQhNRz6xtLWKcxRE4qFT3liU157UedYYw5cSdg0LdA,9945
-runnable-0.23.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.23.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
-runnable-0.23.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.23.0.dist-info/RECORD,,
+runnable/tasks.py,sha256=SYy9eZOs1iCwu1IX5O9WyXk6DMpVsqaruQtMX-YX0bY,29207
+runnable/utils.py,sha256=hJUfRmIgU20weWPmBOHF22F6svBU0A_0nqifRMuXKs0,19822
+runnable-0.25.0.dist-info/METADATA,sha256=bpDSeecHPHb9qCHycgxbAtPFpuEx73t1bO_OAal8dN8,9945
+runnable-0.25.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.25.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
+runnable-0.25.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.25.0.dist-info/RECORD,,