nextpipe 0.6.2.dev3__py3-none-any.whl → 0.6.2.dev5__py3-none-any.whl

nextpipe/__about__.py

@@ -1 +1 @@
-__version__ = "v0.6.2-dev.3"
+__version__ = "v0.6.2-dev.5"

nextpipe/decorators.py

@@ -769,6 +769,11 @@ class App:
         The configuration to apply when running the app.
     polling_options : Optional[nextmv.PollingOptions]
         Options for polling for the results of the app run.
+    bypass_result : typing.Any
+        A value that can be provided to bypass the actual execution of the app.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
     """
 
     def __init__(
@@ -782,6 +787,7 @@ class App:
         run_configuration: nextmv.RunConfiguration = None,
         polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
         bypass_result: typing.Any = None,
+        emit_live_logs: bool = False,
     ):
         """
         Initialize an App object.
@@ -803,6 +809,9 @@ class App:
         bypass_result : typing.Any, optional
             A value that can be provided to bypass the actual execution of the app.
             This is mainly used for testing purposes, by default None.
+        emit_live_logs : bool, optional
+            Whether to emit live logs from the application run. If set to `True`, logs
+            from the sub-run will be relayed to the workflow. Default is False.
         """
 
         if input_type:
@@ -829,6 +838,7 @@ class App:
         self.run_configuration = run_configuration
         self.polling_options = polling_options
         self.bypass_result = bypass_result
+        self.emit_live_logs = emit_live_logs
 
     def __repr__(self):
         """Return a string representation of the app.
@@ -852,6 +862,7 @@ def app(
     run_configuration: nextmv.RunConfiguration = None,
     polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
     bypass_result: typing.Any = None,
+    emit_live_logs: bool = False,
 ):
     """
     Decorator to mark a step as a Nextmv Application (external application)
@@ -894,6 +905,9 @@ def app(
     bypass_result : typing.Any, optional
         A value that can be provided to bypass the actual execution of the app.
         This is mainly used for testing purposes, by default None.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
 
     Example
     -------
@@ -967,6 +981,7 @@ def app(
             run_configuration=run_configuration,
             polling_options=polling_options,
             bypass_result=bypass_result,
+            emit_live_logs=emit_live_logs,
         )
         wrapper.step.type = StepType.APP
 

nextpipe/flow.py

@@ -25,9 +25,11 @@ The module also defines status constants used throughout pipeline execution.
 import ast
 import base64
 import copy
+import datetime
 import inspect
 import io
 import os
+import pathlib
 import random
 import shutil
 import tempfile
@@ -864,12 +866,12 @@ class Runner:
         """
 
         reference: FlowNode = job.reference
-        failed = job.error is not None or reference.error is not None
+        failed = job.error or reference.error
         reference.status = STATUS_SUCCEEDED if not failed else STATUS_FAILED
         reference.result = job.result
         # Check if the job failed and mark the flow as failed if it did
         with self.lock_fail:
-            if job.error is not None and not self.fail:
+            if job.error and not self.fail:
                 self.fail = True
                 self.fail_reason = f"Step {reference.parent.definition.get_id()} failed: {job.error}"
         # Mark the node as done (and its parent if all nodes are done)
@@ -1014,16 +1016,29 @@ class Runner:
             # Run the application
             try:
                 if app_step.bypass_result is None:
+                    # Start the run.
                     run_id = app.new_run(*run_args[0], **run_args[1])
                     console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
                     utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
                     node.run_id = run_id
                     update_dag()  # Update the node with the run_id before we start polling for results
+
+                    # Emit live logs if requested.
+                    if app_step.emit_live_logs:
+                        utils.log_internal(f"Emitting live logs for app step {node.id} ...")
+                        app.run_logs_with_polling(
+                            run_id=run_id,
+                            verbose=True,
+                            log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
+                        )
+
+                    # Wait for run to finish and get the result.
                     result = app.run_result_with_polling(
                         run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
                     )
                 else:
                     utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
+
                     # We wrap the bypass result in a dummy RunResult to ensure consistent behavior.
                     result = nextmv.RunResult(
                         id="bypassed-run",
@@ -1034,8 +1049,8 @@ class Runner:
                             application_id="unavailable",
                             application_instance_id="unavailable",
                             application_version_id="unavailable",
-                            created_at="unavailable",
-                            duration="unavailable",
+                            created_at=datetime.datetime.now(),
+                            duration=0.0,
                             error="unavailable",
                             input_size=0.0,
                             output_size=0.0,
@@ -1048,12 +1063,19 @@ class Runner:
                             ),
                             status_v2=nextmv.StatusV2.succeeded,
                         ),
-                        output=app_step.bypass_result,
                     )
+                    result.output = app_step.bypass_result
+
                     # If the user supplied a bypass result, we need to write it to the temp dir if it's in dir mode
                     if isinstance(app_step.bypass_result, str) and os.path.isdir(app_step.bypass_result):
-                        for filename in os.listdir(app_step.bypass_result):
-                            shutil.copy(os.path.join(app_step.bypass_result, filename), temp_dir)
+                        src = pathlib.Path(app_step.bypass_result)
+                        dst = pathlib.Path(temp_dir)
+                        for item in src.iterdir():
+                            if item.is_file():
+                                shutil.copy2(item, dst / item.name)
+                            elif item.is_dir():
+                                shutil.copytree(item, dst / item.name)
+
             finally:  # Make sure we clean up temp dir on failure too
                 # If the temp dir is empty, remove it
                 dir_result = False

nextpipe/utils.py

@@ -46,7 +46,10 @@ def __get_step_name() -> str:
     return "main"
 
 
-def log(message: str) -> None:
+def log(
+    message: str,
+    step_name: str | None = None,
+) -> None:
     """
     Logs a message using stderr. Furthermore, prepends the name of the calling function if it is a step.
 
@@ -62,6 +65,9 @@ def log(message: str) -> None:
     ----------
     message : str
         The message to log.
+    step_name : str, optional
+        The name of the step to include in the log message. If not provided, it will
+        attempt to determine it automatically.
 
     Examples
     --------
@@ -72,7 +78,7 @@ def log(message: str) -> None:
     >>> # [step_name] Processing data
     """
 
-    step_name = __get_step_name()
+    step_name = step_name or __get_step_name()
     if step_name:
         print(f"[{step_name}] {message}", file=sys.stderr)
     else:

{nextpipe-0.6.2.dev3.dist-info → nextpipe-0.6.2.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.6.2.dev3
+Version: 0.6.2.dev5
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
@@ -105,7 +105,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Requires-Dist: dataclasses-json>=0.6.7
-Requires-Dist: nextmv>=0.40.0
+Requires-Dist: nextmv>=1.2.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
 Requires-Dist: goldie>=0.1.8; extra == 'dev'

{nextpipe-0.6.2.dev3.dist-info → nextpipe-0.6.2.dev5.dist-info}/RECORD

@@ -1,15 +1,15 @@
-nextpipe/__about__.py,sha256=8AMrE0_anvG9euAOawmj8RKxbYUQPQvpyH3k78xhWFM,29
+nextpipe/__about__.py,sha256=mOiwTNqXBqNcb4rptxgI-9gGg1FsAGtcakJZLdc-Im0,29
 nextpipe/__init__.py,sha256=SzRhi2z33zmvskfgqoRHwaKvRnt8G1_JJR1M7BXlda0,694
 nextpipe/config.py,sha256=JSShEaK3fnqieGofalQysKxEJUkyrWIRbwhdyH9yYZM,1659
 nextpipe/constants.py,sha256=ws1_khUbZFau_kShi6nweLhg24_2fjPWyXY6VPVAH7E,667
-nextpipe/decorators.py,sha256=MLTwuTONn30-esVaF7q3hNdOp8AnJ5ksNswQHpFrGlE,25727
-nextpipe/flow.py,sha256=zQsv3OOOaoCl3x3jL-f31fqaL5cB19yvhj1VxloOWMM,42472
+nextpipe/decorators.py,sha256=kLYOVYqx0QWtTbVkgw6US8ZZSa0YijZ5qzMX0mRiO7k,26539
+nextpipe/flow.py,sha256=TAjZ6kuUw_1m6wi-HfB9pgbCVClOBODFXLGNa-3n4Fw,43259
 nextpipe/graph.py,sha256=Oi8HQ4fgvBYxpRK6xBoLtoxftGb-gCOb09BqFwzrwWs,2867
 nextpipe/schema.py,sha256=HsJnzpYhemXUa8IZvFEChPB-_5YXTCbkYSPf5zpPqo8,3263
 nextpipe/threads.py,sha256=5s7jF-hgVF_Eqfxgf6hsF6dhXsRSfqTsq5F0JBy4hNw,9039
 nextpipe/uplink.py,sha256=oW8rIt6byUqKhex5-Nk0KwTOBuPvRAMtuDfP-fmM6_E,11475
-nextpipe/utils.py,sha256=-ni6Em1QfatgVMh95hYSPVCdTboozB8vRKkakti3gls,8063
-nextpipe-0.6.2.dev3.dist-info/METADATA,sha256=MtCbj3EDrUBwU7Hz3Dh0b_qlUnpJe3-qLSN-SiNPDcU,8760
-nextpipe-0.6.2.dev3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-nextpipe-0.6.2.dev3.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
-nextpipe-0.6.2.dev3.dist-info/RECORD,,
+nextpipe/utils.py,sha256=jAeWautNhUZsjuF00K7BVaU5grLRWlBmtLkcNnPYnXg,8279
+nextpipe-0.6.2.dev5.dist-info/METADATA,sha256=2075MaDBj6IyaZP_jiLemxdRw4H-o6-BFF8NBTIuFZM,8759
+nextpipe-0.6.2.dev5.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+nextpipe-0.6.2.dev5.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
+nextpipe-0.6.2.dev5.dist-info/RECORD,,