nextpipe 0.6.2.dev3__tar.gz → 0.6.2.dev4__tar.gz

{nextpipe-0.6.2.dev3 → nextpipe-0.6.2.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.6.2.dev3
+Version: 0.6.2.dev4
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
@@ -105,7 +105,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Requires-Dist: dataclasses-json>=0.6.7
-Requires-Dist: nextmv>=0.40.0
+Requires-Dist: nextmv>=1.2.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
 Requires-Dist: goldie>=0.1.8; extra == 'dev'

nextpipe-0.6.2.dev4/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.6.2-dev.4"

{nextpipe-0.6.2.dev3 → nextpipe-0.6.2.dev4}/nextpipe/decorators.py

@@ -769,6 +769,11 @@ class App:
         The configuration to apply when running the app.
     polling_options : Optional[nextmv.PollingOptions]
         Options for polling for the results of the app run.
+    bypass_result : typing.Any
+        A value that can be provided to bypass the actual execution of the app.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
     """
 
     def __init__(
@@ -782,6 +787,7 @@ class App:
         run_configuration: nextmv.RunConfiguration = None,
         polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
         bypass_result: typing.Any = None,
+        emit_live_logs: bool = False,
     ):
         """
         Initialize an App object.
@@ -803,6 +809,9 @@ class App:
         bypass_result : typing.Any, optional
             A value that can be provided to bypass the actual execution of the app.
             This is mainly used for testing purposes, by default None.
+        emit_live_logs : bool, optional
+            Whether to emit live logs from the application run. If set to `True`, logs
+            from the sub-run will be relayed to the workflow. Default is False.
         """
 
         if input_type:
@@ -829,6 +838,7 @@ class App:
         self.run_configuration = run_configuration
         self.polling_options = polling_options
         self.bypass_result = bypass_result
+        self.emit_live_logs = emit_live_logs
 
     def __repr__(self):
         """Return a string representation of the app.
@@ -852,6 +862,7 @@ def app(
     run_configuration: nextmv.RunConfiguration = None,
     polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
     bypass_result: typing.Any = None,
+    emit_live_logs: bool = False,
 ):
     """
     Decorator to mark a step as a Nextmv Application (external application)
@@ -894,6 +905,9 @@ def app(
     bypass_result : typing.Any, optional
         A value that can be provided to bypass the actual execution of the app.
         This is mainly used for testing purposes, by default None.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
 
     Example
     -------
@@ -967,6 +981,7 @@ def app(
             run_configuration=run_configuration,
             polling_options=polling_options,
             bypass_result=bypass_result,
+            emit_live_logs=emit_live_logs,
         )
         wrapper.step.type = StepType.APP
 

{nextpipe-0.6.2.dev3 → nextpipe-0.6.2.dev4}/nextpipe/flow.py

@@ -25,9 +25,11 @@ The module also defines status constants used throughout pipeline execution.
 import ast
 import base64
 import copy
+import datetime
 import inspect
 import io
 import os
+import pathlib
 import random
 import shutil
 import tempfile
@@ -1014,16 +1016,29 @@ class Runner:
             # Run the application
             try:
                 if app_step.bypass_result is None:
+                    # Start the run.
                     run_id = app.new_run(*run_args[0], **run_args[1])
                     console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
                     utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
                     node.run_id = run_id
                     update_dag()  # Update the node with the run_id before we start polling for results
+
+                    # Emit live logs if requested.
+                    if app_step.emit_live_logs:
+                        utils.log_internal(f"Emitting live logs for app step {node.id} ...")
+                        app.run_logs_with_polling(
+                            run_id=run_id,
+                            verbose=True,
+                            log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
+                        )
+
+                    # Wait for run to finish and get the result.
                     result = app.run_result_with_polling(
                         run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
                     )
                 else:
                     utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
+
                     # We wrap the bypass result in a dummy RunResult to ensure consistent behavior.
                     result = nextmv.RunResult(
                         id="bypassed-run",
@@ -1034,8 +1049,8 @@ class Runner:
                             application_id="unavailable",
                             application_instance_id="unavailable",
                             application_version_id="unavailable",
-                            created_at="unavailable",
-                            duration="unavailable",
+                            created_at=datetime.datetime.now(),
+                            duration=0.0,
                             error="unavailable",
                             input_size=0.0,
                             output_size=0.0,
@@ -1048,12 +1063,19 @@ class Runner:
                             ),
                             status_v2=nextmv.StatusV2.succeeded,
                         ),
-                        output=app_step.bypass_result,
                     )
+                    result.output = app_step.bypass_result
+
                     # If the user supplied a bypass result, we need to write it to the temp dir if it's in dir mode
                     if isinstance(app_step.bypass_result, str) and os.path.isdir(app_step.bypass_result):
-                        for filename in os.listdir(app_step.bypass_result):
-                            shutil.copy(os.path.join(app_step.bypass_result, filename), temp_dir)
+                        src = pathlib.Path(app_step.bypass_result)
+                        dst = pathlib.Path(temp_dir)
+                        for item in src.iterdir():
+                            if item.is_file():
+                                shutil.copy2(item, dst / item.name)
+                            elif item.is_dir():
+                                shutil.copytree(item, dst / item.name)
+
             finally:  # Make sure we clean up temp dir on failure too
                 # If the temp dir is empty, remove it
                 dir_result = False

{nextpipe-0.6.2.dev3 → nextpipe-0.6.2.dev4}/nextpipe/utils.py

@@ -46,7 +46,10 @@ def __get_step_name() -> str:
     return "main"
 
 
-def log(message: str) -> None:
+def log(
+    message: str,
+    step_name: str | None = None,
+) -> None:
     """
     Logs a message using stderr. Furthermore, prepends the name of the calling function if it is a step.
 
@@ -62,6 +65,9 @@ def log(message: str) -> None:
     ----------
     message : str
         The message to log.
+    step_name : str, optional
+        The name of the step to include in the log message. If not provided, it will
+        attempt to determine it automatically.
 
     Examples
     --------
@@ -72,7 +78,7 @@ def log(message: str) -> None:
     >>> # [step_name] Processing data
     """
 
-    step_name = __get_step_name()
+    step_name = step_name or __get_step_name()
     if step_name:
         print(f"[{step_name}] {message}", file=sys.stderr)
     else:

{nextpipe-0.6.2.dev3 → nextpipe-0.6.2.dev4}/pyproject.toml

@@ -17,7 +17,7 @@ classifiers = [
 ]
 dependencies = [
     "requests>=2.31.0",
-    "nextmv>=0.40.0",
+    "nextmv>=1.2.2",
     "dataclasses-json>=0.6.7",
 ]
 description = "Framework for Decision Pipeline modeling and execution"

nextpipe-0.6.2.dev3/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.6.2-dev.3"