nextpipe 0.6.2.dev2__py3-none-any.whl → 0.6.2.dev4__py3-none-any.whl

nextpipe/__about__.py

@@ -1 +1 @@
-__version__ = "v0.6.2-dev.2"
+__version__ = "v0.6.2-dev.4"

nextpipe/constants.py

@@ -1,3 +1,14 @@
+"""
+Constants for nextpipe.
+
+This module provides constants used across the nextpipe package.
+
+Functions
+--------
+_get_run_info
+    Retrieves information about the run from environment variables.
+"""
+
 import os
 
 ENV_APP_ID = "NEXTMV_APP_ID"
@@ -7,7 +18,7 @@ ENV_RUN_ID = "NEXTMV_RUN_ID"
 """Environment variable name for the run ID."""
 
 
-def __get_run_info():
+def _get_run_info():
     """
     Retrieves information about the run from environment variables.
     If the environment variables are not set, it returns None for both values.

nextpipe/decorators.py

@@ -769,6 +769,11 @@ class App:
         The configuration to apply when running the app.
     polling_options : Optional[nextmv.PollingOptions]
         Options for polling for the results of the app run.
+    bypass_result : typing.Any
+        A value that can be provided to bypass the actual execution of the app.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
     """
 
     def __init__(
@@ -782,6 +787,7 @@ class App:
         run_configuration: nextmv.RunConfiguration = None,
         polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
         bypass_result: typing.Any = None,
+        emit_live_logs: bool = False,
     ):
         """
         Initialize an App object.
@@ -803,6 +809,9 @@ class App:
         bypass_result : typing.Any, optional
             A value that can be provided to bypass the actual execution of the app.
             This is mainly used for testing purposes, by default None.
+        emit_live_logs : bool, optional
+            Whether to emit live logs from the application run. If set to `True`, logs
+            from the sub-run will be relayed to the workflow. Default is False.
         """
 
         if input_type:
@@ -829,6 +838,7 @@ class App:
         self.run_configuration = run_configuration
         self.polling_options = polling_options
         self.bypass_result = bypass_result
+        self.emit_live_logs = emit_live_logs
 
     def __repr__(self):
         """Return a string representation of the app.
@@ -852,6 +862,7 @@ def app(
     run_configuration: nextmv.RunConfiguration = None,
     polling_options: nextmv.PollingOptions | None = nextmv.DEFAULT_POLLING_OPTIONS,
     bypass_result: typing.Any = None,
+    emit_live_logs: bool = False,
 ):
     """
     Decorator to mark a step as a Nextmv Application (external application)
@@ -894,6 +905,9 @@ def app(
     bypass_result : typing.Any, optional
         A value that can be provided to bypass the actual execution of the app.
         This is mainly used for testing purposes, by default None.
+    emit_live_logs : bool
+        Whether to emit live logs from the application run. If set to `True`, logs from
+        the sub-run will be relayed to the workflow.
 
     Example
     -------
@@ -967,6 +981,7 @@ def app(
             run_configuration=run_configuration,
             polling_options=polling_options,
             bypass_result=bypass_result,
+            emit_live_logs=emit_live_logs,
         )
         wrapper.step.type = StepType.APP
 

nextpipe/flow.py

@@ -25,9 +25,11 @@ The module also defines status constants used throughout pipeline execution.
 import ast
 import base64
 import copy
+import datetime
 import inspect
 import io
 import os
+import pathlib
 import random
 import shutil
 import tempfile
@@ -40,7 +42,7 @@ from typing import Any
 import nextmv
 from nextmv.cloud import Application, Client
 
-from nextpipe.constants import __get_run_info
+from nextpipe.constants import _get_run_info
 
 from . import config, decorators, graph, schema, threads, uplink, utils
 from .__about__ import __version__
@@ -885,7 +887,7 @@ class Runner:
         """
         Determine name and description to use for sub-runs.
         """
-        wf_app_id, wf_run_id = __get_run_info()
+        wf_app_id, wf_run_id = _get_run_info()
         wf_app_id = wf_app_id or app_step_id  # Fallback if dynamic detection fails
         wf_run_id = wf_run_id or "local"  # Use "local" for non-platform runs
         step_name = step_name[:50]  # Truncate step name to 50 chars to keep overall name within limits
@@ -1014,16 +1016,29 @@ class Runner:
             # Run the application
             try:
                 if app_step.bypass_result is None:
+                    # Start the run.
                     run_id = app.new_run(*run_args[0], **run_args[1])
                     console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
                     utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
                     node.run_id = run_id
                     update_dag()  # Update the node with the run_id before we start polling for results
+
+                    # Emit live logs if requested.
+                    if app_step.emit_live_logs:
+                        utils.log_internal(f"Emitting live logs for app step {node.id} ...")
+                        app.run_logs_with_polling(
+                            run_id=run_id,
+                            verbose=True,
+                            log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
+                        )
+
+                    # Wait for run to finish and get the result.
                     result = app.run_result_with_polling(
                         run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
                     )
                 else:
                     utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
+
                     # We wrap the bypass result in a dummy RunResult to ensure consistent behavior.
                     result = nextmv.RunResult(
                         id="bypassed-run",
@@ -1034,8 +1049,8 @@ class Runner:
                             application_id="unavailable",
                             application_instance_id="unavailable",
                             application_version_id="unavailable",
-                            created_at="unavailable",
-                            duration="unavailable",
+                            created_at=datetime.datetime.now(),
+                            duration=0.0,
                             error="unavailable",
                             input_size=0.0,
                             output_size=0.0,
@@ -1048,12 +1063,19 @@ class Runner:
                             ),
                             status_v2=nextmv.StatusV2.succeeded,
                         ),
-                        output=app_step.bypass_result,
                     )
+                    result.output = app_step.bypass_result
+
                     # If the user supplied a bypass result, we need to write it to the temp dir if it's in dir mode
                     if isinstance(app_step.bypass_result, str) and os.path.isdir(app_step.bypass_result):
-                        for filename in os.listdir(app_step.bypass_result):
-                            shutil.copy(os.path.join(app_step.bypass_result, filename), temp_dir)
+                        src = pathlib.Path(app_step.bypass_result)
+                        dst = pathlib.Path(temp_dir)
+                        for item in src.iterdir():
+                            if item.is_file():
+                                shutil.copy2(item, dst / item.name)
+                            elif item.is_dir():
+                                shutil.copytree(item, dst / item.name)
+
             finally:  # Make sure we clean up temp dir on failure too
                 # If the temp dir is empty, remove it
                 dir_result = False

nextpipe/uplink.py

@@ -34,7 +34,7 @@ from dataclasses import dataclass, field
 from dataclasses_json import config, dataclass_json
 from nextmv.cloud import Client
 
-from nextpipe.constants import __get_run_info
+from nextpipe.constants import _get_run_info
 from nextpipe.utils import log_internal
 
 FAILED_UPDATES_THRESHOLD = 10
@@ -272,7 +272,7 @@ class UplinkClient:
 
         if config is None:
             # Load config from environment
-            app_id, run_id = __get_run_info()
+            app_id, run_id = _get_run_info()
             config = UplinkConfig(application_id=app_id, run_id=run_id)
         self.config = config
         self.inactive = False

nextpipe/utils.py

@@ -46,7 +46,10 @@ def __get_step_name() -> str:
     return "main"
 
 
-def log(message: str) -> None:
+def log(
+    message: str,
+    step_name: str | None = None,
+) -> None:
     """
     Logs a message using stderr. Furthermore, prepends the name of the calling function if it is a step.
 
@@ -62,6 +65,9 @@ def log(message: str) -> None:
     ----------
     message : str
         The message to log.
+    step_name : str, optional
+        The name of the step to include in the log message. If not provided, it will
+        attempt to determine it automatically.
 
     Examples
     --------
@@ -72,7 +78,7 @@ def log(message: str) -> None:
     >>> # [step_name] Processing data
     """
 
-    step_name = __get_step_name()
+    step_name = step_name or __get_step_name()
     if step_name:
         print(f"[{step_name}] {message}", file=sys.stderr)
     else:

{nextpipe-0.6.2.dev2.dist-info → nextpipe-0.6.2.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.6.2.dev2
+Version: 0.6.2.dev4
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
@@ -105,7 +105,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Requires-Dist: dataclasses-json>=0.6.7
-Requires-Dist: nextmv>=0.40.0
+Requires-Dist: nextmv>=1.2.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
 Requires-Dist: goldie>=0.1.8; extra == 'dev'

nextpipe-0.6.2.dev4.dist-info/RECORD

@@ -0,0 +1,15 @@
+nextpipe/__about__.py,sha256=KIn7qdq_AeTnzP-pjx9UBfss83kfsVtubPPrKJOBGes,29
+nextpipe/__init__.py,sha256=SzRhi2z33zmvskfgqoRHwaKvRnt8G1_JJR1M7BXlda0,694
+nextpipe/config.py,sha256=JSShEaK3fnqieGofalQysKxEJUkyrWIRbwhdyH9yYZM,1659
+nextpipe/constants.py,sha256=ws1_khUbZFau_kShi6nweLhg24_2fjPWyXY6VPVAH7E,667
+nextpipe/decorators.py,sha256=kLYOVYqx0QWtTbVkgw6US8ZZSa0YijZ5qzMX0mRiO7k,26539
+nextpipe/flow.py,sha256=zJLYl3ptRYxDPao0umCfMxE1gjpG_xaWNjLqT2JnQtY,43295
+nextpipe/graph.py,sha256=Oi8HQ4fgvBYxpRK6xBoLtoxftGb-gCOb09BqFwzrwWs,2867
+nextpipe/schema.py,sha256=HsJnzpYhemXUa8IZvFEChPB-_5YXTCbkYSPf5zpPqo8,3263
+nextpipe/threads.py,sha256=5s7jF-hgVF_Eqfxgf6hsF6dhXsRSfqTsq5F0JBy4hNw,9039
+nextpipe/uplink.py,sha256=oW8rIt6byUqKhex5-Nk0KwTOBuPvRAMtuDfP-fmM6_E,11475
+nextpipe/utils.py,sha256=jAeWautNhUZsjuF00K7BVaU5grLRWlBmtLkcNnPYnXg,8279
+nextpipe-0.6.2.dev4.dist-info/METADATA,sha256=XFGgOhlOzdhp9Q4eq1h7uTQeg6W4t3Ru5-yU44qyB5Y,8759
+nextpipe-0.6.2.dev4.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+nextpipe-0.6.2.dev4.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
+nextpipe-0.6.2.dev4.dist-info/RECORD,,

nextpipe-0.6.2.dev2.dist-info/RECORD

@@ -1,15 +0,0 @@
-nextpipe/__about__.py,sha256=1-zmKDjeDRIz4nXrUOXUKYbr__bsifB-n1WTe2B-bAc,29
-nextpipe/__init__.py,sha256=SzRhi2z33zmvskfgqoRHwaKvRnt8G1_JJR1M7BXlda0,694
-nextpipe/config.py,sha256=JSShEaK3fnqieGofalQysKxEJUkyrWIRbwhdyH9yYZM,1659
-nextpipe/constants.py,sha256=zvfSKKL4QDZ3D68H5bIW1YicaZf9mKEVTMFeviM3mbo,467
-nextpipe/decorators.py,sha256=MLTwuTONn30-esVaF7q3hNdOp8AnJ5ksNswQHpFrGlE,25727
-nextpipe/flow.py,sha256=bSiAIIMBRLSx1gYErJfm3BdsS64rnjGeW3IDL0HBDzY,42474
-nextpipe/graph.py,sha256=Oi8HQ4fgvBYxpRK6xBoLtoxftGb-gCOb09BqFwzrwWs,2867
-nextpipe/schema.py,sha256=HsJnzpYhemXUa8IZvFEChPB-_5YXTCbkYSPf5zpPqo8,3263
-nextpipe/threads.py,sha256=5s7jF-hgVF_Eqfxgf6hsF6dhXsRSfqTsq5F0JBy4hNw,9039
-nextpipe/uplink.py,sha256=I1gwTcXn-FuIwvQWC_EarrBzHQLgTIShwUQ_aHgmiuE,11477
-nextpipe/utils.py,sha256=-ni6Em1QfatgVMh95hYSPVCdTboozB8vRKkakti3gls,8063
-nextpipe-0.6.2.dev2.dist-info/METADATA,sha256=V842M8UKeo2nlgePGCFRlFTtD-VnTmacGkQWJ2pqB0Q,8760
-nextpipe-0.6.2.dev2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-nextpipe-0.6.2.dev2.dist-info/licenses/LICENSE.md,sha256=FpVScuRcgdbeBNVsMzoPyBbKL2eCB1vFor1vfNNFEj4,4106
-nextpipe-0.6.2.dev2.dist-info/RECORD,,