nextpipe 0.7.4__tar.gz → 0.8.0.dev0__tar.gz

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nextpipe
-Version: 0.7.4
+Version: 0.8.0.dev0
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://docs.nextmv.io/nextpipe

nextpipe-0.8.0.dev0/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.8.0.dev0"

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/nextpipe/__init__.py

@@ -6,9 +6,11 @@ from .decorators import app as app
 from .decorators import foreach as foreach
 from .decorators import join as join
 from .decorators import needs as needs
+from .decorators import on_failure as on_failure
 from .decorators import optional as optional
 from .decorators import repeat as repeat
 from .decorators import step as step
+from .flow import AppRunError as AppRunError
 from .flow import FlowGraph as FlowGraph
 from .flow import FlowSpec as FlowSpec
 from .schema import AppOption as AppOption

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/nextpipe/decorators.py

@@ -18,6 +18,8 @@ Needs
     Represents dependencies between steps.
 Optional
     Represents an optional step condition.
+OnFailure
+    Represents the failure handling of a step.
 Repeat
     Represents a repeating step.
 Foreach
@@ -35,6 +37,8 @@ needs
     Decorator to mark the predecessors of a step.
 optional
     Decorator to mark a step as optional.
+on_failure
+    Decorator to let a step fail without failing the flow.
 repeat
     Decorator to make a step be repeated a number of times.
 foreach
@@ -190,6 +194,18 @@ class Step:
 
         return hasattr(self, "optional")
 
+    def is_on_failure(self):
+        """
+        Check if the step may fail without failing the flow.
+
+        Returns
+        -------
+        bool
+            True if the step has an `on_failure` handler, False otherwise.
+        """
+
+        return hasattr(self, "on_failure")
+
     def skip(self):
         """
         Check if the step should be skipped.
@@ -608,6 +624,129 @@ def optional(condition: Callable[..., bool], skip_downstream: bool = False):
     return decorator
 
 
+class OnFailure:
+    """
+    Represents the failure handling of a step.
+
+    This class is used by the `on_failure` decorator to specify the result that
+    replaces the result of a step if it fails.
+
+    Attributes
+    ----------
+    fallback : Any
+        The result to use instead, or a callable that receives the error and
+        returns it.
+    """
+
+    def __init__(self, fallback: typing.Any = None):
+        """
+        Initialize an OnFailure object.
+
+        Parameters
+        ----------
+        fallback : Any, optional
+            The result to use instead, or a callable that receives the error
+            and returns it, by default None.
+        """
+
+        self.fallback = fallback
+
+    def resolve(self, error: Exception) -> typing.Any:
+        """
+        Determine the result to use for a failed step.
+
+        Parameters
+        ----------
+        error : Exception
+            The error that made the step fail.
+
+        Returns
+        -------
+        Any
+            The fallback result.
+        """
+
+        if callable(self.fallback):
+            return self.fallback(error)
+        return self.fallback
+
+    def __repr__(self):
+        """
+        Return a string representation of the failure handling.
+
+        Returns
+        -------
+        str
+            A string representation of the failure handling.
+        """
+
+        return f"OnFailure({self.fallback})"
+
+
+def on_failure(fallback: typing.Any = None):
+    """
+    Decorator to let a step fail without failing the flow.
+
+    You can import the `on_failure` decorator directly from `nextpipe`:
+
+    ```python
+    from nextpipe import on_failure
+    ```
+
+    By default, a failing step fails the whole flow. With this decorator, a
+    failure (e.g., an exception or a failed app run) only fails the step
+    itself: it has the status `failed`, the flow continues and its successors
+    receive the fallback as its result. If the step has multiple nodes (e.g.,
+    `foreach`), this applies to each node separately.
+
+    Parameters
+    ----------
+    fallback : Any, optional
+        The result to use instead of the result of the failed step. If it is
+        callable, it is called with the error (e.g., an `AppRunError` for a
+        failed app run) and its return value is used. By default, None.
+
+    Example
+    -------
+    In this example the flow still produces an output if the solver app fails.
+
+    ```
+    from nextpipe import FlowSpec, app, log, needs, on_failure, step
+
+
+    class Flow(FlowSpec):
+        @step
+        def prepare(data: dict) -> dict:
+            return data
+
+        @on_failure(fallback=lambda error: {"error": str(error)})
+        @app(app_id="solver")
+        @needs(predecessors=[prepare])
+        @step
+        def solve() -> None:
+            pass
+
+        @needs(predecessors=[solve])
+        @step
+        def report(result: dict) -> dict:
+            if "error" in result:
+                return {"status": "failed", "reason": result["error"]}
+            return {"status": "ok", "solution": result["solution"]}
+
+
+    flow = Flow("DecisionFlow", {"orders": [1, 2, 3]})
+    flow.run()
+    log(flow.get_result(flow.report))
+    ```
+    """
+
+    def decorator(function):
+        function.step.on_failure = OnFailure(fallback)
+        return function
+
+    return decorator
+
+
 class Repeat:
     """
     Represents a repeating step.

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/nextpipe/flow.py

@@ -18,6 +18,8 @@ StepVisitor
     AST visitor for finding step functions in a flow class.
 Runner
     Handles the execution of a flow graph.
+AppRunError
+    Raised when the run of an app step does not succeed.
 
 The module also defines status constants used throughout pipeline execution.
 """
@@ -32,6 +34,7 @@ import os
 import pathlib
 import random
 import shutil
+import signal
 import tempfile
 import threading
 import time
@@ -62,6 +65,54 @@ STATUS_FAILED = "failed"
 STATUS_SKIPPED = "skipped"
 """Status constant indicating a skipped step or node."""
 
+SUB_RUN_CANCEL_TIMEOUT = 5.0
+"""
+Maximum time in seconds to wait for sub-run cancel requests when the flow is aborted.
+
+When a run is canceled on the Nextmv platform, the workflow process is
+interrupted and killed after a grace period of 10 seconds. This timeout, plus
+the final flow update sent afterwards, must fit within that grace period.
+"""
+
+
+class AppRunError(Exception):
+    """
+    Raised when the run of an app step does not succeed.
+
+    You can import the `AppRunError` class directly from `nextpipe`:
+
+    ```python
+    from nextpipe import AppRunError
+    ```
+
+    Attributes
+    ----------
+    step_id : str
+        The ID of the node of the app step.
+    app_id : str
+        The ID of the application that was run.
+    run_id : str
+        The ID of the run.
+    status : str
+        The final status of the run (e.g., `failed` or `canceled`).
+    error : str
+        The error message of the run, if any.
+    console_url : str
+        The URL of the run in the Nextmv Console.
+    """
+
+    def __init__(self, step_id: str, app_id: str, run_id: str, status: str, error: str, console_url: str):
+        self.step_id = step_id
+        self.app_id = app_id
+        self.run_id = run_id
+        self.status = status
+        self.error = error
+        self.console_url = console_url
+        message = f"run {run_id} of app {app_id} {status}"
+        if error:
+            message += f": {error}"
+        super().__init__(f"{message} (see {console_url})")
+
 
 class FlowStep:
     """
@@ -735,6 +786,8 @@ class Runner:
         Whether the flow has failed.
     fail_reason : str or None
         Reason for the flow failure, if any.
+    fail_error : Exception or None
+        Error that made the flow fail, if any.
     lock_fail : threading.Lock
         Lock for updating fail state.
     lock_running : threading.Lock
@@ -771,6 +824,7 @@ class Runner:
         self.node_idxs = {}
         self.fail = False
         self.fail_reason = None
+        self.fail_error = None
         self.lock_fail = threading.Lock()
         self.lock_running = threading.Lock()
 
@@ -882,11 +936,24 @@ class Runner:
         failed = job.error or reference.error
         reference.status = STATUS_SUCCEEDED if not failed else STATUS_FAILED
         reference.result = job.result
+        error = job.error
+        if error:
+            reference.error = str(error)
+            definition = reference.parent.definition
+            if definition.is_on_failure() and not reference.cancel:
+                # The step may fail, continue with the fallback as its result
+                try:
+                    reference.result = definition.on_failure.resolve(error)
+                    utils.log_internal(f"Node {reference.id} failed, continuing with fallback: {error}")
+                    error = None
+                except Exception as e:
+                    error = e
         # Check if the job failed and mark the flow as failed if it did
         with self.lock_fail:
-            if job.error and not self.fail:
+            if error and not self.fail:
                 self.fail = True
-                self.fail_reason = f"Step {reference.parent.definition.get_id()} failed: {job.error}"
+                self.fail_reason = f"Step {reference.parent.definition.get_id()} failed: {error}"
+                self.fail_error = error
         # Mark the node as done (and its parent if all nodes are done)
         reference.done = True
         with reference.parent.lock:
@@ -964,8 +1031,9 @@ class Runner:
 
         polling_options = copy.deepcopy(app_step.polling_options)
         polling_options.initial_delay = random.uniform(0, 5)
-        if polling_options.stop is None:
-            polling_options.stop = lambda: node.cancel
+        # Always stop polling when the node gets cancelled, in addition to any user-defined stop condition.
+        user_stop = polling_options.stop
+        polling_options.stop = lambda: node.cancel or (user_stop is not None and user_stop())
 
         run_kwargs: dict[str, Any] = {
             "options": options,
@@ -1004,6 +1072,9 @@ class Runner:
             utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
             node.run_id = run_id
             update_dag()  # Update node with run_id before polling
+            if node.cancel:
+                # The flow was aborted while the run was being started, cancel it right away.
+                Runner.__cancel_app_run(node, app)
 
             if app_step.emit_live_logs:
                 utils.log_internal(f"Emitting live logs for app step {node.id} ...")
@@ -1013,7 +1084,25 @@ class Runner:
                     log_func=lambda msg: utils.log(message=msg.log, step_name=node.id),
                 )
 
-            return app.run_result_with_polling(run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir)
+            def polling_func() -> tuple[Any, bool]:
+                information = app.run_information(run_id=run_id)
+                return information, information.metadata.run_is_finalized()
+
+            information = nextmv.poll(polling_options=polling_options, polling_func=polling_func)
+            if information is None:
+                raise RuntimeError(f"Stopped waiting for app step {node.id} run {run_id}")
+            # Only fetch the result of succeeded runs, failed ones may not have any output
+            status = information.metadata.status_v2
+            if status != nextmv.StatusV2.succeeded:
+                raise AppRunError(
+                    step_id=node.id,
+                    app_id=app_step.app_id,
+                    run_id=run_id,
+                    status=getattr(status, "value", str(status)),
+                    error=information.metadata.error,
+                    console_url=console_url,
+                )
+            return app.run_result(run_id=run_id, output_dir_path=temp_dir)
 
         utils.log_internal(f"Bypassing app step {node.id} run, using bypass result")
 
@@ -1232,31 +1321,126 @@ class Runner:
             reference=node,
         )
 
-    def run(self):
+    @staticmethod
+    def __cancel_app_run(node: FlowNode, app: Application) -> None:
         """
-        Run the flow.
+        Request the cancellation of the remote run of an app step node.
 
-        This method starts the uplink communication, executes the flow steps
-        in the correct order based on dependencies, and handles failures.
+        This is best effort: failures are logged, but not raised.
 
-        Raises
-        ------
-        RuntimeError
-            If the flow execution fails.
+        Parameters
+        ----------
+        node : FlowNode
+            The node whose run should be cancelled.
+        app : Application
+            The application the run belongs to.
         """
 
-        # Start communicating updates to the platform
         try:
-            self.uplink.submit_update(self.graph._to_uplink_dto())
-            self.uplink.run_async()
+            app.cancel_run(node.run_id)
+            utils.log_internal(f"Requested cancellation of app step {node.id} run {node.run_id}")
         except Exception as e:
-            self.uplink.terminate()
-            utils.log_internal(f"Failed to update graph with platform: {e}")
+            utils.log_internal(f"Failed to cancel app step {node.id} run {node.run_id}: {e}")
 
-        # Start running the flow
-        open_steps: set[FlowStep] = set(self.graph.start_steps)
-        running_steps: set[FlowStep] = set()
-        closed_steps: set[FlowStep] = set()
+    def __cancel_running(self, running_steps: set[FlowStep]) -> None:
+        """
+        Cancel all nodes of the given running steps.
+
+        Flags the nodes as cancelled (which stops polling for their results) and
+        requests the cancellation of the remote runs of app steps that are still
+        in progress. The cancel requests are sent in parallel and waited for at
+        most `SUB_RUN_CANCEL_TIMEOUT` seconds in total.
+
+        Parameters
+        ----------
+        running_steps : set[FlowStep]
+            The steps that are currently running.
+        """
+
+        cancel_threads: list[threading.Thread] = []
+        for step in running_steps:
+            for node in step.nodes:
+                node.cancel = True
+                node.status = STATUS_FAILED
+                if node.done or node.run_id is None or not step.definition.is_app():
+                    continue
+                app = Application(client=self.spec.client, id=step.definition.app.app_id)
+                thread = threading.Thread(
+                    target=Runner.__cancel_app_run,
+                    args=(node, app),
+                    name=f"{utils.THREAD_NAME_PREFIX}cancel-{node.id}",
+                    daemon=True,
+                )
+                thread.start()
+                cancel_threads.append(thread)
+
+        deadline = time.monotonic() + SUB_RUN_CANCEL_TIMEOUT
+        for thread in cancel_threads:
+            thread.join(timeout=max(0.0, deadline - time.monotonic()))
+
+    @staticmethod
+    def __install_sigterm_handler() -> Any:
+        """
+        Install a SIGTERM handler that raises `SystemExit`.
+
+        This allows handling a termination request the same way as a keyboard
+        interrupt, i.e., by cancelling the running steps. Signal handlers can only
+        be installed from the main thread, otherwise this is a no-op.
+
+        Returns
+        -------
+        Any
+            The previous SIGTERM handler, or None if no handler was installed.
+        """
+
+        if threading.current_thread() is not threading.main_thread():
+            return None
+
+        def handler(signum, frame):
+            raise SystemExit(128 + signum)
+
+        return signal.signal(signal.SIGTERM, handler)
+
+    @staticmethod
+    def __restore_sigterm_handler(previous_handler: Any) -> None:
+        """
+        Restore the SIGTERM handler that was active before running the flow.
+
+        Parameters
+        ----------
+        previous_handler : Any
+            The previous handler as returned by `__install_sigterm_handler`.
+        """
+
+        if previous_handler is not None:
+            signal.signal(signal.SIGTERM, previous_handler)
+
+    def __run_steps(
+        self,
+        open_steps: set[FlowStep],
+        running_steps: set[FlowStep],
+        closed_steps: set[FlowStep],
+    ) -> None:
+        """
+        Execute the steps of the flow until all of them are done.
+
+        The given sets are updated in place, so that the caller can react to
+        interruptions (e.g., cancel the steps still running).
+
+        Parameters
+        ----------
+        open_steps : set[FlowStep]
+            Steps that are ready to be considered for execution.
+        running_steps : set[FlowStep]
+            Steps that are currently running.
+        closed_steps : set[FlowStep]
+            Steps that are done.
+
+        Raises
+        ------
+        RuntimeError
+            If the flow execution fails.
+        """
 
         # Run the steps in parallel
         while open_steps or running_steps:
@@ -1315,16 +1499,57 @@ class Runner:
                 # Raise an exception if the flow failed
                 with self.lock_fail:
                     if self.fail:
-                        # Issue cancel to all nodes
-                        for step in running_steps:
-                            for node in step.nodes:
-                                node.cancel = True
-                                node.status = STATUS_FAILED
+                        # Issue cancel to all nodes (including their remote runs)
+                        self.__cancel_running(running_steps)
                         # Submitting the final state and terminating uplink causes the last
                         # update to be send to the platform (reflecting the final state).
                         self.uplink.submit_update(self.graph._to_uplink_dto())
                         self.uplink.terminate()  # This will issue the final update.
-                        raise RuntimeError(f"Flow failed: {self.fail_reason}")
+                        # Failed app runs are fully described by their message, other
+                        # errors keep their traceback (e.g., to debug the step function).
+                        cause = None if isinstance(self.fail_error, AppRunError) else self.fail_error
+                        raise RuntimeError(f"Flow failed: {self.fail_reason}") from cause
+
+    def run(self):
+        """
+        Run the flow.
+
+        This method starts the uplink communication, executes the flow steps
+        in the correct order based on dependencies, and handles failures.
+
+        Raises
+        ------
+        RuntimeError
+            If the flow execution fails.
+        """
+
+        # Start communicating updates to the platform
+        try:
+            self.uplink.submit_update(self.graph._to_uplink_dto())
+            self.uplink.run_async()
+        except Exception as e:
+            self.uplink.terminate()
+            utils.log_internal(f"Failed to update graph with platform: {e}")
+
+        # Start running the flow
+        open_steps: set[FlowStep] = set(self.graph.start_steps)
+        running_steps: set[FlowStep] = set()
+        closed_steps: set[FlowStep] = set()
+
+        # Translate SIGTERM (e.g., a cancelled run on the platform) into an
+        # exception, so that it is handled like SIGINT (ctrl+c).
+        previous_sigterm_handler = Runner.__install_sigterm_handler()
+
+        try:
+            self.__run_steps(open_steps, running_steps, closed_steps)
+        except (KeyboardInterrupt, SystemExit):
+            utils.log_internal("Flow was interrupted, cancelling running steps")
+            self.__cancel_running(running_steps)
+            self.uplink.submit_update(self.graph._to_uplink_dto())
+            self.uplink.terminate()
+            raise
+        finally:
+            Runner.__restore_sigterm_handler(previous_sigterm_handler)
 
         # Terminate uplink
         self.uplink.terminate()

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/nextpipe/threads.py

@@ -225,8 +225,8 @@ class Pool:
                     job.start_callback(job)
                 job.run()
             except Exception as e:
+                # Keep the error on the job, it is up to the done callback to handle it
                 job.error = e
-                raise RuntimeError(f"Error in thread {thread_id}: {e}") from e
             finally:
                 with self.lock:
                     self.running.pop(thread_id, None)

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/nextpipe/uplink.py

@@ -286,6 +286,7 @@ class UplinkClient:
         self.changed = False
         self._terminate = False
         self._terminated = False
+        self._wake = threading.Event()
         self._updates_failed = 0
 
     def _post_node_update(self):
@@ -375,8 +376,8 @@ class UplinkClient:
                                 self._terminate = True
                 else:
                     self._updates_failed = 0
-                # Sleep
-                time.sleep(UPDATE_INTERVAL)
+                # Sleep (woken up early on termination)
+                self._wake.wait(UPDATE_INTERVAL)
 
             # Signal termination
             self._terminated = True
@@ -399,6 +400,7 @@ class UplinkClient:
 
         # Terminate the client
         self._terminate = True
+        self._wake.set()
         while not self._terminated:
             time.sleep(0.1)
 

nextpipe-0.8.0.dev0/tests/test_cancel.py

@@ -0,0 +1,138 @@
+import os
+import signal
+import sys
+import threading
+import time
+import unittest
+from unittest import mock
+
+from nextpipe import FlowSpec, app, needs, step
+from nextpipe.flow import STATUS_FAILED
+
+# Add the parent directory to the sys.path to allow imports from the main package. This
+# is meant to help VS Code testing features.
+sys.path.append(os.path.dirname(sys.path[0]))
+
+
+class _FakeApplication:
+    """Stand-in for a nextmv Application whose runs only end when cancelled."""
+
+    instances: list["_FakeApplication"] = []
+    lock = threading.Lock()
+    started = threading.Event()
+    fail_cancel = False
+
+    def __init__(self, client=None, id=None, **kwargs):
+        self.id = id
+        self.default_instance_id = None
+        self.cancelled: list[str] = []
+        with _FakeApplication.lock:
+            _FakeApplication.instances.append(self)
+
+    @classmethod
+    def reset(cls):
+        cls.instances = []
+        cls.started = threading.Event()
+        cls.fail_cancel = False
+
+    @classmethod
+    def cancelled_runs(cls) -> list[str]:
+        with cls.lock:
+            return sorted(r for a in cls.instances for r in a.cancelled)
+
+    def new_run(self, **kwargs) -> str:
+        _FakeApplication.started.set()
+        return f"run-{self.id}"
+
+    def run_information(self, run_id):
+        # The run never finishes on its own, so polling only ends on the stop condition.
+        time.sleep(0.01)
+        return mock.Mock(metadata=mock.Mock(run_is_finalized=lambda: False))
+
+    def cancel_run(self, run_id: str) -> None:
+        if _FakeApplication.fail_cancel:
+            raise RuntimeError("cancel failed")
+        with _FakeApplication.lock:
+            self.cancelled.append(run_id)
+
+
+class SubRunFlow(FlowSpec):
+    @step
+    def prepare(data: dict) -> dict:
+        return data
+
+    @app(app_id="solver-a")
+    @needs(predecessors=[prepare])
+    @step
+    def solve_a() -> None:
+        pass
+
+    @app(app_id="solver-b")
+    @needs(predecessors=[prepare])
+    @step
+    def solve_b() -> None:
+        pass
+
+    @needs(predecessors=[prepare])
+    @step
+    def broken(data: dict) -> None:
+        if data.get("fail"):
+            _FakeApplication.started.wait(5)
+            time.sleep(0.2)  # Give both app steps the chance to start their runs
+            raise ValueError("boom")
+        _FakeApplication.started.wait(5)
+
+
+def _interrupt_when_started(sig: int):
+    """Send the given signal to this process once the sub-runs have been started."""
+
+    def target():
+        _FakeApplication.started.wait(5)
+        time.sleep(0.2)
+        os.kill(os.getpid(), sig)
+
+    threading.Thread(target=target, daemon=True).start()
+
+
+@mock.patch("nextpipe.flow.Application", _FakeApplication)
+class TestCascadingCancel(unittest.TestCase):
+    def setUp(self):
+        _FakeApplication.reset()
+
+    def test_failure_cancels_sub_runs(self):
+        flow = SubRunFlow("SubRun", {"fail": True})
+        with self.assertRaises(RuntimeError):
+            flow.run()
+        self.assertEqual(_FakeApplication.cancelled_runs(), ["run-solver-a", "run-solver-b"])
+
+    def test_keyboard_interrupt_cancels_sub_runs(self):
+        flow = SubRunFlow("SubRun", {})
+        _interrupt_when_started(signal.SIGINT)
+        with self.assertRaises(KeyboardInterrupt):
+            flow.run()
+        self.assertEqual(_FakeApplication.cancelled_runs(), ["run-solver-a", "run-solver-b"])
+        for node in flow.graph.get_step(SubRunFlow.solve_a.step).nodes:
+            self.assertTrue(node.cancel)
+            self.assertEqual(node.status, STATUS_FAILED)
+
+    def test_sigterm_cancels_sub_runs(self):
+        previous = signal.getsignal(signal.SIGTERM)
+        flow = SubRunFlow("SubRun", {})
+        _interrupt_when_started(signal.SIGTERM)
+        with self.assertRaises(SystemExit):
+            flow.run()
+        self.assertEqual(_FakeApplication.cancelled_runs(), ["run-solver-a", "run-solver-b"])
+        # The previous SIGTERM handler is restored after the flow ran
+        self.assertIs(signal.getsignal(signal.SIGTERM), previous)
+
+    def test_failing_cancel_request_does_not_mask_interrupt(self):
+        _FakeApplication.fail_cancel = True
+        flow = SubRunFlow("SubRun", {})
+        _interrupt_when_started(signal.SIGINT)
+        with self.assertRaises(KeyboardInterrupt):
+            flow.run()
+        self.assertEqual(_FakeApplication.cancelled_runs(), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

nextpipe-0.8.0.dev0/tests/test_on_failure.py

@@ -0,0 +1,217 @@
+import io
+import os
+import sys
+import threading
+import unittest
+from contextlib import redirect_stderr
+from unittest import mock
+
+import nextmv
+
+from nextpipe import AppRunError, FlowSpec, app, foreach, join, needs, on_failure, step
+from nextpipe.flow import STATUS_FAILED, STATUS_SUCCEEDED
+
+# Add the parent directory to the sys.path to allow imports from the main package. This
+# is meant to help VS Code testing features.
+sys.path.append(os.path.dirname(sys.path[0]))
+
+
+class _FakeApplication:
+    """Stand-in for a nextmv Application whose runs end with a given status."""
+
+    status = nextmv.StatusV2.failed
+    error = "solver crashed"
+
+    def __init__(self, client=None, id=None, **kwargs):
+        self.id = id
+        self.default_instance_id = None
+
+    def new_run(self, **kwargs) -> str:
+        return f"run-{self.id}"
+
+    def run_information(self, run_id):
+        status = _FakeApplication.status
+        metadata = mock.Mock(status_v2=status, error=_FakeApplication.error, run_is_finalized=lambda: True)
+        return mock.Mock(metadata=metadata)
+
+    def run_result(self, run_id, output_dir_path):
+        return mock.Mock(output={"solution": "ok"}, metadata=mock.Mock(error=""))
+
+
+class FallbackFlow(FlowSpec):
+    @step
+    def prepare(data: dict) -> dict:
+        return data
+
+    @on_failure(fallback=lambda error: {"error": str(error)})
+    @needs(predecessors=[prepare])
+    @step
+    def broken(data: dict) -> dict:
+        raise ValueError("boom")
+
+    @needs(predecessors=[broken])
+    @step
+    def report(result: dict) -> dict:
+        return {"report": result}
+
+
+class DefaultFallbackFlow(FlowSpec):
+    @on_failure()
+    @step
+    def broken(data: dict) -> dict:
+        raise ValueError("boom")
+
+    @needs(predecessors=[broken])
+    @step
+    def report(result: dict | None) -> str:
+        return "no result" if result is None else "result"
+
+
+class FailingFallbackFlow(FlowSpec):
+    @on_failure(fallback=lambda error: 1 / 0)
+    @step
+    def broken(data: dict) -> dict:
+        raise ValueError("boom")
+
+
+class ForeachFallbackFlow(FlowSpec):
+    @foreach()
+    @step
+    def split(data: dict) -> list[int]:
+        return [1, 2, 3]
+
+    @on_failure(fallback=-1)
+    @needs(predecessors=[split])
+    @step
+    def work(item: int) -> int:
+        if item == 2:
+            raise ValueError("boom")
+        return item * 10
+
+    @join()
+    @needs(predecessors=[work])
+    @step
+    def collect(results: list[list[int]]) -> list[int]:
+        return [r[0] for r in results]
+
+
+class AppFlow(FlowSpec):
+    @step
+    def prepare(data: dict) -> dict:
+        return data
+
+    @app(app_id="solver")
+    @needs(predecessors=[prepare])
+    @step
+    def solve() -> None:
+        pass
+
+    @needs(predecessors=[solve])
+    @step
+    def report(result: dict) -> dict:
+        return result
+
+
+class AppFallbackFlow(FlowSpec):
+    @step
+    def prepare(data: dict) -> dict:
+        return data
+
+    @on_failure(fallback=lambda error: error)
+    @app(app_id="solver")
+    @needs(predecessors=[prepare])
+    @step
+    def solve() -> None:
+        pass
+
+    @needs(predecessors=[solve])
+    @step
+    def report(result) -> object:
+        return result
+
+
+class TestOnFailure(unittest.TestCase):
+    def test_fallback_is_passed_to_successors(self):
+        flow = FallbackFlow("Fallback", {})
+        flow.run()
+        self.assertEqual(flow.get_result(flow.report), {"report": {"error": "boom"}})
+        node = flow.graph.get_step(FallbackFlow.broken.step).nodes[0]
+        self.assertEqual(node.status, STATUS_FAILED)
+        self.assertEqual(node.error, "boom")
+
+    def test_handled_failure_does_not_print_traceback(self):
+        unhandled = []
+        previous_hook = threading.excepthook
+        threading.excepthook = lambda args: unhandled.append(args.exc_value)
+        try:
+            stderr = io.StringIO()
+            with redirect_stderr(stderr):
+                FallbackFlow("Fallback", {}).run()
+        finally:
+            threading.excepthook = previous_hook
+        self.assertEqual(unhandled, [])
+        self.assertNotIn("Traceback", stderr.getvalue())
+        self.assertIn("Node broken_0 failed, continuing with fallback: boom", stderr.getvalue())
+
+    def test_failure_keeps_cause(self):
+        flow = FailingFallbackFlow("FailingFallback", {})
+        with self.assertRaises(RuntimeError) as ctx:
+            flow.run()
+        self.assertIsInstance(ctx.exception.__cause__, ZeroDivisionError)
+
+    def test_default_fallback_is_none(self):
+        flow = DefaultFallbackFlow("DefaultFallback", {})
+        flow.run()
+        self.assertEqual(flow.get_result(flow.report), "no result")
+
+    def test_failing_fallback_fails_flow(self):
+        flow = FailingFallbackFlow("FailingFallback", {})
+        with self.assertRaises(RuntimeError) as ctx:
+            flow.run()
+        self.assertIn("division by zero", str(ctx.exception))
+
+    def test_fallback_per_node(self):
+        flow = ForeachFallbackFlow("ForeachFallback", {})
+        flow.run()
+        self.assertEqual(flow.get_result(flow.collect), [10, -1, 30])
+        statuses = [n.status for n in flow.graph.get_step(ForeachFallbackFlow.work.step).nodes]
+        self.assertEqual(statuses, [STATUS_SUCCEEDED, STATUS_FAILED, STATUS_SUCCEEDED])
+
+
+@mock.patch("nextpipe.flow.Application", _FakeApplication)
+@mock.patch("nextpipe.flow.random.uniform", lambda a, b: 0)
+class TestAppRunFailure(unittest.TestCase):
+    def setUp(self):
+        _FakeApplication.status = nextmv.StatusV2.failed
+        _FakeApplication.error = "solver crashed"
+
+    def test_failed_run_fails_flow(self):
+        flow = AppFlow("App", {}, client=mock.Mock(console_url="https://console"))
+        with self.assertRaises(RuntimeError) as ctx:
+            flow.run()
+        message = str(ctx.exception)
+        self.assertIn("Step solve failed", message)
+        self.assertIn("run run-solver of app solver failed: solver crashed", message)
+        self.assertIn("https://console/app/solver/run/run-solver", message)
+        self.assertIsNone(flow.get_result(flow.report))
+        # The error message describes the failed run, no traceback of nextpipe internals
+        self.assertIsNone(ctx.exception.__cause__)
+
+    def test_failed_run_with_fallback(self):
+        flow = AppFallbackFlow("AppFallback", {}, client=mock.Mock(console_url="https://console"))
+        flow.run()
+        error = flow.get_result(flow.report)
+        self.assertIsInstance(error, AppRunError)
+        self.assertEqual(error.run_id, "run-solver")
+        self.assertEqual(error.status, "failed")
+        self.assertEqual(error.error, "solver crashed")
+
+    def test_succeeded_run(self):
+        _FakeApplication.status = nextmv.StatusV2.succeeded
+        flow = AppFlow("App", {}, client=mock.Mock(console_url="https://console"))
+        flow.run()
+        self.assertEqual(flow.get_result(flow.report), {"solution": "ok"})
+
+
+if __name__ == "__main__":
+    unittest.main()

{nextpipe-0.7.4 → nextpipe-0.8.0.dev0}/tests/test_uplink.py

@@ -7,7 +7,7 @@ import unittest
 import nextmv.cloud
 
 from nextpipe import FlowSpec, app, needs, step
-from nextpipe.uplink import FlowDTO, FlowUpdateDTO, NodeDTO, StepDTO, UplinkClient
+from nextpipe.uplink import UPDATE_INTERVAL, FlowDTO, FlowUpdateDTO, NodeDTO, StepDTO, UplinkClient, UplinkConfig
 
 # Add the parent directory to the sys.path to allow imports from the main package. This
 # is meant to help VS Code testing features.
@@ -107,3 +107,19 @@ class TestLogger(unittest.TestCase):
         time.sleep(0.5)
         uplink.terminate()
         time.sleep(0.5)
+
+    def test_terminate_does_not_wait_for_update_interval(self):
+        """Terminating must not wait out the sleep between updates (e.g., on cancellation)."""
+
+        client = nextmv.cloud.Client(api_key="unavailable", url="https://unavailable.url")
+        uplink = UplinkClient(client=client, config=UplinkConfig(application_id="app", run_id="run"))
+        posted = []
+        uplink._post_node_update = lambda: posted.append(time.monotonic())
+        uplink.run_async()
+        uplink.submit_update(_create_example_flow())
+        # Wait for the first update, the update thread is sleeping afterwards.
+        while not posted:
+            time.sleep(0.01)
+        start = time.monotonic()
+        uplink.terminate()
+        self.assertLess(time.monotonic() - start, UPDATE_INTERVAL / 2)

nextpipe-0.7.4/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.7.4"