nextpipe 0.7.3.dev0__tar.gz → 0.7.4.dev0__tar.gz

{nextpipe-0.7.3.dev0 → nextpipe-0.7.4.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nextpipe
-Version: 0.7.3-dev.0
+Version: 0.7.4.dev0
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://docs.nextmv.io/nextpipe

nextpipe-0.7.4.dev0/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.7.4.dev0"

{nextpipe-0.7.3.dev0 → nextpipe-0.7.4.dev0}/nextpipe/decorators.py

@@ -45,6 +45,7 @@ app
     Decorator to mark a step as a Nextmv Application.
 """
 
+import inspect
 import typing
 from collections.abc import Callable
 from enum import Enum
@@ -177,17 +178,37 @@ class Step:
 
         return hasattr(self, "needs")
 
+    def is_optional(self):
+        """
+        Check if the step is optional.
+
+        Returns
+        -------
+        bool
+            True if the step is optional, False otherwise.
+        """
+
+        return hasattr(self, "optional")
+
     def skip(self):
         """
         Check if the step should be skipped.
 
+        Deprecated: skipping is decided by the runner, which also passes the
+        step's inputs to the condition and propagates skips. Use
+        `is_optional()` to check whether a step has a condition.
+
         Returns
         -------
         bool
             True if the step should be skipped, False otherwise.
         """
 
-        return hasattr(self, "optional") and not self.optional.condition(self)
+        deprecated(
+            "Step.skip",
+            "Step.skip() is deprecated and will be removed in a future release. Use Step.is_optional() instead.",
+        )
+        return self.is_optional() and not self.optional.condition(self)
 
     def is_repeat(self):
         """
@@ -435,9 +456,12 @@ class Optional:
     condition : callable
         A function that takes a step and returns a boolean indicating
         whether the step should be executed or not.
+    skip_downstream : bool
+        Whether skipping this step also skips the steps that depend only on
+        it.
     """
 
-    def __init__(self, condition: callable):
+    def __init__(self, condition: callable, skip_downstream: bool = False):
         """
         Initialize an Optional object.
 
@@ -446,9 +470,45 @@ class Optional:
         condition : callable
             A function that takes a step and returns a boolean indicating
             whether the step should be executed or not.
+        skip_downstream : bool, optional
+            Whether skipping this step also skips the steps that depend only
+            on it, by default False.
         """
 
         self.condition = condition
+        self.skip_downstream = skip_downstream
+
+    def evaluate(self, step: Step, inputs: list[typing.Any]) -> bool:
+        """
+        Evaluate the condition.
+
+        Conditions with a single required parameter only receive the step.
+        Conditions with more required parameters (or `*args`) additionally
+        receive the inputs of the step (the results of its predecessors, or the
+        flow input if it has none).
+
+        Parameters
+        ----------
+        step : Step
+            The step the condition belongs to.
+        inputs : list[Any]
+            The inputs of the step.
+
+        Returns
+        -------
+        bool
+            True if the step should be executed, False otherwise.
+        """
+
+        # Parameters with defaults are not counted, and `self` of bound methods
+        # is not part of the signature.
+        params = inspect.signature(self.condition).parameters.values()
+        positional = (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        required = [p for p in params if p.kind in positional and p.default is inspect.Parameter.empty]
+        varargs = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params)
+        if len(required) <= 1 and not varargs:
+            return self.condition(step)
+        return self.condition(step, *inputs)
 
     def __repr__(self):
         """
@@ -460,10 +520,10 @@ class Optional:
             A string representation of the optional condition.
         """
 
-        return f"StepOnlyIf({self.condition})"
+        return f"StepOnlyIf({self.condition}, skip_downstream={self.skip_downstream})"
 
 
-def optional(condition: Callable[[Step], bool]):
+def optional(condition: Callable[..., bool], skip_downstream: bool = False):
     """
     Decorator to mark a step as optional.
 
@@ -474,41 +534,75 @@ def optional(condition: Callable[[Step], bool]):
     ```
 
     This is used to determine whether the step should be executed or not. The
-    condition is a callable that takes the step as an argument and returns a
-    boolean indicating whether the step should be executed or not. The
-    condition is evaluated at runtime, so it can depend on the runtime state of
-    the pipeline.
+    condition is evaluated at runtime, right before the step would run. It
+    receives the step and, if it accepts more arguments, the inputs of the step
+    (the results of its predecessors, or the flow input if it has none).
+
+    A skipped step has the status `skipped` and its result is `None`. By
+    default, only the step itself is skipped: its successors still run and
+    receive `None` as its result.
+
+    With `skip_downstream=True`, the skip propagates: a step whose
+    predecessors were all skipped this way is skipped as well (and so on). A
+    step with at least one other predecessor still runs and receives `None`
+    for each skipped one. This allows modeling if/else branches that are
+    merged again by a common successor.
 
     Parameters
     ----------
-    condition : Callable[[Step], bool]
+    condition : Callable[..., bool]
         The condition to evaluate. This is a callable that takes the step
-        as an argument and returns a boolean indicating whether the step
-        should be executed or not.
+        (and optionally the inputs of the step) as arguments and returns a
+        boolean indicating whether the step should be executed or not.
+    skip_downstream : bool, optional
+        Whether skipping this step also skips the steps that depend only on
+        it, by default False.
 
     Example
     -------
-    In this example the step `step1` is executed given that the condition is
-    true.
+    In this example either `solve_small` or `solve_large` (and its successor
+    `polish`) is executed, depending on the input. `report` merges both branches.
 
     ```
-    from nextpipe import FlowSpec, log, optional, step
+    from nextpipe import FlowSpec, log, needs, optional, step
 
 
     class Flow(FlowSpec):
-        @optional(condition=lambda step: step.get_id() == "step1")
         @step
-        def step1() -> None:
-            log("Execute optional step 1")
+        def prepare(data: dict) -> dict:
+            return data
 
+        @optional(condition=lambda step, data: len(data["orders"]) <= 10, skip_downstream=True)
+        @needs(predecessors=[prepare])
+        @step
+        def solve_small(data: dict) -> dict:
+            return {"solver": "small"}
 
-    flow = Flow("DecisionFlow", None)
+        @optional(condition=lambda step, data: len(data["orders"]) > 10, skip_downstream=True)
+        @needs(predecessors=[prepare])
+        @step
+        def solve_large(data: dict) -> dict:
+            return {"solver": "large"}
+
+        @needs(predecessors=[solve_large])
+        @step
+        def polish(result: dict) -> dict:  # skipped if solve_large is skipped
+            return result | {"polished": True}
+
+        @needs(predecessors=[solve_small, polish])
+        @step
+        def report(small: dict | None, large: dict | None) -> dict:
+            return small or large
+
+
+    flow = Flow("DecisionFlow", {"orders": [1, 2, 3]})
     flow.run()
+    log(flow.get_result(flow.report))  # {'solver': 'small'}
     ```
     """
 
     def decorator(function):
-        function.step.optional = Optional(condition)
+        function.step.optional = Optional(condition, skip_downstream)
         return function
 
     return decorator

{nextpipe-0.7.3.dev0 → nextpipe-0.7.4.dev0}/nextpipe/flow.py

@@ -59,6 +59,9 @@ STATUS_SUCCEEDED = "succeeded"
 STATUS_FAILED = "failed"
 """Status constant indicating a failed step or node."""
 
+STATUS_SKIPPED = "skipped"
+"""Status constant indicating a skipped step or node."""
+
 
 class FlowStep:
     """
@@ -89,6 +92,10 @@ class FlowStep:
         Thread lock for this step.
     done : bool
         Whether this step has completed execution.
+    skipped : bool
+        Whether this step was skipped.
+    skip_downstream : bool
+        Whether the skip of this step propagates to its successors.
     successors : list[FlowStep]
         Steps that depend on this step.
     predecessors : list[FlowStep]
@@ -121,6 +128,8 @@ class FlowStep:
         self.docstring = docstring
         self.lock = threading.Lock()
         self.done = False
+        self.skipped = False
+        self.skip_downstream = False
         self.successors: list[FlowStep] = []
         self.predecessors: list[FlowStep] = []
         self.nodes: list[FlowNode] = []
@@ -162,7 +171,7 @@ class FlowNode:
     id : str
         Unique identifier for this node.
     status : str
-        Current execution status (pending, running, succeeded, failed).
+        Current execution status (pending, running, succeeded, failed, skipped).
     error : str or None
         Error message if the node execution failed.
     predecessors : list[FlowNode]
@@ -804,6 +813,10 @@ class Runner:
         predecessor_inputs = {}
         for predecessor in step.predecessors:
             predecessor_results = [res.result for res in predecessor.nodes]
+            if predecessor.skipped:
+                # Skipped predecessors contribute a single None input.
+                predecessor_inputs[predecessor] = [None]
+                continue
             if predecessor.definition.is_foreach():
                 # Make sure the result is in fact a list.
                 if len(predecessor_results) != 1 or not isinstance(predecessor_results[0], list):
@@ -1138,6 +1151,53 @@ class Runner:
                 output = node.parent.definition.function(*inputs)
             return output
 
+    @staticmethod
+    def __skipped_by_predecessors(step: FlowStep) -> bool:
+        """
+        Check whether a step is skipped because of its predecessors.
+
+        This is the case if all of its predecessors were skipped with
+        `skip_downstream`. A step with at least one other predecessor still runs
+        and receives `None` for each skipped one (e.g., merging branches).
+
+        Parameters
+        ----------
+        step : FlowStep
+            The step to check. All its predecessors must be done.
+
+        Returns
+        -------
+        bool
+            True if the step is skipped because of its predecessors.
+        """
+
+        return bool(step.predecessors) and all(p.skipped and p.skip_downstream for p in step.predecessors)
+
+    def __should_skip(self, step: FlowStep) -> bool:
+        """
+        Determine whether a step should be skipped.
+
+        A step is skipped if it is skipped because of its predecessors (see
+        `__skipped_by_predecessors`) or if it is decorated with `@optional` and
+        its condition is not met.
+
+        Parameters
+        ----------
+        step : FlowStep
+            The step to check. All its predecessors must be done.
+
+        Returns
+        -------
+        bool
+            True if the step should be skipped, False otherwise.
+        """
+
+        if self.__skipped_by_predecessors(step):
+            return True
+        if not step.definition.is_optional():
+            return False
+        return not step.definition.optional.evaluate(step.definition, self.spec._get_inputs(step))
+
     def __create_job(self, node: FlowNode, inputs: list[Any] | Any) -> threads.Job:
         """
         Create a job for executing a step.
@@ -1207,14 +1267,21 @@ class Runner:
                     # No more steps to run at this point. Wait for the remaining tasks to finish.
                     break
                 open_steps.remove(step)
-                # Skip the step if it is optional and the condition is not met
-                if step.definition.skip():
+                # Skip the step if its predecessors were skipped (with skip_downstream) or its condition is not met
+                if self.__should_skip(step):
                     utils.log_internal(f"Skipping step {step.definition.get_id()}")
                     # Create dummy node
                     node = FlowNode(step, 0)
-                    node.status = STATUS_SUCCEEDED
+                    node.status = STATUS_SKIPPED
                     node.result = None
+                    node.done = True
                     step.nodes.append(node)
+                    step.skipped = True
+                    # Propagated skips keep propagating, own skips only if requested
+                    step.skip_downstream = (
+                        self.__skipped_by_predecessors(step) or step.definition.optional.skip_downstream
+                    )
+                    step.done = True
                     closed_steps.add(step)
                     open_steps.update(step.successors)
                     self.uplink.submit_update(self.graph._to_uplink_dto())

{nextpipe-0.7.3.dev0 → nextpipe-0.7.4.dev0}/pyproject.toml

@@ -70,7 +70,7 @@ path = "nextpipe/__about__.py"
 
 [dependency-groups]
 dev = [
-    "goldie==0.1.8",
-    "pytest==9.0.2",
+    "goldie==0.1.10",
+    "pytest==9.1.1",
     "ruff==0.16.3",
 ]

nextpipe-0.7.4.dev0/tests/pipelines/foreach-2-pred.json.golden

@@ -0,0 +1,80 @@
+[
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=0"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        },
+        {
+            "stats": {
+                "count": 1
+            }
+        }
+    ],
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=1"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        },
+        {
+            "stats": {
+                "count": 1
+            }
+        }
+    ],
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=2"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        },
+        {
+            "stats": {
+                "count": 1
+            }
+        }
+    ]
+]

nextpipe-0.7.4.dev0/tests/pipelines/foreach.json.golden

@@ -0,0 +1,65 @@
+[
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=0"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        }
+    ],
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=1"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        }
+    ],
+    [
+        {
+            "options": {},
+            "solution": {
+                "echo": {
+                    "data": {
+                        "hello": "world!"
+                    },
+                    "args": [
+                        "-param=2"
+                    ]
+                }
+            },
+            "assets": [],
+            "metrics": {
+                "run": {
+                    "duration": 0.123
+                }
+            }
+        }
+    ]
+]

{nextpipe-0.7.3.dev0 → nextpipe-0.7.4.dev0}/tests/test_integration.py

@@ -147,9 +147,10 @@ class TestExample(unittest.TestCase):
         # FOREACH
         config.comparison_configuration.json_processing_config = goldie.ConfigProcessJson(
             replacements=[
-                goldie.JsonReplacement(path="$[0].metrics.run.duration", value=0.123),
-                goldie.JsonReplacement(path="$[1].metrics.run.duration", value=0.123),
-                goldie.JsonReplacement(path="$[2].metrics.run.duration", value=0.123),
+                # A join step receives one entry per parallel run, each holding one result per predecessor
+                goldie.JsonReplacement(path="$[0][0].metrics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[1][0].metrics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[2][0].metrics.run.duration", value=0.123),
             ],
         )
         goldie.run_file_unittest(