nextpipe 0.1.0.dev6__tar.gz → 0.1.0.dev8__tar.gz

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/.github/workflows/test.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.11", "3.13"]
     steps:
       - name: git clone
         uses: actions/checkout@v4
@@ -23,4 +23,6 @@ jobs:
       - name: unit tests
         env:
           NEXTMV_API_KEY_NEXTPIPE: ${{ secrets.NEXTMV_API_KEY_NEXTPIPE }}
-        run: python -m unittest
+        run: |
+          export NEXTMV_API_KEY=$NEXTMV_API_KEY_NEXTPIPE
+          python -m unittest

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.1.0.dev6
+Version: 0.1.0.dev8
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://www.nextmv.io/docs
@@ -109,7 +109,7 @@ Requires-Dist: nextmv>=0.13.1
 Requires-Dist: pathos>=0.3.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
-Requires-Dist: goldie>=0.1.6; extra == 'dev'
+Requires-Dist: goldie>=0.1.7; extra == 'dev'
 Requires-Dist: ruff>=0.6.4; extra == 'dev'
 Description-Content-Type: text/markdown
 

nextpipe-0.1.0.dev8/examples/apps/echo/input.json

@@ -0,0 +1 @@
+{ "hello": "world!" }

{nextpipe-0.1.0.dev6/examples/pipeline-complex → nextpipe-0.1.0.dev8/examples/pipeline-chain}/app.yaml

@@ -1,7 +1,6 @@
 type: python
 runtime: ghcr.io/nextmv-io/runtime/python:3.11
 files:
-    - main.py
-    - key.json
+  - main.py
 python:
-    pip-requirements: requirements.txt
+  pip-requirements: requirements.txt

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-chain/main.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 import nextmv
 
@@ -31,10 +30,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Load input data
     input = nextmv.load_local()
 

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-complex/README.md

@@ -40,5 +40,5 @@ graph LR
 
 ```bash
 nextmv app push -a <app-id>
-echo '{}' | nextmv app run -a <app-id>
+curl "https://gist.githubusercontent.com/merschformann/a90959b87d1360b604e4a9f6457340ca/raw/661e631376bdf78a07548a3cd136c1fc6e47c639/muenster.json" | nextmv app run -a <app-id>
 ```

{nextpipe-0.1.0.dev6/examples/pipeline-ensemble → nextpipe-0.1.0.dev8/examples/pipeline-complex}/app.yaml

@@ -1,7 +1,6 @@
 type: python
 runtime: ghcr.io/nextmv-io/runtime/python:3.11
 files:
-    - main.py
-    - key.json
+  - main.py
 python:
-    pip-requirements: requirements.txt
+  pip-requirements: requirements.txt

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-complex/main.py

@@ -1,8 +1,7 @@
+import copy
 import json
-import os
 
 import nextmv
-import requests
 
 from nextpipe import FlowSpec, app, needs, repeat, step
 
@@ -10,29 +9,39 @@ from nextpipe import FlowSpec, app, needs, repeat, step
 # >>> Workflow definition
 class Flow(FlowSpec):
     @step
-    def fetch_data(_):
-        """Fetches data from the database."""
-        file_url = "https://gist.githubusercontent.com/merschformann/a90959b87d1360b604e4a9f6457340ca/raw/661e631376bdf78a07548a3cd136c1fc6e47c639/muenster.json"
-        response = requests.get(file_url)
-        return response.json()
+    def prepare(input: dict):
+        """Prepares the data."""
+        return input
 
-    @repeat(repetitions=3)
+    @needs(predecessors=[prepare])
+    @step
+    def convert(input: dict):
+        """Converts the data."""
+        clone = copy.deepcopy(input)
+        if "defaults" in clone and "stops" in clone["defaults"] and "quantity" in clone["defaults"]["stops"]:
+            clone["defaults"]["stops"]["quantity"] *= -1
+        for stop in clone["stops"]:
+            if "quantity" in stop:
+                stop["quantity"] *= -1
+        return clone
+
+    @repeat(repetitions=2)
     @app(app_id="routing-nextroute")
-    @needs(predecessors=[fetch_data])
+    @needs(predecessors=[prepare])
     @step
     def run_nextroute():
         """Runs the model."""
         pass
 
     @app(app_id="routing-ortools")
-    @needs(predecessors=[fetch_data])
+    @needs(predecessors=[convert])
     @step
     def run_ortools():
         """Runs the model."""
         pass
 
     @app(app_id="routing-pyvroom")
-    @needs(predecessors=[fetch_data])
+    @needs(predecessors=[convert])
     @step
     def run_pyvroom():
         """Runs the model."""
@@ -56,16 +65,17 @@ class Flow(FlowSpec):
         values.sort()
         nextmv.log(f"Values: {values}")
 
-        return results[best_solution_idx]
+        # For test stability reasons, we always return the or-tools result
+        _ = results.pop(best_solution_idx)
+        return result_ortools
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
+    # Load input data
+    input = nextmv.load_local()
 
     # Run workflow
-    flow = Flow("DecisionFlow", None)
+    flow = Flow("DecisionFlow", input.data)
     flow.run()
     result = flow.get_result(flow.pick_best)
     print(json.dumps(result))

nextpipe-0.1.0.dev8/examples/pipeline-ensemble/app.yaml

@@ -0,0 +1,6 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+  - main.py
+python:
+  pip-requirements: requirements.txt

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-ensemble/main.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 import nextmv
 
@@ -37,10 +36,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Load input data
     input = nextmv.load_local()
 

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-foreach/README.md

@@ -16,7 +16,7 @@ graph LR
 
 ## Pre-requisites
 
-- Push the echo app as described in the [echo app README](../apps/echo/README.md)`
+- Push the echo app as described in the [echo app README](../apps/echo/README.md)
 
 ## Usage
 

nextpipe-0.1.0.dev8/examples/pipeline-foreach/app.yaml

@@ -0,0 +1,6 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+  - main.py
+python:
+  pip-requirements: requirements.txt

nextpipe-0.1.0.dev8/examples/pipeline-foreach/main.py

@@ -0,0 +1,58 @@
+import copy
+import json
+
+import nextmv
+import nextmv.cloud
+
+from nextpipe import AppOption, AppRunConfig, FlowSpec, app, foreach, join, needs, step
+
+
+class Flow(FlowSpec):
+    @foreach()  # Run the successor step for each item in the result list of this step
+    @step
+    def fanout(data: dict):
+        """
+        Creates 3 copies of the input and configures them for 3 different app parameters.
+        """
+        inputs = [copy.deepcopy(data) for _ in range(3)]
+        run_configs = [AppRunConfig(input, [AppOption("param", i)]) for i, input in enumerate(inputs)]
+        return run_configs
+
+    @step
+    def stats(data: dict):
+        """
+        Calculates some statistics to put on the output as well.
+        """
+        return {"stats": {"count": len(data)}}
+
+    @app(app_id="echo")
+    @needs(predecessors=[fanout])
+    @step
+    def solve():
+        """
+        Runs the model.
+        """
+        pass
+
+    @needs(predecessors=[solve, stats])
+    @join()  # Collect the results from the previous 'foreach' step and combine them into a list passed as the arg
+    @step
+    def merge(results: list):
+        """Merges the results."""
+        return results
+
+
+def main():
+    # Load input data
+    input = nextmv.load_local()
+
+    # Run workflow
+    flow = Flow("DecisionFlow", input.data)
+    flow.run()
+
+    # Write out the result
+    print(json.dumps(flow.get_result(flow.merge)))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev8/examples/pipeline-preprocess/app.yaml

@@ -0,0 +1,6 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+  - main.py
+python:
+  pip-requirements: requirements.txt

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/examples/pipeline-preprocess/main.py

@@ -1,6 +1,5 @@
 import csv
 import json
-import os
 
 import requests
 from nextmv.logger import log
@@ -90,10 +89,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Run workflow
     flow = Flow("DecisionFlow", None)
     flow.run()

nextpipe-0.1.0.dev8/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.1.0.dev8"

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/nextpipe/config.py

@@ -1,8 +1,24 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from dataclasses_json import dataclass_json
 
 
+@dataclass_json
+@dataclass
+class AppPollingOptions:
+    """Options for polling the platform for the status of an app."""
+
+    timeout: float = 1800
+    """
+    Timeout in seconds for polling the platform.
+    This is used for example when waiting for results of an app run.
+    """
+    max_backoff: float = 30
+    """
+    Maximum backoff time in seconds.
+    """
+
+
 @dataclass_json
 @dataclass
 class Configuration:
@@ -20,3 +36,7 @@ class Configuration:
     of inputs used when a step has multiple predecessors which are themselves repeated or
     foreach steps.
     """
+    app_polling: AppPollingOptions = field(default_factory=AppPollingOptions)
+    """
+    Options for polling the platform for the status of an app.
+    """

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/nextpipe/flow.py

@@ -288,8 +288,8 @@ class Runner:
         inputs = [list(item) for item in product(*predecessor_inputs.values())]
         # If the steps is a 'join' step, we need to combine the inputs from all predecessors.
         if step.definition.is_join():
-            # Unwrap the inputs from the list of lists.
-            inputs = [item[0] for item in inputs]
+            # Make sure that we only pass one list as the input.
+            inputs = [[inputs]]
         # If the step is a 'repeat' step, repeat the inputs for each repetition.
         if step.definition.is_repeat():
             inputs = inputs * step.definition.get_repetitions()

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/nextpipe/uplink.py

@@ -12,7 +12,7 @@ from nextpipe.utils import log
 
 FAILED_UPDATES_THRESHOLD = 10
 
-ENV_APPLICATION_ID = "NEXTMV_APPLICATION_ID"
+ENV_APP_ID = "NEXTMV_APP_ID"
 ENV_RUN_ID = "NEXTMV_RUN_ID"
 
 
@@ -96,7 +96,7 @@ class UplinkClient:
         if config is None:
             # Load config from environment
             config = UplinkConfig(
-                application_id=os.environ.get(ENV_APPLICATION_ID),
+                application_id=os.environ.get(ENV_APP_ID),
                 run_id=os.environ.get(ENV_RUN_ID),
             )
         self.config = config

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/nextpipe/utils.py

@@ -36,15 +36,15 @@ _INFINITE_TIMEOUT = sys.maxsize
 def wait_for_runs(
     app: Application,
     run_ids: list[str],
-    timeout: int = _INFINITE_TIMEOUT,
-    max_backoff: int = 30,
+    timeout: float = _INFINITE_TIMEOUT,
+    max_backoff: float = 30,
 ) -> list[RunResult]:
     """
     Wait until all runs with the given IDs are finished.
     """
     # Wait until all runs are finished or the timeout is reached
     missing = set(run_ids)
-    backoff = 1
+    backoff = 2
     start_time = time.time()
     while missing and time.time() - start_time < timeout:
         time.sleep(backoff)

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/pyproject.toml

@@ -72,5 +72,5 @@ path = "nextpipe/__about__.py"
 [project.optional-dependencies]
 dev = [
     "ruff>=0.6.4",
-    "goldie>=0.1.6",
+    "goldie>=0.1.7",
 ]

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/tests/pipelines/chain.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 import nextmv
 
@@ -31,10 +30,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Load input data
     input = nextmv.load_local()
 

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/tests/pipelines/complex.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 import nextmv
 
@@ -59,10 +58,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Load input data
     input = nextmv.load_local()
 

nextpipe-0.1.0.dev8/tests/pipelines/foreach-2-pred.json

@@ -0,0 +1,3 @@
+{
+  "hello": "world!"
+}

nextpipe-0.1.0.dev8/tests/pipelines/foreach-2-pred.json.golden

@@ -0,0 +1,65 @@
+[
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=0"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": [],
+        "stats": {
+            "count": 1
+        }
+    },
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=1"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": [],
+        "stats": {
+            "count": 1
+        }
+    },
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=2"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": [],
+        "stats": {
+            "count": 1
+        }
+    }
+]

nextpipe-0.1.0.dev8/tests/pipelines/foreach-2-pred.py

@@ -0,0 +1,58 @@
+import copy
+import json
+
+import nextmv
+import nextmv.cloud
+
+from nextpipe import AppOption, AppRunConfig, FlowSpec, app, foreach, join, needs, step
+
+
+class Flow(FlowSpec):
+    @foreach()  # Run the successor step for each item in the result list of this step
+    @step
+    def fanout(data: dict):
+        """
+        Creates 3 copies of the input and configures them for 3 different app parameters.
+        """
+        inputs = [copy.deepcopy(data) for _ in range(3)]
+        run_configs = [AppRunConfig(input, [AppOption("param", i)]) for i, input in enumerate(inputs)]
+        return run_configs
+
+    @step
+    def stats(data: dict):
+        """
+        Calculates some statistics to put on the output as well.
+        """
+        return {"stats": {"count": len(data)}}
+
+    @app(app_id="echo")
+    @needs(predecessors=[fanout])
+    @step
+    def solve():
+        """
+        Runs the model.
+        """
+        pass
+
+    @needs(predecessors=[solve, stats])
+    @join()  # Collect the results from the previous 'foreach' step and combine them into a list passed as the arg
+    @step
+    def merge(results: list):
+        """Merges the results."""
+        return results
+
+
+def main():
+    # Load input data
+    input = nextmv.load_local()
+
+    # Run workflow
+    flow = Flow("DecisionFlow", input.data)
+    flow.run()
+
+    # Write out the result
+    print(json.dumps(flow.get_result(flow.merge)))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev8/tests/pipelines/foreach.json

@@ -0,0 +1,3 @@
+{
+  "hello": "world!"
+}

nextpipe-0.1.0.dev8/tests/pipelines/foreach.json.golden

@@ -0,0 +1,56 @@
+[
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=0"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": []
+    },
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=1"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": []
+    },
+    {
+        "solution": {
+            "echo": {
+                "data": {
+                    "hello": "world!"
+                },
+                "args": [
+                    "-param=2"
+                ]
+            }
+        },
+        "statistics": {
+            "run": {
+                "duration": 0.123
+            }
+        },
+        "assets": []
+    }
+]

nextpipe-0.1.0.dev6/examples/pipeline-foreach/main.py → nextpipe-0.1.0.dev8/tests/pipelines/foreach.py

@@ -1,6 +1,5 @@
 import copy
 import json
-import os
 
 import nextmv
 import nextmv.cloud
@@ -37,10 +36,6 @@ class Flow(FlowSpec):
 
 
 def main():
-    # Read API key from file (until secrets management support)
-    with open("key.json") as f:
-        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
-
     # Load input data
     input = nextmv.load_local()
 

{nextpipe-0.1.0.dev6 → nextpipe-0.1.0.dev8}/tests/test_integration.py

@@ -2,7 +2,6 @@ import os
 import os.path
 import random
 import unittest
-from dataclasses import replace
 
 import goldie
 from nextmv import cloud
@@ -91,9 +90,42 @@ class TestExample(unittest.TestCase):
             configuration=config,
         )
 
+        # FOREACH
+        config.comparison_configuration.json_processing_config = goldie.ConfigProcessJson(
+            replacements=[
+                goldie.JsonReplacement(path="$[0].statistics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[1].statistics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[2].statistics.run.duration", value=0.123),
+            ],
+        )
+        goldie.run_file_unittest(
+            test=self,
+            td=goldie.TestDefinition(
+                input_file=os.path.join(path, "foreach.json"),
+                extra_args=[("pipeline", os.path.join(path, "foreach.py"))],
+            ),
+            configuration=config,
+        )
+
+        # FOREACH 2 PREDECESSORS
+        config.comparison_configuration.json_processing_config = goldie.ConfigProcessJson(
+            replacements=[
+                goldie.JsonReplacement(path="$[0][0].statistics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[1][0].statistics.run.duration", value=0.123),
+                goldie.JsonReplacement(path="$[2][0].statistics.run.duration", value=0.123),
+            ],
+        )
+        goldie.run_file_unittest(
+            test=self,
+            td=goldie.TestDefinition(
+                input_file=os.path.join(path, "foreach-2-pred.json"),
+                extra_args=[("pipeline", os.path.join(path, "foreach-2-pred.py"))],
+            ),
+            configuration=config,
+        )
+
         # COMPLEX
-        config_complex = replace(config)
-        config_complex.comparison_configuration.json_processing_config = goldie.ConfigProcessJson(
+        config.comparison_configuration.json_processing_config = goldie.ConfigProcessJson(
             replacements=[
                 goldie.JsonReplacement(path="$.statistics.result.duration", value="0.123"),
                 goldie.JsonReplacement(path="$.statistics.run.duration", value="0.123"),
@@ -105,7 +137,7 @@ class TestExample(unittest.TestCase):
                 input_file=os.path.join(path, "complex.json"),
                 extra_args=[("pipeline", os.path.join(path, "complex.py"))],
             ),
-            configuration=config_complex,
+            configuration=config,
         )
 
 

nextpipe-0.1.0.dev6/examples/pipeline-foreach/app.yaml

@@ -1,7 +0,0 @@
-type: python
-runtime: ghcr.io/nextmv-io/runtime/python:3.11
-files:
-    - main.py
-    - key.json
-python:
-    pip-requirements: requirements.txt

nextpipe-0.1.0.dev6/examples/pipeline-preprocess/app.yaml

@@ -1,7 +0,0 @@
-type: python
-runtime: ghcr.io/nextmv-io/runtime/python:3.11
-files:
-    - main.py
-    - key.json
-python:
-    pip-requirements: requirements.txt

nextpipe-0.1.0.dev6/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.1.0.dev6"

nextpipe-0.1.0.dev6/tests/deploy/app.yaml

@@ -1,7 +0,0 @@
-type: python
-runtime: ghcr.io/nextmv-io/runtime/python:3.11
-files:
-    - main.py
-    - key.json
-python:
-    pip-requirements: requirements.txt