nextpipe 0.1.0.dev1__tar.gz → 0.1.0.dev3__tar.gz

{nextpipe-0.1.0.dev1 → nextpipe-0.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: nextpipe
-Version: 0.1.0.dev1
+Version: 0.1.0.dev3
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://www.nextmv.io/docs
@@ -104,7 +104,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
-Requires-Dist: nextmv>=0.12.0
+Requires-Dist: nextmv>=0.12.1.dev2
 Requires-Dist: pathos>=0.3.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev

nextpipe-0.1.0.dev3/examples/pipeline-chain/README.md

@@ -0,0 +1,21 @@
+# Chain example
+
+A simple chain pipeline.
+
+## Graph
+
+```mermaid
+graph TD
+  prepare(prepare)
+  prepare --> solve
+  solve(solve)
+  solve --> enhance
+  enhance(enhance)
+```
+
+## Usage
+
+```bash
+nextmv app push -a <app-id>
+echo '{"hello": "world!"}' | nextmv app run -a <app-id> -e "8c16gb12h"
+```

nextpipe-0.1.0.dev3/examples/pipeline-chain/app.yaml

@@ -0,0 +1,7 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+    - main.py
+    - key.json
+python:
+    pip-requirements: requirements.txt

nextpipe-0.1.0.dev3/examples/pipeline-chain/main.py

@@ -0,0 +1,50 @@
+import json
+import os
+
+import nextmv
+
+from nextpipe import FlowSpec, app, needs, step
+
+
+# >>> Workflow definition
+class Flow(FlowSpec):
+    @step
+    def prepare(input: dict):
+        """Prepares the data."""
+        input["prepared"] = True
+        return input
+
+    @app(app_id="echo")
+    @needs(predecessors=[prepare])
+    @step
+    def solve():
+        """Runs the model."""
+        pass
+
+    @needs(predecessors=[solve])
+    @step
+    def enhance(result: dict):
+        """Enhances the result."""
+        output = result["solution"]  # Unwrap the solution
+        output["echo"]["enhanced"] = True
+        return output
+
+
+def main():
+    # Read API key from file (until secrets management support)
+    with open("key.json") as f:
+        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
+
+    # Load input data
+    input = nextmv.load_local()
+
+    # Run workflow
+    flow = Flow("DecisionFlow", input.data)
+    flow.run()
+
+    # Write out the result
+    print(json.dumps(flow.get_result(flow.enhance)))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev3/examples/pipeline-chain/requirements.txt

@@ -0,0 +1 @@
+nextpipe==0.1.0.dev2

nextpipe-0.1.0.dev3/examples/pipeline-complex/README.md

@@ -0,0 +1,37 @@
+# Complex example
+
+A more complex pipeline combining some concepts.
+
+## Graph
+
+```mermaid
+graph TD
+  fetch_data(fetch_data)
+  fetch_data --> run_nextroute
+  fetch_data --> run_ortools
+  fetch_data --> run_pyvroom
+  run_nextroute{ }
+  run_nextroute_join{ }
+  run_nextroute_0(run_nextroute_0)
+  run_nextroute --> run_nextroute_0
+  run_nextroute_0 --> run_nextroute_join
+  run_nextroute_1(run_nextroute_1)
+  run_nextroute --> run_nextroute_1
+  run_nextroute_1 --> run_nextroute_join
+  run_nextroute_2(run_nextroute_2)
+  run_nextroute --> run_nextroute_2
+  run_nextroute_2 --> run_nextroute_join
+  run_nextroute_join --> pick_best
+  run_ortools(run_ortools)
+  run_ortools --> pick_best
+  run_pyvroom(run_pyvroom)
+  run_pyvroom --> pick_best
+  pick_best(pick_best)
+```
+
+## Usage
+
+```bash
+nextmv app push -a <app-id>
+echo '{}' | nextmv app run -a <app-id> -e "8c16gb12h"
+```

nextpipe-0.1.0.dev3/examples/pipeline-complex/app.yaml

@@ -0,0 +1,7 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+    - main.py
+    - key.json
+python:
+    pip-requirements: requirements.txt

nextpipe-0.1.0.dev3/examples/pipeline-complex/main.py

@@ -0,0 +1,75 @@
+import json
+import os
+
+import nextmv
+import requests
+
+from nextpipe import FlowSpec, app, needs, repeat, step
+
+
+# >>> Workflow definition
+class Flow(FlowSpec):
+    @step
+    def fetch_data(_):
+        """Fetches data from the database."""
+        file_url = "https://gist.githubusercontent.com/merschformann/a90959b87d1360b604e4a9f6457340ca/raw/661e631376bdf78a07548a3cd136c1fc6e47c639/muenster.json"
+        response = requests.get(file_url)
+        return response.json()
+
+    @repeat(repetitions=3)
+    @app(app_id="routing-nextroute")
+    @needs(predecessors=[fetch_data])
+    @step
+    def run_nextroute():
+        """Runs the model."""
+        pass
+
+    @app(app_id="routing-ortools")
+    @needs(predecessors=[fetch_data])
+    @step
+    def run_ortools():
+        """Runs the model."""
+        pass
+
+    @app(app_id="routing-pyvroom")
+    @needs(predecessors=[fetch_data])
+    @step
+    def run_pyvroom():
+        """Runs the model."""
+        pass
+
+    @needs(predecessors=[run_nextroute, run_ortools, run_pyvroom])
+    @step
+    def pick_best(
+        results_nextroute: list[dict],
+        result_ortools: dict,
+        result_pyvroom: dict,
+    ):
+        """Aggregates the results."""
+        results = results_nextroute + [result_ortools, result_pyvroom]
+        best_solution_idx = min(
+            range(len(results)),
+            key=lambda i: results[i]["statistics"]["result"]["value"],
+        )
+
+        values = [result["statistics"]["result"]["value"] for result in results]
+        values.sort()
+        nextmv.log(f"Values: {values}")
+
+        return results[best_solution_idx]
+
+
+def main():
+    # Read API key from file (until secrets management support)
+    with open("key.json") as f:
+        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
+
+    # Run workflow
+    flow = Flow("DecisionFlow", None)
+    flow.run()
+    result = flow.get_result(flow.pick_best)
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev3/examples/pipeline-complex/requirements.txt

@@ -0,0 +1 @@
+nextpipe==0.1.0.dev2

nextpipe-0.1.0.dev3/examples/pipeline-ensemble/README.md

@@ -0,0 +1,29 @@
+# Ensemble example
+
+A basic ensemble pipeline.
+
+## Graph
+
+```mermaid
+graph TD
+  run_nextroute{ }
+  run_nextroute_join{ }
+  run_nextroute_0(run_nextroute_0)
+  run_nextroute --> run_nextroute_0
+  run_nextroute_0 --> run_nextroute_join
+  run_nextroute_1(run_nextroute_1)
+  run_nextroute --> run_nextroute_1
+  run_nextroute_1 --> run_nextroute_join
+  run_nextroute_2(run_nextroute_2)
+  run_nextroute --> run_nextroute_2
+  run_nextroute_2 --> run_nextroute_join
+  run_nextroute_join --> pick_best
+  pick_best(pick_best)
+```
+
+## Usage
+
+```bash
+nextmv app push -a <app-id>
+cat /path/to/routing/input.json | nextmv app run -a <app-id> -e "8c16gb12h"
+```

nextpipe-0.1.0.dev3/examples/pipeline-ensemble/app.yaml

@@ -0,0 +1,7 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+    - main.py
+    - key.json
+python:
+    pip-requirements: requirements.txt

nextpipe-0.1.0.dev3/examples/pipeline-ensemble/main.py

@@ -0,0 +1,49 @@
+import json
+import os
+
+import nextmv
+
+from nextpipe import FlowSpec, app, needs, repeat, step
+
+
+# >>> Workflow definition
+class Flow(FlowSpec):
+    @app(
+        app_id="routing-nextroute",
+        instance_id="v171-5s",
+        parameters={"model.constraints.enable.cluster": True},
+    )
+    @repeat(repetitions=3)
+    @step
+    def run_nextroute():
+        """Runs the model."""
+        pass
+
+    @needs(predecessors=[run_nextroute])
+    @step
+    def pick_best(results: list[dict]):
+        """Aggregates the results."""
+        best_solution_idx = min(
+            range(len(results)),
+            key=lambda i: results[i]["statistics"]["result"]["value"],
+        )
+        return results[best_solution_idx]
+
+
+def main():
+    # Read API key from file (until secrets management support)
+    with open("key.json") as f:
+        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
+
+    # Load input data
+    input = nextmv.load_local()
+
+    # Run workflow
+    flow = Flow("DecisionFlow", input.data)
+    flow.run()
+    result = flow.get_result(flow.pick_best)
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev3/examples/pipeline-ensemble/requirements.txt

@@ -0,0 +1 @@
+nextpipe==0.1.0.dev2

nextpipe-0.1.0.dev3/examples/pipeline-preprocess/README.md

@@ -0,0 +1,39 @@
+# CSV to JSON multi-solver ensemble example
+
+An example of a pipeline fetching CSV data, converting it to JSON, ensembling across multiple solvers and repetitions and picking the best result.
+
+## Graph
+
+```mermaid
+graph TD
+  fetch(fetch)
+  fetch --> convert
+  convert(convert)
+  convert --> solve_nextroute
+  convert --> solve_vroom
+  convert --> solve_ortools
+  solve_nextroute{ }
+  solve_nextroute_join{ }
+  solve_nextroute_0(solve_nextroute_0)
+  solve_nextroute --> solve_nextroute_0
+  solve_nextroute_0 --> solve_nextroute_join
+  solve_nextroute_1(solve_nextroute_1)
+  solve_nextroute --> solve_nextroute_1
+  solve_nextroute_1 --> solve_nextroute_join
+  solve_nextroute_2(solve_nextroute_2)
+  solve_nextroute --> solve_nextroute_2
+  solve_nextroute_2 --> solve_nextroute_join
+  solve_nextroute_join --> pick_best
+  solve_vroom(solve_vroom)
+  solve_vroom --> pick_best
+  solve_ortools(solve_ortools)
+  solve_ortools --> pick_best
+  pick_best(pick_best)
+```
+
+## Usage
+
+```bash
+nextmv app push -a <app-id>
+echo '{}' | nextmv app run -a <app-id> -e "8c16gb12h"
+```

nextpipe-0.1.0.dev3/examples/pipeline-preprocess/app.yaml

@@ -0,0 +1,7 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+    - main.py
+    - key.json
+python:
+    pip-requirements: requirements.txt

nextpipe-0.1.0.dev3/examples/pipeline-preprocess/main.py

@@ -0,0 +1,106 @@
+import csv
+import json
+import os
+
+import requests
+from nextmv.logger import log
+
+from nextpipe import FlowSpec, app, needs, repeat, step
+
+
+# >>> Workflow definition
+class Flow(FlowSpec):
+    # >>> Fetch CSV data from external source
+    @step
+    def fetch(_: dict):
+        file_url = "https://gist.githubusercontent.com/merschformann/5dc2d06b246f924f8c65e93dd3c646e2/raw/fca52de5b7b9a1396e8d9525f8006488ac3467c9/muenster-stops.csv"
+        response = requests.get(file_url)
+        return response.text
+
+    # >>> Convert CSV data to JSON
+    @needs(predecessors=[fetch])
+    @step
+    def convert(input_csv: str):
+        reader = csv.reader(input_csv.splitlines())
+        next(reader)  # skip header
+        return json.dumps(
+            {
+                "vehicles": [
+                    {
+                        "id": f"vehicle-{i}",
+                        "start_location": {"lon": 7.62558, "lat": 51.96223},
+                        "end_location": {"lon": 7.62558, "lat": 51.96223},
+                        "start_time": "2024-09-04T11:00:00+00:00",
+                        "speed": 10,
+                        "capacity": 27,
+                    }
+                    for i in range(20)
+                ],
+                "stops": [
+                    {
+                        "id": row[0],
+                        "location": {"lon": float(row[1]), "lat": float(row[2])},
+                        "quantity": -1,
+                        "unplanned_penalty": 2000000000,
+                    }
+                    for row in reader
+                ],
+            }
+        )
+
+    # >>> Solve the problem using different solvers
+    @app(
+        app_id="routing-nextroute",
+        parameters={"solve.duration": "30s"},
+    )
+    @repeat(repetitions=3)
+    @needs(predecessors=[convert])
+    @step
+    def solve_nextroute():
+        pass
+
+    @app(app_id="routing-pyvroom")
+    @needs(predecessors=[convert])
+    @step
+    def solve_vroom():
+        pass
+
+    @app(app_id="routing-ortools")
+    @needs(predecessors=[convert])
+    @step
+    def solve_ortools():
+        pass
+
+    # >>> Pick the best solution
+    @needs(predecessors=[solve_nextroute, solve_vroom, solve_ortools])
+    @step
+    def pick_best(
+        nextroute_results: list[dict],
+        vroom_result: dict,
+        ortools_result: dict,
+    ):
+        results = nextroute_results + [vroom_result, ortools_result]
+        best_solution_idx = min(
+            range(len(results)),
+            key=lambda i: results[i]["statistics"]["result"]["value"],
+        )
+        for result in results:
+            log(f"{result['statistics']['run']['custom']['solver']}: " + f"{result['statistics']['result']['value']}")
+        return results[best_solution_idx]
+
+
+def main():
+    # Read API key from file (until secrets management support)
+    with open("key.json") as f:
+        os.environ["NEXTMV_API_KEY"] = json.load(f)["nextmv_api_key"]
+
+    # Run workflow
+    flow = Flow("DecisionFlow", None)
+    flow.run()
+
+    # Write out the result
+    print(json.dumps(flow.get_result(flow.pick_best)))
+
+
+if __name__ == "__main__":
+    main()

nextpipe-0.1.0.dev3/examples/pipeline-preprocess/requirements.txt

@@ -0,0 +1 @@
+nextpipe==0.1.0.dev2

nextpipe-0.1.0.dev3/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.1.0.dev3"

{nextpipe-0.1.0.dev1 → nextpipe-0.1.0.dev3}/pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
 dependencies = [
     "requests>=2.31.0",
     "pathos>=0.3.2",
-    "nextmv>=0.12.0",
+    "nextmv>=0.12.1.dev2",
 ]
 description = "Framework for Decision Pipeline modeling and execution"
 dynamic = [

nextpipe-0.1.0.dev1/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.1.0.dev1"