nextpipe 0.1.0.dev4__tar.gz → 0.1.0.dev5__tar.gz

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: nextpipe
-Version: 0.1.0.dev4
+Version: 0.1.0.dev5
 Summary: Framework for Decision Pipeline modeling and execution
 Project-URL: Homepage, https://www.nextmv.io
 Project-URL: Documentation, https://www.nextmv.io/docs
@@ -104,7 +104,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
-Requires-Dist: nextmv>=0.12.1.dev2
+Requires-Dist: dataclasses-json>=0.6.7
+Requires-Dist: nextmv>=0.13.1
 Requires-Dist: pathos>=0.3.2
 Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
@@ -113,4 +114,8 @@ Description-Content-Type: text/markdown
 
 # nextpipe
 
-Framework for Decision Pipeline modeling and execution
+Framework for Decision Pipeline modeling and execution.
+
+## Examples
+
+You can find examples of how to use `nextpipe` in the [examples](./examples) directory.

nextpipe-0.1.0.dev5/README.md

@@ -0,0 +1,7 @@
+# nextpipe
+
+Framework for Decision Pipeline modeling and execution.
+
+## Examples
+
+You can find examples of how to use `nextpipe` in the [examples](./examples) directory.

nextpipe-0.1.0.dev5/examples/README.md

@@ -0,0 +1,3 @@
+# Examples
+
+This directory contains examples of how to use `nextpipe`. Each example typically makes use of other apps that get executed in the example pipeline. If these apps are not already available via [community-apps](https://github.com/nextmv-io/community-apps) (via CLI: `nextmv community clone -a <app-id>`), you can find them in the `./apps` directory.

nextpipe-0.1.0.dev5/examples/apps/echo/.gitignore

@@ -0,0 +1 @@
+*.json

nextpipe-0.1.0.dev5/examples/apps/echo/README.md

@@ -0,0 +1,17 @@
+# JSON echo
+
+This is a sample app that reads JSON from the input and writes it to the output.
+Furthermore, it adds a statistics section to the output.
+
+## Usage
+
+```bash
+echo '{"hello": "world!"}' | python main.py
+```
+
+## Push to Nextmv
+
+```bash
+nextmv app create -a echo -n "JSON echo" -d "This is a sample app that reads JSON from the input and writes it to the output."
+nextmv app push -a echo
+```

nextpipe-0.1.0.dev5/examples/apps/echo/app.yaml

@@ -0,0 +1,6 @@
+type: python
+runtime: ghcr.io/nextmv-io/runtime/python:3.11
+files:
+    - main.py
+python:
+    pip-requirements: requirements.txt

nextpipe-0.1.0.dev5/examples/apps/echo/main.py

@@ -0,0 +1,11 @@
+import time
+
+import nextmv
+
+before = time.time()
+input = nextmv.load_local()
+output = nextmv.Output(
+    solution={"echo": input.data},
+    statistics={"run": {"duration": time.time() - before}},
+)
+nextmv.write_local(output)

nextpipe-0.1.0.dev5/examples/apps/echo/requirements.txt

@@ -0,0 +1 @@
+nextmv==0.13.1

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/examples/pipeline-chain/README.md

@@ -13,6 +13,10 @@ graph TD
   enhance(enhance)
 ```
 
+## Pre-requisites
+
+- Push the echo app as described in the [echo app README](../apps/echo/README.md)
+
 ## Usage
 
 ```bash

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/examples/pipeline-complex/README.md

@@ -29,6 +29,13 @@ graph TD
   pick_best(pick_best)
 ```
 
+## Pre-requisites
+
+- Subscribe to the following marketplace apps and name them as follows:
+  - _Nextmv Routing_: `routing-nextroute`
+  - _OR-Tools Routing_: `routing-ortools`
+  - _PyVroom Routing_: `routing-pyvroom`
+
 ## Usage
 
 ```bash

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/examples/pipeline-ensemble/README.md

@@ -21,9 +21,14 @@ graph TD
   pick_best(pick_best)
 ```
 
+## Pre-requisites
+
+- Subscribe to the following marketplace apps and name them as follows:
+  - _Nextmv Routing_: `routing-nextroute`
+
 ## Usage
 
 ```bash
 nextmv app push -a <app-id>
-cat /path/to/routing/input.json | nextmv app run -a <app-id> -e "8c16gb12h"
+cat /path/to/routing/input.json | nextmv app run -a <app-id> -e "8c16gb12h" -o 'instance=v171-5s'
 ```

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/examples/pipeline-ensemble/main.py

@@ -5,12 +5,18 @@ import nextmv
 
 from nextpipe import FlowSpec, app, needs, repeat, step
 
+# Define the options for the workflow
+parameters = [
+    nextmv.Parameter("instance", str, "devint", "App instance to use. Default is devint.", False),
+]
+options = nextmv.Options(*parameters)
+
 
 # >>> Workflow definition
 class Flow(FlowSpec):
     @app(
         app_id="routing-nextroute",
-        instance_id="v171-5s",
+        instance_id=options.instance,
         parameters={"model.constraints.enable.cluster": True},
     )
     @repeat(repetitions=3)

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/examples/pipeline-preprocess/README.md

@@ -31,6 +31,13 @@ graph TD
   pick_best(pick_best)
 ```
 
+## Pre-requisites
+
+- Subscribe to the following marketplace apps and name them as follows:
+  - _Nextmv Routing_: `routing-nextroute`
+  - _OR-Tools Routing_: `routing-ortools`
+  - _PyVroom Routing_: `routing-pyvroom`
+
 ## Usage
 
 ```bash

nextpipe-0.1.0.dev5/nextpipe/__about__.py

@@ -0,0 +1 @@
+__version__ = "v0.1.0.dev5"

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/nextpipe/decorators.py

@@ -21,6 +21,8 @@ class Step:
     def __init__(self, function: callable):
         self.function = function
         self.type = StepType.DEFAULT
+        self.state = "pending"
+        self.run_ids = []
         self._inputs = {}
         self._output = None
 
@@ -37,6 +39,12 @@ class Step:
     def get_name(self):
         return self.function.__name__
 
+    def set_state(self, state: str):
+        self.state = state
+
+    def get_state(self):
+        return self.state
+
     def is_needs(self):
         return hasattr(self, "needs")
 
@@ -49,6 +57,12 @@ class Step:
     def is_app(self):
         return self.type == StepType.APP
 
+    def set_run_ids(self, run_ids: List[str]):
+        self.run_ids = run_ids
+
+    def get_run_ids(self):
+        return self.run_ids
+
 
 class Needs:
     def __init__(self, predecessors: List[callable]):

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/nextpipe/flow.py

@@ -4,26 +4,16 @@ import collections
 import inspect
 import io
 import time
+from importlib.metadata import version
 from typing import List, Optional, Union
 
-import nextmv
 from nextmv.cloud import Application, Client, StatusV2
-from pathos.multiprocessing import ProcessingPool as Pool
 
-from . import utils
-
-
-class FlowParameter(nextmv.Parameter):
-    """
-    Parameter that is used in a `FlowSpec`.
-    """
-
-    def __init__(self, name, type, default=None, description=None, required=False):
-        super().__init__(name, type, default, description, required)
+from . import decorators, schema, threads, utils
 
 
 class DAGNode:
-    def __init__(self, step_function, step_definition, docstring):
+    def __init__(self, step_function: callable, step_definition: decorators.Step, docstring: str):
         self.step_function = step_function
         self.step = step_definition
         self.docstring = docstring
@@ -89,45 +79,51 @@ class FlowSpec:
 
         # Run the nodes in parallel
         tasks = {}
-        with Pool(8) as pool:
-            while open_nodes:
-                while True:
-                    # Get the first node from the open nodes which has all its predecessors done
-                    node = next(
-                        iter(
-                            filter(
-                                lambda n: all(p in closed_nodes for p in n.predecessors),
-                                open_nodes,
-                            )
-                        ),
-                        None,
-                    )
-                    if node is None:
-                        # No more nodes to run at this point. Wait for the remaining tasks to finish.
-                        break
-                    open_nodes.remove(node)
-                    # Run the node asynchronously
-                    tasks[node] = pool.apipe(
-                        self.__run_node,
-                        node,
-                        self._get_inputs(node),
-                        self.client,
-                    )
-
-                # Wait until at least one task is done
-                task_done = False
-                while not task_done:
-                    time.sleep(0.1)
-                    # Check if any tasks are done, if not, keep waiting
-                    for node, task in list(tasks.items()):
-                        if task.ready():
-                            # Remove task and mark successors as ready by adding them to the open list.
-                            result = task.get()
-                            self.set_result(node, result)
-                            del tasks[node]
-                            task_done = True
-                            closed_nodes.add(node)
-                            open_nodes.update(node.successors)
+        pool = threads.Pool(8)
+        while open_nodes:
+            while True:
+                # Get the first node from the open nodes which has all its predecessors done
+                node = next(
+                    iter(
+                        filter(
+                            lambda n: all(p in closed_nodes for p in n.predecessors),
+                            open_nodes,
+                        )
+                    ),
+                    None,
+                )
+                if node is None:
+                    # No more nodes to run at this point. Wait for the remaining tasks to finish.
+                    break
+                open_nodes.remove(node)
+                # Skip the node if it is optional and the condition is not met
+                if node.step.skip():
+                    utils.log(f"Skipping node {node.step.get_name()}")
+                    node.step.set_state("skipped")
+                    utils.log("NEXTPIPE_DAG_UPDATE=" + self.graph._persist_dag_update(node))
+                    continue
+                # Run the node asynchronously
+                job = threads.Job(self.__run_node, (node, self._get_inputs(node), self.client))
+                pool.run(job)
+                tasks[node] = job
+                node.step.set_state("running")
+                utils.log("NEXTPIPE_DAG_UPDATE=" + self.graph._persist_dag_update(node))
+
+            # Wait until at least one task is done
+            task_done = False
+            while not task_done:
+                time.sleep(0.1)
+                # Check if any tasks are done, if not, keep waiting
+                for node, job in list(tasks.items()):
+                    if job.done:
+                        # Remove task and mark successors as ready by adding them to the open list.
+                        self.set_result(node, job.result)
+                        node.step.set_state("succeeded")
+                        utils.log("NEXTPIPE_DAG_UPDATE=" + self.graph._persist_dag_update(node))
+                        del tasks[node]
+                        task_done = True
+                        closed_nodes.add(node)
+                        open_nodes.update(node.successors)
 
     def set_result(self, step: callable, result: object):
         self.results[step.step] = result
@@ -146,11 +142,6 @@ class FlowSpec:
     def __run_node(node: DAGNode, inputs: List[object], client: Client) -> Union[List[object], object, None]:
         utils.log(f"Running node {node.step.get_name()}")
 
-        # Skip the node if it is optional and the condition is not met
-        if node.step.skip():
-            utils.log(f"Skipping node {node.step.get_name()}")
-            return
-
         # Run the step
         if node.step.is_app():
             app_step = node.step.app
@@ -176,6 +167,7 @@ class FlowSpec:
             app = Application(client=client, id=app_step.app_id, default_instance_id=app_step.instance_id)
             # Run the app (or multiple runs if it is a repeat step)
             run_ids = [app.new_run(*i[0], **i[1]) for i in inputs]
+            node.step.set_run_ids(run_ids)
             outputs = utils.wait_for_runs(app=app, run_ids=run_ids)
             # Check if all runs were successful
             for output in outputs:
@@ -202,11 +194,14 @@ class FlowGraph:
     def __init__(self, flow_spec):
         self.flow_spec = flow_spec
         self.__create_graph(flow_spec)
+        self.__debug_print_head()
         self.__debug_print_graph()
+        # Print the DAG in persistence format
+        utils.log("NEXTPIPE_DAG=" + self._persist_dag())
         # Create a Mermaid diagram of the graph and log it
-        mermaid = self.__to_mermaid()
+        mermaid = self._to_mermaid()
         utils.log(mermaid)
-        mermaid_url = f'https://mermaid.ink/svg/{base64.b64encode(mermaid.encode("utf8")).decode("ascii")}?theme=dark'
+        mermaid_url = f"https://mermaid.ink/svg/{base64.b64encode(mermaid.encode('utf8')).decode('ascii')}?theme=dark"
         utils.log(f"Mermaid URL: {mermaid_url}")
 
     def __create_graph(self, flow_spec):
@@ -250,7 +245,30 @@ class FlowGraph:
         if cycle:
             raise Exception(f"Cycle detected in the flow graph, cycle nodes: {cycle_nodes}")
 
-    def __to_mermaid(self):
+    def _persist_dag(self) -> str:
+        dto = schema.DAGDTO(
+            nodes=[
+                schema.NodeDTO(
+                    id=node.step.get_name(),
+                    app_id=node.step.app.app_id if node.step.is_app() else "",
+                    step_name=node.step.get_name(),
+                    docs=node.docstring,
+                    successors=[s.step.get_name() for s in node.successors],
+                )
+                for node in self.nodes
+            ]
+        )
+        return schema.serialize_dag(dto)
+
+    def _persist_dag_update(self, node: DAGNode) -> str:
+        dto = schema.NodeUpdateDTO(
+            node_id=node.step.get_name(),
+            state=node.step.get_state(),
+            run_ids=node.step.get_run_ids(),
+        )
+        return schema.serialize_node_update(dto)
+
+    def _to_mermaid(self):
         """Convert the graph to a Mermaid diagram."""
         out = io.StringIO()
         out.write("graph TD\n")
@@ -272,6 +290,12 @@ class FlowGraph:
                     out.write(f"  {node_name} --> {successor.step.get_name()}\n")
         return out.getvalue()
 
+    def __debug_print_head(self):
+        utils.log(f"Flow: {self.flow_spec.__name__}")
+        utils.log(f"nextpipe: {version('nextpipe')}")
+        utils.log(f"nextmv: {version('nextmv')}")
+        utils.log("Flow graph nodes:")
+
     def __debug_print_graph(self):
         for node in self.nodes:
             utils.log("Node:")

nextpipe-0.1.0.dev5/nextpipe/schema.py

@@ -0,0 +1,50 @@
+import base64
+import dataclasses
+from typing import List
+
+import dataclasses_json
+
+
+@dataclasses_json.dataclass_json
+@dataclasses.dataclass
+class NodeUpdateDTO:
+    node_id: str
+    """The ID of the node to update."""
+    state: str
+    """The state of the node."""
+    run_ids: List[str] = dataclasses.field(default_factory=list)
+    """The ID of the associated run (if any)."""
+
+
+@dataclasses_json.dataclass_json
+@dataclasses.dataclass
+class NodeDTO:
+    id: str
+    """The ID of the node."""
+    app_id: str
+    """The ID of the app this step represents (if any)."""
+    step_name: str
+    """The name of the step (func)."""
+    docs: str
+    """The doc string of the step."""
+    successors: List[str]
+    """The IDs of the nodes that depend on this node."""
+
+
+@dataclasses_json.dataclass_json
+@dataclasses.dataclass
+class DAGDTO:
+    nodes: List[NodeDTO]
+    """The nodes in the DAG."""
+
+
+def serialize_dag(dag: DAGDTO) -> str:
+    """Serialize the DAG for transmission."""
+    j_str = dag.to_json(separators=(",", ":"))
+    return base64.b64encode(j_str.encode()).decode()
+
+
+def serialize_node_update(node_update: NodeUpdateDTO) -> str:
+    """Serialize the node update for transmission."""
+    j_str = node_update.to_json(separators=(",", ":"))
+    return base64.b64encode(j_str.encode()).decode()

nextpipe-0.1.0.dev5/nextpipe/threads.py

@@ -0,0 +1,75 @@
+import threading
+import time
+from typing import Callable, Optional
+
+
+class Job:
+    def __init__(self, target: Callable, args: Optional[tuple] = None):
+        self.target = target
+        self.args = args
+        self.done = False
+        self.result = None
+
+    def run(self):
+        if self.args:
+            self.result = self.target(*self.args)
+        else:
+            self.result = self.target()
+        self.done = True
+
+
+class Pool:
+    def __init__(self, max_threads: int):
+        self.max_threads = max_threads
+        self.counter = 0  # Used to assign unique IDs to threads
+        self.waiting = {}
+        self.running = {}
+        self.lock = threading.Lock()
+        self.cond = threading.Condition(self.lock)
+
+    def run(self, job: Job) -> None:
+        with self.lock:
+            self.counter += 1
+            thread_id = self.counter
+            self.waiting[thread_id] = job
+
+        def worker(job: Job, thread_id: int):
+            try:
+                job.run()
+            finally:
+                with self.lock:
+                    self.running.pop(thread_id, None)
+                    self.cond.notify_all()  # Notify others that a thread is available
+
+        while True:
+            with self.lock:
+                if len(self.running) < self.max_threads:
+                    # Move job from waiting to running
+                    thread = threading.Thread(target=worker, args=(job, thread_id))
+                    self.running[thread_id] = thread
+                    self.waiting.pop(thread_id, None)
+                    thread.start()
+                    break  # Successfully assigned the job to a thread
+                else:
+                    self.cond.wait()  # Wait until a thread is available
+
+    def join(self) -> None:
+        with self.cond:
+            while self.waiting or self.running:
+                self.cond.wait()  # Wait until all jobs are finished
+
+
+def test_pool():
+    def target(*args):
+        print(f"Running job with args: {args}")
+        time.sleep(0.5)  # Simulate work
+
+    pool = Pool(2)
+    for i in range(1, 7):  # Submit 6 jobs
+        pool.run(Job(target, (i,)))
+    pool.join()
+    print("All jobs completed.")
+
+
+if __name__ == "__main__":
+    test_pool()

{nextpipe-0.1.0.dev4 → nextpipe-0.1.0.dev5}/pyproject.toml

@@ -18,7 +18,8 @@ classifiers = [
 dependencies = [
     "requests>=2.31.0",
     "pathos>=0.3.2",
-    "nextmv>=0.12.1.dev2",
+    "nextmv>=0.13.1",
+    "dataclasses-json>=0.6.7",
 ]
 description = "Framework for Decision Pipeline modeling and execution"
 dynamic = [

nextpipe-0.1.0.dev4/README.md

@@ -1,3 +0,0 @@
-# nextpipe
-
-Framework for Decision Pipeline modeling and execution

nextpipe-0.1.0.dev4/nextpipe/__about__.py

@@ -1 +0,0 @@
-__version__ = "v0.1.0.dev4"