dv-flow-mgr 0.0.1.12919555073a1__tar.gz → 0.0.1.12922457458a1__tar.gz

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12919555073a1
+Version: 0.0.1.12922457458a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12919555073a1"
+version = "0.0.1.12922457458a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/package_def.py

@@ -215,7 +215,7 @@ class PackageDef(BaseModel):
             if len(task.params) > 0:
                 ctor_t.params = task.params
             if len(task.depends) > 0:
-                ctor_t.depends.extends(task.depends)
+                ctor_t.depends.extend(task.depends)
 
         return ctor_t
 

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
 
         depends = []
 
+        for dep in ctor_t.depends:
+            if not dep in self._task_m.keys():
+                task = self._mkTaskGraph(dep, rundir)
+                self._task_m[dep] = task
+                pass
+            depends.append(self._task_m[dep])
+
         # The returned task should have all param references resolved
         print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
         task = ctor_t.task_ctor(

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow/mgr/task_graph_runner_local.py

@@ -23,7 +23,8 @@ import asyncio
 import os
 import yaml
 import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List, Union
+from toposort import toposort
+from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
 from .fragment_def import FragmentDef
 from .package import Package
 from .pkg_rgy import PkgRgy
@@ -37,7 +38,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
     """Session manages execution of a task graph"""
 
     rundir : str
-    nproc : int = -1
+    nproc : int = 4
+    done_task_m : Dict = dc.field(default_factory=dict)
     _workers : List = dc.field(default_factory=list)
 
     _inst : ClassVar['TaskGraphRunner'] = None
@@ -62,15 +64,62 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
             task = [task]
         else:
             unwrap = False
-        
-        run_o = list(t.do_run() for t in task)
 
-        ret = await asyncio.gather(*run_o)
+        dep_m = {}
+        task_m = {}
+
+        for t in task:
+            self._mkDeps(dep_m, task_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l : List[Tuple[Task,Coroutine]]= []
+        # Now, iterate over the concurrent sets
+        for active_s in order:
+
+            # Check to see if all tasks are complete
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                self.done_task_m[tt.name] = tt
+                                active_task_l.pop(i)
+                                break
+                if t not in self.done_task_m.keys():
+                    task = task_m[t]
+                    coro = asyncio.Task(task.do_run())
+                    active_task_l.append((task, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+#        print("order: %s" % str(order))
+#        
+#        run_o = list(t.do_run() for t in task)
+
+#        ret = await asyncio.gather(*run_o)
+        ret = None
 
         if unwrap:
-            return ret[0]
+            return task.output
         else:
-            return ret
+            return list(t.output for t in task)
+        
+    def _mkDeps(self, dep_m, task_m, task):
+        if task.name not in dep_m.keys():
+            task_m[task.name] = task
+            dep_m[task.name] = set(t.name for t in task.depends)
+            for d in task.depends:
+                self._mkDeps(dep_m, task_m, d)
     
     async def runTask(self, task : Task) -> 'TaskData':
         return await task.do_run()
@@ -82,6 +131,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
 @dc.dataclass
 class LocalRunnerWorker(object):
     runner : TaskGraphRunnerLocal
-    pass
+    task_s : List = dc.field(default_factory=list)
+
 
 

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12919555073a1
+Version: 0.0.1.12922457458a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12922457458a1}/tests/unit/test_pyclass.py

@@ -193,3 +193,52 @@ class foo(Task):
 
     task = builder.mkTaskGraph("pkg1.foo2")
     output = asyncio.run(runner.run(task))
+
+def test_broad_parallel(tmpdir):
+    # Test that we can 
+    flow = """
+package:
+  name: pkg1
+  tasks:
+  - name: foo
+    pyclass: my_module.foo
+    with:
+      param1:
+        type: str
+        value: "1"
+"""
+
+    count = 1000
+
+    for t in range(count):
+        flow += "  - name: foo_%02d\n" % t
+        flow += "    uses: foo\n"
+        flow += "    with:\n"
+        flow += "      param1: \"%d\"\n" % t
+    flow += "  - name: final\n"
+    flow += "    needs: [" + ",".join("foo_%02d" % t for t in range(count)) + "]\n"
+
+    module = """
+from dv_flow.mgr import Task, TaskData
+
+class foo(Task):
+    async def run(self, input : TaskData) -> TaskData:
+        print("foo::run", flush=True)
+        print("params: %s" % str(self.params), flush=True)
+        return input
+"""
+
+    with open(os.path.join(tmpdir, "my_module.py"), "w") as f:
+        f.write(module)
+    with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
+        f.write(flow)
+
+    pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
+    builder = TaskGraphBuilder(
+        root_pkg=pkg_def,
+        rundir=os.path.join(tmpdir, "rundir"))
+    runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
+
+    task = builder.mkTaskGraph("pkg1.final")
+    output = asyncio.run(runner.run(task))
+