dv-flow-mgr 0.0.1.12919555073a1__tar.gz → 0.0.1.12931082290a1__tar.gz

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12919555073a1
+Version: 0.0.1.12931082290a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12919555073a1"
+version = "0.0.1.12931082290a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/src/dv_flow/mgr/package_def.py

@@ -215,7 +215,7 @@ class PackageDef(BaseModel):
             if len(task.params) > 0:
                 ctor_t.params = task.params
             if len(task.depends) > 0:
-                ctor_t.depends.extends(task.depends)
+                ctor_t.depends.extend(task.depends)
 
         return ctor_t
 

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/src/dv_flow/mgr/task.py

@@ -68,12 +68,8 @@ class Task(object):
     session : 'TaskGraphRunner' = None
     basedir : str = None
     memento : TaskMemento = None
-    depend_refs : List['TaskSpec'] = dc.field(default_factory=list)
-    depends : List[int] = dc.field(default_factory=list)
-    running : bool = False
-    output_set : bool = False
+    depends : List['Task'] = dc.field(default_factory=list)
     output : Any = None
-    output_ev : Any = asyncio.Event()
 
     # Implementation data below
     basedir : str = dc.field(default=None)
@@ -107,23 +103,16 @@ class Task(object):
     async def do_run(self) -> TaskData:
         print("do_run: %s - %d depends" % (self.name, len(self.depends)))
         if len(self.depends) > 0:
-            awaitables = [dep.waitOutput() for dep in self.depends]
-            deps_o = await asyncio.gather(*awaitables)
+            deps_o = []
+            for d in self.depends:
+                dep_o = d.getOutput()
+                if dep_o is None:
+                    raise Exception("Null output for %s" % d.name)
+                deps_o.append(dep_o)
 
             # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
-
-#            print("deps_m: %s" % str(deps_m))
-
-            # Merge the output of the dependencies into a single input data
-#            if len(self.depends) > 1:
-#                raise Exception("TODO: handle >1 inputs")
-
-            # Now that we have a clean input object, we need
-            # to build the dep map
-
-#            input = self.depends[0].output.copy()
             input = TaskData.merge(deps_o)
             input.src = self.name
             input.deps[self.name] = list(inp.name for inp in self.depends)
@@ -137,29 +126,17 @@ class Task(object):
 
         self.init_rundir()
 
-        result = await self.run(input)
-
-        if not self.output_set:
-            if result is None:
-                result = TaskData()
+        self.output = await self.run(input)
 
-            # We perform an auto-merge algorithm if the task 
-            # doesn't take control
-#            for dep_o in deps_o:
-#                result.deps.append(dep_o.clone())
-
-            self.setOutput(result)
-        else:
-            # The task has taken control of the output
-            result = self.getOutput()
+        if self.output is None:
+            raise Exception("No output produced by %s" % self.name)
+            result = TaskData()
 
         # Write-back the memento, if specified
         self.save_memento()
 
-        self.running = False
-
         # Combine data from the deps to produce a result
-        return result
+        return self.output
 
     async def run(self, input : TaskData) -> TaskData:
         raise NotImplementedError("TaskImpl.run() not implemented")
@@ -173,24 +150,6 @@ class Task(object):
             with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
                 fp.write(self.memento.model_dump_json(indent=2))
     
-    def setOutput(self, output : TaskData):
-        self.output_set = True
-        output.src = self.name
-        self.output = output
-        self.output_ev.set()
-
-    async def waitOutput(self) -> TaskData:
-        if not self.output_set:
-            if self.running:
-                # Task is already running
-                print("wait")
-                await self.output_ev.wait()
-            else:
-                self.running = True
-                print("start task")
-                await self.do_run()
-        return self.output
-    
     def getOutput(self) -> TaskData:
         return self.output
 

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
 
         depends = []
 
+        for dep in ctor_t.depends:
+            if not dep in self._task_m.keys():
+                task = self._mkTaskGraph(dep, rundir)
+                self._task_m[dep] = task
+                pass
+            depends.append(self._task_m[dep])
+
         # The returned task should have all param references resolved
         print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
         task = ctor_t.task_ctor(
@@ -80,13 +87,6 @@ class TaskGraphBuilder(object):
             rundir=rundir,
             srcdir=ctor_t.srcdir)
         
-        for i,d in enumerate(task.depend_refs):
-            if d in self._task_m.keys():
-                task.depends.append(self._task_m[d])
-            else:
-                print("mkTaskGraph: %s" % d)
-                task.depends.append(self._mkTaskGraph(d, parent_rundir))
-
         self._task_m[task.name] = task
 
         self._pkg_s.pop()

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/src/dv_flow/mgr/task_graph_runner_local.py

@@ -23,7 +23,8 @@ import asyncio
 import os
 import yaml
 import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List, Union
+from toposort import toposort
+from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
 from .fragment_def import FragmentDef
 from .package import Package
 from .pkg_rgy import PkgRgy
@@ -37,7 +38,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
     """Session manages execution of a task graph"""
 
     rundir : str
-    nproc : int = -1
+    nproc : int = 4
+    done_task_m : Dict = dc.field(default_factory=dict)
     _workers : List = dc.field(default_factory=list)
 
     _inst : ClassVar['TaskGraphRunner'] = None
@@ -62,15 +64,62 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
             task = [task]
         else:
             unwrap = False
-        
-        run_o = list(t.do_run() for t in task)
 
-        ret = await asyncio.gather(*run_o)
+        dep_m = {}
+        task_m = {}
+
+        for t in task:
+            self._mkDeps(dep_m, task_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l : List[Tuple[Task,Coroutine]]= []
+        # Now, iterate over the concurrent sets
+        for active_s in order:
+
+            # Check to see if all tasks are complete
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                self.done_task_m[tt.name] = tt
+                                active_task_l.pop(i)
+                                break
+                if t not in self.done_task_m.keys():
+                    task = task_m[t]
+                    coro = asyncio.Task(task.do_run())
+                    active_task_l.append((task, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+#        print("order: %s" % str(order))
+#        
+#        run_o = list(t.do_run() for t in task)
+
+#        ret = await asyncio.gather(*run_o)
+        ret = None
 
         if unwrap:
-            return ret[0]
+            return task.output
         else:
-            return ret
+            return list(t.output for t in task)
+        
+    def _mkDeps(self, dep_m, task_m, task):
+        if task.name not in dep_m.keys():
+            task_m[task.name] = task
+            dep_m[task.name] = set(t.name for t in task.depends)
+            for d in task.depends:
+                self._mkDeps(dep_m, task_m, d)
     
     async def runTask(self, task : Task) -> 'TaskData':
         return await task.do_run()
@@ -82,6 +131,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
 @dc.dataclass
 class LocalRunnerWorker(object):
     runner : TaskGraphRunnerLocal
-    pass
+    task_s : List = dc.field(default_factory=list)
+
 
 

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12919555073a1
+Version: 0.0.1.12931082290a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12919555073a1 → dv_flow_mgr-0.0.1.12931082290a1}/tests/unit/test_pyclass.py

@@ -193,3 +193,52 @@ class foo(Task):
 
     task = builder.mkTaskGraph("pkg1.foo2")
     output = asyncio.run(runner.run(task))
+
+def test_broad_parallel(tmpdir):
+    # Test that we can 
+    flow = """
+package:
+  name: pkg1
+  tasks:
+  - name: foo
+    pyclass: my_module.foo
+    with:
+      param1:
+        type: str
+        value: "1"
+"""
+
+    count = 1000
+
+    for t in range(count):
+        flow += "  - name: foo_%02d\n" % t
+        flow += "    uses: foo\n"
+        flow += "    with:\n"
+        flow += "      param1: \"%d\"\n" % t
+    flow += "  - name: final\n"
+    flow += "    needs: [" + ",".join("foo_%02d" % t for t in range(count)) + "]\n"
+
+    module = """
+from dv_flow.mgr import Task, TaskData
+
+class foo(Task):
+    async def run(self, input : TaskData) -> TaskData:
+        print("foo::run", flush=True)
+        print("params: %s" % str(self.params), flush=True)
+        return input
+"""
+
+    with open(os.path.join(tmpdir, "my_module.py"), "w") as f:
+        f.write(module)
+    with open(os.path.join(tmpdir, "flow.dv"), "w") as f:
+        f.write(flow)
+
+    pkg_def = PackageDef.load(os.path.join(tmpdir, "flow.dv"))
+    builder = TaskGraphBuilder(
+        root_pkg=pkg_def,
+        rundir=os.path.join(tmpdir, "rundir"))
+    runner = TaskGraphRunnerLocal(rundir=os.path.join(tmpdir, "rundir"))
+
+    task = builder.mkTaskGraph("pkg1.final")
+    output = asyncio.run(runner.run(task))
+