dv-flow-mgr 0.0.1.12919555073a1__py3-none-any.whl → 0.0.1.12931082290a1__py3-none-any.whl

dv_flow/mgr/package_def.py

@@ -215,7 +215,7 @@ class PackageDef(BaseModel):
             if len(task.params) > 0:
                 ctor_t.params = task.params
             if len(task.depends) > 0:
-                ctor_t.depends.extends(task.depends)
+                ctor_t.depends.extend(task.depends)
 
         return ctor_t
 

dv_flow/mgr/task.py

@@ -68,12 +68,8 @@ class Task(object):
     session : 'TaskGraphRunner' = None
     basedir : str = None
     memento : TaskMemento = None
-    depend_refs : List['TaskSpec'] = dc.field(default_factory=list)
-    depends : List[int] = dc.field(default_factory=list)
-    running : bool = False
-    output_set : bool = False
+    depends : List['Task'] = dc.field(default_factory=list)
     output : Any = None
-    output_ev : Any = asyncio.Event()
 
     # Implementation data below
     basedir : str = dc.field(default=None)
@@ -107,23 +103,16 @@ class Task(object):
     async def do_run(self) -> TaskData:
         print("do_run: %s - %d depends" % (self.name, len(self.depends)))
         if len(self.depends) > 0:
-            awaitables = [dep.waitOutput() for dep in self.depends]
-            deps_o = await asyncio.gather(*awaitables)
+            deps_o = []
+            for d in self.depends:
+                dep_o = d.getOutput()
+                if dep_o is None:
+                    raise Exception("Null output for %s" % d.name)
+                deps_o.append(dep_o)
 
             # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
-
-#            print("deps_m: %s" % str(deps_m))
-
-            # Merge the output of the dependencies into a single input data
-#            if len(self.depends) > 1:
-#                raise Exception("TODO: handle >1 inputs")
-
-            # Now that we have a clean input object, we need
-            # to build the dep map
-
-#            input = self.depends[0].output.copy()
             input = TaskData.merge(deps_o)
             input.src = self.name
             input.deps[self.name] = list(inp.name for inp in self.depends)
@@ -137,29 +126,17 @@ class Task(object):
 
         self.init_rundir()
 
-        result = await self.run(input)
-
-        if not self.output_set:
-            if result is None:
-                result = TaskData()
+        self.output = await self.run(input)
 
-            # We perform an auto-merge algorithm if the task 
-            # doesn't take control
-#            for dep_o in deps_o:
-#                result.deps.append(dep_o.clone())
-
-            self.setOutput(result)
-        else:
-            # The task has taken control of the output
-            result = self.getOutput()
+        if self.output is None:
+            raise Exception("No output produced by %s" % self.name)
+            result = TaskData()
 
         # Write-back the memento, if specified
         self.save_memento()
 
-        self.running = False
-
         # Combine data from the deps to produce a result
-        return result
+        return self.output
 
     async def run(self, input : TaskData) -> TaskData:
         raise NotImplementedError("TaskImpl.run() not implemented")
@@ -173,24 +150,6 @@ class Task(object):
             with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
                 fp.write(self.memento.model_dump_json(indent=2))
     
-    def setOutput(self, output : TaskData):
-        self.output_set = True
-        output.src = self.name
-        self.output = output
-        self.output_ev.set()
-
-    async def waitOutput(self) -> TaskData:
-        if not self.output_set:
-            if self.running:
-                # Task is already running
-                print("wait")
-                await self.output_ev.wait()
-            else:
-                self.running = True
-                print("start task")
-                await self.do_run()
-        return self.output
-    
     def getOutput(self) -> TaskData:
         return self.output
 

dv_flow/mgr/task_graph_builder.py

@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
 
         depends = []
 
+        for dep in ctor_t.depends:
+            if not dep in self._task_m.keys():
+                task = self._mkTaskGraph(dep, rundir)
+                self._task_m[dep] = task
+                pass
+            depends.append(self._task_m[dep])
+
         # The returned task should have all param references resolved
         print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
         task = ctor_t.task_ctor(
@@ -80,13 +87,6 @@ class TaskGraphBuilder(object):
             rundir=rundir,
             srcdir=ctor_t.srcdir)
         
-        for i,d in enumerate(task.depend_refs):
-            if d in self._task_m.keys():
-                task.depends.append(self._task_m[d])
-            else:
-                print("mkTaskGraph: %s" % d)
-                task.depends.append(self._mkTaskGraph(d, parent_rundir))
-
         self._task_m[task.name] = task
 
         self._pkg_s.pop()

dv_flow/mgr/task_graph_runner_local.py

@@ -23,7 +23,8 @@ import asyncio
 import os
 import yaml
 import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List, Union
+from toposort import toposort
+from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
 from .fragment_def import FragmentDef
 from .package import Package
 from .pkg_rgy import PkgRgy
@@ -37,7 +38,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
     """Session manages execution of a task graph"""
 
     rundir : str
-    nproc : int = -1
+    nproc : int = 4
+    done_task_m : Dict = dc.field(default_factory=dict)
     _workers : List = dc.field(default_factory=list)
 
     _inst : ClassVar['TaskGraphRunner'] = None
@@ -62,15 +64,62 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
             task = [task]
         else:
             unwrap = False
-        
-        run_o = list(t.do_run() for t in task)
 
-        ret = await asyncio.gather(*run_o)
+        dep_m = {}
+        task_m = {}
+
+        for t in task:
+            self._mkDeps(dep_m, task_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l : List[Tuple[Task,Coroutine]]= []
+        # Now, iterate over the concurrent sets
+        for active_s in order:
+
+            # Check to see if all tasks are complete
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                self.done_task_m[tt.name] = tt
+                                active_task_l.pop(i)
+                                break
+                if t not in self.done_task_m.keys():
+                    task = task_m[t]
+                    coro = asyncio.Task(task.do_run())
+                    active_task_l.append((task, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+#        print("order: %s" % str(order))
+#        
+#        run_o = list(t.do_run() for t in task)
+
+#        ret = await asyncio.gather(*run_o)
+        ret = None
 
         if unwrap:
-            return ret[0]
+            return task.output
         else:
-            return ret
+            return list(t.output for t in task)
+        
+    def _mkDeps(self, dep_m, task_m, task):
+        if task.name not in dep_m.keys():
+            task_m[task.name] = task
+            dep_m[task.name] = set(t.name for t in task.depends)
+            for d in task.depends:
+                self._mkDeps(dep_m, task_m, d)
     
     async def runTask(self, task : Task) -> 'TaskData':
         return await task.do_run()
@@ -82,6 +131,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
 @dc.dataclass
 class LocalRunnerWorker(object):
     runner : TaskGraphRunnerLocal
-    pass
+    task_s : List = dc.field(default_factory=list)
+
 
 

{dv_flow_mgr-0.0.1.12919555073a1.dist-info → dv_flow_mgr-0.0.1.12931082290a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12919555073a1
+Version: 0.0.1.12931082290a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12919555073a1.dist-info → dv_flow_mgr-0.0.1.12931082290a1.dist-info}/RECORD

@@ -4,16 +4,16 @@ dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow/mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
 dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
 dv_flow/mgr/package.py,sha256=k6gaDun9mJeGwGsFP5YOYOaFLmKb3KyPZy5wGRgJd_E,1965
-dv_flow/mgr/package_def.py,sha256=hT8wr4ghH61rjK6VEovqTKVUuO_EwrCB5JQuFR-whiU,11455
+dv_flow/mgr/package_def.py,sha256=fdPzhoCqZ-w336ktDVWZ1HKKFk9Zi6iehCcS-iW_EvU,11454
 dv_flow/mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
 dv_flow/mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
 dv_flow/mgr/pkg_rgy.py,sha256=y7HOBZAgKHL4ItnFvjT0lWC_ne81qJi4lJGNLt69au0,2581
-dv_flow/mgr/task.py,sha256=gaMlOiV4pKmEBzewTkVle3RN2zfKPkSdYW3oJX0j_Jg,6421
+dv_flow/mgr/task.py,sha256=4ktSrDJOR7N477u8HWOBfeer1Wwc4mwqMYode1vkApY,5144
 dv_flow/mgr/task_data.py,sha256=-6Dqa3oUI7RJc1Js2SRSnhxNTcASkamXFYMN6UiknZQ,10376
 dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
-dv_flow/mgr/task_graph_builder.py,sha256=wDBRf6g0ILN1lBYgHI4nOGFqSObUQBsJHRR9HX983-g,7131
+dv_flow/mgr/task_graph_builder.py,sha256=1b636njhFJ7gu-re2zBzb1g4GMKQkQmmCqk51niYnU0,7097
 dv_flow/mgr/task_graph_runner.py,sha256=x2e-wpJFhZ4Zkov30PtC8CuKyOK6htA1SKzmrcVBI9M,2154
-dv_flow/mgr/task_graph_runner_local.py,sha256=PeVtJDUgOcolWhB0dir1Twco8Nx3s8WzbsZpgQbgNq0,2585
+dv_flow/mgr/task_graph_runner_local.py,sha256=QDCUyFj_m4hJTUsZD5nCUCnPu3hysdk65-gVA8eJ894,4546
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
 dv_flow/mgr/util.py,sha256=WKwMF4vwYdte5wzieSTDpZTmZU0sjQro3Ofi9moCayE,613
 dv_flow/mgr/cmds/cmd_run.py,sha256=PqAbPMwqovaaq14tnNrCvP7-De8lMI09X0R7d6RIbwY,2691
@@ -23,9 +23,9 @@ dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
 dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
 dv_flow/mgr/std/task_fileset.py,sha256=UzTYONvK0X9rgy3rP9LiX4giBU8SyCCJav0LSNUJ1Qg,3140
 dv_flow/mgr/std/task_null.py,sha256=UEJ3fIoIMYWVsagiQC7GHD23UES7WoH4wtq94b4tcs4,265
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/METADATA,sha256=8UjxIrdl3PJYhr7p9u-WRmRpOkg0IAYf7I_SGMI0ESc,13276
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.12919555073a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/METADATA,sha256=_K8BKKre8AzPLV2GnOy2xkn21sti9eA-liuG63-qP6E,13276
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.12931082290a1.dist-info/RECORD,,