dv-flow-mgr 0.0.1.12911707440a1__py3-none-any.whl → 0.0.1.12922457458a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -3,4 +3,7 @@ from .package_def import *
 from .task_graph_runner import *
 from .task import *
 from .task_data import *
+from .task_graph_runner import TaskGraphRunner
+from .task_graph_runner_local import TaskGraphRunnerLocal
+from .task_graph_builder import TaskGraphBuilder
 

dv_flow/mgr/package_def.py

@@ -215,7 +215,7 @@ class PackageDef(BaseModel):
             if len(task.params) > 0:
                 ctor_t.params = task.params
             if len(task.depends) > 0:
-                ctor_t.depends.extends(task.depends)
+                ctor_t.depends.extend(task.depends)
 
         return ctor_t
 

dv_flow/mgr/task_graph_builder.py

@@ -70,6 +70,13 @@ class TaskGraphBuilder(object):
 
         depends = []
 
+        for dep in ctor_t.depends:
+            if not dep in self._task_m.keys():
+                task = self._mkTaskGraph(dep, rundir)
+                self._task_m[dep] = task
+                pass
+            depends.append(self._task_m[dep])
+
         # The returned task should have all param references resolved
         print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
         task = ctor_t.task_ctor(
@@ -99,7 +106,6 @@ class TaskGraphBuilder(object):
         print("getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
         if len(self._pkg_spec_s) > 0:
             pkg_spec = self._pkg_spec_s[-1]
-            print("pkg_spec: %s ; root_pkg: %s" % (pkg_spec.name, self.root_pkg.name))
             if self.root_pkg.name == pkg_spec.name:
                 pkg_def = self.root_pkg
             else:

dv_flow/mgr/task_graph_runner_local.py

@@ -1,5 +1,5 @@
 #****************************************************************************
-#* session.py
+#* task_graph_runner_local.py
 #*
 #* Copyright 2023 Matthew Ballance and Contributors
 #*
@@ -23,7 +23,8 @@ import asyncio
 import os
 import yaml
 import dataclasses as dc
-from typing import Any, Callable, ClassVar, Dict, List, Union
+from toposort import toposort
+from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
 from .fragment_def import FragmentDef
 from .package import Package
 from .pkg_rgy import PkgRgy
@@ -37,7 +38,8 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
     """Session manages execution of a task graph"""
 
     rundir : str
-    nproc : int = -1
+    nproc : int = 4
+    done_task_m : Dict = dc.field(default_factory=dict)
     _workers : List = dc.field(default_factory=list)
 
     _inst : ClassVar['TaskGraphRunner'] = None
@@ -58,11 +60,66 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
 
     async def run(self, task : Union[Task,List[Task]]) -> List['TaskData']:
         if isinstance(task, Task):
+            unwrap = True
             task = [task]
-        
-        run_o = list(t.do_run() for t in task)
+        else:
+            unwrap = False
+
+        dep_m = {}
+        task_m = {}
+
+        for t in task:
+            self._mkDeps(dep_m, task_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l : List[Tuple[Task,Coroutine]]= []
+        # Now, iterate over the concurrent sets
+        for active_s in order:
 
-        return await asyncio.gather(*run_o)
+            # Check to see if all tasks are complete
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in self.done_task_m.keys():
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                self.done_task_m[tt.name] = tt
+                                active_task_l.pop(i)
+                                break
+                if t not in self.done_task_m.keys():
+                    task = task_m[t]
+                    coro = asyncio.Task(task.do_run())
+                    active_task_l.append((task, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+#        print("order: %s" % str(order))
+#        
+#        run_o = list(t.do_run() for t in task)
+
+#        ret = await asyncio.gather(*run_o)
+        ret = None
+
+        if unwrap:
+            return task.output
+        else:
+            return list(t.output for t in task)
+        
+    def _mkDeps(self, dep_m, task_m, task):
+        if task.name not in dep_m.keys():
+            task_m[task.name] = task
+            dep_m[task.name] = set(t.name for t in task.depends)
+            for d in task.depends:
+                self._mkDeps(dep_m, task_m, d)
     
     async def runTask(self, task : Task) -> 'TaskData':
         return await task.do_run()
@@ -74,6 +131,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
 @dc.dataclass
 class LocalRunnerWorker(object):
     runner : TaskGraphRunnerLocal
-    pass
+    task_s : List = dc.field(default_factory=list)
+
 
 

{dv_flow_mgr-0.0.1.12911707440a1.dist-info → dv_flow_mgr-0.0.1.12922457458a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12911707440a1
+Version: 0.0.1.12922457458a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License
@@ -206,6 +206,7 @@ License:                                  Apache License
            limitations under the License.
         
 License-File: LICENSE
+Requires-Dist: pydantic
 Requires-Dist: pyyaml
 Requires-Dist: toposort
 Requires-Dist: svdep

{dv_flow_mgr-0.0.1.12911707440a1.dist-info → dv_flow_mgr-0.0.1.12922457458a1.dist-info}/RECORD

@@ -1,19 +1,19 @@
-dv_flow/mgr/__init__.py,sha256=Ms7IJnAbW08GwaanCI64ad_QWZd_XPGfua0WhYr57Jg,107
+dv_flow/mgr/__init__.py,sha256=VtQfTiskn-fBaWU18s8odFPMDJ1Lx2DD4Y4Uy5EJ0Qs,261
 dv_flow/mgr/__main__.py,sha256=pHXC_c-XSPUYQsBDxk5y7LtQ5kA5pKGhhzu4ko5jh7k,513
 dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow/mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
 dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
 dv_flow/mgr/package.py,sha256=k6gaDun9mJeGwGsFP5YOYOaFLmKb3KyPZy5wGRgJd_E,1965
-dv_flow/mgr/package_def.py,sha256=hT8wr4ghH61rjK6VEovqTKVUuO_EwrCB5JQuFR-whiU,11455
+dv_flow/mgr/package_def.py,sha256=fdPzhoCqZ-w336ktDVWZ1HKKFk9Zi6iehCcS-iW_EvU,11454
 dv_flow/mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
 dv_flow/mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
 dv_flow/mgr/pkg_rgy.py,sha256=y7HOBZAgKHL4ItnFvjT0lWC_ne81qJi4lJGNLt69au0,2581
 dv_flow/mgr/task.py,sha256=gaMlOiV4pKmEBzewTkVle3RN2zfKPkSdYW3oJX0j_Jg,6421
 dv_flow/mgr/task_data.py,sha256=-6Dqa3oUI7RJc1Js2SRSnhxNTcASkamXFYMN6UiknZQ,10376
 dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
-dv_flow/mgr/task_graph_builder.py,sha256=qx-59f14-oJJi2qx_zYTaP3PvvxxQCNyYqQ6jDjx_OM,7218
+dv_flow/mgr/task_graph_builder.py,sha256=8G9Q-eC0OZmHw0V3F1CxCkXzDZWrYnfVFJ9hQET9RAY,7376
 dv_flow/mgr/task_graph_runner.py,sha256=x2e-wpJFhZ4Zkov30PtC8CuKyOK6htA1SKzmrcVBI9M,2154
-dv_flow/mgr/task_graph_runner_local.py,sha256=-69HoOGkEPHL897RJGB7gItjqXDL_8eam267af4G77E,2420
+dv_flow/mgr/task_graph_runner_local.py,sha256=QDCUyFj_m4hJTUsZD5nCUCnPu3hysdk65-gVA8eJ894,4546
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
 dv_flow/mgr/util.py,sha256=WKwMF4vwYdte5wzieSTDpZTmZU0sjQro3Ofi9moCayE,613
 dv_flow/mgr/cmds/cmd_run.py,sha256=PqAbPMwqovaaq14tnNrCvP7-De8lMI09X0R7d6RIbwY,2691
@@ -23,9 +23,9 @@ dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
 dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
 dv_flow/mgr/std/task_fileset.py,sha256=UzTYONvK0X9rgy3rP9LiX4giBU8SyCCJav0LSNUJ1Qg,3140
 dv_flow/mgr/std/task_null.py,sha256=UEJ3fIoIMYWVsagiQC7GHD23UES7WoH4wtq94b4tcs4,265
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/METADATA,sha256=tjOZgtwJrZwNzN011TtAKggCWYSik3b60ZXkBNB3MbI,13252
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.12911707440a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/METADATA,sha256=Rsa9WygYLLdeeSpss1f5HJ4ZGwGjrlobtza7ZjNnXIs,13276
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.12922457458a1.dist-info/RECORD,,