dv-flow-mgr 0.0.1.13478128278a1__py3-none-any.whl → 0.0.1.13489806632a1__py3-none-any.whl

dv_flow/mgr/task_node.py

@@ -0,0 +1,80 @@
+
+import dataclasses as dc
+import logging
+from typing import Any, Callable, ClassVar, Dict, List
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+
+@dc.dataclass
+class TaskNode(object):
+    """Executable view of a task"""
+    # Ctor fields -- must specify on construction
+    name : str
+    srcdir : str
+    # This can be the resolved parameters
+    params : TaskParamsCtor 
+
+    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
+
+    # Runtime fields -- these get populated during execution
+    changed : bool = False
+    needs : List['TaskNode'] = dc.field(default_factory=list)
+    rundir : str = dc.field(default=None)
+    output : TaskDataOutput = dc.field(default=None)
+
+    _log : ClassVar = logging.getLogger("TaskNode")
+
+    def __hash__(self):
+        return id(self)
+    
+@staticmethod
+def task(paramT):
+    def wrapper(T):
+        ctor = TaskNodeCtorWrapper(T.__name__, T, paramT)
+        return ctor
+    return wrapper
+
+@dc.dataclass
+class TaskNodeCtor(object):
+    """
+    Factory for a specific task type
+    - Produces a task parameters object, applying value-setting instructions
+    - Produces a TaskNode
+    """
+    name : str
+
+
+    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
+        raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
+
+    def mkTaskParams(self, params : Dict) -> Any:
+        raise NotImplementedError("mkTaskParams in type %s" % str(type(self)))
+
+@dc.dataclass
+class TaskNodeCtorWrapper(TaskNodeCtor):
+    T : Any
+    paramT : Any
+
+    def __call__(self, 
+                 srcdir, 
+                 name=None, 
+                 params=None, 
+                 needs=None,
+                 **kwargs):
+        """Convenience method for direct creation of tasks"""
+        if params is None:
+            params = self.mkTaskParams(kwargs)
+        
+        node = self.mkTaskNode(srcdir, params, name)
+        if needs is not None:
+            node.needs.extend(needs)
+        return node
+
+    def mkTaskNode(self, srcdir, params, name=None) -> TaskNode:
+        node = TaskNode(name, srcdir, params, self.T)
+        return node
+
+    def mkTaskParams(self, params : Dict) -> Any:
+        obj = self.paramT()
+        # TODO: apply user-specified params
+        return obj

dv_flow/mgr/task_runner.py

@@ -1,6 +1,9 @@
+import asyncio
 import dataclasses as dc
-from .task_data import TaskDataInput, TaskDataResult
-from typing import Any, Callable, List, Tuple
+from toposort import toposort
+from typing import Any, Callable, List, Tuple, Union
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_node import TaskNode
 
 @dc.dataclass
 class TaskRunner(object):
@@ -19,6 +22,98 @@ class TaskRunner(object):
                   memento : Any = None) -> 'TaskDataResult':
         pass
 
+@dc.dataclass
+class TaskSetRunner(TaskRunner):
+    nproc : int = 8
+
+    async def run(self, task : Union[TaskNode,List[TaskNode]]):
+        # First, build a depedency map
+        tasks = task if isinstance(task, list) else [task]
+        dep_m = {}
+        for t in tasks:
+            self._buildDepMap(dep_m, t)
+
+        print("dep_m: %s" % str(dep_m))
+
+        order = list(toposort(dep_m))
+
+        active_task_l = []
+        done_task_s = set()
+        for active_s in order:
+            done = True
+            for t in active_s:
+                while len(active_task_l) >= self.nproc and t not in done_task_s:
+                    # Wait for at least one job to complete
+                    done, pending = await asyncio.wait(at[1] for at in active_task_l)
+                    for d in done:
+                        for i in range(len(active_task_l)):
+                            if active_task_l[i][1] == d:
+                                tt = active_task_l[i][0]
+                                done_task_s.add(tt)
+                                active_task_l.pop(i)
+                                break
+                if t not in done_task_s:
+                    coro = asyncio.Task(self.do_run(t))
+                    active_task_l.append((t, coro))
+               
+            # Now, wait for tasks to complete
+            if len(active_task_l):
+                coros = list(at[1] for at in active_task_l)
+                res = await asyncio.gather(*coros)
+
+
+        pass
+
+    async def do_run(self, 
+                  task : 'TaskNode',
+                  memento : Any = None) -> 'TaskDataResult':
+        changed = False
+        for dep in task.needs:
+            changed |= dep.changed
+
+        # TODO: create an evaluator for substituting param values
+        eval = None
+
+        for field in dc.fields(task.params):
+            print("Field: %s" % field.name)
+
+        input = TaskDataInput(
+            changed=changed,
+            srcdir=task.srcdir,
+            rundir=self.rundir,
+            params=task.params,
+            memento=memento)
+
+        # TODO: notify of task start
+        ret : TaskDataResult = await task.task(self, input)
+        # TODO: notify of task complete
+
+        # Store the result
+        task.output = TaskDataOutput(
+            changed=ret.changed,
+            output=ret.output.copy())
+
+        # # By definition, none of this have run, since we just ran        
+        # for dep in task.dependents:
+        #     is_sat = True
+        #     for need in dep.needs:
+        #         if need.output is None:
+        #             is_sat = False
+        #             break
+            
+        #     if is_sat:
+        #         # TODO: queue task for evaluation
+        #     pass
+        # TODO: 
+
+        return ret
+
+    def _buildDepMap(self, dep_m, task : TaskNode):
+        if task not in dep_m.keys():
+            dep_m[task] = set(task.needs)
+            for need in task.needs:
+                self._buildDepMap(dep_m, need)
+
 @dc.dataclass
 class SingleTaskRunner(TaskRunner):
 
@@ -32,15 +127,36 @@ class SingleTaskRunner(TaskRunner):
         # TODO: create an evaluator for substituting param values
         eval = None
 
-        params = task.params.mk(eval)
+        for field in dc.fields(task.params):
+            print("Field: %s" % field.name)
 
         input = TaskDataInput(
             changed=changed,
             srcdir=task.srcdir,
             rundir=self.rundir,
-            params=params,
+            params=task.params,
             memento=memento)
-        
-        ret = await task.run(self, input)
+
+        # TODO: notify of task start
+        ret : TaskDataResult = await task.task(self, input)
+        # TODO: notify of task complete
+
+        # Store the result
+        task.output = TaskDataOutput(
+            changed=ret.changed,
+            output=ret.output.copy())
+
+        # # By definition, none of this have run, since we just ran        
+        # for dep in task.dependents:
+        #     is_sat = True
+        #     for need in dep.needs:
+        #         if need.output is None:
+        #             is_sat = False
+        #             break
+            
+        #     if is_sat:
+        #         # TODO: queue task for evaluation
+        #     pass
+        # TODO: 
 
         return ret

{dv_flow_mgr-0.0.1.13478128278a1.dist-info → dv_flow_mgr-0.0.1.13489806632a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13478128278a1
+Version: 0.0.1.13489806632a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13478128278a1.dist-info → dv_flow_mgr-0.0.1.13489806632a1.dist-info}/RECORD

@@ -21,9 +21,10 @@ dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1Jip
 dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
+dv_flow/mgr/task_node.py,sha256=wcvxdtrRNwPwV494DerrB79PFYAv6bPdWZDvRGQ0V6Q,2318
 dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
 dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
-dv_flow/mgr/task_runner.py,sha256=XrVYZ13qIbGDkLgDT-NAq_qd2plCqVjHKEe25MD-7xE,1272
+dv_flow/mgr/task_runner.py,sha256=5vCKM2UFxdHB9OslUVIx5Ipc0r04Cizvrc2YSKGMqqI,5055
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
 dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
@@ -33,9 +34,9 @@ dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,27
 dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
 dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
 dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/METADATA,sha256=HCQCME5rxfNs5HNZ0WmoQtIQMTWNtZzkGrfwhXoO09I,13276
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13478128278a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/METADATA,sha256=iGQC00bCbcchWcesU4kXOBNBn-ggHWurn1YqLBKB610,13276
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13489806632a1.dist-info/RECORD,,