dv-flow-mgr 0.0.1.13862597957a1__py3-none-any.whl → 0.0.1.13885203883a1__py3-none-any.whl

dv_flow/mgr/task_def.py

@@ -30,6 +30,11 @@ from .task_output import TaskOutput
 class TaskSpec(object):
     name : str
 
+@dc.dataclass
+class NeedSpec(object):
+    name : str
+    block : bool = False
+
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
     name : str
@@ -39,7 +44,7 @@ class TaskDef(BaseModel):
     pytask : str = dc.Field(default=None)
     desc : str = dc.Field(default="")
     doc : str = dc.Field(default="")
-    needs : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
+    needs : List[Union[str,NeedSpec,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Union[str,list,ParamDef]] = dc.Field(default_factory=dict, alias="with")
     passthrough: bool = dc.Field(default=False)
     consumes : List[Any] = dc.Field(default_factory=list)

dv_flow/mgr/task_node.py

@@ -25,7 +25,7 @@ import dataclasses as dc
 import pydantic.dataclasses as pdc
 import logging
 import toposort
-from typing import Any, Callable, ClassVar, Dict, List
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
 from .task_params_ctor import TaskParamsCtor
 from .param_ref_eval import ParamRefEval
@@ -46,7 +46,7 @@ class TaskNode(object):
     changed : bool = False
     passthrough : bool = False
     consumes : List[Any] = dc.field(default_factory=list)
-    needs : List['TaskNode'] = dc.field(default_factory=list)
+    needs : List[Tuple['TaskNode',bool]] = dc.field(default_factory=list)
     rundir : str = dc.field(default=None)
     output : TaskDataOutput = dc.field(default=None)
     result : TaskDataResult = dc.field(default=None)
@@ -58,6 +58,10 @@ class TaskNode(object):
     def __post_init__(self):
         if self.needs is None:
             self.needs = []
+        else:
+            for i,need in enumerate(self.needs):
+                if not isinstance(need, tuple):
+                    self.needs[i] = (need, False)
 
     async def do_run(self, 
                   runner,
@@ -65,7 +69,7 @@ class TaskNode(object):
                   memento : Any = None) -> 'TaskDataResult':
         self._log.debug("--> do_run: %s" % self.name)
         changed = False
-        for dep in self.needs:
+        for dep,_ in self.needs:
             changed |= dep.changed
 
         self.rundir = rundir
@@ -73,22 +77,30 @@ class TaskNode(object):
         # TODO: Form dep-map from inputs
 
         dep_m = {}
-        for need in self.needs:
+        for need,block in self.needs:
             self._log.debug("dep %s dep_m: %s" % (need.name, str(dep_m)))
-            for subdep in need.output.dep_m.keys():
-                if subdep not in dep_m.keys():
-                    dep_m[subdep] = []
-                dep_m[subdep].extend(need.output.dep_m[subdep])
+            if not block:
+                for subdep in need.output.dep_m.keys():
+                    if subdep not in dep_m.keys():
+                        dep_m[subdep] = []
+                    for dep in need.output.dep_m[subdep]:
+                        if dep not in dep_m[subdep]:
+                            dep_m[subdep].append(dep)
         self._log.debug("input dep_m: %s %s" % (self.name, str(dep_m)))
 
         sorted = toposort.toposort(dep_m)
 
         in_params_m = {}
-        for need in self.needs:
-            for p in need.output.output:
-                if p.src not in in_params_m.keys():
-                    in_params_m[p.src] = []
-                in_params_m[p.src].append(p)
+        added_srcs = set()
+        for need,block in self.needs:
+            if not block:
+                for p in need.output.output:
+                    # Avoid adding parameters from a single task more than once
+                    if p.src not in added_srcs:
+                        added_srcs.add(p.src)
+                        if p.src not in in_params_m.keys():
+                            in_params_m[p.src] = []
+                        in_params_m[p.src].append(p)
 
         # in_params holds parameter sets ordered by dependency
         in_params = []
@@ -126,8 +138,6 @@ class TaskNode(object):
                     if elem.find("${{") != -1:
                         new_val = eval.eval(elem)
                         value[i] = new_val
-            else:
-                raise Exception("Unhandled param type: %s" % str(value))
 
         input = TaskDataInput(
             name=self.name,
@@ -150,43 +160,28 @@ class TaskNode(object):
 
         # Pass-through all dependencies
         # Add an entry for ourselves
-        dep_m[self.name] = list(need.name for need in self.needs)
+        dep_m[self.name] = list(need.name for need,_ in self.needs)
 
         if self.passthrough:
             self._log.debug("passthrough: %s" % self.name)
 
             if self.consumes is None and len(self.consumes):
                 self._log.debug("Propagating all input parameters to output")
-                for need in self.needs:
-                    output.extend(need.output.output)
+                for need,block in self.needs:
+                    if not block:
+                        output.extend(need.output.output)
             else:
                 # Filter out parameter sets that were consumed
                 self._log.debug("Propagating non-consumed input parameters to output")
                 self._log.debug("consumes: %s" % str(self.consumes))
-                for need in self.needs:
-                    for out in need.output.output:
-                        if not self._matches(out, self.consumes):
-                            self._log.debug("Propagating type %s from %s" % (
-                                getattr(out, "type", "<unknown>"),
-                                getattr(out, "src", "<unknown>")))
-                            output.append(out)
-
-                        # consumed = False
-                        # for c in self.consumes:
-                        #     match = False
-                        #     for k,v in c.items():
-                        #         self._log.debug("k,v: %s,%s" % (k,v))
-                        #         if hasattr(out, k):
-                        #             self._log.debug("has attribute: %s" % str(getattr(out ,k)))
-                        #             if getattr(out, k) == v:
-                        #                 self._log.debug("match")
-                        #                 match = True
-                        #                 break
-                        #     if match:
-                        #         consumed = True
-                        #         break
-                        
-                        # if not consumed:
+                for need,block in self.needs:
+                    if not block:
+                        for out in need.output.output:
+                            if not self._matches(out, self.consumes):
+                                self._log.debug("Propagating type %s from %s" % (
+                                    getattr(out, "type", "<unknown>"),
+                                    getattr(out, "src", "<unknown>")))
+                                output.append(out)
         else:
             self._log.debug("non-passthrough: %s (only local outputs propagated)" % self.name)
             # empty dependency map

dv_flow/mgr/task_runner.py

@@ -194,8 +194,8 @@ class TaskSetRunner(TaskRunner):
             self._anon_tid += 1
 
         if task not in dep_m.keys():
-            dep_m[task] = set(task.needs)
-            for need in task.needs:
+            dep_m[task] = set(task[0] for task in task.needs)
+            for need,block in task.needs:
                 self._buildDepMap(dep_m, need)
 
 @dc.dataclass
@@ -205,7 +205,7 @@ class SingleTaskRunner(TaskRunner):
                   task : 'Task',
                   memento : Any = None) -> 'TaskDataResult':
         changed = False
-        for dep in task.needs:
+        for dep,_ in task.needs:
             changed |= dep.changed
 
         # TODO: create an evaluator for substituting param values

{dv_flow_mgr-0.0.1.13862597957a1.dist-info → dv_flow_mgr-0.0.1.13885203883a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13862597957a1
+Version: 0.0.1.13885203883a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13862597957a1.dist-info → dv_flow_mgr-0.0.1.13885203883a1.dist-info}/RECORD

@@ -17,7 +17,7 @@ dv_flow/mgr/pkg_rgy.py,sha256=U5LgBsdCpVJ7tXjlTsn_Qjf9OoEEOjyWsYNsOecS2uQ,5337
 dv_flow/mgr/task.py,sha256=BmS0wd51l4tpTrsit3-ugCB5VGnwpMJNl0dfJfHS4DQ,5981
 dv_flow/mgr/task_ctor.py,sha256=YDYb5uW9Js7MnA8GqbYJljNCbeJFKGveZ2v-IJRgIa8,2221
 dv_flow/mgr/task_data.py,sha256=ysawiUcbQu-ImNSgU5LsfOWlBKdfBt_8juwSacnAqAQ,11437
-dv_flow/mgr/task_def.py,sha256=qdBEEHYVvYC4kBKHxcx-0wfHeQtyse3XnLCvbpAIAyg,1992
+dv_flow/mgr/task_def.py,sha256=BJKZZE8yO_kcKaf36HnzTkhYaC0DPIya4IKlKNM4s5I,2080
 dv_flow/mgr/task_exec_data.py,sha256=-mh9G2-iq6ioB7v7QrHiQAOEwd_hDKhnVf2Mb_IHRtg,1483
 dv_flow/mgr/task_graph_builder.py,sha256=EYFsMZCDDHMEFWZOVSpPb7KBVE6N6XyL4KFzJVrPSPE,10106
 dv_flow/mgr/task_graph_runner.py,sha256=LZ5cTXLQlcK348AxTYKqfpsFFmxA3aseGfholvQW39s,2187
@@ -25,10 +25,10 @@ dv_flow/mgr/task_graph_runner_local.py,sha256=quAxmF1go2iVhBDieksg3brAZukIvpvF-8
 dv_flow/mgr/task_impl_data.py,sha256=jpO7kQ6AjYcOOfpgFIPCEYA1pQbVwFcIY_SY3Q53eGk,1163
 dv_flow/mgr/task_listener_log.py,sha256=b7caUHERGkN6eT8IinBJTdZpwASUs8xAqDmtEymMKzQ,3529
 dv_flow/mgr/task_memento.py,sha256=CV3d1J4ADhMpqvYPfWvUI7duSFCbLEzY41y2GZBHIhM,1101
-dv_flow/mgr/task_node.py,sha256=Xk-q3gYkFer2Brse_xXNC3SfIC_oBMFBm2NBf_wmlN8,15066
+dv_flow/mgr/task_node.py,sha256=xPCarkVSj9rZ1v12WiHtMOe-nMKHbkaUpP9-edQOXek,14828
 dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
 dv_flow/mgr/task_params_ctor.py,sha256=BPkbnoCtzhCxc1g8CJ6VimCcm5UAu92PXeDMhQ4lYsQ,1957
-dv_flow/mgr/task_runner.py,sha256=t0n-9sJ_DyJ3uQDjeou1gpoSLg29zQ3jAYHfOudb7AI,8762
+dv_flow/mgr/task_runner.py,sha256=TB4feCEItxufUyyBSwNNY3OkQ0yjSO0fUfjga0u7rsg,8790
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=NDeyplKrPnWwEv4yHkhXEMK9d9j39b5MOeLB-1Mchqo,1095
 dv_flow/mgr/util.py,sha256=BLp-25Ph9roqo1O3VHI5sKhwxQ0wUU7znFwcSKt0hpA,1442
@@ -40,9 +40,9 @@ dv_flow/mgr/std/fileset.py,sha256=5IxS6T-x0wzA6fdEQcfHZ9kNP8IpH9hfJ3UMhL4A6Iw,39
 dv_flow/mgr/std/flow.dv,sha256=CcQOvSfT0JgsaNf7dH8gfbG8lPvQQXw9W9gqrT52Pw0,2025
 dv_flow/mgr/std/message.py,sha256=_Gox9oBzL9ZYG1JfJ-WYPXojVLqGNaKxZ9tpLVT0LO0,1035
 dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/METADATA,sha256=fg7IS51Mhq7PUfBEkJHau71mbQfQpv8PjdJBIqPLfqA,13314
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13862597957a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/METADATA,sha256=lH34dG2edpLv69XsxqBilTInQ_C4sfWeBkvy_3xM-Cw,13314
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13885203883a1.dist-info/RECORD,,