dv-flow-mgr 1.0.0.14370893826a1__py3-none-any.whl → 1.0.0.14392445998a1__py3-none-any.whl

dv_flow/mgr/package_loader.py

@@ -605,7 +605,7 @@ class PackageLoader(object):
                 rundir = base_t.rundir
 
         if passthrough is None:
-            passthrough = PassthroughE.No
+            passthrough = PassthroughE.Unused
         if consumes is None:
             consumes = ConsumesE.All
 

dv_flow/mgr/task_graph_builder.py

@@ -71,6 +71,7 @@ class TaskGraphBuilder(object):
     _ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
     _compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
     _task_rundir_s : List[List[str]] = dc.field(default_factory=list)
+    _task_node_s : List[TaskNode] = dc.field(default_factory=list)
     _uses_count : int = 0
 
     _log : logging.Logger = None
@@ -203,51 +204,7 @@ class TaskGraphBuilder(object):
 
     def mkTaskGraph(self, task : str, rundir=None) -> TaskNode:
         return self.mkTaskNode(task, rundir=rundir)
-        # self._task_node_m.clear()
-
-        # if rundir is not None:
-        #     self._rundir_s.append(rundir)
-
-        # ret = self._mkTaskGraph(task)
-
-        # if rundir is not None:
-        #     self._rundir_s.pop()
-
-        # return ret
-        
-    def _mkTaskGraph(self, task : str) -> TaskNode:
-        if task in self.root_pkg.task_m.keys():
-            task_t = self.root_pkg.task_m[task]
-        else:
-            pass
-
-        if task_t is None:
-            raise Exception("Failed to find task %s" % task)
-
-        ctor = self._getTaskCtor(task_t)
-
-        params = ctor.mkTaskParams()
-
-        needs = []
-
-        for need in task_t.needs:
-            need_n = self.findTask(need.name)
-            if need_n is None:
-                raise Exception("Failed to find need %s" % need.name)
-            needs.append(need_n)
-
-        task = ctor.mkTaskNode(
-            builder=self,
-            params=params,
-            name=task,
-            needs=needs)
-        task.rundir = self.get_rundir(task.name)
-#        task.rundir = rundir
         
-        self._task_node_m[task.name] = task
-
-        return task
-    
     def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
         self._log.debug("--> mkTaskNode: %s" % task_t)
 
@@ -282,37 +239,59 @@ class TaskGraphBuilder(object):
                     break
         return task
     
-    def _mkTaskNode(self, task : Task, hierarchical=False):
+    def _mkTaskNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False):
 
         if not hierarchical:
             self._task_rundir_s.append([])
 
         # Determine how to build this node
-        if task.subtasks is not None and len(task.subtasks):
-            ret = self._mkTaskCompoundNode(task)
+        if self._isCompound(task):
+            ret = self._mkTaskCompoundNode(
+                task, 
+                name=name,
+                srcdir=srcdir,
+                params=params,
+                hierarchical=hierarchical)
         else:
-            ret = self._mkTaskLeafNode(task)
+            ret = self._mkTaskLeafNode(
+                task, 
+                name=name,
+                srcdir=srcdir,
+                params=params,
+                hierarchical=hierarchical)
 
         if not hierarchical:
             self._task_rundir_s.pop()
 
         return ret        
     
+    def _isCompound(self, task):
+        if task.subtasks is not None and len(task.subtasks):
+            return True
+        elif task.uses is not None:
+            return self._isCompound(task.uses)
+    
     def _getTaskNode(self, name):
         if name in self._task_node_m.keys():
             return self._task_node_m[name]
         else:
             return self.mkTaskNode(name)
     
-    def _mkTaskLeafNode(self, task : Task, name=None) -> TaskNode:
+    def _mkTaskLeafNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
         self._log.debug("--> _mkTaskLeafNode %s" % task.name)
-        srcdir = os.path.dirname(task.srcinfo.file)
+
+        if name is None:
+            name = task.name
+
+        if srcdir is None:
+            srcdir = os.path.dirname(task.srcinfo.file)
+        
+        if params is None:
+            params = task.paramT()
 
         if task.rundir == RundirE.Unique:
             self.enter_rundir(task.name)
 
-        if name is None:
-            name = task.name
 
         callable = None
         if task.run is not None:
@@ -328,19 +307,19 @@ class TaskGraphBuilder(object):
         node = TaskNodeLeaf(
             name=name,
             srcdir=srcdir,
-            params=task.paramT(),
+            params=params,
             passthrough=task.passthrough,
             consumes=task.consumes,
             task=callable(task.run))
         self._task_node_m[name] = node
         node.rundir = self.get_rundir()
 
+        if len(self._task_node_s):
+            node.parent = self._task_node_s[-1]
+
         # Now, link up the needs
         self._log.debug("--> processing needs")
-        for n in task.needs:
-            self._log.debug("-- need %s" % n.name)
-            nn = self._getTaskNode(n.name)
-            node.needs.append((nn, False))
+        self._gatherNeeds(task, node)
         self._log.debug("<-- processing needs")
 
         if task.rundir == RundirE.Unique:
@@ -349,23 +328,46 @@ class TaskGraphBuilder(object):
         self._log.debug("<-- _mkTaskLeafNode %s" % task.name)
         return node
     
-    def _mkTaskCompoundNode(self, task : Task, name=None) -> TaskNode:
+    def _mkTaskCompoundNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
         self._log.debug("--> _mkTaskCompoundNode %s" % task.name)
-        srcdir = os.path.dirname(task.srcinfo.file)
 
         if name is None:
             name = task.name
 
+        if srcdir is None:
+            srcdir = os.path.dirname(task.srcinfo.file)
+
+        if params is None:
+            params = task.paramT()
+
         if task.rundir == RundirE.Unique:
             self.enter_rundir(task.name)
 
-        # Node represents the terminal node of the sub-DAG
-        node = TaskNodeCompound(
-            name=name,
-            srcdir=srcdir,
-            params=task.paramT()
-        )
+        if task.uses is not None:
+            # This is a compound task that is based on
+            # another. Create the base implementation
+            node = self._mkTaskNode(
+                task.uses,
+                name=name, 
+                srcdir=srcdir,
+                params=params,
+                hierarchical=True)
+            
+            if not isinstance(node, TaskNodeCompound):
+                # TODO: need to enclose the leaf node in a compound wrapper
+                raise Exception("Task %s is not compound" % task.uses)
+        else:
+            # Node represents the terminal node of the sub-DAG
+            node = TaskNodeCompound(
+                name=name,
+                srcdir=srcdir,
+                params=params)
+
+        if len(self._task_node_s):
+            node.parent = self._task_node_s[-1]
+
         self._task_node_m[name] = node
+        self._task_node_s.append(node)
 
         node.rundir = self.get_rundir()
 
@@ -374,11 +376,13 @@ class TaskGraphBuilder(object):
         node.input.rundir = self.get_rundir()
         self.leave_rundir()
 
-        self._log.debug("--> processing needs")
+        self._log.debug("--> processing needs (%s)" % task.name)
         for need in task.needs:
-            self._log.debug("-- need: %s" % need.name)
-            nn = self._getTaskNode(need.name)
-            node.input.needs.append((nn, False))
+            need_n = self._getTaskNode(need.name)
+            self._log.debug("Add need %s" % need_n.name)
+            if need_n is None:
+                raise Exception("Failed to find need %s" % need.name)
+            node.input.needs.append((need_n, False))
         self._log.debug("<-- processing needs")
 
         # TODO: handle strategy
@@ -387,11 +391,17 @@ class TaskGraphBuilder(object):
         # For now, build out local tasks and link up the needs
         tasks = []
         for t in task.subtasks:
-            nn = self._mkTaskNode(t, True)
-            tasks.append((t, self._getTaskNode(t.name)))
+            nn = self._mkTaskNode(t, hierarchical=True)
+            node.tasks.append(nn)
+#            tasks.append((t, self._getTaskNode(t.name)))
+            tasks.append((t, nn))
+
+        # Pop the node stack, since we're done constructing the body
+        self._task_node_s.pop()
 
         # Fill in 'needs'
         for t, tn in tasks:
+            self._log.debug("Process node %s" % t.name)
 
             referenced = None
             for tt in task.subtasks:
@@ -400,7 +410,9 @@ class TaskGraphBuilder(object):
                     break
 
             refs_internal = None
-            for nn,_ in tn.needs:
+            # Assess how this task is connected to others in the compound node
+            for nn,_ in tn.first.needs:
+                self._log.debug("Need: %s" % nn.name)
                 for _,tnn in tasks:
                     if nn == tnn:
                         refs_internal = tnn
@@ -414,7 +426,7 @@ class TaskGraphBuilder(object):
                 self._log.debug("Node %s doesn't reference any internal node" % t.name)
                 tn.needs.append((node.input, False))
             else:
-                self._log.debug("Node references internal node %s" % refs_internal.name)
+                self._log.debug("Node %s references internal node %s" % (t.name, refs_internal.name))
 
             if referenced is not None:
                 # Add this task as a dependency of the output
@@ -429,35 +441,18 @@ class TaskGraphBuilder(object):
 
         return node
 
+    def _gatherNeeds(self, task_t, node):
+        self._log.debug("--> _gatherNeeds %s" % task_t.name)
+        if task_t.uses is not None:
+            self._gatherNeeds(task_t.uses, node)
+
+        for need in task_t.needs:
+            need_n = self._getTaskNode(need.name)
+            if need_n is None:
+                raise Exception("Failed to find need %s" % need.name)
+            node.needs.append((need_n, False))
+        self._log.debug("<-- _gatherNeeds %s" % task_t.name)
         
-    def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskNodeCtor':
-        from .task_def import TaskSpec
-        if type(spec) == str:
-            spec = TaskSpec(spec)
-
-        self._log.debug("--> getTaskCtor %s" % spec.name)
-        spec_e = spec.name.split(".")
-        task_name = spec_e[-1]
-
-        # if len(spec_e) == 1:
-        #     # Just have a task name. Use the current package
-        #     if len(self._pkg_s) == 0:
-        #         raise Exception("No package context for task %s" % spec.name)
-        #     pkg = self._pkg_s[-1]
-        # else:
-        #     pkg_name = ".".join(spec_e[0:-1])
-
-        #     try:
-        #         pkg = self.getPackage(PackageSpec(pkg_name))
-        #     except Exception as e:
-        #         self._log.critical("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
-        #         raise e
-
-        ctor = pkg.getTaskCtor(task_name)
-
-        self._log.debug("<-- getTaskCtor %s" % spec.name)
-        return ctor
-    
     def error(self, msg, loc=None):
         if loc is not None:
             marker = TaskMarker(msg=msg, severity=SeverityE.Error, loc=loc)
@@ -468,217 +463,3 @@ class TaskGraphBuilder(object):
     def marker(self, marker):
         self.marker_l(marker)
 
-    def _getTaskCtor(self, task : Task) -> TaskNodeCtor:
-        if task in self._task_ctor_m.keys():
-            ctor = self._task_ctor_m[task]
-        else:
-            ctor = self._mkTaskCtor(task)
-            self._task_ctor_m[task] = ctor
-        return ctor
-
-    def _mkTaskCtor(self, task):
-        srcdir = os.path.dirname(task.srcinfo.file)
-        self._log.debug("--> mkTaskCtor %s (srcdir: %s)" % (task.name, srcdir))
-
-        if len(task.subtasks) > 0:
-            self._log.debug("Task has a body")
-            # Compound task
-            self._log.debug("Task specifies sub-task implementation")
-            ctor = self._mkCompoundTaskCtor(task)
-        else:
-            self._log.debug("Task doesn't specify a body")
-            # Shell task or 'null'
-            ctor = self._mkLeafTaskCtor(task)
-
-        if ctor is None:
-            raise Exception()
-
-        return ctor
-
-    def _mkLeafTaskCtor(self, task) -> TaskNodeCtor:
-        self._log.debug("--> _mkLeafTaskCtor")
-        srcdir = os.path.dirname(task.srcinfo.file)
-        base_ctor_t : TaskNodeCtor = None
-        ctor_t : TaskNodeCtor = None
-        base_params = None
-        callable = None
-#        fullname = self.name + "." + task.name
-#        rundir = task.rundir
-
-        # TODO: should we have the ctor look this up itself?
-        # Want to confirm that the value can be found.
-        # Defer final resolution until actual graph building (post-config)
-        if task.uses is not None:
-            self._log.debug("Uses: %s" % task.uses.name)
-
-            base_ctor_t = self._getTaskCtor(task.uses)
-
-            if base_ctor_t is None:
-                self._log.error("Failed to load task ctor %s" % task.uses)
-#            base_params = base_ctor_t.mkTaskParams()
-        else:
-            self._log.debug("No 'uses' specified %s" % task.name)
-
-        self._log.debug("%d needs" % len(task.needs))
-
-        # Determine the implementation constructor first
-        if task.run is not None:
-            shell = task.shell if task.shell is not None else "shell"
-
-            if shell in self._shell_m.keys():
-                self._log.debug("Use shell implementation")
-                callable = self._shell_m[shell]
-            else:
-                self._log.debug("Shell %s not found" % shell)
-                raise Exception("Shell %s not found" % shell)
-
-            # if taskdef.body.pytask is not None:
-            #     # Built-in impl
-            #     # Now, lookup the class
-            #     self._log.debug("Use PyTask implementation")
-            #     last_dot = taskdef.body.pytask.rfind('.')
-            #     clsname = taskdef.body.pytask[last_dot+1:]
-            #     modname = taskdef.body.pytask[:last_dot]
-
-            #     try:
-            #         if modname not in sys.modules:
-            #             if srcdir not in sys.path:
-            #                 sys.path.append(srcdir)
-            #             mod = importlib.import_module(modname)
-            #         else:
-            #             mod = sys.modules[modname]
-            #     except ModuleNotFoundError as e:
-            #         raise Exception("Failed to import module %s (_basedir=%s): %s" % (
-            #             modname, self._basedir, str(e)))
-                
-            #     if not hasattr(mod, clsname):
-            #         raise Exception("Method %s not found in module %s" % (clsname, modname))
-            #     callable = getattr(mod, clsname)
-            # elif taskdef.body.run is not None:
-            #     callable = self._getRunCallable(taskdef)
-        else:
-            # TODO: use null task
-            pass
-
-        # Determine if we need to use a new 
-        if task.paramT is None:
-            raise Exception()
-        paramT = task.paramT
-        needs = []
-
-        # TODO:
-        rundir : RundirE = task.rundir
-        
-        if callable is not None:
-            ctor_t = TaskNodeCtorTask(
-                name=task.name,
-                srcdir=srcdir,
-                paramT=task.paramT, # TODO: need to determine the parameter type
-                passthrough=task.passthrough,
-                consumes=task.consumes,
-                needs=needs, # TODO: need to determine the needs
-                rundir=rundir,
-                task=callable)
-        elif base_ctor_t is not None:
-            # Use the existing (base) to create the implementation
-            ctor_t = TaskNodeCtorProxy(
-                name=task.name,
-                srcdir=srcdir,
-                paramT=task.paramT, # TODO: need to determine the parameter type
-                passthrough=task.passthrough,
-                consumes=task.consumes,
-                needs=needs,
-                rundir=rundir,
-                uses=base_ctor_t)
-        else:
-            self._log.debug("Use 'Null' as the class implementation")
-            ctor_t = TaskNodeCtorTask(
-                name=task.name,
-                srcdir=srcdir,
-                paramT=paramT,
-                passthrough=task.passthrough,
-                consumes=task.consumes,
-                needs=needs,
-                rundir=rundir,
-                task=TaskNull)
-
-        self._log.debug("<-- mkTaskCtor %s" % task.name)
-        return ctor_t
-    
-    def _getRunCallable(self, task):
-        self._log.debug("--> _getRunCallable %s" % task.name)
-        callable = None
-        if task.run is not None and task.shell == "python":
-            # Evaluate a Python script
-            pass
-        else:
-            # run a shell script
-            shell = None
-            body = task.run.strip()
-
-            callable = ShellCallable(body=body, shell=shell)
-            pass
-        return callable
-
-    def _mkCompoundTaskCtor(self, task) -> TaskNodeCtor:
-        self._log.debug("--> _mkCompoundTaskCtor %s" % task.name)
-        srcdir = os.path.dirname(task.srcinfo.file)
-        base_ctor_t : TaskNodeCtor = None
-        ctor_t : TaskNodeCtor = None
-        base_params = None
-        callable = None
-
-#        fullname = self._getScopeFullname()
-        fullname = task.name
-
-        if task.uses is not None:
-            self._log.debug("Uses: %s" % task.uses)
-            base_ctor_t = task.uses.ctor
-            base_params = base_ctor_t.mkTaskParams()
-
-            if base_ctor_t is None:
-                self._log.error("Failed to load task ctor %s" % task.uses)
-
-        # TODO: should build during loading
-#        passthrough, consumes, needs = self._getPTConsumesNeeds(taskdef, base_ctor_t)
-        passthrough = []
-        consumes = []
-        needs = []
-
-        # Determine if we need to use a new 
-#        paramT = self._getParamT(taskdef, base_params)
-        paramT = task.paramT
-
-        if base_ctor_t is not None:
-            ctor_t = TaskNodeCtorCompoundProxy(
-                name=fullname,
-                srcdir=srcdir,
-                paramT=paramT,
-                passthrough=passthrough,
-                consumes=consumes,
-                needs=needs,
-                task=task,
-                uses=base_ctor_t)
-        else:
-            self._log.debug("No 'uses' specified")
-            ctor_t = TaskNodeCtorCompound(
-                name=fullname,
-                srcdir=srcdir,
-                paramT=paramT,
-                passthrough=passthrough,
-                consumes=consumes,
-                needs=needs,
-                task=task)
-            
-        for st in task.subtasks:
-            ctor = self._getTaskCtor(st)
-            if ctor is None:
-                raise Exception("ctor for %s is None" % st.name)
-            ctor_t.tasks.append(st)
-
-#        for t in task.subtasks:
-#            ctor_t.tasks.append(self._mkTaskCtor(t, srcdir))
-
-        
-        self._log.debug("<-- mkCompoundTaskCtor %s (%d)" % (task.name, len(ctor_t.tasks)))
-        return ctor_t

dv_flow/mgr/task_graph_dot_writer.py

@@ -1,7 +1,7 @@
 import dataclasses as dc
 import logging
 import sys
-from typing import ClassVar, Dict, TextIO
+from typing import ClassVar, Dict, Set, TextIO
 from .task_node import TaskNode
 from .task_node_compound import TaskNodeCompound
 
@@ -10,6 +10,7 @@ class TaskGraphDotWriter(object):
     fp : TextIO = dc.field(default=None)
     _ind : str = ""
     _node_id_m : Dict[TaskNode, str] = dc.field(default_factory=dict)
+    _processed_needs : Set[TaskNode] = dc.field(default_factory=set)
     _node_id : int = 1
     _cluster_id : int = 1
     _log : ClassVar = logging.getLogger("TaskGraphDotWriter")
@@ -22,48 +23,107 @@ class TaskGraphDotWriter(object):
         else:
             self.fp = open(filename, "w")
         self.println("digraph G {")
-        self.process_node(node)
+        # First, build-out all nodes
+        self.build_node(node)
+        self.process_needs(node)
         self.println("}")
 
         self.fp.close()
         self._log.debug("<-- TaskGraphDotWriter::write")
 
-    def process_node(self, node):
-        self._log.debug("--> process_node %s (%d)" % (node.name, len(node.needs),))
-        node_id = self._node_id
-        self._node_id += 1
-        node_name = "n%d" % self._node_id
-        self._node_id_m[node] = node_name
+    def build_node(self, node):
+        self._log.debug("--> build_node %s (%d)" % (node.name, len(node.needs),))
 
         if isinstance(node, TaskNodeCompound):
-            self.println("subgraph cluster_%d {" % self._cluster_id)
-            self._cluster_id += 1
-            self.inc_ind()
-            self.println("label=\"%s\";" % node.name)
-            self.println("color=blue;")
-            self.println("style=dashed;")
-            self.process_node(node.input)
-
-            self.println("%s[label=\"%s.out\"];" % (
-                node_name,
-                node.name))
+            self._log.debug("-- compound node")
+            # Find the root and build out any expanded sub-nodes
+            root = node
+            while root.parent is not None:
+                root = root.parent
+            self.build_compound_node(root)
         else:
-            self.println("%s[label=\"%s\"];" % (
-                node_name,
-                node.name))
+            # Leaf node
+            self._log.debug("-- leaf node")
+            node_id = self._node_id
+            self._node_id += 1
+            node_name = "n%d" % node_id
+            self._node_id_m[node] = node_name
+            self.println("%s[label=\"%s\"];" % (node_name, node.name))
+        self._log.debug("<-- build_node %s (%d)" % (node.name, len(node.needs),))
+
+    def process_needs(self, node):
+        self._log.debug("--> process_needs %s (%d)" % (node.name, len(node.needs),))
+
+        # if isinstance(node, TaskNodeCompound):
+        #     self.println("subgraph cluster_%d {" % self._cluster_id)
+        #     self._cluster_id += 1
+        #     self.inc_ind()
+        #     self.println("label=\"%s\";" % node.name)
+        #     self.println("color=blue;")
+        #     self.println("style=dashed;")
+        #     self.process_node(node.input)
 
-        for dep in node.needs:
-            if dep[0] not in self._node_id_m.keys():
-                self.process_node(dep[0])
+        #     self.println("%s[label=\"%s.out\"];" % (
+        #         node_name,
+        #         node.name))
+        # else:
+        #     self.println("%s[label=\"%s\"];" % (
+        #         node_name,
+        #         node.name))
+
+        for dep,_ in node.needs:
+            if dep not in self._node_id_m.keys():
+                self.build_node(dep)
+            if dep not in self._node_id_m.keys():
+                self._log.error("Dep-node not built: %s" % dep.name)
+            if node not in self._node_id_m.keys():
+                self.build_node(node)
+            if node not in self._node_id_m.keys():
+                self._log.error("Dep-node not built: %s" % node.name)
             self.println("%s -> %s;" % (
-                self._node_id_m[dep[0]],
+                self._node_id_m[dep],
                 self._node_id_m[node]))
+            if dep not in self._processed_needs:
+                self._processed_needs.add(dep)
+                self.process_needs(dep)
             
-        if isinstance(node, TaskNodeCompound):
-            self.dec_ind()
-            self.println("}")
+        self._log.debug("<-- process_needs %s (%d)" % (node.name, len(node.needs),))
+
+    def build_compound_node(self, node):
+        """Hierarchical build of a compound root node"""
+
+        self._log.debug("--> build_compound_node %s (%d)" % (node.name, len(node.tasks),))
+
+        id = self._cluster_id
+        self._cluster_id += 1
+        self.println("subgraph cluster_%d {" % id)
+        self.inc_ind()
+        self.println("label=\"%s\";" % node.name)
+        self.println("color=blue;")
+        self.println("style=dashed;")
+
+        task_node_id = self._node_id
+        self._node_id += 1
+        task_node_name = "n%d" % task_node_id
+        self.println("%s[label=\"%s\"];" % (task_node_name, node.name))
+        self._node_id_m[node] = task_node_name
+
+        for n in node.tasks:
+            if isinstance(n, TaskNodeCompound):
+                # Recurse
+                self.build_compound_node(n)
+            else:
+                # Leaf node
+                node_id = self._node_id
+                self._node_id += 1
+                node_name = "n%d" % node_id
+                self._node_id_m[n] = node_name
+                leaf_name = n.name[n.name.rfind(".") + 1:]
+                self.println("%s[label=\"%s\"];" % (node_name, leaf_name))
+        self.dec_ind()
+        self.println("}")
 
-        self._log.debug("<-- process_node %s (%d)" % (node.name, len(node.needs),))
+        self._log.debug("<-- build_compound_node %s (%d)" % (node.name, len(node.tasks),))
 
     def println(self, l):
         self.fp.write("%s%s\n" % (self._ind, l))

dv_flow/mgr/task_node.py

@@ -56,6 +56,7 @@ class TaskNode(object):
     start : float = dc.field(default=None)
     end : float = dc.field(default=None)
     save_exec_data : bool = dc.field(default=True)
+    parent : 'TaskNode' = dc.field(default=None)
 
     _log : ClassVar = logging.getLogger("TaskNode")
 
@@ -69,6 +70,10 @@ class TaskNode(object):
                 if not isinstance(need, tuple):
                     self.needs[i] = (need, False)
 
+    @property
+    def first(self):
+        return self
+
     async def do_run(self, 
                   runner,
                   rundir,

dv_flow/mgr/task_node_compound.py

@@ -46,8 +46,14 @@ class TaskNodeCompound(TaskNode):
             srcdir=self.srcdir,
             params=NullParams())
         self.input.task = null_run
+        self.tasks.append(self.input)
+
         return super().__post_init__()
 
+    @property
+    def first(self):
+        return self.input
+    
     async def do_run(self, 
                      runner : TaskRunner, 
                      rundir, 

dv_flow/mgr/task_node_leaf.py

@@ -98,7 +98,7 @@ class TaskNodeLeaf(TaskNode):
         else:
             self._log.debug("consumes(unknown): %s" % str(self.consumes))
 
-        for name,field in self.params.model_fields.items():
+        for name,field in type(self.params).model_fields.items():
             value = getattr(self.params, name)
             if type(value) == str:
                 if value.find("${{") != -1:
@@ -170,7 +170,9 @@ class TaskNodeLeaf(TaskNode):
                 for need,block in self.needs:
                     if not block:
                         output.extend(need.output.output)
-            else:
+            elif self.consumes == ConsumesE.All:
+                self._log.debug("All inputs are consumed, so not propagating any")
+            elif isinstance(self.consumes, list):
                 # Filter out parameter sets that were consumed
                 self._log.debug("Propagating non-consumed input parameters to output")
                 self._log.debug("consumes: %s" % str(self.consumes))

{dv_flow_mgr-1.0.0.14370893826a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 1.0.0.14370893826a1
+Version: 1.0.0.14392445998a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-1.0.0.14370893826a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/RECORD

@@ -18,7 +18,7 @@ dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=PvOuNfuy-SlaiMDzT5SU48OlPJrhS9vpNCgX06KNyTo,2336
 dv_flow/mgr/package_def.py,sha256=-UyeFb_0Sj16RtS2vxtIxTfl-oW7adJjM1I3ZSjcRpc,5729
 dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
-dv_flow/mgr/package_loader.py,sha256=LsPAcCKfAiEapC0I3m3Jv6Ra6YsQCXAqJe3_ETE6m_U,25071
+dv_flow/mgr/package_loader.py,sha256=Z80md4ZhJk6TPVlvYAWlnH1aJvATlB9skIyb1pZ1P8U,25075
 dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
 dv_flow/mgr/param_def.py,sha256=hOBBRLiXJ5DakXkhrLCBAQ9GPlgq-QS52r0aflmIgbg,1832
 dv_flow/mgr/param_ref_eval.py,sha256=5yH37oIX6f2qmk7GfRgNT5qZx0jm3CJFgB9lLDZZ1yQ,1981
@@ -31,11 +31,11 @@ dv_flow/mgr/srcinfo.py,sha256=xrfp-relVr7hYNbxOjY5jqX4H0nNMm9HK5AAM6_Piyk,366
 dv_flow/mgr/task.py,sha256=Xe1sCCTN4yJwhB2Aw_rywdlmVPY2CDnuwZisULuOXhw,2036
 dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
 dv_flow/mgr/task_def.py,sha256=VAjkFay7KxXfZoloZfbiPr6WhURztV2Dxm6-REMQweo,4912
-dv_flow/mgr/task_graph_builder.py,sha256=SMGuYXWGwTXchd_ObKFNzbpXmlOTAWr0_YWN4XLbMWI,24178
-dv_flow/mgr/task_graph_dot_writer.py,sha256=GxqiYwQJbFgUQdnPCS9vpIYmYFbSXwnXCSbGmjbxC3M,2418
+dv_flow/mgr/task_graph_builder.py,sha256=h0jiGY4VQI-wLJevawHl40wTILiMulA2eZpi0jFJ82A,16739
+dv_flow/mgr/task_graph_dot_writer.py,sha256=fZlNR-ZUFsI5FpH1KFCWJcSpyLSKUh8IhHO9Eavc60c,4925
 dv_flow/mgr/task_listener_log.py,sha256=QH7hAVO1agYJoOx16XqjWv1c-jk5zb6p6A-hic7OMU0,4150
-dv_flow/mgr/task_node.py,sha256=JXq2QimCZKxfhhkdhM-HKk4JbxpIlpW65xUJ0hVlQc0,4981
-dv_flow/mgr/task_node_compound.py,sha256=mNu4nf9hVqu2698ue5fpE3FeAOkvJH0Ke2W9V0G3-As,2975
+dv_flow/mgr/task_node.py,sha256=hPlCcped3nCNgOYj4oysVcxu4cHOIUCBUqAevOr20Lc,5086
+dv_flow/mgr/task_node_compound.py,sha256=kmhfcHeqNwYE-XIRh8dsdU_zvLpAzHwVTdr4S3eAtN8,3080
 dv_flow/mgr/task_node_ctor.py,sha256=YsoVMX5WbpbzcHvEK7ps_ZRV-J7MZ3F8NNozQw7vbog,4418
 dv_flow/mgr/task_node_ctor_compound.py,sha256=290JdcTnL3b3Gv7s_wRLjdM02ezKhc9QnxZE0mv72i8,4379
 dv_flow/mgr/task_node_ctor_compound_proxy.py,sha256=D8x54nD8Pd-2-_mr1syhqVeSFfIVf100ldi3bdzmSfI,2073
@@ -43,7 +43,7 @@ dv_flow/mgr/task_node_ctor_def_base.py,sha256=_8QQHKDkONio_ve0Z409yxC0AMO8ocNBPD
 dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v2PA6VA,1906
 dv_flow/mgr/task_node_ctor_task.py,sha256=d49g90TyPCMFR8BuWWqp4ym-MW5vGSdDR0V47Ru28JY,2232
 dv_flow/mgr/task_node_ctor_wrapper.py,sha256=Nb1CVcPHZofnb-iLWDHQWAxlTOdbRrnR9DdSxY8yOec,3626
-dv_flow/mgr/task_node_leaf.py,sha256=wp0wCstQ8eyEJwJcbCqdpei6FvLE0C3-EEBPMjGv3-Y,8488
+dv_flow/mgr/task_node_leaf.py,sha256=QZru3SisLZwykpfATJPo8qQJbRwArw0GBvRlLtLB8bs,8658
 dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
 dv_flow/mgr/task_params_ctor.py,sha256=qlrzibGAFHmbqOu88jEoh1wOSFHu68Gwdgc259-50e8,1915
 dv_flow/mgr/task_run_ctxt.py,sha256=w1EOG9R16j0eT8fNcOCqlfyFrw277onsSur607eNprY,2811
@@ -66,9 +66,9 @@ dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,
 dv_flow/mgr/util/util.py,sha256=cBNt3JJ0PGLlUQFTtBLi12i2j_9gNgSBtKdwS3VfF5Y,1566
 dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/util/cmds/cmd_schema.py,sha256=IJzZdxCSEgIQ79LpYiM7UqJ9RJ-7yraqmBN2XVgAgXA,1752
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/METADATA,sha256=gLrvvQKKh0wI5KalUtQj2i8IRWSVpW5mmCob-EqwoxY,13336
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-1.0.0.14370893826a1.dist-info/RECORD,,
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/METADATA,sha256=B60agWgNKhS9gxExLemROgmyWl_S8SpiF2OqDjCyAKg,13336
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-1.0.0.14392445998a1.dist-info/RECORD,,