dv-flow-mgr 0.0.2.14180132375a1__py3-none-any.whl → 1.0.0.14370600369a1__py3-none-any.whl

dv_flow/mgr/task_graph_builder.py

@@ -22,14 +22,26 @@
 import os
 import dataclasses as dc
 import logging
-from typing import Callable
+from typing import Callable, Any, Dict, List, Union
 from .package import Package
 from .package_def import PackageDef, PackageSpec
-from .pkg_rgy import PkgRgy
+from .ext_rgy import ExtRgy
+from .task import Task
 from .task_def import RundirE
+from .task_data import TaskMarker, TaskMarkerLoc, SeverityE
 from .task_node import TaskNode
 from .task_node_ctor import TaskNodeCtor
-from typing import Dict, List, Union
+from .task_node_ctor_compound import TaskNodeCtorCompound
+from .task_node_ctor_compound_proxy import TaskNodeCtorCompoundProxy
+from .task_node_ctor_proxy import TaskNodeCtorProxy
+from .task_node_ctor_task import TaskNodeCtorTask
+from .task_node_ctor_wrapper import TaskNodeCtorWrapper
+from .task_node_compound import TaskNodeCompound
+from .task_node_leaf import TaskNodeLeaf
+from .std.task_null import TaskNull
+from .exec_callable import ExecCallable
+from .null_callable import NullCallable
+from .shell_callable import ShellCallable
 
 @dc.dataclass
 class TaskNamespaceScope(object):
@@ -46,85 +58,68 @@ class CompoundTaskCtxt(object):
 @dc.dataclass
 class TaskGraphBuilder(object):
     """The Task-Graph Builder knows how to discover packages and construct task graphs"""
-    root_pkg : PackageDef
+    root_pkg : Package
     rundir : str
-    pkg_rgy : PkgRgy = None
     marker_l : Callable = lambda *args, **kwargs: None
-    _pkg_s : List[Package] = dc.field(default_factory=list)
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
-    _task_m : Dict['TaskSpec',TaskNode] = dc.field(default_factory=dict)
+    _shell_m : Dict[str,Callable] = dc.field(default_factory=dict)
+    _task_m : Dict[str,Task] = dc.field(default_factory=dict)
+    _task_node_m : Dict['TaskSpec',TaskNode] = dc.field(default_factory=dict)
+    _task_ctor_m : Dict[Task,TaskNodeCtor] = dc.field(default_factory=dict)
     _override_m : Dict[str,str] = dc.field(default_factory=dict)
     _ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
     _compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
-    _rundir_s : List[str] = dc.field(default_factory=list)
+    _task_rundir_s : List[List[str]] = dc.field(default_factory=list)
     _uses_count : int = 0
 
-    _logger : logging.Logger = None
+    _log : logging.Logger = None
 
     def __post_init__(self):
-        if self.pkg_rgy is None:
-            self.pkg_rgy = PkgRgy.inst().copy()
-
         # Initialize the overrides from the global registry
-        self._override_m.update(self.pkg_rgy.getOverrides())
-
-        self._logger = logging.getLogger(type(self).__name__)
+        self._log = logging.getLogger(type(self).__name__)
+        self._shell_m.update(ExtRgy._inst._shell_m)
+        self._task_rundir_s.append([])
 
         if self.root_pkg is not None:
-            self._logger.debug("TaskGraphBuilder: root_pkg: %s" % str(self.root_pkg))
-
-            # Register package definitions found during loading
-            visited = set()
-            self._registerPackages(self.root_pkg, visited)
-
-            self._pkg_spec_s.append(self.root_pkg)
-            pkg = self.root_pkg.mkPackage(self)
-            self._pkg_spec_s.pop()
-
-            # Allows us to find ourselves
-            self._pkg_m[PackageSpec(self.root_pkg.name)] = pkg
-
-    def loadPkg(self, pkgfile : str):
-        pkg = PackageDef.load(pkgfile)
-        visited = set()
-        self._registerPackages(pkg, visited)
+            # Collect all the tasks
+            pkg_s = set()
+            self._addPackageTasks(self.root_pkg, pkg_s)
+
+    def _addPackageTasks(self, pkg, pkg_s):
+        if pkg not in pkg_s:
+            pkg_s.add(pkg)
+            for task in pkg.task_m.values():
+                self._addTask(task)
+            for subpkg in pkg.pkg_m.values():
+                self._addPackageTasks(subpkg, pkg_s)
+
+    def _addTask(self, task):
+        if task.name not in self._task_m.keys():
+            self._task_m[task.name] = task
+            for st in task.subtasks:
+                self._addTask(st)
 
     def addOverride(self, key : str, val : str):
         self._override_m[key] = val
 
-    def _registerPackages(self, pkg : PackageDef, visited):
-        self._logger.debug("Packages: %s" % str(pkg))
-        if pkg.name not in visited:
-            visited.add(pkg.name)
-            self._logger.debug("Registering package %s" % pkg.name)
-            self.pkg_rgy.registerPackage(pkg)
-            for subpkg in pkg.subpkg_m.values():
-                self._registerPackages(subpkg, visited)
-
-    
-    def push_package(self, pkg : Package, add=False):
-        self._pkg_s.append(pkg)
-        if add:
-            self._pkg_m[PackageSpec(pkg.name, pkg.params)] = pkg
-
-    def pop_package(self, pkg : Package):
-        self._pkg_s.pop()
-
-    def package(self):
-        return self._pkg_s[-1]
+    def enter_package(self, pkg : PackageDef):
+        pass
 
     def enter_rundir(self, rundir : str):
-        self._rundir_s.append(rundir)
+        self._log.debug("enter_rundir: %s (%d)" % (rundir, len(self._task_rundir_s[-1])))
+        self._task_rundir_s[-1].append(rundir)
 
     def get_rundir(self, rundir=None):
-        ret = self._rundir_s.copy()
+        ret = self._task_rundir_s[-1].copy()
         if rundir is not None:
             ret.append(rundir)
+        self._log.debug("get_rundir: %s" % str(ret))
         return ret
     
     def leave_rundir(self):
-        self._rundir_s.pop()
+        self._log.debug("leave_rundir")
+        self._task_rundir_s[-1].pop()
 
     def enter_uses(self):
         self._uses_count += 1
@@ -142,15 +137,6 @@ class TaskGraphBuilder(object):
         if rundir is None or rundir == RundirE.Unique:
             self._rundir_s.append(task.name)
 
-    def get_name_prefix(self):
-        if len(self._compound_task_ctxt_s) > 0:
-            # Use the compound scope name
-            name = ".".join(c.task.name for c in self._compound_task_ctxt_s)
-        else:
-            name = self._pkg_s[-1].name
-
-        return name
-
     def enter_compound_uses(self):
         self._compound_task_ctxt_s[-1].uses_s.append({})
 
@@ -159,7 +145,7 @@ class TaskGraphBuilder(object):
             # Propagate the items up the stack, appending 'super' to 
             # the names
             for k,v in self._compound_task_ctxt_s[-1].uses_s[-1].items():
-                self._compound_task_ctxt_s[-1].uses[-2]["super.%s" % k] = v
+                self._compound_task_ctxt_s[-1].uses_s[-2]["super.%s" % k] = v
         else:
             # Propagate the items to the compound namespace, appending
             # 'super' to the names
@@ -171,18 +157,18 @@ class TaskGraphBuilder(object):
         return len(self._compound_task_ctxt_s) > 0 and len(self._compound_task_ctxt_s[-1].uses_s) != 0
 
     def addTask(self, name, task : TaskNode):
-        self._logger.debug("--> addTask: %s" % name)
+        self._log.debug("--> addTask: %s" % name)
 
         if len(self._compound_task_ctxt_s) == 0:
-            self._task_m[name] = task
+            self._task_node_m[name] = task
         else:
             if len(self._compound_task_ctxt_s[-1].uses_s) > 0:
                 self._compound_task_ctxt_s[-1].uses_s[-1][name] = task
             else:
                 self._compound_task_ctxt_s[-1].task_m[name] = task
-        self._logger.debug("<-- addTask: %s" % name)
+        self._log.debug("<-- addTask: %s" % name)
 
-    def findTask(self, name):
+    def findTask(self, name, create=True):
         task = None
 
         if len(self._compound_task_ctxt_s) > 0:
@@ -191,8 +177,22 @@ class TaskGraphBuilder(object):
                     task = self._compound_task_ctxt_s[-1].uses_s[-1][name]
             if task is None and name in self._compound_task_ctxt_s[-1].task_m.keys():
                 task = self._compound_task_ctxt_s[-1].task_m[name]
-        if task is None and name in self._task_m.keys():
-            task = self._task_m[name]
+        if task is None and name in self._task_node_m.keys():
+            task = self._task_node_m[name]
+
+        if task is None and create:
+            if name in self.root_pkg.task_m.keys():
+                task = self.mkTaskGraph(name)
+                self._log.debug("Found task %s in root package" % name)
+            else:
+                raise Exception("Failed to find task %s" % name)
+                pass
+            # Go search type definitions
+            pass
+
+            # Check the current package
+#            if len(self._pkg_s) > 0 and name in self._pkg_s[-1].task_m.keys():
+#                task = self._pkg_s[-1].task_m[name]
         
         return task
 
@@ -202,220 +202,483 @@ class TaskGraphBuilder(object):
             self._rundir_s.pop()
 
     def mkTaskGraph(self, task : str, rundir=None) -> TaskNode:
-        self._pkg_s.clear()
-        self._task_m.clear()
+        return self.mkTaskNode(task, rundir=rundir)
+        # self._task_node_m.clear()
 
-        if rundir is not None:
-            self._rundir_s.append(rundir)
+        # if rundir is not None:
+        #     self._rundir_s.append(rundir)
 
-        ret = self._mkTaskGraph(task)
+        # ret = self._mkTaskGraph(task)
 
-        if rundir is not None:
-            self._rundir_s.pop()
+        # if rundir is not None:
+        #     self._rundir_s.pop()
 
-        return ret
+        # return ret
         
     def _mkTaskGraph(self, task : str) -> TaskNode:
-
-        elems = task.split(".")
-
-        pkg_name = ".".join(elems[0:-1])
-        task_name = elems[-1]
-
-        if pkg_name == "":
-            if len(self._pkg_spec_s) == 0:
-                raise Exception("No package context for %s" % task)
-            pkg_spec = self._pkg_spec_s[-1]
-            pkg_name = pkg_spec.name
+        if task in self.root_pkg.task_m.keys():
+            task_t = self.root_pkg.task_m[task]
         else:
-            pkg_spec = PackageSpec(pkg_name)
+            pass
 
-        self._logger.debug("pkg_spec: %s" % str(pkg_spec))
-        self._pkg_spec_s.append(pkg_spec)
-        pkg = self.getPackage(pkg_spec)
-        
-        self._pkg_s.append(pkg)
+        if task_t is None:
+            raise Exception("Failed to find task %s" % task)
 
-        ctor_t : TaskNodeCtor = pkg.getTaskCtor(task_name)
+        ctor = self._getTaskCtor(task_t)
 
-        self._logger.debug("ctor_t: %s" % ctor_t.name)
+        params = ctor.mkTaskParams()
 
         needs = []
 
-        for need_def in ctor_t.getNeeds():
-            # Resolve the full name of the need
-            need_fullname = self._resolveNeedRef(need_def)
-            self._logger.debug("Searching for qualifed-name task %s" % need_fullname)
-            if not need_fullname in self._task_m.keys():
-                # Go back to the root from a rundir perspective
-                rundir_s = self._rundir_s
-                self._rundir_s = [need_fullname]
-                need_t = self._mkTaskGraph(need_fullname)
-                self._rundir_s = rundir_s
-                self._task_m[need_fullname] = need_t
-            needs.append(self._task_m[need_fullname])
-
-        # The returned task should have all param references resolved
-        params = ctor_t.mkTaskParams()
-
-        if params is None:
-            raise Exception("ctor %s returned None for params" % str(ctor_t))
-
-        task = ctor_t.mkTaskNode(
+        for need in task_t.needs:
+            need_n = self.findTask(need.name)
+            if need_n is None:
+                raise Exception("Failed to find need %s" % need.name)
+            needs.append(need_n)
+
+        task = ctor.mkTaskNode(
             builder=self,
             params=params,
             name=task,
             needs=needs)
+        task.rundir = self.get_rundir(task.name)
 #        task.rundir = rundir
         
-        self._task_m[task.name] = task
-
-        self._pkg_s.pop()
-        self._pkg_spec_s.pop()
+        self._task_node_m[task.name] = task
 
         return task
+    
+    def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
+        self._log.debug("--> mkTaskNode: %s" % task_t)
 
-    def _resolveNeedRef(self, need_def) -> str:
-        if need_def.find(".") == -1:
-            # Need is a local task. Prefix to avoid ambiguity
-            return self._pkg_s[-1].name + "." + need_def
+        if task_t in self._task_m.keys():
+            task = self._task_m[task_t]
         else:
-            return need_def
-
-    def getPackage(self, spec : PackageSpec) -> Package:
-        # Obtain the active package definition
-        self._logger.debug("--> getPackage: %s len: %d" % (spec.name, len(self._pkg_spec_s)))
-        if len(self._pkg_spec_s) > 0:
-            pkg_spec = self._pkg_spec_s[-1]
-            if self.root_pkg is not None and self.root_pkg.name == pkg_spec.name:
-                pkg_def = self.root_pkg
+            raise Exception("task_t (%s) not present" % str(task_t))
+        
+        ret = self._mkTaskNode(task)
+
+        if needs is not None:
+            for need in needs:
+                ret.needs.append((need, False))
+
+        for k,v in kwargs.items():
+            if hasattr(ret.params, k):
+                setattr(ret.params, k, v)
             else:
-                pkg_def = self.pkg_rgy.getPackage(pkg_spec.name)
+                raise Exception("Task %s parameters do not include %s" % (task.name, k))
+
+        self._log.debug("<-- mkTaskNode: %s" % task_t)
+        return ret
+    
+    def _findTask(self, pkg, name):
+        task = None
+        if name in pkg.task_m.keys():
+            task = pkg.task_m[name]
         else:
-            pkg_def = None
-
-        # Need a stack to track which package we are currently in
-        # Need a map to get a concrete package from a name with parameterization
-
-        self._logger.debug("pkg_s: %d %s" % (
-            len(self._pkg_s), (self._pkg_s[-1].name if len(self._pkg_s) else "<unknown>")))
-
-        # First, check the active pkg_def to see if any aliases
-        # Should be considered
-        pkg_name = spec.name
-        if pkg_def is not None:
-            # Look for an import alias
-            self._logger.debug("Search package %s for import alias %s" % (
-                pkg_def.name, pkg_spec.name))
-            for imp in pkg_def.imports:
-                if type(imp) != str:
-                    self._logger.debug("imp: %s" % str(imp))
-                    if imp.alias is not None and imp.alias == spec.name:
-                        # Found the alias name. Just need to get an instance of this package
-                        self._logger.debug("Found alias %s -> %s" % (imp.alias, imp.name))
-                        pkg_name = imp.name
-                        break
+            for subpkg in pkg.pkg_m.values():
+                task = self._findTask(subpkg, name)
+                if task is not None:
+                    break
+        return task
+    
+    def _mkTaskNode(self, task : Task, hierarchical=False):
+
+        if not hierarchical:
+            self._task_rundir_s.append([])
 
-        # Note: _pkg_m needs to be context specific, such that imports from
-        # one package don't end up visible in another
-        spec.name = pkg_name
-
-        if spec in self._pkg_m.keys():
-            self._logger.debug("Found cached package instance")
-            pkg = self._pkg_m[spec]
-        elif self.pkg_rgy.hasPackage(spec.name):
-            self._logger.debug("Registry has a definition")
-            p_def =  self.pkg_rgy.getPackage(spec.name)
-
-            self._pkg_spec_s.append(p_def)
-            pkg = p_def.mkPackage(self)
-            self._pkg_spec_s.pop()
-            self._pkg_m[spec] = pkg
+        # Determine how to build this node
+        if task.subtasks is not None and len(task.subtasks):
+            ret = self._mkTaskCompoundNode(task)
         else:
-            self.error("Failed to find package %s" % spec.name)
-            raise Exception("Failed to find definition of package %s" % spec.name)
+            ret = self._mkTaskLeafNode(task)
 
-        self._logger.debug("<-- getPackage: %s" % str(pkg))
+        if not hierarchical:
+            self._task_rundir_s.pop()
 
-        return pkg
+        return ret        
     
-    def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
-        self._logger.debug("--> mkTaskNode: %s" % task_t)
-
-        if task_t in self._override_m.keys():
-            self._logger.debug("Overriding task %s with %s" % (task_t, self._override_m[task_t]))
-            task_t = self._override_m[task_t]
+    def _getTaskNode(self, name):
+        if name in self._task_node_m.keys():
+            return self._task_node_m[name]
         else:
-            dot_idx = task_t.rfind(".")
-            if dot_idx != -1:
-                pkg = task_t[0:dot_idx]
-                tname = task_t[dot_idx+1:]
-
-                if pkg in self._override_m.keys():
-                    self._logger.debug("Overriding package %s with %s" % (pkg, self._override_m[pkg]))
-                    task_t = self._override_m[pkg] + "." + tname
-
-        dot_idx = task_t.rfind(".")
-        pkg = task_t[0:dot_idx]
-        self._pkg_s.append(self.getPackage(PackageSpec(pkg)))
-
-        ctor = self.getTaskCtor(task_t)
-        if ctor is not None:
-            if needs is None:
-                needs = []
-            for need_def in ctor.getNeeds():
-                # Resolve the full name of the need
-                need_fullname = self._resolveNeedRef(need_def)
-                self._logger.debug("Searching for qualifed-name task %s" % need_fullname)
-                if not need_fullname in self._task_m.keys():
-                    rundir_s = self._rundir_s
-                    self._rundir_s = [need_fullname]
-                    need_t = self._mkTaskGraph(need_fullname)
-                    self._rundir_s = rundir_s
-                    self._task_m[need_fullname] = need_t
-                needs.append(self._task_m[need_fullname])
-
-            self._logger.debug("ctor: %s" % ctor.name)
-            params = ctor.mkTaskParams(kwargs)
-            ret = ctor.mkTaskNode(
-                self,
-                params=params,
-                name=name, 
-                srcdir=srcdir, 
-                needs=needs)
-            ret.rundir = self.get_rundir(name)
+            return self.mkTaskNode(name)
+    
+    def _mkTaskLeafNode(self, task : Task, name=None) -> TaskNode:
+        self._log.debug("--> _mkTaskLeafNode %s" % task.name)
+        srcdir = os.path.dirname(task.srcinfo.file)
+
+        if task.rundir == RundirE.Unique:
+            self.enter_rundir(task.name)
+
+        if name is None:
+            name = task.name
+
+        callable = None
+        if task.run is not None:
+            shell = task.shell if task.shell is not None else "shell"
+            if shell in self._shell_m.keys():
+                self._log.debug("Use shell implementation")
+                callable = self._shell_m[shell]
+            else:
+                raise Exception("Shell %s not found" % shell)
         else:
-            raise Exception("Failed to find ctor for task %s" % task_t)
-        self._pkg_s.pop()
-        self._logger.debug("<-- mkTaskNode: %s" % task_t)
-        return ret
+            callable = NullCallable
+
+        node = TaskNodeLeaf(
+            name=name,
+            srcdir=srcdir,
+            params=task.paramT(),
+            passthrough=task.passthrough,
+            consumes=task.consumes,
+            task=callable(task.run))
+        self._task_node_m[name] = node
+        node.rundir = self.get_rundir()
+
+        # Now, link up the needs
+        self._log.debug("--> processing needs")
+        for n in task.needs:
+            self._log.debug("-- need %s" % n.name)
+            nn = self._getTaskNode(n.name)
+            node.needs.append((nn, False))
+        self._log.debug("<-- processing needs")
+
+        if task.rundir == RundirE.Unique:
+            self.leave_rundir()
+
+        self._log.debug("<-- _mkTaskLeafNode %s" % task.name)
+        return node
+    
+    def _mkTaskCompoundNode(self, task : Task, name=None) -> TaskNode:
+        self._log.debug("--> _mkTaskCompoundNode %s" % task.name)
+        srcdir = os.path.dirname(task.srcinfo.file)
+
+        if name is None:
+            name = task.name
+
+        if task.rundir == RundirE.Unique:
+            self.enter_rundir(task.name)
+
+        # Node represents the terminal node of the sub-DAG
+        node = TaskNodeCompound(
+            name=name,
+            srcdir=srcdir,
+            params=task.paramT()
+        )
+        self._task_node_m[name] = node
+
+        node.rundir = self.get_rundir()
+
+        # Put the input node inside the compound task's rundir
+        self.enter_rundir(task.name + ".in")
+        node.input.rundir = self.get_rundir()
+        self.leave_rundir()
+
+        self._log.debug("--> processing needs")
+        for need in task.needs:
+            self._log.debug("-- need: %s" % need.name)
+            nn = self._getTaskNode(need.name)
+            node.input.needs.append((nn, False))
+        self._log.debug("<-- processing needs")
+
+        # TODO: handle strategy
+
+        # Need a local map of name -> task 
+        # For now, build out local tasks and link up the needs
+        tasks = []
+        for t in task.subtasks:
+            nn = self._mkTaskNode(t, True)
+            tasks.append((t, self._getTaskNode(t.name)))
+
+        # Fill in 'needs'
+        for t, tn in tasks:
+
+            referenced = None
+            for tt in task.subtasks:
+                if tt in t.needs:
+                    referenced = tt
+                    break
+
+            refs_internal = None
+            for nn,_ in tn.needs:
+                for _,tnn in tasks:
+                    if nn == tnn:
+                        refs_internal = tnn
+                        break
+                if refs_internal is not None:
+                    break
+            
+            if not refs_internal:
+                # Any node that doesn't depend on an internal
+                # task is a top-level task
+                self._log.debug("Node %s doesn't reference any internal node" % t.name)
+                tn.needs.append((node.input, False))
+            else:
+                self._log.debug("Node references internal node %s" % refs_internal.name)
+
+            if referenced is not None:
+                # Add this task as a dependency of the output
+                # node (the root one)
+                self._log.debug("Add node %s as a top-level dependency" % tn.name)
+                node.needs.append((tn, False))
+            else:
+                self._log.debug("Node %s has internal needs" % tn.name)
+
+        if task.rundir == RundirE.Unique:
+            self.leave_rundir()
+
+        return node
+
         
     def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskNodeCtor':
         from .task_def import TaskSpec
         if type(spec) == str:
             spec = TaskSpec(spec)
 
-        self._logger.debug("--> getTaskCtor %s" % spec.name)
+        self._log.debug("--> getTaskCtor %s" % spec.name)
         spec_e = spec.name.split(".")
         task_name = spec_e[-1]
 
-        if len(spec_e) == 1:
-            # Just have a task name. Use the current package
-            if len(self._pkg_s) == 0:
-                raise Exception("No package context for task %s" % spec.name)
-            pkg = self._pkg_s[-1]
-        else:
-            pkg_name = ".".join(spec_e[0:-1])
+        # if len(spec_e) == 1:
+        #     # Just have a task name. Use the current package
+        #     if len(self._pkg_s) == 0:
+        #         raise Exception("No package context for task %s" % spec.name)
+        #     pkg = self._pkg_s[-1]
+        # else:
+        #     pkg_name = ".".join(spec_e[0:-1])
 
-            try:
-                pkg = self.getPackage(PackageSpec(pkg_name))
-            except Exception as e:
-                self._logger.critical("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
-                raise e
+        #     try:
+        #         pkg = self.getPackage(PackageSpec(pkg_name))
+        #     except Exception as e:
+        #         self._log.critical("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
+        #         raise e
 
         ctor = pkg.getTaskCtor(task_name)
 
-        self._logger.debug("--> getTaskCtor %s" % spec.name)
+        self._log.debug("<-- getTaskCtor %s" % spec.name)
         return ctor
     
+    def error(self, msg, loc=None):
+        if loc is not None:
+            marker = TaskMarker(msg=msg, severity=SeverityE.Error, loc=loc)
+        else:
+            marker = TaskMarker(msg=msg, severity=SeverityE.Error)
+        self.marker(marker)
+
+    def marker(self, marker):
+        self.marker_l(marker)
+
+    def _getTaskCtor(self, task : Task) -> TaskNodeCtor:
+        if task in self._task_ctor_m.keys():
+            ctor = self._task_ctor_m[task]
+        else:
+            ctor = self._mkTaskCtor(task)
+            self._task_ctor_m[task] = ctor
+        return ctor
+
+    def _mkTaskCtor(self, task):
+        srcdir = os.path.dirname(task.srcinfo.file)
+        self._log.debug("--> mkTaskCtor %s (srcdir: %s)" % (task.name, srcdir))
+
+        if len(task.subtasks) > 0:
+            self._log.debug("Task has a body")
+            # Compound task
+            self._log.debug("Task specifies sub-task implementation")
+            ctor = self._mkCompoundTaskCtor(task)
+        else:
+            self._log.debug("Task doesn't specify a body")
+            # Shell task or 'null'
+            ctor = self._mkLeafTaskCtor(task)
+
+        if ctor is None:
+            raise Exception()
 
+        return ctor
+
+    def _mkLeafTaskCtor(self, task) -> TaskNodeCtor:
+        self._log.debug("--> _mkLeafTaskCtor")
+        srcdir = os.path.dirname(task.srcinfo.file)
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
+        base_params = None
+        callable = None
+#        fullname = self.name + "." + task.name
+#        rundir = task.rundir
+
+        # TODO: should we have the ctor look this up itself?
+        # Want to confirm that the value can be found.
+        # Defer final resolution until actual graph building (post-config)
+        if task.uses is not None:
+            self._log.debug("Uses: %s" % task.uses.name)
+
+            base_ctor_t = self._getTaskCtor(task.uses)
+
+            if base_ctor_t is None:
+                self._log.error("Failed to load task ctor %s" % task.uses)
+#            base_params = base_ctor_t.mkTaskParams()
+        else:
+            self._log.debug("No 'uses' specified %s" % task.name)
+
+        self._log.debug("%d needs" % len(task.needs))
+
+        # Determine the implementation constructor first
+        if task.run is not None:
+            shell = task.shell if task.shell is not None else "shell"
+
+            if shell in self._shell_m.keys():
+                self._log.debug("Use shell implementation")
+                callable = self._shell_m[shell]
+            else:
+                self._log.debug("Shell %s not found" % shell)
+                raise Exception("Shell %s not found" % shell)
+
+            # if taskdef.body.pytask is not None:
+            #     # Built-in impl
+            #     # Now, lookup the class
+            #     self._log.debug("Use PyTask implementation")
+            #     last_dot = taskdef.body.pytask.rfind('.')
+            #     clsname = taskdef.body.pytask[last_dot+1:]
+            #     modname = taskdef.body.pytask[:last_dot]
+
+            #     try:
+            #         if modname not in sys.modules:
+            #             if srcdir not in sys.path:
+            #                 sys.path.append(srcdir)
+            #             mod = importlib.import_module(modname)
+            #         else:
+            #             mod = sys.modules[modname]
+            #     except ModuleNotFoundError as e:
+            #         raise Exception("Failed to import module %s (_basedir=%s): %s" % (
+            #             modname, self._basedir, str(e)))
+                
+            #     if not hasattr(mod, clsname):
+            #         raise Exception("Method %s not found in module %s" % (clsname, modname))
+            #     callable = getattr(mod, clsname)
+            # elif taskdef.body.run is not None:
+            #     callable = self._getRunCallable(taskdef)
+        else:
+            # TODO: use null task
+            pass
+
+        # Determine if we need to use a new 
+        if task.paramT is None:
+            raise Exception()
+        paramT = task.paramT
+        needs = []
+
+        # TODO:
+        rundir : RundirE = task.rundir
+        
+        if callable is not None:
+            ctor_t = TaskNodeCtorTask(
+                name=task.name,
+                srcdir=srcdir,
+                paramT=task.paramT, # TODO: need to determine the parameter type
+                passthrough=task.passthrough,
+                consumes=task.consumes,
+                needs=needs, # TODO: need to determine the needs
+                rundir=rundir,
+                task=callable)
+        elif base_ctor_t is not None:
+            # Use the existing (base) to create the implementation
+            ctor_t = TaskNodeCtorProxy(
+                name=task.name,
+                srcdir=srcdir,
+                paramT=task.paramT, # TODO: need to determine the parameter type
+                passthrough=task.passthrough,
+                consumes=task.consumes,
+                needs=needs,
+                rundir=rundir,
+                uses=base_ctor_t)
+        else:
+            self._log.debug("Use 'Null' as the class implementation")
+            ctor_t = TaskNodeCtorTask(
+                name=task.name,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=task.passthrough,
+                consumes=task.consumes,
+                needs=needs,
+                rundir=rundir,
+                task=TaskNull)
+
+        self._log.debug("<-- mkTaskCtor %s" % task.name)
+        return ctor_t
+    
+    def _getRunCallable(self, task):
+        self._log.debug("--> _getRunCallable %s" % task.name)
+        callable = None
+        if task.run is not None and task.shell == "python":
+            # Evaluate a Python script
+            pass
+        else:
+            # run a shell script
+            shell = None
+            body = task.run.strip()
+
+            callable = ShellCallable(body=body, shell=shell)
+            pass
+        return callable
+
+    def _mkCompoundTaskCtor(self, task) -> TaskNodeCtor:
+        self._log.debug("--> _mkCompoundTaskCtor %s" % task.name)
+        srcdir = os.path.dirname(task.srcinfo.file)
+        base_ctor_t : TaskNodeCtor = None
+        ctor_t : TaskNodeCtor = None
+        base_params = None
+        callable = None
+
+#        fullname = self._getScopeFullname()
+        fullname = task.name
+
+        if task.uses is not None:
+            self._log.debug("Uses: %s" % task.uses)
+            base_ctor_t = task.uses.ctor
+            base_params = base_ctor_t.mkTaskParams()
+
+            if base_ctor_t is None:
+                self._log.error("Failed to load task ctor %s" % task.uses)
+
+        # TODO: should build during loading
+#        passthrough, consumes, needs = self._getPTConsumesNeeds(taskdef, base_ctor_t)
+        passthrough = []
+        consumes = []
+        needs = []
+
+        # Determine if we need to use a new 
+#        paramT = self._getParamT(taskdef, base_params)
+        paramT = task.paramT
+
+        if base_ctor_t is not None:
+            ctor_t = TaskNodeCtorCompoundProxy(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task=task,
+                uses=base_ctor_t)
+        else:
+            self._log.debug("No 'uses' specified")
+            ctor_t = TaskNodeCtorCompound(
+                name=fullname,
+                srcdir=srcdir,
+                paramT=paramT,
+                passthrough=passthrough,
+                consumes=consumes,
+                needs=needs,
+                task=task)
+            
+        for st in task.subtasks:
+            ctor = self._getTaskCtor(st)
+            if ctor is None:
+                raise Exception("ctor for %s is None" % st.name)
+            ctor_t.tasks.append(st)
+
+#        for t in task.subtasks:
+#            ctor_t.tasks.append(self._mkTaskCtor(t, srcdir))
+
+        
+        self._log.debug("<-- mkCompoundTaskCtor %s (%d)" % (task.name, len(ctor_t.tasks)))
+        return ctor_t