dv-flow-mgr 1.0.0.14528489065a1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dv_flow/mgr/parsetab.py

@@ -6,9 +6,9 @@ _tabversion = '3.10'
 
 _lr_method = 'LALR'
 
-_lr_signature = 'leftPLUSMINUSPIPEleftTIMESDIVIDECOMMA DIVIDE ID LPAREN MINUS NUMBER PIPE PLUS RPAREN STRING1 STRING2 TIMESexpression : ID LPAREN RPAREN \n                      | ID LPAREN args RPARENargs : expression \n                | args COMMA expressionexpression : expression PLUS expression\n                      | expression MINUS expression\n                      | expression TIMES expression\n                      | expression PIPE expression\n                      | expression DIVIDE expressionexpression : LPAREN expression RPARENexpression : NUMBERexpression : IDexpression : STRING1expression : STRING2'
+_lr_signature = 'leftPLUSMINUSPIPEleftTIMESDIVIDECOMMA DIVIDE DOT ID LPAREN MINUS NUMBER PIPE PLUS RPAREN STRING1 STRING2 TIMESexpression : ID LPAREN RPAREN \n                      | ID LPAREN args RPARENargs : expression \n                | args COMMA expressionexpression : expression PLUS expression\n                      | expression MINUS expression\n                      | expression TIMES expression\n                      | expression PIPE expression\n                      | expression DIVIDE expressionexpression : LPAREN expression RPARENexpression : NUMBERexpression : IDexpression : hier_idhier_id : ID DOT hier_id \n                   | IDexpression : STRING1expression : STRING2'
     
-_lr_action_items = {'ID':([0,3,7,8,9,10,11,12,24,],[2,2,2,2,2,2,2,2,2,]),'LPAREN':([0,2,3,7,8,9,10,11,12,24,],[3,12,3,3,3,3,3,3,3,3,]),'NUMBER':([0,3,7,8,9,10,11,12,24,],[4,4,4,4,4,4,4,4,4,]),'STRING1':([0,3,7,8,9,10,11,12,24,],[5,5,5,5,5,5,5,5,5,]),'STRING2':([0,3,7,8,9,10,11,12,24,],[6,6,6,6,6,6,6,6,6,]),'$end':([1,2,4,5,6,14,15,16,17,18,19,22,23,],[0,-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,-10,-2,]),'PLUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[7,-12,-11,-13,-14,7,-5,-6,-7,-8,-9,-1,7,-10,-2,7,]),'MINUS':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[8,-12,-11,-13,-14,8,-5,-6,-7,-8,-9,-1,8,-10,-2,8,]),'TIMES':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[9,-12,-11,-13,-14,9,9,9,-7,9,-9,-1,9,-10,-2,9,]),'PIPE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[10,-12,-11,-13,-14,10,-5,-6,-7,-8,-9,-1,10,-10,-2,10,]),'DIVIDE':([1,2,4,5,6,13,14,15,16,17,18,19,21,22,23,25,],[11,-12,-11,-13,-14,11,11,11,-7,11,-9,-1,11,-10,-2,11,]),'RPAREN':([2,4,5,6,12,13,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,19,22,-5,-6,-7,-8,-9,-1,23,-3,-10,-2,-4,]),'COMMA':([2,4,5,6,14,15,16,17,18,19,20,21,22,23,25,],[-12,-11,-13,-14,-5,-6,-7,-8,-9,-1,24,-3,-10,-2,-4,]),}
+_lr_action_items = {'ID':([0,3,8,9,10,11,12,13,14,28,],[2,2,2,2,2,2,2,2,24,2,]),'LPAREN':([0,2,3,8,9,10,11,12,13,28,],[3,13,3,3,3,3,3,3,3,3,]),'NUMBER':([0,3,8,9,10,11,12,13,28,],[4,4,4,4,4,4,4,4,4,]),'STRING1':([0,3,8,9,10,11,12,13,28,],[6,6,6,6,6,6,6,6,6,]),'STRING2':([0,3,8,9,10,11,12,13,28,],[7,7,7,7,7,7,7,7,7,]),'$end':([1,2,4,5,6,7,16,17,18,19,20,21,24,25,26,27,],[0,-12,-11,-13,-16,-17,-5,-6,-7,-8,-9,-1,-15,-14,-10,-2,]),'PLUS':([1,2,4,5,6,7,15,16,17,18,19,20,21,23,24,25,26,27,29,],[8,-12,-11,-13,-16,-17,8,-5,-6,-7,-8,-9,-1,8,-15,-14,-10,-2,8,]),'MINUS':([1,2,4,5,6,7,15,16,17,18,19,20,21,23,24,25,26,27,29,],[9,-12,-11,-13,-16,-17,9,-5,-6,-7,-8,-9,-1,9,-15,-14,-10,-2,9,]),'TIMES':([1,2,4,5,6,7,15,16,17,18,19,20,21,23,24,25,26,27,29,],[10,-12,-11,-13,-16,-17,10,10,10,-7,10,-9,-1,10,-15,-14,-10,-2,10,]),'PIPE':([1,2,4,5,6,7,15,16,17,18,19,20,21,23,24,25,26,27,29,],[11,-12,-11,-13,-16,-17,11,-5,-6,-7,-8,-9,-1,11,-15,-14,-10,-2,11,]),'DIVIDE':([1,2,4,5,6,7,15,16,17,18,19,20,21,23,24,25,26,27,29,],[12,-12,-11,-13,-16,-17,12,12,12,-7,12,-9,-1,12,-15,-14,-10,-2,12,]),'RPAREN':([2,4,5,6,7,13,15,16,17,18,19,20,21,22,23,24,25,26,27,29,],[-12,-11,-13,-16,-17,21,26,-5,-6,-7,-8,-9,-1,27,-3,-15,-14,-10,-2,-4,]),'COMMA':([2,4,5,6,7,16,17,18,19,20,21,22,23,24,25,26,27,29,],[-12,-11,-13,-16,-17,-5,-6,-7,-8,-9,-1,28,-3,-15,-14,-10,-2,-4,]),'DOT':([2,24,],[14,14,]),}
 
 _lr_action = {}
 for _k, _v in _lr_action_items.items():
@@ -17,7 +17,7 @@ for _k, _v in _lr_action_items.items():
       _lr_action[_x][_k] = _y
 del _lr_action_items
 
-_lr_goto_items = {'expression':([0,3,7,8,9,10,11,12,24,],[1,13,14,15,16,17,18,21,25,]),'args':([12,],[20,]),}
+_lr_goto_items = {'expression':([0,3,8,9,10,11,12,13,28,],[1,15,16,17,18,19,20,23,29,]),'hier_id':([0,3,8,9,10,11,12,13,14,28,],[5,5,5,5,5,5,5,5,25,5,]),'args':([13,],[22,]),}
 
 _lr_goto = {}
 for _k, _v in _lr_goto_items.items():
@@ -27,18 +27,21 @@ for _k, _v in _lr_goto_items.items():
 del _lr_goto_items
 _lr_productions = [
   ("S' -> expression","S'",1,None,None,None),
-  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',180),
-  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',181),
-  ('args -> expression','args',1,'p_args','expr_parser.py',185),
-  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',186),
-  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',194),
-  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',195),
-  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',196),
-  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',197),
-  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',198),
-  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',209),
-  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',213),
-  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',217),
-  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',221),
-  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',225),
+  ('expression -> ID LPAREN RPAREN','expression',3,'p_call','expr_parser.py',208),
+  ('expression -> ID LPAREN args RPAREN','expression',4,'p_call','expr_parser.py',209),
+  ('args -> expression','args',1,'p_args','expr_parser.py',213),
+  ('args -> args COMMA expression','args',3,'p_args','expr_parser.py',214),
+  ('expression -> expression PLUS expression','expression',3,'p_expression_binop','expr_parser.py',222),
+  ('expression -> expression MINUS expression','expression',3,'p_expression_binop','expr_parser.py',223),
+  ('expression -> expression TIMES expression','expression',3,'p_expression_binop','expr_parser.py',224),
+  ('expression -> expression PIPE expression','expression',3,'p_expression_binop','expr_parser.py',225),
+  ('expression -> expression DIVIDE expression','expression',3,'p_expression_binop','expr_parser.py',226),
+  ('expression -> LPAREN expression RPAREN','expression',3,'p_expression_group','expr_parser.py',237),
+  ('expression -> NUMBER','expression',1,'p_expression_number','expr_parser.py',241),
+  ('expression -> ID','expression',1,'p_expression_name','expr_parser.py',245),
+  ('expression -> hier_id','expression',1,'p_expression_hid','expr_parser.py',249),
+  ('hier_id -> ID DOT hier_id','hier_id',3,'p_hier_id','expr_parser.py',253),
+  ('hier_id -> ID','hier_id',1,'p_hier_id','expr_parser.py',254),
+  ('expression -> STRING1','expression',1,'p_expression_string1','expr_parser.py',259),
+  ('expression -> STRING2','expression',1,'p_expression_string2','expr_parser.py',263),
 ]

dv_flow/mgr/std/flow.dv

@@ -82,6 +82,12 @@ package:
         type: str
       content:
         type: str
+  - name: IncDirs
+    shell: pytask
+    run: dv_flow.mgr.std.incdirs.IncDirs
+    doc: |
+      Creates a list of include directories from a set of
+      input files.
   - name: Exec
     shell: pytask
     run: dv_flow.mgr.std.exec.Exec
@@ -113,66 +119,38 @@ package:
   types:
   - name: DataItem
     with:
-    - name: type
-      type: str
+      type:
+        type: str
   - name: FileSet
     uses: std.DataItem
     with:
-    - name: filetype
-      type: str
-      value: "FileSet"
-    - name: basedir
-      type: str
-    - name: files
-      type: list
-      item:
+      filetype:
         type: str
-
-  # - name: TaskDataItem
-  #   doc: |
-  #     Base type for a data element produced by a task.
-  #     TaskDataItem is never used directly
-  #   with:
-  #   - name: kind
-  #     type: str
-  #     kind: metadata
-  #     doc: |
-  #       Kind of data item
-  #   - name: src
-  #     type: str
-  #     kind: metadata
-  #     doc: |
-  #       Name of the task that produced this item
-  #   - name: id
-  #     type: str
-  #     kind: metadata
-  #     doc: |
-  #       Unique identiifer 
+        value: ""
+      basedir:
+        type: str
+      files:
+        type:
+          list:
+            item: str
+      incdirs:
+        type:
+          list:
+            item: str
+      defines:
+        type:
+          list:
+            item: str
 
   - name: Env
     doc: |
       Environment variables
     with:
-    - name: doc
-
-  - name: FileSet
-    uses: TaskDataItem
-    with:
-    - name: kind
-      value: "FileSet"
-    - name: base
-      type: str
-    - name: fileType
-      type: str
-    - name: files
-      type: 
-        list:
-          item:
-            type: str
+      vals:
+        type:
+          map:
+            key:
+              type: str
+            val:
+              type: str
 
-    # type:
-    #   list: (str,int)
-    #     - complex type
-    # type:
-    #   object:
-    #     

dv_flow/mgr/std/incdirs.py

@@ -0,0 +1,18 @@
+from dv_flow.mgr import TaskDataResult, TaskDataInput, TaskRunCtxt, FileSet
+
+async def IncDirs(ctxt : TaskRunCtxt, input : TaskDataInput) -> TaskDataResult:
+    """
+    IncDirs - create a list of include directories
+    """
+    outputs = []
+
+    for inp in input.inputs:
+        if hasattr(inp, "basedir") and hasattr(inp, "filetype"):
+            outputs.append(FileSet(
+                filetype=inp.filetype,
+                basedir=inp.basedir,
+                incdirs=[inp.basedir]))
+    return TaskDataResult(
+        status=0,
+        changed=input.changed,
+        output=outputs)

dv_flow/mgr/task.py

@@ -23,6 +23,7 @@ class Task(object):
     doc : str = ""
     paramT : Any = None
     uses : 'Task' = None
+    iff : str = None
     needs : List[str] = dc.field(default_factory=list)
     consumes : Union[ConsumesE, List[Dict[str, Any]]] = dc.field(default=None)
     passthrough : Union[PassthroughE, List[Dict[str, Any]]] = dc.field(default=None)

dv_flow/mgr/task_def.py

@@ -103,6 +103,10 @@ class TaskDef(BaseModel):
     body: List['TaskDef'] = dc.Field(
         default_factory=list,
         description="Sub-tasks")
+    iff : Union[str, bool, Any] = dc.Field(
+        default=None,
+        title="Task enable condition",
+        description="Condition that must be true for this task to run")
     pytask : str = dc.Field(
         default=None,
         description="Python-based implementation (deprecated)")
@@ -125,7 +129,7 @@ class TaskDef(BaseModel):
     needs : List[Union[str]] = dc.Field(
         default_factory=list, 
         description="List of tasks that this task depends on")
-    params: Dict[str,Union[str,list,ParamDef]] = dc.Field(
+    params: Dict[str,Union[str,list,bool,ParamDef]] = dc.Field(
         default_factory=dict, 
         alias="with",
         description="Parameters for the task")

dv_flow/mgr/task_graph_builder.py

@@ -22,10 +22,12 @@
 import os
 import dataclasses as dc
 import logging
+import pydantic
 from typing import Callable, Any, Dict, List, Union
 from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .package_loader import PackageLoader
+from .param_ref_eval import ParamRefEval
 from .ext_rgy import ExtRgy
 from .task import Task
 from .task_def import RundirE
@@ -38,7 +40,9 @@ from .task_node_ctor_proxy import TaskNodeCtorProxy
 from .task_node_ctor_task import TaskNodeCtorTask
 from .task_node_ctor_wrapper import TaskNodeCtorWrapper
 from .task_node_compound import TaskNodeCompound
+from .task_node_ctxt import TaskNodeCtxt
 from .task_node_leaf import TaskNodeLeaf
+from .type import Type
 from .std.task_null import TaskNull
 from .exec_callable import ExecCallable
 from .null_callable import NullCallable
@@ -64,16 +68,21 @@ class TaskGraphBuilder(object):
     loader : PackageLoader = None
     marker_l : Callable = lambda *args, **kwargs: None
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
+    _pkg_params_m : Dict[str,Any] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
     _shell_m : Dict[str,Callable] = dc.field(default_factory=dict)
     _task_m : Dict[str,Task] = dc.field(default_factory=dict)
+    _type_m : Dict[str,Type] = dc.field(default_factory=dict)
     _task_node_m : Dict['TaskSpec',TaskNode] = dc.field(default_factory=dict)
+    _type_node_m : Dict[str,Any] = dc.field(default_factory=dict)
     _task_ctor_m : Dict[Task,TaskNodeCtor] = dc.field(default_factory=dict)
     _override_m : Dict[str,str] = dc.field(default_factory=dict)
     _ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
     _compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
     _task_rundir_s : List[List[str]] = dc.field(default_factory=list)
     _task_node_s : List[TaskNode] = dc.field(default_factory=list)
+    _eval : ParamRefEval = dc.field(default_factory=ParamRefEval)
+    _ctxt : TaskNodeCtxt = None
     _uses_count : int = 0
 
     _log : logging.Logger = None
@@ -82,18 +91,63 @@ class TaskGraphBuilder(object):
         # Initialize the overrides from the global registry
         self._log = logging.getLogger(type(self).__name__)
         self._shell_m.update(ExtRgy.inst()._shell_m)
-        self._task_rundir_s.append([])
+        self._task_rundir_s.append([self.rundir])
+
+        self._eval.set("env", os.environ)
+
+
 
         if self.root_pkg is not None:
             # Collect all the tasks
             pkg_s = set()
+
+            self._ctxt = TaskNodeCtxt(
+                root_pkgdir=self.root_pkg.basedir,
+                root_rundir=self.rundir)
+
+            self._eval.set("root", {
+                "dir": self.root_pkg.basedir
+            })
+
+            params = self.root_pkg.paramT()
+
+            self._expandParams(params, self._eval)
+
+            for key in self.root_pkg.paramT.model_fields.keys():
+                self._eval.set(key, getattr(params, key))
+
+            self._pkg_params_m[self.root_pkg.name] = params
+
             self._addPackageTasks(self.root_pkg, pkg_s)
+        else:
+            self._ctxt = TaskNodeCtxt(
+                root_pkgdir=None,
+                root_rundir=self.rundir)
+
+
+    def setParam(self, name, value):
+        if self.root_pkg is None:
+            raise Exception("No root package")
+        params = self._pkg_params_m[self.root_pkg.name]
+
+        if not hasattr(params, name):
+            raise Exception("Package %s does not have parameter %s" % (self.root_pkg.name, name))
+        setattr(params, name, value)
 
     def _addPackageTasks(self, pkg, pkg_s):
+
+        # Build out the package parameters
+        params = pkg.paramT()
+        self._expandParams(params, self._eval)
+        self._pkg_params_m[pkg.name] = params
+        self._eval.set(pkg.name, params)
+
         if pkg not in pkg_s:
             pkg_s.add(pkg)
             for task in pkg.task_m.values():
                 self._addTask(task)
+            for tt in pkg.type_m.values():
+                self._addType(tt)
             for subpkg in pkg.pkg_m.values():
                 self._addPackageTasks(subpkg, pkg_s)
 
@@ -103,6 +157,10 @@ class TaskGraphBuilder(object):
             for st in task.subtasks:
                 self._addTask(st)
 
+    def _addType(self, tt):
+        if tt.name not in self._type_m.keys():
+            self._type_m[tt.name] = tt
+
     def addOverride(self, key : str, val : str):
         self._override_m[key] = val
 
@@ -220,10 +278,14 @@ class TaskGraphBuilder(object):
         else:
             raise Exception("task_t (%s) not present" % str(task_t))
         
+        # TODO: need to obtain the package
+
+        
         ret = self._mkTaskNode(
             task, 
             name=name, 
-            srcdir=srcdir)
+            srcdir=srcdir,
+            eval=self._eval)
 
         if needs is not None:
             for need in needs:
@@ -238,6 +300,73 @@ class TaskGraphBuilder(object):
         self._log.debug("<-- mkTaskNode: %s (%d needs)" % (task_t, len(ret.needs)))
         return ret
     
+    def mkDataItem(self, name, **kwargs):
+        self._log.debug("--> mkDataItem: %s" % name)
+
+        if name in self._type_m.keys():
+            tt = self._type_m[name]
+        else:
+            raise Exception("Type %s does not exist" % name)
+        
+        if tt in self._type_node_m.keys():
+            tn = self._type_node_m[tt]
+        else:
+#            tn = self._mkDataItem(tt)
+            tn = tt.paramT
+            self._type_node_m[tt] = tn
+
+        ret = tn()
+
+        for k, v in kwargs.items():
+            if hasattr(ret, k):
+                setattr(ret, k, v)
+            else:
+                raise Exception("Data item %s parameters do not include %s" % (name, k))
+
+        self._log.debug("<-- mkDataItem: %s" % name)
+        return ret
+    
+    def _findType(self, pkg, name):
+        tt = None
+        if name in pkg.type_m.keys():
+            tt = pkg.type_m[name]
+        else:
+            for subpkg in pkg.pkg_m.values():
+                tt = self._findType(subpkg, name)
+                if tt is not None:
+                    break
+        return tt
+    
+    def _mkDataItem(self, tt : Type):
+        field_m = {}
+
+        # Save the type name in each instance 
+        field_m["type"] = (str, tt.name)
+        exclude_s = set()
+        exclude_s.add("type")
+
+        self._mkDataItemI(tt, field_m, exclude_s)
+
+        ret = pydantic.create_model(tt.name, **field_m)
+
+        return ret
+    
+    def _mkDataItemI(self, tt : Type, field_m, exclude_s):
+        # First, identify cases where the value is set
+        for pt in tt.params.values():
+            if pt.name not in exclude_s:
+                if pt.type is not None:
+                    # Defining a new attribute
+                    field_m[pt.name] = (str, pt.value)
+                else:
+                    # TODO: determine whether 
+                    field_m[pt.name] = (str, None)
+        if tt.uses is not None:
+            self._mkDataItemI(tt.uses, field_m, exclude_s)
+
+    def _applyParameterOverrides(self, obj, **kwargs):
+        pass
+    
     def _findTask(self, pkg, name):
         task = None
         if name in pkg.task_m.keys():
@@ -249,26 +378,69 @@ class TaskGraphBuilder(object):
                     break
         return task
     
-    def _mkTaskNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False):
+    def _mkTaskNode(self, 
+                    task : Task, 
+                    name=None, 
+                    srcdir=None, 
+                    params=None, 
+                    hierarchical=False,
+                    eval=None):
 
         if not hierarchical:
-            self._task_rundir_s.append([])
+            self._task_rundir_s.append([self.rundir])
+
+        # If the task has an enable condition, evaluate
+        # that now
+        iff = True
+        if task.iff is not None:
+            self._log.debug("Evaluate iff condition \"%s\"" % task.iff)
+            iff = self._expandParam(task.iff, eval)
+
+            if iff:
+                self._log.debug("Condition \"%s\" is true" % task.iff)
+            else:
+                self._log.debug("Condition \"%s\" is false" % task.iff)
 
         # Determine how to build this node
-        if self._isCompound(task):
-            ret = self._mkTaskCompoundNode(
-                task, 
-                name=name,
-                srcdir=srcdir,
-                params=params,
-                hierarchical=hierarchical)
+        if iff:
+            if self._isCompound(task):
+                ret = self._mkTaskCompoundNode(
+                    task, 
+                    name=name,
+                    srcdir=srcdir,
+                    params=params,
+                    hierarchical=hierarchical,
+                    eval=eval)
+            else:
+                ret = self._mkTaskLeafNode(
+                    task, 
+                    name=name,
+                    srcdir=srcdir,
+                    params=params,
+                    hierarchical=hierarchical,
+                    eval=eval)
         else:
-            ret = self._mkTaskLeafNode(
-                task, 
+            if name is None:
+                name = task.name
+            
+            if params is None:
+                params = task.paramT()
+
+            if srcdir is None:
+                srcdir = os.path.dirname(task.srcinfo.file)
+
+            # Create a null task
+            ret = TaskNodeLeaf(
                 name=name,
                 srcdir=srcdir,
                 params=params,
-                hierarchical=hierarchical)
+                passthrough=task.passthrough,
+                consumes=task.consumes,
+                task=NullCallable(task.run),
+                ctxt=None,
+                iff=False)
+            self._task_node_m[name] = ret
+
 
         if not hierarchical:
             self._task_rundir_s.pop()
@@ -287,7 +459,12 @@ class TaskGraphBuilder(object):
         else:
             return self.mkTaskNode(name)
     
-    def _mkTaskLeafNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
+    def _mkTaskLeafNode(self, 
+                        task : Task, name=None, 
+                        srcdir=None, 
+                        params=None, 
+                        hierarchical=False,
+                        eval=None) -> TaskNode:
         self._log.debug("--> _mkTaskLeafNode %s" % task.name)
 
         if name is None:
@@ -299,9 +476,26 @@ class TaskGraphBuilder(object):
         if params is None:
             params = task.paramT()
 
+            eval.set("rundir", "/".join([str(e) for e in self.get_rundir()]))
+
+#            self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
+#            eval.setVar("in", in_params)
+#            eval.setVar("rundir", rundir)
+
+            # Set variables from the inputs
+            # for need in self.needs:
+            #     for name,value in {"rundir" : need[0].rundir}.items():
+            #         eval.setVar("%s.%s" % (need[0].name, name), value)
+
+            # expand any variable references
+            self._expandParams(params, eval)
+
+
         if task.rundir == RundirE.Unique:
             self.enter_rundir(name)
 
+        # TODO: handle callable in light of overrides
+
 
         callable = None
         if task.run is not None:
@@ -318,6 +512,7 @@ class TaskGraphBuilder(object):
             name=name,
             srcdir=srcdir,
             params=params,
+            ctxt=self._ctxt,
             passthrough=task.passthrough,
             consumes=task.consumes,
             task=callable(task.run))
@@ -338,7 +533,13 @@ class TaskGraphBuilder(object):
         self._log.debug("<-- _mkTaskLeafNode %s" % task.name)
         return node
     
-    def _mkTaskCompoundNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
+    def _mkTaskCompoundNode(self, 
+                            task : Task, 
+                            name=None, 
+                            srcdir=None, 
+                            params=None, 
+                            hierarchical=False,
+                            eval=None) -> TaskNode:
         self._log.debug("--> _mkTaskCompoundNode %s" % task.name)
 
         if name is None:
@@ -350,28 +551,41 @@ class TaskGraphBuilder(object):
         if params is None:
             params = task.paramT()
 
+            # expand any variable references
+            self._expandParams(params, eval)
+
         if task.rundir == RundirE.Unique:
             self.enter_rundir(name)
 
         if task.uses is not None:
             # This is a compound task that is based on
             # another. Create the base implementation
+            task_uses = task.uses
+
+            if not self.in_uses():
+                # Determine whether this task is overridden
+                task_uses = self._findOverride(task_uses)
+
+            self.enter_uses()
             node = self._mkTaskNode(
-                task.uses,
+                task_uses,
                 name=name, 
                 srcdir=srcdir,
                 params=params,
-                hierarchical=True)
+                hierarchical=True,
+                eval=eval)
+            self.leave_uses()
             
             if not isinstance(node, TaskNodeCompound):
                 # TODO: need to enclose the leaf node in a compound wrapper
-                raise Exception("Task %s is not compound" % task.uses)
+                raise Exception("Task %s is not compound" % task_uses)
         else:
             # Node represents the terminal node of the sub-DAG
             node = TaskNodeCompound(
                 name=name,
                 srcdir=srcdir,
-                params=params)
+                params=params,
+                ctxt=self._ctxt)
 
         if len(self._task_node_s):
             node.parent = self._task_node_s[-1]
@@ -391,9 +605,11 @@ class TaskGraphBuilder(object):
             need_n = self._getTaskNode(need.name)
             if need_n is None:
                 raise Exception("Failed to find need %s" % need.name)
-            self._log.debug("Add need %s with %d dependencies" % (need_n.name, len(need_n.needs)))
-            node.input.needs.append((need_n, False))
-#            node.needs.append((need_n, False))
+            elif need_n.iff:
+                self._log.debug("Add need %s with %d dependencies" % (need_n.name, len(need_n.needs)))
+                node.input.needs.append((need_n, False))
+            else:
+                self._log.debug("Needed node %s is not enabled" % need_n.name)
         self._log.debug("<-- processing needs")
 
         # TODO: handle strategy
@@ -402,7 +618,7 @@ class TaskGraphBuilder(object):
         # For now, build out local tasks and link up the needs
         tasks = []
         for t in task.subtasks:
-            nn = self._mkTaskNode(t, hierarchical=True)
+            nn = self._mkTaskNode(t, hierarchical=True, eval=eval)
             node.tasks.append(nn)
 #            tasks.append((t, self._getTaskNode(t.name)))
             tasks.append((t, nn))
@@ -455,6 +671,28 @@ class TaskGraphBuilder(object):
 
         return node
 
+    def _expandParams(self, params, eval):
+        for name in type(params).model_fields.keys():
+            value = getattr(params, name)
+            new_val = self._expandParam(value, eval)
+            setattr(params, name, new_val)
+
+    def _expandParam(self, value, eval):
+        new_val = value
+        if type(value) == str:
+            if value.find("${{") != -1:
+                print("Expr: %s" % str(value), flush=True)
+                new_val = eval.eval(value)
+                self._log.debug("Param: Evaluate expression \"%s\" => \"%s\"" % (value, new_val))
+        elif isinstance(value, list):
+            new_val = []
+            for i,elem in enumerate(value):
+                if elem.find("${{") != -1:
+                    new_val.append(eval.eval(elem))
+                else:
+                    new_val.append(elem)
+        return new_val
+
     def _gatherNeeds(self, task_t, node):
         self._log.debug("--> _gatherNeeds %s (%d)" % (task_t.name, len(task_t.needs)))
         if task_t.uses is not None:
@@ -477,3 +715,6 @@ class TaskGraphBuilder(object):
     def marker(self, marker):
         self.marker_l(marker)
 
+    def _findOverride(self, task):
+        return task
+