dv-flow-mgr 0.0.1.13979842530a1__py3-none-any.whl → 0.0.1.14097297609a1__py3-none-any.whl

dv_flow/mgr/task_data.py

@@ -48,20 +48,16 @@ class TaskMarker(BaseModel):
     severity : SeverityE
     loc : TaskMarkerLoc = dc.Field(default=None)
 
-class TaskParameterSet(BaseModel):
-    type : str = None
-    task : str = None # Name of the task that produced this param set
-    seq : int  = -1 # Order in which the param-set must appear
-
 class TaskDataInput(BaseModel):
     """
-    Input data to a task
-    - name - name of the task
-    - changed - indicates whether any of this task's dependencies have changed
-    - rundir - directory in which the task is to be run
-    - params - parameters to the task
-    - inputs - list of parameter sets 'consumed' by this task
-    - memento - memento data previously returned by this task. None if no memento is available
+    Input data to a task:
+
+    * name - name of the task
+    * changed - indicates whether any of this task's dependencies have changed
+    * rundir - directory in which the task is to be run
+    * params - parameters to the task
+    * inputs - list of `TaskDataItem` that are consumed' by this task
+    * memento - memento data previously returned by this task. None if no memento is available
     """
     name : str
     changed : bool
@@ -73,12 +69,13 @@ class TaskDataInput(BaseModel):
 
 class TaskDataResult(BaseModel):
     """
-    Result data from a task
-    - changed - indicates whether the task modified its result data
-    - output - list of output parameter sets
-    - memento - memento data to be passed to the next invocation of the task
-    - markers - list of markers produced by the task
-    - status - status code (0=success, non-zero=failure)
+    Result data from a task:
+
+    * changed - indicates whether the task modified its result data
+    * output - list of output parameter sets
+    * memento - memento data to be passed to the next invocation of the task
+    * markers - list of markers produced by the task
+    * status - status code (0=success, non-zero=failure)
     """
     changed : bool = dc.Field(default=True)
     output : List[Any] = dc.Field(default_factory=list)
@@ -113,9 +110,16 @@ class TaskDataParam(BaseModel):
     ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
 
 class TaskDataItem(BaseModel):
+    """
+    Base class for task data items
+
+    * type - Name of the data item type
+    * src - Name of the task that produced this item
+    * seq - Sequence number of the item within the task
+    """
     type : str
-    src : str
-    id : str
+    src : str = None
+    seq : int = -1
 
 class TaskData(BaseModel):
     src : str = None

dv_flow/mgr/task_def.py

@@ -24,7 +24,6 @@ import enum
 from pydantic import BaseModel
 from typing import Any, Dict, List, Union, Tuple
 from .param_def import ParamDef
-from .task import Task
 from .task_output import TaskOutput
 
 @dc.dataclass

dv_flow/mgr/task_graph_builder.py

@@ -25,10 +25,21 @@ import logging
 from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .pkg_rgy import PkgRgy
-from .task import Task
-from .task_node import TaskNodeCtor
+from .task_node import TaskNode
+from .task_node_ctor import TaskNodeCtor
 from typing import Dict, List, Union
 
+@dc.dataclass
+class TaskNamespaceScope(object):
+    task_m : Dict[str,TaskNode] = dc.field(default_factory=dict)
+
+@dc.dataclass
+class CompoundTaskCtxt(object):
+    parent : 'TaskGraphBuilder'
+    task : 'TaskNode'
+    task_m : Dict[str,TaskNode] = dc.field(default_factory=dict)
+    uses_s : List[Dict[str, TaskNode]] = dc.field(default_factory=list)
+
 @dc.dataclass
 class TaskGraphBuilder(object):
     """The Task-Graph Builder knows how to discover packages and construct task graphs"""
@@ -38,8 +49,12 @@ class TaskGraphBuilder(object):
     _pkg_s : List[Package] = dc.field(default_factory=list)
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
-    _task_m : Dict['TaskSpec',Task] = dc.field(default_factory=dict)
+    _task_m : Dict['TaskSpec',TaskNode] = dc.field(default_factory=dict)
     _override_m : Dict[str,str] = dc.field(default_factory=dict)
+    _ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
+    _compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
+    _uses_count : int = 0
+
     _logger : logging.Logger = None
 
     def __post_init__(self):
@@ -94,13 +109,85 @@ class TaskGraphBuilder(object):
     def package(self):
         return self._pkg_s[-1]
 
-    def mkTaskGraph(self, task : str) -> Task:
+    def enter_uses(self):
+        self._uses_count += 1
+
+    def in_uses(self):
+        return (self._uses_count > 0)
+    
+    def leave_uses(self):
+        self._uses_count -= 1
+    
+    def enter_compound(self, task : TaskNode):
+        self._compound_task_ctxt_s.append(CompoundTaskCtxt(parent=self, task=task))
+
+    def get_name_prefix(self):
+        if len(self._compound_task_ctxt_s) > 0:
+            # Use the compound scope name
+            name = ".".join(c.task.name for c in self._compound_task_ctxt_s)
+        else:
+            name = self._pkg_s[-1].name
+
+        return name
+
+    def enter_compound_uses(self):
+        self._compound_task_ctxt_s[-1].uses_s.append({})
+
+    def leave_compound_uses(self):
+        if len(self._compound_task_ctxt_s[-1].uses_s) > 1:
+            # Propagate the items up the stack, appending 'super' to 
+            # the names
+            for k,v in self._compound_task_ctxt_s[-1].uses_s[-1].items():
+                self._compound_task_ctxt_s[-1].uses[-2]["super.%s" % k] = v
+        else:
+            # Propagate the items to the compound namespace, appending
+            # 'super' to the names
+            for k,v in self._compound_task_ctxt_s[-1].uses_s[-1].items():
+                self._compound_task_ctxt_s[-1].task_m["super.%s" % k] = v
+        self._compound_task_ctxt_s[-1].uses_s.pop()
+
+    def is_compound_uses(self):
+        return len(self._compound_task_ctxt_s) > 0 and len(self._compound_task_ctxt_s[-1].uses_s) != 0
+
+    def addTask(self, name, task : TaskNode):
+        self._logger.debug("--> addTask: %s" % name)
+        if len(self._compound_task_ctxt_s) == 0:
+            self._task_m[name] = task
+        else:
+            if len(self._compound_task_ctxt_s[-1].uses_s) > 0:
+                self._compound_task_ctxt_s[-1].uses_s[-1][name] = task
+            else:
+                self._compound_task_ctxt_s[-1].task_m[name] = task
+        self._logger.debug("<-- addTask: %s" % name)
+
+    def findTask(self, name):
+        task = None
+
+        if len(self._compound_task_ctxt_s) > 0:
+            if len(self._compound_task_ctxt_s[-1].uses_s) > 0:
+                if name in self._compound_task_ctxt_s[-1].uses_s[-1].keys():
+                    task = self._compound_task_ctxt_s[-1].uses_s[-1][name]
+            if task is None and name in self._compound_task_ctxt_s[-1].task_m.keys():
+                task = self._compound_task_ctxt_s[-1].task_m[name]
+        if task is None and name in self._task_m.keys():
+            task = self._task_m[name]
+        
+        # if task is None:
+        #     # TODO: Look for a def that hasn't yet been constructed
+        #     task = self._mkTaskGraph(name, self.rundir)
+
+        return task
+
+    def leave_compound(self, task : TaskNode):
+        self._compound_task_ctxt_s.pop()
+
+    def mkTaskGraph(self, task : str) -> TaskNode:
         self._pkg_s.clear()
         self._task_m.clear()
 
         return self._mkTaskGraph(task, self.rundir)
         
-    def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
+    def _mkTaskGraph(self, task : str, parent_rundir : str) -> TaskNode:
 
         elems = task.split(".")
 
@@ -145,6 +232,7 @@ class TaskGraphBuilder(object):
             raise Exception("ctor %s returned None for params" % str(ctor_t))
 
         task = ctor_t.mkTaskNode(
+            builder=self,
             params=params,
             name=task,
             needs=needs)
@@ -236,15 +324,34 @@ class TaskGraphBuilder(object):
                     self._logger.debug("Overriding package %s with %s" % (pkg, self._override_m[pkg]))
                     task_t = self._override_m[pkg] + "." + tname
 
+        dot_idx = task_t.rfind(".")
+        pkg = task_t[0:dot_idx]
+        self._pkg_s.append(self.getPackage(PackageSpec(pkg)))
 
         ctor = self.getTaskCtor(task_t)
-        self._logger.debug("ctor: %s" % ctor.name)
-        params = ctor.mkTaskParams(kwargs)
-        ret = ctor.mkTaskNode(
-            params=params,
-            name=name, 
-            srcdir=srcdir, 
-            needs=needs)
+        if ctor is not None:
+            if needs is None:
+                needs = []
+            for need_def in ctor.getNeeds():
+                # Resolve the full name of the need
+                need_fullname = self._resolveNeedRef(need_def)
+                self._logger.debug("Searching for qualifed-name task %s" % need_fullname)
+                if not need_fullname in self._task_m.keys():
+                    need_t = self._mkTaskGraph(need_fullname, self.rundir)
+                    self._task_m[need_fullname] = need_t
+                needs.append(self._task_m[need_fullname])
+
+            self._logger.debug("ctor: %s" % ctor.name)
+            params = ctor.mkTaskParams(kwargs)
+            ret = ctor.mkTaskNode(
+                self,
+                params=params,
+                name=name, 
+                srcdir=srcdir, 
+                needs=needs)
+        else:
+            raise Exception("Failed to find ctor for task %s" % task_t)
+        self._pkg_s.pop()
         self._logger.debug("<-- mkTaskNode: %s" % task_t)
         return ret
         
@@ -275,3 +382,5 @@ class TaskGraphBuilder(object):
 
         self._logger.debug("--> getTaskCtor %s" % spec.name)
         return ctor
+    
+

dv_flow/mgr/task_graph_dot_writer.py

@@ -0,0 +1,78 @@
+import dataclasses as dc
+import logging
+import sys
+from typing import ClassVar, Dict, TextIO
+from .task_node import TaskNode
+from .task_node_compound import TaskNodeCompound
+
+@dc.dataclass
+class TaskGraphDotWriter(object):
+    fp : TextIO = dc.field(default=None)
+    _ind : str = ""
+    _node_id_m : Dict[TaskNode, str] = dc.field(default_factory=dict)
+    _node_id : int = 1
+    _cluster_id : int = 1
+    _log : ClassVar = logging.getLogger("TaskGraphDotWriter")
+
+    def write(self, node, filename):
+        self._log.debug("--> TaskGraphDotWriter::write")
+
+        if filename == "-":
+            self.fp = sys.stdout
+        else:
+            self.fp = open(filename, "w")
+        self.println("digraph G {")
+        self.process_node(node)
+        self.println("}")
+
+        self.fp.close()
+        self._log.debug("<-- TaskGraphDotWriter::write")
+
+    def process_node(self, node):
+        self._log.debug("--> process_node %s (%d)" % (node.name, len(node.needs),))
+        node_id = self._node_id
+        self._node_id += 1
+        node_name = "n%d" % self._node_id
+        self._node_id_m[node] = node_name
+
+        if isinstance(node, TaskNodeCompound):
+            self.println("subgraph cluster_%d {" % self._cluster_id)
+            self._cluster_id += 1
+            self.inc_ind()
+            self.println("label=\"%s\";" % node.name)
+            self.println("color=blue;")
+            self.println("style=dashed;")
+            self.process_node(node.input)
+
+            self.println("%s[label=\"%s.out\"];" % (
+                node_name,
+                node.name))
+        else:
+            self.println("%s[label=\"%s\"];" % (
+                node_name,
+                node.name))
+
+        for dep in node.needs:
+            if dep[0] not in self._node_id_m.keys():
+                self.process_node(dep[0])
+            self.println("%s -> %s;" % (
+                self._node_id_m[dep[0]],
+                self._node_id_m[node]))
+            
+        if isinstance(node, TaskNodeCompound):
+            self.dec_ind()
+            self.println("}")
+
+        self._log.debug("<-- process_node %s (%d)" % (node.name, len(node.needs),))
+
+    def println(self, l):
+        self.fp.write("%s%s\n" % (self._ind, l))
+    
+    def inc_ind(self):
+        self._ind += "  "
+    
+    def dec_ind(self):
+        if len(self._ind) > 4:
+            self._ind = self._ind[4:]
+        else:
+            self._ind = ""

dv_flow/mgr/task_node.py

@@ -27,9 +27,7 @@ import pydantic.dataclasses as pdc
 import logging
 import toposort
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
-from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
-from .task_params_ctor import TaskParamsCtor
-from .param_ref_eval import ParamRefEval
+from .task_data import TaskDataOutput, TaskDataResult
 from .param import Param
 
 class RundirE(enum.Enum):
@@ -43,9 +41,7 @@ class TaskNode(object):
     name : str
     srcdir : str
     # This can be the resolved parameters
-    params : TaskParamsCtor 
-
-    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
+    params : Any
 
     # Runtime fields -- these get populated during execution
     changed : bool = False
@@ -73,147 +69,7 @@ class TaskNode(object):
                   runner,
                   rundir,
                   memento : Any = None) -> 'TaskDataResult':
-        self._log.debug("--> do_run: %s" % self.name)
-        changed = False
-        for dep,_ in self.needs:
-            changed |= dep.changed
-
-        self.rundir = rundir
-
-        # TODO: Form dep-map from inputs
-
-        dep_m = {}
-        for need,block in self.needs:
-            self._log.debug("dep %s dep_m: %s" % (need.name, str(dep_m)))
-            if not block:
-                for subdep in need.output.dep_m.keys():
-                    if subdep not in dep_m.keys():
-                        dep_m[subdep] = []
-                    for dep in need.output.dep_m[subdep]:
-                        if dep not in dep_m[subdep]:
-                            dep_m[subdep].append(dep)
-        self._log.debug("input dep_m: %s %s" % (self.name, str(dep_m)))
-
-        sorted = toposort.toposort(dep_m)
-
-        in_params_m = {}
-        added_srcs = set()
-        for need,block in self.needs:
-            if not block:
-                for p in need.output.output:
-                    # Avoid adding parameters from a single task more than once
-                    if p.src not in added_srcs:
-                        added_srcs.add(p.src)
-                        if p.src not in in_params_m.keys():
-                            in_params_m[p.src] = []
-                        in_params_m[p.src].append(p)
-
-        # in_params holds parameter sets ordered by dependency
-        in_params = []
-        for sorted_s in sorted:
-            self._log.debug("sorted_s: %s" % str(sorted_s))
-            for dep in sorted_s:
-                if dep in in_params_m.keys():
-                    self._log.debug("(%s) Extend with: %s" % (dep, str(in_params_m[dep])))
-                    in_params.extend(in_params_m[dep])
-
-        self._log.debug("in_params[1]: %s" % ",".join(p.src for p in in_params))
-
-        # Create an evaluator for substituting param values
-        eval = ParamRefEval()
-
-        self._log.debug("in_params[2]: %s" % ",".join(p.src for p in in_params))
-        eval.setVar("in", in_params)
-        eval.setVar("rundir", rundir)
-
-        # Set variables from the inputs
-        for need in self.needs:
-            for name,value in {"rundir" : need[0].rundir}.items():
-                eval.setVar("%s.%s" % (need[0].name, name), value)
-
-        # Default inputs is the list of parameter sets that match 'consumes'
-        inputs = []
-        if self.consumes is not None and len(self.consumes):
-            for in_p in in_params:
-                if self._matches(in_p, self.consumes):
-                    inputs.append(in_p)
-
-        for name,field in self.params.model_fields.items():
-            value = getattr(self.params, name)
-            if type(value) == str:
-                if value.find("${{") != -1:
-                    new_val = eval.eval(value)
-                    self._log.debug("Param %s: Evaluate expression \"%s\" => \"%s\"" % (name, value, new_val))
-                    setattr(self.params, name, new_val)
-            elif isinstance(value, list):
-                for i,elem in enumerate(value):
-                    if elem.find("${{") != -1:
-                        new_val = eval.eval(elem)
-                        value[i] = new_val
-
-        input = TaskDataInput(
-            name=self.name,
-            changed=changed,
-            srcdir=self.srcdir,
-            rundir=rundir,
-            params=self.params,
-            inputs=inputs,
-            memento=memento)
-
-        self._log.debug("--> Call task method %s" % str(self.task))
-        self.result : TaskDataResult = await self.task(self, input)
-        self._log.debug("<-- Call task method %s" % str(self.task))
-
-        output=self.result.output.copy()
-        for out in output:
-            out.src = self.name
-
-        self._log.debug("output[1]: %s" % str(output))
-
-        # Pass-through all dependencies
-        # Add an entry for ourselves
-        dep_m[self.name] = list(need.name for need,_ in self.needs)
-
-        if self.passthrough:
-            self._log.debug("passthrough: %s" % self.name)
-
-            if self.consumes is None and len(self.consumes):
-                self._log.debug("Propagating all input parameters to output")
-                for need,block in self.needs:
-                    if not block:
-                        output.extend(need.output.output)
-            else:
-                # Filter out parameter sets that were consumed
-                self._log.debug("Propagating non-consumed input parameters to output")
-                self._log.debug("consumes: %s" % str(self.consumes))
-                for need,block in self.needs:
-                    if not block:
-                        for out in need.output.output:
-                            if not self._matches(out, self.consumes):
-                                self._log.debug("Propagating type %s from %s" % (
-                                    getattr(out, "type", "<unknown>"),
-                                    getattr(out, "src", "<unknown>")))
-                                output.append(out)
-        else:
-            self._log.debug("non-passthrough: %s (only local outputs propagated)" % self.name)
-            # empty dependency map
-#            dep_m = {
-#                self.name : []
-#            }
-
-        self._log.debug("output dep_m: %s %s" % (self.name, str(dep_m)))
-        self._log.debug("output[2]: %s" % str(output))
-
-        # Store the result
-        self.output = TaskDataOutput(
-            changed=self.result.changed,
-            dep_m=dep_m,
-            output=output)
-
-        # TODO: 
-        self._log.debug("<-- do_run: %s" % self.name)
-
-        return self.result
+        pass
 
     def __hash__(self):
         return id(self)
@@ -240,186 +96,7 @@ class TaskNode(object):
                 break
         self._log.debug("<-- _matches: %s %s" % (self.name, consumed))
         return consumed
-
-    
-
-@dc.dataclass
-class TaskNodeCtor(object):
-    """
-    Factory for a specific task type
-    - Produces a task parameters object, applying value-setting instructions
-    - Produces a TaskNode
-    """
-    name : str
-    srcdir : str
-    paramT : Any
-    passthrough : bool
-    consumes : List[Any]
-
-    def __call__(self, 
-                 name=None,
-                 srcdir=None,
-                 params=None,
-                 needs=None,
-                 passthrough=None,
-                 consumes=None,
-                 **kwargs):
-        """Convenience method for direct creation of tasks"""
-        if params is None:
-            params = self.mkTaskParams(kwargs)
-        
-        node = self.mkTaskNode(
-            srcdir=srcdir, 
-            params=params, 
-            name=name, 
-            needs=needs)
-        if passthrough is not None:
-            node.passthrough = passthrough
-        else:
-            node.passthrough = self.passthrough
-        if consumes is not None:
-            if node.consumes is None:
-                node.consumes = consumes
-            else:
-                node.consumes.extend(consumes)
-        else:
-            if node.consumes is None:
-                node.consumes = self.consumes
-            else:
-                node.consumes.extend(consumes)
-
-        return node
-
-    def getNeeds(self) -> List[str]:
-        return []
-
-    def mkTaskNode(self,
-                   params,
-                   srcdir=None,
-                   name=None,
-                   needs=None) -> TaskNode:
-        raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
-
-    def mkTaskParams(self, params : Dict = None) -> Any:
-        obj = self.paramT()
-
-        # Apply user-specified params
-        if params is not None:
-            for key,value in params.items():
-                if not hasattr(obj, key):
-                    raise Exception("Parameters class %s does not contain field %s" % (
-                        str(type(obj)),
-                        key))
-                else:
-                    if isinstance(value, Param):
-                        if value.append is not None:
-                            ex_value = getattr(obj, key, [])
-                            ex_value.extend(value.append)
-                            setattr(obj, key, ex_value)
-                        elif value.prepend is not None:
-                            ex_value = getattr(obj, key, [])
-                            value = value.copy()
-                            value.extend(ex_value)
-                            setattr(obj, key, value)
-                            pass
-                        else:
-                            raise Exception("Unhandled value spec: %s" % str(value))
-                    else:
-                        setattr(obj, key, value)
-        return obj
-
-@dc.dataclass
-class TaskNodeCtorDefBase(TaskNodeCtor):
-    """Task defines its own needs, that will need to be filled in"""
-    needs : List['str']
-
-    def __post_init__(self):
-        if self.needs is None:
-            self.needs = []
-
-    def getNeeds(self) -> List[str]:
-        return self.needs
-
-@dc.dataclass
-class TaskNodeCtorProxy(TaskNodeCtorDefBase):
-    """Task has a 'uses' clause, so we delegate creation of the node"""
-    uses : TaskNodeCtor
-
-    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
-        if srcdir is None:
-            srcdir = self.srcdir
-        node = self.uses.mkTaskNode(params=params, srcdir=srcdir, name=name, needs=needs)
-        node.passthrough = self.passthrough
-        node.consumes = self.consumes
-        return node
     
-@dc.dataclass
-class TaskNodeCtorTask(TaskNodeCtorDefBase):
-    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
 
-    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
-        if srcdir is None:
-            srcdir = self.srcdir
-
-        node = TaskNode(name, srcdir, params, self.task, needs=needs)
-        node.passthrough = self.passthrough
-        node.consumes = self.consumes
-        node.task = self.task
-
-        return node
-
-@dc.dataclass
-class TaskNodeCtorWrapper(TaskNodeCtor):
-    T : Any
-
-
-
-    def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
-        node = TaskNode(name, srcdir, params, self.T, needs=needs)
-        node.passthrough = self.passthrough
-        node.consumes = self.consumes
-        return node
-
-    def mkTaskParams(self, params : Dict = None) -> Any:
-        obj = self.paramT()
-
-        # Apply user-specified params
-        for key,value in params.items():
-            if not hasattr(obj, key):
-                raise Exception("Parameters class %s does not contain field %s" % (
-                    str(type(obj)),
-                    key))
-            else:
-                if isinstance(value, Param):
-                    if value.append is not None:
-                        ex_value = getattr(obj, key, [])
-                        ex_value.extend(value.append)
-                        setattr(obj, key, ex_value)
-                    elif value.prepend is not None:
-                        ex_value = getattr(obj, key, [])
-                        value = value.copy()
-                        value.extend(ex_value)
-                        setattr(obj, key, value)
-                        pass
-                    else:
-                        raise Exception("Unhandled value spec: %s" % str(value))
-                else:
-                    setattr(obj, key, value)
-        return obj
-    
-def task(paramT,passthrough=False,consumes=None):
-    """Decorator to wrap a task method as a TaskNodeCtor"""
-    def wrapper(T):
-        task_mname = T.__module__
-        task_module = sys.modules[task_mname]
-        ctor = TaskNodeCtorWrapper(
-            name=T.__name__, 
-            srcdir=os.path.dirname(os.path.abspath(task_module.__file__)), 
-            paramT=paramT,
-            passthrough=passthrough,
-            consumes=consumes,
-            T=T)
-        return ctor
-    return wrapper