dv-flow-mgr 0.0.1.13644197178a1__py3-none-any.whl → 0.0.1.13657597614a1__py3-none-any.whl

dv_flow/mgr/package_def.py

@@ -36,7 +36,7 @@ from .package_import_spec import PackageImportSpec, PackageSpec
 from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
 from .task_ctor import TaskCtor
 from .task_def import TaskDef, TaskSpec
-from .std.task_null import TaskNull
+from .std.task_null import TaskNull, TaskNullParams
 from .type_def import TypeDef
 
 
@@ -122,96 +122,13 @@ class PackageDef(BaseModel):
             ctor_t = tasks_m[task_name][2]
         return ctor_t
 
-    def handleParams(self, task, ctor_t):
-        self._log.debug("--> handleParams %s params=%s" % (task.name, str(task.params)))
-
-        if task.params is not None and len(task.params) > 0:
-            decl_params = False
-
-            # First, add in a parameter-setting stage 
-            ctor_t = TaskCtorParam(
-                name=ctor_t.name,
-                uses=ctor_t,
-                srcdir=ctor_t.srcdir)
-#            ctor_t.params.update(task.params)
-
-            for value in task.params.values():
-                self._log.debug("value: %s" % str(value))
-                if type(value) == dict and "type" in value.keys():
-                    decl_params = True
-                    break
-
-            field_m = {}
-            # First, add parameters from the base class
-            base_o = ctor_t.mkParams()
-            for fname,info in base_o.model_fields.items():
-                self._log.debug("Field: %s (%s)" % (fname, info.default))
-                field_m[fname] = (info.annotation, info.default)
-
-            if decl_params:
-                self._log.debug("Type declares new parameters")
-                # We need to combine base parameters with new parameters
-
-                for p in task.params.keys():
-                    param = task.params[p]
-                    self._log.debug("param: %s" % str(param))
-                    if type(param) == dict and "type" in param.keys():
-                        ptype_s = param["type"]
-                        if ptype_s not in ptype_m.keys():
-                            raise Exception("Unknown type %s" % ptype_s)
-                        ptype = ptype_m[ptype_s]
-
-                        if p in field_m.keys():
-                            raise Exception("Duplicate field %s" % p)
-                        if "value" in param.keys():
-                            field_m[p] = (ptype, param["value"])
-                        else:
-                            field_m[p] = (ptype, pdflt_m[ptype_s])
-                    else:
-                        if p not in field_m.keys():
-                            raise Exception("Field %s not found" % p)
-                        if type(param) != dict:
-                            value = param
-                        elif "value" in param.keys():
-                            value = param["value"]
-                        else:
-                            raise Exception("No value specified for param %s: %s" % (
-                                p, str(param)))
-                        field_m[p] = (field_m[p][0], value)
-                self._log.debug("field_m: %s" % str(field_m))
-                param_t = pydantic.create_model(
-                    "Task%sParams" % task.name, **field_m)
-                ctor_t = TaskCtorParamCls(
-                    name=ctor_t.name,
-                    uses=ctor_t,
-                    params_ctor=param_t)
-            else: # no new parameters declared
-                self._log.debug("Type only overrides existing parameters")
-                for p in task.params.keys():
-                    param = task.params[p]
-                    if p not in field_m.keys():
-                        raise Exception("Field %s not found" % p)
-                    if type(param) != dict:
-                        value = param
-                    elif "value" in param.keys():
-                        value = param["value"]
-                    else:
-                        raise Exception("No value specified for param %s: %s" % (
-                            p, str(param)))
-                    field_m[p] = (field_m[p][0], value)
-                    self._log.debug("Set param=%s to %s" % (p, str(value)))
-                    ctor_t.params[p] = value
-
-        self._log.debug("<-- handleParams %s" % task.name)
-
-        return ctor_t
-
     def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
         self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
         base_ctor_t : TaskCtor = None
         ctor_t : TaskCtor = None
         base_params : BaseModel = None
         callable = None
+        passthrough = False
         needs = [] if task.needs is None else task.needs.copy()
 
         if task.uses is not None:
@@ -239,7 +156,7 @@ class PackageDef(BaseModel):
                     modname, self.basedir, str(e)))
                 
             if not hasattr(mod, clsname):
-                raise Exception("Class %s not found in module %s" % (clsname, modname))
+                raise Exception("Method %s not found in module %s" % (clsname, modname))
             callable = getattr(mod, clsname)
 
         # Determine if we need to use a new 
@@ -250,6 +167,7 @@ class PackageDef(BaseModel):
                 name=task.name,
                 srcdir=srcdir,
                 paramT=paramT, # TODO: need to determine the parameter type
+                passthrough=passthrough,
                 needs=needs, # TODO: need to determine the needs
                 task=callable)
         elif base_ctor_t is not None:
@@ -258,18 +176,18 @@ class PackageDef(BaseModel):
                 name=task.name,
                 srcdir=srcdir,
                 paramT=paramT, # TODO: need to determine the parameter type
+                passthrough=passthrough,
                 needs=needs,
                 uses=base_ctor_t)
         else:
             self._log.debug("Use 'Null' as the class implementation")
-            ctor_t = TaskCtorCls(
+            ctor_t = TaskNodeCtorTask(
                 name=task.name,
-                task_ctor=TaskNull,
-                srcdir=srcdir)
-
-# TODO:
-#        ctor_t = self.handleParams(task, ctor_t)
-#        ctor_t.depends.extend(task.depends)
+                srcdir=srcdir,
+                paramT=TaskNullParams,
+                passthrough=passthrough,
+                needs=needs,
+                task=TaskNull)
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t

dv_flow/mgr/std/exec.py

@@ -0,0 +1,19 @@
+import asyncio
+import logging
+from dv_flow.mgr import TaskDataResult
+
+_log = logging.getLogger("Exec")
+
+async def Exec(runner, input) -> TaskDataResult:
+    _log.debug("TaskExec run: %s: cmd=%s" % (input.name, input.params.command))
+
+
+    proc = await asyncio.create_subprocess_shell(
+        input.params.command,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE)
+    
+    await proc.wait()
+
+    return TaskDataResult()
+

dv_flow/mgr/std/fileset.py

@@ -4,75 +4,72 @@ import fnmatch
 import glob
 import logging
 import pydantic.dataclasses as dc
+from pydantic import BaseModel
 from typing import ClassVar, List, Tuple
-from dv_flow.mgr import Task, TaskData, TaskMemento
+from dv_flow.mgr import TaskDataResult
 from dv_flow.mgr import FileSet as _FileSet
 
-class TaskFileSetMemento(TaskMemento):
+class TaskFileSetMemento(BaseModel):
     files : List[Tuple[str,float]] = dc.Field(default_factory=list)
 
-class FileSet(Task):
+_log = logging.getLogger("FileSet")
 
-    _log : ClassVar = logging.getLogger("FileSet")
+async def FileSet(runner, input) -> TaskDataResult:
+    _log.debug("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
+        input.name,
+        input.srcdir,
+        input.params.base, input.params.type, str(input.params.include)
+    ))
 
-    async def run(self, input : TaskData) -> TaskData:
-        self._log.debug("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
-            self.name,
-            self.srcdir,
-            self.params.base, self.params.type, str(self.params.include)
-        ))
 
+    changed = False
+    ex_memento = input.memento
+    memento = TaskFileSetMemento()
 
-        ex_memento = self.getMemento(TaskFileSetMemento)
-        memento = TaskFileSetMemento()
+    _log.debug("ex_memento: %s" % str(ex_memento))
+    _log.debug("params: %s" % str(input.params))
 
-        self._log.debug("ex_memento: %s" % str(ex_memento))
-        self._log.debug("params: %s" % str(self.params))
+    if input.params is not None:
+        glob_root = os.path.join(input.srcdir, input.params.base)
+        glob_root = glob_root.strip()
 
-        if self.params is not None:
-            glob_root = os.path.join(self.srcdir, self.params.base)
-            glob_root = glob_root.strip()
+        if glob_root[-1] == '/' or glob_root == '\\':
+            glob_root = glob_root[:-1]
 
-            if glob_root[-1] == '/' or glob_root == '\\':
-                glob_root = glob_root[:-1]
+        _log.debug("glob_root: %s" % glob_root)
 
-            self._log.debug("glob_root: %s" % glob_root)
-
-            fs = _FileSet(
-                src=self.name, 
-                type=self.params.type,
+        fs = _FileSet(
+                src=input.name, 
+                type=input.params.type,
                 basedir=glob_root)
 
-            if not isinstance(self.params.include, list):
-                self.params.include = [self.params.include]
-
-            included_files = []
-            for pattern in self.params.include:
-                included_files.extend(glob.glob(os.path.join(glob_root, pattern), recursive=False))
-
-            self._log.debug("included_files: %s" % str(included_files))
-
-            for file in included_files:
-                if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in self.params.exclude):
-                    memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))
-                    fs.files.append(file[len(glob_root)+1:])
-
-        # Check to see if the filelist or fileset have changed
-        # Only bother doing this if the upstream task data has not changed
-        if ex_memento is not None and not input.changed:
-            ex_memento.files.sort(key=lambda x: x[0])
-            memento.files.sort(key=lambda x: x[0])
-            self._log.debug("ex_memento.files: %s" % str(ex_memento.files))
-            self._log.debug("memento.files: %s" % str(memento.files))
-            input.changed = ex_memento != memento
-        else:
-            input.changed = True
-
-        self.setMemento(memento)
-
-        if fs is not None:
-            input.addFileSet(fs)
-
-        return input
-
-    pass
+        if not isinstance(input.params.include, list):
+            input.params.include = [input.params.include]
+
+        included_files = []
+        for pattern in input.params.include:
+            included_files.extend(glob.glob(os.path.join(glob_root, pattern), recursive=False))
+
+        _log.debug("included_files: %s" % str(included_files))
+
+        for file in included_files:
+            if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in input.params.exclude):
+                memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))
+                fs.files.append(file[len(glob_root)+1:])
+
+    # Check to see if the filelist or fileset have changed
+    # Only bother doing this if the upstream task data has not changed
+    if ex_memento is not None and not input.changed:
+        ex_memento.files.sort(key=lambda x: x[0])
+        memento.files.sort(key=lambda x: x[0])
+        _log.debug("ex_memento.files: %s" % str(ex_memento.files))
+        _log.debug("memento.files: %s" % str(memento.files))
+        changed = ex_memento != memento
+    else:
+        changed = True
+
+    return TaskDataResult(
+        memento=memento,
+        changed=changed,
+        output=[fs]
+    )

dv_flow/mgr/std/flow.dv

@@ -1,16 +1,19 @@
 
+# yaml-language-server: $schema=https://dv-flow.github.io/dv-flow.schema.json
+
 package:
   name: std
 
   tasks:
   - name: Message
-    pyclass: dv_flow.mgr.std.message.Message
+    pytask: dv_flow.mgr.std.message.Message
     with:
       msg:
         type: str
         value: ""
   - name: FileSet
-    pyclass: dv_flow.mgr.std.fileset.FileSet
+    pytask: dv_flow.mgr.std.fileset.FileSet
+    passthrough: true
     with:
       base:
         type: str
@@ -24,9 +27,12 @@ package:
       exclude:
         type: str
         value: ""
-#  - name: Exec
-#    pyclass: dv_flow.mgr.std.exec.Exec
-#    with: {}
+  - name: Exec
+    pytask: dv_flow.mgr.std.exec.Exec
+    with: 
+      command:
+        type: str
+        value: ""
   types:
   # - name: TaskDataItem
   #   doc: |

dv_flow/mgr/std/message.py

@@ -1,7 +1,6 @@
 
-from dv_flow.mgr import Task, TaskData
+from dv_flow.mgr import Task, TaskDataResult
 
-class Message(Task):
-    async def run(self, input : TaskData) -> TaskData:
-        print("%s: %s" % (self.name, self.params.msg))
-        return input
+async def Message(runner, input) -> TaskDataResult:
+        print("%s: %s" % (input.name, input.params.msg))
+        return TaskDataResult()

dv_flow/mgr/std/task_null.py

@@ -1,11 +1,9 @@
-from ..task import Task
-from ..task_data import TaskData
+from pydantic import BaseModel
+from ..task_data import TaskDataResult
 
-class TaskNull(Task):
-    """The Null task simply propagates its input to its output"""
+class TaskNullParams(BaseModel):
+    pass
 
-    async def run(self, input : TaskData) -> TaskData:
-        # No memento ; data pass-through
-        self._log.debug("%s: TaskNull.run" % self.name)
-        return input
+async def TaskNull(runner, input) -> TaskDataResult:
+    return TaskDataResult()
 

dv_flow/mgr/task_data.py

@@ -42,6 +42,7 @@ class TaskParameterSet(BaseModel):
     seq : int  = -1 # Order in which the param-set must appear
 
 class TaskDataInput(BaseModel):
+    name : str
     changed : bool
     srcdir : str
     rundir : str

dv_flow/mgr/task_def.py

@@ -40,6 +40,7 @@ class TaskDef(BaseModel):
     doc : str = dc.Field(default="")
     needs : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Union[str,ParamDef]] = dc.Field(default_factory=dict, alias="with")
+    passthrough: bool = dc.Field(default=False)
 #    out: List[TaskOutput] = dc.Field(default_factory=list)
 
     def copy(self) -> 'TaskDef':

dv_flow/mgr/task_graph_builder.py

@@ -26,6 +26,7 @@ from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .pkg_rgy import PkgRgy
 from .task import Task
+from .task_node import TaskNodeCtor
 from typing import Dict, List
 
 @dc.dataclass
@@ -92,18 +93,17 @@ class TaskGraphBuilder(object):
         
         self._pkg_s.append(pkg)
 
-        ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
+        ctor_t : TaskNodeCtor = pkg.getTaskCtor(task_name)
 
         self._logger.debug("ctor_t: %s" % ctor_t.name)
 
         needs = []
 
-        # for dep in ctor_t.depends:
-        #     if not dep in self._task_m.keys():
-        #         task = self._mkTaskGraph(dep, rundir)
-        #         self._task_m[dep] = task
-        #         pass
-        #     depends.append(self._task_m[dep])
+        for need_def in ctor_t.getNeeds():
+            if not need_def in self._task_m.keys():
+                task = self._mkTaskGraph(need_def, rundir)
+                self._task_m[need_def] = task
+            needs.append(self._task_m[need_def])
 
         # The returned task should have all param references resolved
         params = ctor_t.mkTaskParams()

dv_flow/mgr/task_listener_log.py

@@ -0,0 +1,15 @@
+import dataclasses as dc
+
+class TaskListenerLog(object):
+
+    def event(self, task : 'Task', reason : 'Reason'):
+        if reason == 'enter':
+            print("> Task %s" % task.name, flush=True)
+        elif reason == 'leave':
+            for m in task.result.markers:
+                print("  %s" % m)
+            print("< Task %s" % task.name, flush=True)
+        else:
+            print("- Task %s" % task.name, flush=True)
+        pass
+

dv_flow/mgr/task_node.py

@@ -22,9 +22,11 @@ class TaskNode(object):
 
     # Runtime fields -- these get populated during execution
     changed : bool = False
+    passthrough : bool = False
     needs : List['TaskNode'] = dc.field(default_factory=list)
     rundir : str = dc.field(default=None)
     output : TaskDataOutput = dc.field(default=None)
+    result : TaskDataResult = dc.field(default=None)
 
     _log : ClassVar = logging.getLogger("TaskNode")
 
@@ -46,7 +48,7 @@ class TaskNode(object):
         for need in self.needs:
             in_params.extend(need.output.output)
 
-        # TODO: create an evaluator for substituting param values
+        # Create an evaluator for substituting param values
         eval = ParamRefEval()
 
         eval.setVar("in", in_params)
@@ -66,6 +68,7 @@ class TaskNode(object):
                 raise Exception("Unhandled param type: %s" % str(value))
 
         input = TaskDataInput(
+            name=self.name,
             changed=changed,
             srcdir=self.srcdir,
             rundir=rundir,
@@ -73,7 +76,7 @@ class TaskNode(object):
             memento=memento)
 
         # TODO: notify of task start
-        ret : TaskDataResult = await self.task(self, input)
+        self.result : TaskDataResult = await self.task(self, input)
         # TODO: notify of task complete
 
         # TODO: form a dep map from the outgoing param sets
@@ -81,13 +84,13 @@ class TaskNode(object):
 
         # Store the result
         self.output = TaskDataOutput(
-            changed=ret.changed,
+            changed=self.result.changed,
             dep_m=dep_m,
-            output=ret.output.copy())
+            output=self.result.output.copy())
 
         # TODO: 
 
-        return ret
+        return self.result
 
     def __hash__(self):
         return id(self)
@@ -103,6 +106,7 @@ class TaskNodeCtor(object):
     name : str
     srcdir : str
     paramT : Any
+    passthrough : bool
 
     def getNeeds(self) -> List[str]:
         return []
@@ -163,6 +167,7 @@ class TaskNodeCtorProxy(TaskNodeCtorDefBase):
         if srcdir is None:
             srcdir = self.srcdir
         node = self.uses.mkTaskNode(params=params, srcdir=srcdir, name=name, needs=needs)
+        node.passthrough = self.passthrough
         return node
     
 @dc.dataclass
@@ -174,6 +179,7 @@ class TaskNodeCtorTask(TaskNodeCtorDefBase):
             srcdir = self.srcdir
 
         node = TaskNode(name, srcdir, params, self.task, needs=needs)
+        node.passthrough = self.passthrough
         node.task = self.task
 
         return node
@@ -187,6 +193,7 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                  srcdir=None,
                  params=None,
                  needs=None,
+                 passthrough=None,
                  **kwargs):
         """Convenience method for direct creation of tasks"""
         if params is None:
@@ -197,11 +204,16 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
             params=params, 
             name=name, 
             needs=needs)
+        if passthrough is not None:
+            node.passthrough = passthrough
+        else:
+            node.passthrough = self.passthrough
 
         return node
 
     def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
         node = TaskNode(name, srcdir, params, self.T, needs=needs)
+        node.passthrough = self.passthrough
         return node
 
     def mkTaskParams(self, params : Dict = None) -> Any:
@@ -231,16 +243,17 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                     setattr(obj, key, value)
         return obj
 
-def task(paramT):
+def task(paramT,passthrough=False):
     """Decorator to wrap a task method as a TaskNodeCtor"""
     def wrapper(T):
         task_mname = T.__module__
         task_module = sys.modules[task_mname]
         ctor = TaskNodeCtorWrapper(
-            T.__name__, 
-            os.path.dirname(os.path.abspath(task_module.__file__)), 
-            paramT,
-            T)
+            name=T.__name__, 
+            srcdir=os.path.dirname(os.path.abspath(task_module.__file__)), 
+            paramT=paramT,
+            passthrough=passthrough,
+            T=T)
         return ctor
     return wrapper
 

dv_flow/mgr/task_runner.py

@@ -1,7 +1,11 @@
 import asyncio
+import json
+import os
+import re
 import dataclasses as dc
+import logging
 from toposort import toposort
-from typing import Any, Callable, List, Tuple, Union
+from typing import Any, Callable, ClassVar, List, Tuple, Union
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
 from .task_node import TaskNode
 
@@ -12,6 +16,15 @@ class TaskRunner(object):
     # List of [Listener:Callable[Task],Recurisve:bool]
     listeners : List[Tuple[Callable['Task','Reason'], bool]] = dc.field(default_factory=list)
 
+    _log : ClassVar = logging.getLogger("TaskRunner")
+
+    def add_listener(self, l, recursive=False):
+        self.listeners.append((l, recursive))
+
+    def _notify(self, task : 'Task', reason : 'Reason'):
+        for listener in self.listeners:
+            listener[0](task, reason)
+
     async def do_run(self, 
                   task : 'Task',
                   memento : Any = None) -> 'TaskDataResult':
@@ -26,17 +39,52 @@ class TaskRunner(object):
 class TaskSetRunner(TaskRunner):
     nproc : int = 8
 
+    _anon_tid : int = 1
+
+    _log : ClassVar = logging.getLogger("TaskSetRunner")
+
     async def run(self, task : Union[TaskNode,List[TaskNode]]):
+        # Ensure that the rundir exists or can be created
+
+        if not os.path.isdir(self.rundir):
+            os.makedirs(self.rundir)
+
+        if not os.path.isdir(os.path.join(self.rundir, "cache")):
+            os.makedirs(os.path.join(self.rundir, "cache"))
+
+        src_memento = None
+        dst_memento = {}
+        if os.path.isfile(os.path.join(self.rundir, "cache", "mementos.json")):
+            try:
+                with open(os.path.join(self.rundir, "cache", "mementos.json"), "r") as f:
+                    src_memento = json.load(f)
+            except Exception as e:
+                src_memento = {}
+        else:
+            src_memento = {}
+
+
         # First, build a depedency map
         tasks = task if isinstance(task, list) else [task]
         dep_m = {}
+        self._anon_tid = 1
         for t in tasks:
             self._buildDepMap(dep_m, t)
 
-        print("dep_m: %s" % str(dep_m))
+        if self._log.isEnabledFor(logging.DEBUG):
+            self._log.debug("Deps:")
+            for t,value in dep_m.items():
+                self._log.debug("  Task: %s", str(t.name))
+                for v in value:
+                    self._log.debug("  - %s", str(v.name))
 
         order = list(toposort(dep_m))
 
+        if self._log.isEnabledFor(logging.DEBUG):
+            self._log.debug("Order:")
+            for active_s in order:
+                self._log.debug("- {%s}", ",".join(t.name for t in active_s))
+
         active_task_l = []
         done_task_s = set()
         for active_s in order:
@@ -49,25 +97,56 @@ class TaskSetRunner(TaskRunner):
                         for i in range(len(active_task_l)):
                             if active_task_l[i][1] == d:
                                 tt = active_task_l[i][0]
+                                if tt.result.memento is not None:
+                                    dst_memento[tt.name] = tt.result.memento.model_dump()
+                                else:
+                                    dst_memento[tt.name] = None
+                                self._notify(tt, "leave")
                                 done_task_s.add(tt)
                                 active_task_l.pop(i)
                                 break
                 if t not in done_task_s:
+                    memento = src_memento.get(t.name, None)
+                    dirname = t.name
+                    invalid_chars_pattern = r'[\/:*?"<>|#%&{}\$\\!\'`;=@+]'
+                    # Replace invalid characters with the replacement string.
+                    dirname = re.sub(invalid_chars_pattern, '_', dirname)
+
+                    rundir = os.path.join(self.rundir, dirname)
+
+                    if not os.path.isdir(rundir):
+                        os.makedirs(rundir, exist_ok=True)
+
+                    self._notify(t, "enter")
                     coro = asyncio.Task(t.do_run(
                         self,
-                        self.rundir, # TODO
-                        None)) # TODO: memento
+                        rundir,
+                        memento)) 
                     active_task_l.append((t, coro))
                
             # Now, wait for tasks to complete
             if len(active_task_l):
+                # TODO: Shouldn't gather here -- reach to each completion
                 coros = list(at[1] for at in active_task_l)
                 res = await asyncio.gather(*coros)
+                for tt in active_task_l:
+                    if tt[0].result.memento is not None:
+                        dst_memento[tt[0].name] = tt[0].result.memento.model_dump()
+                    else:
+                        dst_memento[tt[0].name] = None
+                    self._notify(tt[0], "leave")
+                active_task_l.clear()
 
+        with open(os.path.join(self.rundir, "cache", "mementos.json"), "w") as f:
+            json.dump(dst_memento, f)
 
         pass
 
     def _buildDepMap(self, dep_m, task : TaskNode):
+        if task.name is None:
+            task.name = "anon_%d" % self._anon_tid
+            self._anon_tid += 1
+
         if task not in dep_m.keys():
             dep_m[task] = set(task.needs)
             for need in task.needs:
@@ -86,8 +165,8 @@ class SingleTaskRunner(TaskRunner):
         # TODO: create an evaluator for substituting param values
         eval = None
 
-        for field in dc.fields(task.params):
-            print("Field: %s" % field.name)
+#        for field in dc.fields(task.params):
+#            print("Field: %s" % field.name)
 
         input = TaskDataInput(
             changed=changed,

{dv_flow_mgr-0.0.1.13644197178a1.dist-info → dv_flow_mgr-0.0.1.13657597614a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13644197178a1
+Version: 0.0.1.13657597614a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13644197178a1.dist-info → dv_flow_mgr-0.0.1.13657597614a1.dist-info}/RECORD

@@ -7,7 +7,7 @@ dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=878twhPD-E1pFlDNUtuyeFEgJ_Y89b560og4St-Iwrs,1679
-dv_flow/mgr/package_def.py,sha256=m3vnoEcV4jEhOg_ijwYPTplrF4YN5dVJYJhVBTL5AgM,17408
+dv_flow/mgr/package_def.py,sha256=EdKwyxqvWT8xQmALGa6BbNv0PMbDLZTr3A27L9coIzc,13761
 dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
 dv_flow/mgr/param.py,sha256=3BY-ucig6JRw73FhjyJQL-vpd57qhAzixgZ8I5FoUpw,553
 dv_flow/mgr/param_def.py,sha256=gLua-EQiY8V2CFX-2svLRIlrs8PEeGh4-EPtn4a2Mng,712
@@ -16,30 +16,32 @@ dv_flow/mgr/parsetab.py,sha256=enSOnMQ-woIsMEzHyeYiefvhAl8enxfX9Ct_o8-jkqs,3780
 dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
 dv_flow/mgr/task.py,sha256=kLQSvnVwj9ROIDtxq8lLu-4mJizTxOqvUeogmgN6QAA,5976
 dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
-dv_flow/mgr/task_data.py,sha256=9c5NrVZWECHFXrCPOpwZBQXRhr1O3ni5aFeP2I1P5Rw,11398
-dv_flow/mgr/task_def.py,sha256=gOqh-sUE7MKC1fsobFxcO2jlF-jtjBJRFzWB2iDYlZY,1831
+dv_flow/mgr/task_data.py,sha256=gzs7BfwTLPKUEpaiGMwvA3MNfebTKHv_wDFppvWHo8A,11413
+dv_flow/mgr/task_def.py,sha256=PORXrUBoynoj_oYAVISR5NW53OZevZ6hL4T7TutkkHo,1879
 dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
-dv_flow/mgr/task_graph_builder.py,sha256=iprAMw0LVROOxWa563OHg8KTEm4IOS9hZVymjDeCnEo,7243
+dv_flow/mgr/task_graph_builder.py,sha256=sswBPUZg71dFT8kaB0towbWIadhNdbTy5gLgvC2uiVA,7276
 dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
 dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
+dv_flow/mgr/task_listener_log.py,sha256=B9-LEgSF2QwcUREBoUjEsS0rRYg1_ilFHBUKlNeT5YA,444
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/task_node.py,sha256=J_i53KntyljY7gwSqBq-lK_Gbu0jryNp4Xq_jUsa0PA,7903
+dv_flow/mgr/task_node.py,sha256=iQKiDTC8Oz8tQwLzJF4NpsREg8JPXI-rju5tG7P6dfw,8448
 dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
 dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
-dv_flow/mgr/task_runner.py,sha256=JmerE6CyXiJJK7-n6P65OzwW9UTYFQqV8Cck7eHe0B8,3882
+dv_flow/mgr/task_runner.py,sha256=Xv5bPwAKV793R9W5Ksu1u_WBoRUzlpfHpGtGLXdsIlY,6940
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
 dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
 dv_flow/mgr/cmds/cmd_run.py,sha256=eths8kT7mBmpZqwOuMtpKAaux4rg-f7hPBxxTHbpKT4,2903
 dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
-dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,2710
-dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
-dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
-dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/METADATA,sha256=yOdqkLx8LGFA6PcCcaQ3eMIwC3_lM0iup5Tx0iEra4U,13276
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13644197178a1.dist-info/RECORD,,
+dv_flow/mgr/std/exec.py,sha256=ETx9xSxhdCD_iw6pcmhrafDCJ-41AneyEAPwQf3q-3w,452
+dv_flow/mgr/std/fileset.py,sha256=r2s2H45FuBhTLsjvjqn26Zb6EsR-psvV00ObMIyEGNA,2486
+dv_flow/mgr/std/flow.dv,sha256=jlFOh3xVECOzHws7x6YvJ9eCIGHM5gsPeEnheiGOukY,1553
+dv_flow/mgr/std/message.py,sha256=CWrBKImbXKe2d7hJ223U3Ifuxo54zLpFPJviE8BUJvk,188
+dv_flow/mgr/std/task_null.py,sha256=UKwUnqwFPBY8BO44ZAPcgehQB59kHZFa1qyZc1TwUqE,196
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/METADATA,sha256=HVQUNoijJOwkTY6DfEDvnWzRDQ3JVzQFeR-9brAOQus,13276
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13657597614a1.dist-info/RECORD,,