dv-flow-mgr 0.0.1.12968982426a1__py3-none-any.whl → 0.0.1.12971126211a1__py3-none-any.whl

dv_flow/mgr/__main__.py

@@ -20,10 +20,17 @@
 #*
 #****************************************************************************
 import argparse
+import logging
 from .cmds.cmd_run import CmdRun
 
 def get_parser():
     parser = argparse.ArgumentParser(description='dv_flow_mgr')
+    parser.add_argument("-d", "--debug", 
+                        help="Enable debug",
+                        action="store_true")
+    parser.add_argument("-v", "--verbose", 
+                        help="Enable verbose output",
+                        action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
     run_parser = subparsers.add_parser('run', help='run a flow')
@@ -35,6 +42,12 @@ def get_parser():
 def main():
     parser = get_parser()
     args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
     args.func(args)
 
 if __name__ == "__main__":

dv_flow/mgr/cmds/cmd_run.py

@@ -1,5 +1,7 @@
 import asyncio
 import os
+import logging
+from typing import ClassVar
 from ..task_graph_runner import TaskGraphRunner
 from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
@@ -8,6 +10,7 @@ from ..pkg_rgy import PkgRgy
 
 
 class CmdRun(object):
+    _log : ClassVar = logging.getLogger("CmdRun")
 
     def __call__(self, args):
 
@@ -17,7 +20,7 @@ class CmdRun(object):
         if pkg is None:
             raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
 
-        print("pkg: %s" % pkg.name)
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
 
         if len(args.tasks) > 0:
             pass

dv_flow/mgr/package.py

@@ -20,7 +20,8 @@
 #*
 #****************************************************************************
 import dataclasses as dc
-from typing import Any, Dict
+import logging
+from typing import Any, ClassVar, Dict
 from .task import TaskCtor
 
 @dc.dataclass
@@ -30,8 +31,10 @@ class Package(object):
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
     tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    _log : ClassVar = logging.getLogger("Package")
 
     def getTaskCtor(self, name : str) -> TaskCtor:
+        self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))
         return self.tasks[name]

dv_flow/mgr/package_def.py

@@ -32,7 +32,7 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-from .task import TaskCtor, TaskParams
+from .task import TaskCtor, TaskCtorProxy, TaskCtorCls, TaskCtorParam, TaskCtorParamCls
 from .task_def import TaskDef, TaskSpec
 from .std.task_null import TaskNull
 
@@ -90,53 +90,131 @@ class PackageDef(BaseModel):
         self._log.debug("<-- mkPackage %s" % self.name)
         return ret
     
-    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
-        ctor_t : TaskCtor = None
+    def getTaskCtor(self, session, task_name, tasks_m):
+        self._log.debug("--> getTaskCtor")
+        # Find package (not package_def) that implements this task
+        # Insert an indirect reference to that tasks's constructor
+        last_dot = task_name.rfind('.')
+
+        if last_dot != -1:
+            pkg_name = task_name[:last_dot]
+            task_name = task_name[last_dot+1:]
+        else:
+            pkg_name = None
+
+        if pkg_name is not None:
+            self._log.debug("Package-qualified 'uses'")
+            pkg = session.getPackage(PackageSpec(pkg_name))
+            if pkg is None:
+                raise Exception("Failed to find package %s" % pkg_name)
+            ctor_t = pkg.getTaskCtor(task_name)
+        else:
+            self._log.debug("Unqualified 'uses'")
+            if task_name not in tasks_m.keys():
+                raise Exception("Failed to find task %s" % task_name)
+            if len(tasks_m[task_name]) != 3:
+                raise Exception("Task %s not fully defined" % task_name)
 
-        if task.uses is not None:
-            # Find package (not package_def) that implements this task
-            # Insert an indirect reference to that tasks's constructor
-            last_dot = task.uses.rfind('.')
+            ctor_t = tasks_m[task_name][2]
+        return ctor_t
 
-            if last_dot != -1:
-                pkg_name = task.uses[:last_dot]
-                task_name = task.uses[last_dot+1:]
-            else:
-                pkg_name = None
-                task_name = task.uses
-
-            if pkg_name is not None:
-                pkg = session.getPackage(PackageSpec(pkg_name))
-                if pkg is None:
-                    raise Exception("Failed to find package %s" % pkg_name)
-                ctor_t = pkg.getTaskCtor(task_name)
-                ctor_t = ctor_t.copy()
-                ctor_t.srcdir = srcdir
-            else:
-                if task_name not in tasks_m.keys():
-                    raise Exception("Failed to find task %s" % task_name)
-                if len(tasks_m[task_name]) == 3:
-                    ctor_t = tasks_m[task_name][2].copy()
-                    ctor_t.srcdir = srcdir
-                else:
-                    task_i = tasks_m[task_name]
-                    ctor_t = self.mkTaskCtor(
-                        session, 
-                        task=task_i[0], 
-                        srcdir=srcdir,
-                        tasks_m=tasks_m)
-                    tasks_m[task_name] = ctor_t
-
-        if ctor_t is None:
-            # Provide a default implementation
-            ctor_t = TaskCtor(
-                task_ctor=TaskNull,
-                param_ctor=TaskParams,
-                srcdir=srcdir)
+    def handleParams(self, task, ctor_t):
+
+        if task.params is not None and len(task.params) > 0:
+            decl_params = False
+
+            # First, add in a parameter-setting stage 
+            ctor_t = TaskCtorParam(
+                name=ctor_t.name,
+                uses=ctor_t,
+                srcdir=ctor_t.srcdir)
+#            ctor_t.params.update(task.params)
+
+            for value in task.params.values():
+                if "type" in value:
+                    decl_params = True
+                    break
+
+            if decl_params:
+                # We need to combine base parameters with new parameters
+                field_m = {}
+                # First, add parameters from the base class
+                base_o = ctor_t.mkParams()
+                for fname,info in base_o.model_fields.items():
+                    self._log.debug("Field: %s (%s)" % (fname, info.default))
+                    field_m[fname] = (info.annotation, info.default)
+                ptype_m = {
+                    "str" : str,
+                    "int" : int,
+                    "float" : float,
+                    "bool" : bool,
+                    "list" : List
+                }
+                pdflt_m = {
+                    "str" : "",
+                    "int" : 0,
+                    "float" : 0.0,
+                    "bool" : False,
+                    "list" : []
+                }
+                for p in task.params.keys():
+                    param = task.params[p]
+                    if type(param) == dict and "type" in param.keys():
+                        ptype_s = param["type"]
+                        if ptype_s not in ptype_m.keys():
+                            raise Exception("Unknown type %s" % ptype_s)
+                        ptype = ptype_m[ptype_s]
+
+                        if p in field_m.keys():
+                            raise Exception("Duplicate field %s" % p)
+                        if "value" in param.keys():
+                            field_m[p] = (ptype, param["value"])
+                        else:
+                            field_m[p] = (ptype, pdflt_m[ptype_s])
+                    else:
+                        if p not in field_m.keys():
+                            raise Exception("Field %s not found" % p)
+                        if type(param) != dict:
+                            value = param
+                        elif "value" in param.keys():
+                            value = param["value"]
+                        else:
+                            raise Exception("No value specified for param %s: %s" % (
+                                p, str(param)))
+                        field_m[p] = (field_m[p][0], value)
+                self._log.debug("field_m: %s" % str(field_m))
+                param_t = pydantic.create_model(
+                    "Task%sParams" % task.name, **field_m)
+                ctor_t = TaskCtorParamCls(
+                    name=ctor_t.name,
+                    uses=ctor_t,
+                    params_ctor=param_t)
+            else: # no new parameters declared
+                for p in task.params.keys():
+                    param = task.params[p]
+                    if p not in field_m.keys():
+                        raise Exception("Field %s not found" % p)
+                    if type(param) != dict:
+                        value = param
+                    elif "value" in param.keys():
+                        value = param["value"]
+                    else:
+                        raise Exception("No value specified for param %s: %s" % (
+                            p, str(param)))
+                    field_m[p] = (field_m[p][0], value)
+                ctor_t.params[p] = value
 
+        return ctor_t
+
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+        self._log.debug("--> %s::mkTaskCtor %s" % (self.name, task.name))
+        ctor_t : TaskCtor = None
+
+        # Determine the implementation constructor first
         if task.pyclass is not None:
             # Built-in impl
             # Now, lookup the class
+            self._log.debug("Use PyClass implementation")
             last_dot = task.pyclass.rfind('.')
             clsname = task.pyclass[last_dot+1:]
             modname = task.pyclass[:last_dot]
@@ -154,72 +232,36 @@ class PackageDef(BaseModel):
                 
             if not hasattr(mod, clsname):
                 raise Exception("Class %s not found in module %s" % (clsname, modname))
-            ctor_t.task_ctor = getattr(mod, clsname)
-
-            if task.uses is None:
-                ctor_t.param_ctor = TaskParams
-
-        decl_params = False
-        for value in task.params.values():
-            if "type" in value:
-                decl_params = True
-                break
-        
-        if decl_params:
-            # We need to combine base parameters with new parameters
-            field_m = {}
-            # First, add parameters from the base class
-            for fname,info in ctor_t.param_ctor.model_fields.items():
-                self._log.debug("Field: %s (%s)" % (fname, info.default))
-                field_m[fname] = (info.annotation, info.default)
-            ptype_m = {
-                "str" : str,
-                "int" : int,
-                "float" : float,
-                "bool" : bool,
-                "list" : List
-            }
-            pdflt_m = {
-                "str" : "",
-                "int" : 0,
-                "float" : 0.0,
-                "bool" : False,
-                "list" : []
-            }
-            for p in task.params.keys():
-                param = task.params[p]
-                if type(param) == dict and "type" in param.keys():
-                    ptype_s = param["type"]
-                    if ptype_s not in ptype_m.keys():
-                        raise Exception("Unknown type %s" % ptype_s)
-                    ptype = ptype_m[ptype_s]
-
-                    if p in field_m.keys():
-                        raise Exception("Duplicate field %s" % p)
-                    if "value" in param.keys():
-                        field_m[p] = (ptype, param["value"])
-                    else:
-                        field_m[p] = (ptype, pdflt_m[ptype_s])
-                else:
-                    if p not in field_m.keys():
-                        raise Exception("Field %s not found" % p)
-                    if type(param) != dict:
-                        value = param
-                    elif "value" in param.keys():
-                        value = param["value"]
-                    else:
-                        raise Exception("No value specified for param %s: %s" % (
-                            p, str(param)))
-                    field_m[p] = (field_m[p][0], value)
-            self._log.debug("field_m: %s" % str(field_m))
-            ctor_t.param_ctor = pydantic.create_model(
-                "Task%sParams" % task.name, **field_m)
+            task_ctor = getattr(mod, clsname)
+
+            # Determine if we need to use a new 
+
+            if task.uses is not None:
+                uses = self.getTaskCtor(session, task.uses, tasks_m)
+            else:
+                uses = None
+
+            ctor_t = TaskCtorCls(
+                name=task.name,
+                uses=uses,
+                task_ctor=task_ctor,
+                srcdir=srcdir)
+        elif task.uses is not None:
+            # Use the existing (base) to create the implementation
+            ctor_t = TaskCtor(
+                name=task.name,
+                uses=self.getTaskCtor(session, task.uses, tasks_m),
+                srcdir=srcdir)
         else:
-            if len(task.params) > 0:
-                ctor_t.params = task.params
-            if len(task.depends) > 0:
-                ctor_t.depends.extend(task.depends)
+            self._log.debug("Use 'Null' as the class implementation")
+            ctor_t = TaskCtorCls(
+                name=task.name,
+                task_ctor=TaskNull,
+                srcdir=srcdir)
+
+        ctor_t = self.handleParams(task, ctor_t)
 
+        self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
 
     @staticmethod

dv_flow/mgr/std/fileset.py

@@ -2,8 +2,9 @@
 import os
 import fnmatch
 import glob
+import logging
 import pydantic.dataclasses as dc
-from typing import List, Tuple
+from typing import ClassVar, List, Tuple
 from dv_flow.mgr import Task, TaskData, TaskMemento
 from dv_flow.mgr import FileSet as _FileSet
 
@@ -12,8 +13,10 @@ class TaskFileSetMemento(TaskMemento):
 
 class FileSet(Task):
 
+    _log : ClassVar = logging.getLogger("FileSet")
+
     async def run(self, input : TaskData) -> TaskData:
-        print("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
+        self._log.debug("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
             self.name,
             self.srcdir,
             self.params.base, self.params.type, str(self.params.include)
@@ -52,8 +55,8 @@ class FileSet(Task):
         if ex_memento is not None and not input.changed:
             ex_memento.files.sort(key=lambda x: x[0])
             memento.files.sort(key=lambda x: x[0])
-            print("ex_memento.files: %s" % str(ex_memento.files))
-            print("memento.files: %s" % str(memento.files))
+            self._log.debug("ex_memento.files: %s" % str(ex_memento.files))
+            self._log.debug("memento.files: %s" % str(memento.files))
             input.changed = ex_memento != memento
         else:
             input.changed = True

dv_flow/mgr/std/task_null.py

@@ -6,5 +6,6 @@ class TaskNull(Task):
 
     async def run(self, input : TaskData) -> TaskData:
         # No memento ; data pass-through
+        self._log.debug("%s: TaskNull.run" % self.name)
         return input
 

dv_flow/mgr/task.py

@@ -21,10 +21,10 @@
 #****************************************************************************
 import os
 import json
-import asyncio
 import dataclasses as dc
+import logging
 from pydantic import BaseModel
-from typing import Any, Callable, Dict, List, Tuple
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskData
 from .task_memento import TaskMemento
 
@@ -35,30 +35,148 @@ class TaskSpec(object):
 class TaskParams(BaseModel):
     pass
 
+
 @dc.dataclass
 class TaskCtor(object):
-    task_ctor : Callable
-    param_ctor : Callable
-    params : Dict[str,Any] = None
+    name : str
+    uses : 'TaskCtor' = None
     srcdir : str = None
     depends : List[TaskSpec] = dc.field(default_factory=list)
 
-    def copy(self):
-        return TaskCtor(
-            task_ctor=self.task_ctor,
-            param_ctor=self.param_ctor,
-            params=self.params,
-            srcdir=self.srcdir,
-            depends=self.depends.copy())
+    _log : ClassVar = logging.getLogger("TaskCtor")
 
+    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
+        if srcdir is None:
+            srcdir = self.srcdir
+        if params is None:
+            params = self.mkParams()
+
+        if self.uses is not None:
+            return self.uses.mkTask(name, depends, rundir, srcdir, params)
+        else:
+            raise NotImplementedError("TaskCtor.mkTask() not implemented for %s" % str(type(self)))
+    
     def mkParams(self):
-        print("mkParams: %s" % str(self.params))
-        ret = self.param_ctor()
+        self._log.debug("--> %s::mkParams" % self.name)
+        if self.uses is not None:
+            params = self.uses.mkParams()
+        else:
+            params = TaskParams()
+        self._log.debug("<-- %s::mkParams: %s" % (self.name, str(params)))
+
+        return params
+
+    def applyParams(self, params):
+        if self.uses is not None:
+            self.uses.applyParams(params)
+
+
+@dc.dataclass
+class TaskCtorParam(TaskCtor):
+    params : Dict[str,Any] = dc.field(default_factory=dict)
+
+    _log : ClassVar = logging.getLogger("TaskCtorParam")
+
+    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
+        self._log.debug("--> %s::mkTask" % self.name)
+        if params is None:
+            params = self.mkParams()
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        ret = self.uses.mkTask(name, depends, rundir, srcdir, params)
+
+        self.applyParams(ret.params)
+        self._log.debug("<-- %s::mkTask" % self.name)
+
+        return ret
+
+    def applyParams(self, params):
+        self._log.debug("--> %s::applyParams: %s %s" % (self.name, str(type(self.params)), str(type(params))))
         if self.params is not None:
             for k,v in self.params.items():
-                setattr(ret, k, v)
+                self._log.debug("  change %s %s=>%s" % (
+                    k, 
+                    str(getattr(params, k)),
+                    str(v)))
+                setattr(params, k, v)
+        else:
+            self._log.debug("  no params")
+        self._log.debug("<-- %s::applyParams: %s" % (self.name, str(self.params)))
+
+@dc.dataclass
+class TaskCtorParamCls(TaskCtor):
+    params_ctor : Callable = None
+
+    _log : ClassVar = logging.getLogger("TaskCtorParamType")
+
+    def mkParams(self):
+        self._log.debug("--> %s::mkParams" % str(self.name))
+        params = self.params_ctor()
+        self._log.debug("<-- %s::mkParams: %s" % (str(self.name), str(type(params))))
+        return params
+
+@dc.dataclass
+class TaskCtorCls(TaskCtor):
+    task_ctor : Callable = None
+
+    _log : ClassVar = logging.getLogger("TaskCtorCls")
+
+    def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
+        self._log.debug("--> %s::mkTask (%s)" % (self.name, str(self.task_ctor)))
+
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        if params is None:
+            params = self.mkParams()
+
+        ret = self.task_ctor(
+            name=name, 
+            depends=depends, 
+            rundir=rundir, 
+            srcdir=srcdir, 
+            params=params)
+        ret.srcdir = self.srcdir
+
+        # Update parameters on the way back
+        self.applyParams(ret.params)
+
+        self._log.debug("<-- %s::mkTask" % self.name)
+        return ret
+
+@dc.dataclass
+class TaskCtorProxy(TaskCtor):
+    task_ctor : TaskCtor = None
+    param_ctor : Callable = None
+
+    _log : ClassVar = logging.getLogger("TaskCtorProxy")
+
+    def mkTask(self, *args, **kwargs):
+        self._log.debug("--> %s::mkTask" % self.name)
+        ret = self.task_ctor.mkTask(*args, **kwargs)
+        self._log.debug("<-- %s::mkTask" % self.name)
         return ret
 
+    def mkParams(self, params=None):
+        self._log.debug("--> %s::mkParams: %s" % (self.name, str(self.params)))
+
+        if params is None and self.param_ctor is not None:
+            params = self.param_ctor()
+
+        params = self.task_ctor.mkParams(params)
+
+        if self.params is not None:
+            for k,v in self.params.items():
+                self._log.debug("  change %s %s=>%s" % (
+                    k, 
+                    str(getattr(params, k)),
+                    str(v)))
+                setattr(params, k, v)
+        self._log.debug("<-- %s::mkParams: %s" % (self.name, str(self.params)))
+        return params
+
+
 @dc.dataclass
 class Task(object):
     """Executable view of a task"""
@@ -78,6 +196,8 @@ class Task(object):
     body: Dict[str,Any] = dc.field(default_factory=dict)
     impl_t : Any = None
 
+    _log : ClassVar = logging.getLogger("Task")
+
     def init(self, runner, basedir):
         self.session = runner
         self.basedir = basedir
@@ -89,7 +209,7 @@ class Task(object):
                     data = json.load(fp)
                     self.memento = T(**data)
                 except Exception as e:
-                    print("Failed to load memento %s: %s" % (
+                    self._log.critical("Failed to load memento %s: %s" % (
                         os.path.join(self.rundir, "memento.json"), str(e)))
                     os.unlink(os.path.join(self.rundir, "memento.json"))
         return self.memento
@@ -100,8 +220,14 @@ class Task(object):
     async def isUpToDate(self, memento) -> bool:
         return False
 
-    async def do_run(self) -> TaskData:
-        print("do_run: %s - %d depends" % (self.name, len(self.depends)))
+    async def do_run(self, session) -> TaskData:
+        self._log.info("--> %s (%s) do_run - %d depends" % (
+            self.name, 
+            str(type(self)),
+            len(self.depends)))
+
+        self.session = session
+
         if len(self.depends) > 0:
             deps_o = []
             for d in self.depends:
@@ -115,7 +241,6 @@ class Task(object):
             input.deps[self.name] = list(inp.name for inp in self.depends)
         else:
             input = TaskData()
-        
 
 
         # Mark the source of this data as being this task
@@ -133,6 +258,10 @@ class Task(object):
         self.save_memento()
 
         # Combine data from the deps to produce a result
+        self._log.info("<-- %s (%s) do_run - %d depends" % (
+            self.name, 
+            str(type(self)),
+            len(self.depends)))
         return self.output
 
     async def run(self, input : TaskData) -> TaskData:

dv_flow/mgr/task_data.py

@@ -20,9 +20,10 @@
 #*
 #****************************************************************************
 import enum
+import logging
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
-from typing import Any, Dict, Set, List, Tuple
+from typing import Any, ClassVar, Dict, Set, List, Tuple
 from .fileset import FileSet
 from toposort import toposort
 
@@ -54,6 +55,8 @@ class TaskData(BaseModel):
     filesets : List[FileSet] = dc.Field(default_factory=list)
     changed : bool = False
 
+    _log : ClassVar = logging.getLogger("TaskData")
+
     def hasParam(self, name: str) -> bool:
         return name in self.params
     
@@ -69,29 +72,29 @@ class TaskData(BaseModel):
     def getFileSets(self, type=None, order=True) -> List[FileSet]:
         ret = []
 
-        print("getFileSets: filesets=%s" % str(self.filesets))
+        self._log.debug("getFileSets: filesets=%s" % str(self.filesets))
 
         if order:
             # The deps map specifies task dependencies
 
             candidate_fs = []
             for fs in self.filesets:
-                print("fs: %s" % str(fs))
+                self._log.debug("fs: %s" % str(fs))
                 if type is None or fs.type in type:
                     candidate_fs.append(fs)
-            print("self.deps: %s" % str(self.deps))
+            self._log.debug("self.deps: %s" % str(self.deps))
             order = toposort(self.deps)
 
-            print("order: %s" % str(order))
+            self._log.debug("order: %s" % str(order))
 
             for order_s in order:
-                print("order_s: %s" % str(order_s))
+                self._log.debug("order_s: %s" % str(order_s))
                 i = 0
                 while i < len(candidate_fs):
                     fs = candidate_fs[i]
-                    print("fs.src: %s" % fs.src)
+                    self._log.debug("fs.src: %s" % fs.src)
                     if fs.src in order_s:
-                            print("Add fileset")
+                            self._log.debug("Add fileset")
                             ret.append(fs)
                             candidate_fs.pop(i)
                     else:

dv_flow/mgr/task_graph_builder.py

@@ -94,6 +94,8 @@ class TaskGraphBuilder(object):
 
         ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
 
+        self._logger.debug("ctor_t: %s" % ctor_t.name)
+
         depends = []
 
         for dep in ctor_t.depends:
@@ -104,13 +106,10 @@ class TaskGraphBuilder(object):
             depends.append(self._task_m[dep])
 
         # The returned task should have all param references resolved
-        self._logger.debug("task_ctor=%s" % str(ctor_t.task_ctor))
-        task = ctor_t.task_ctor(
+        task = ctor_t.mkTask(
             name=task_name,
-            params=ctor_t.mkParams(),
             depends=depends,
-            rundir=rundir,
-            srcdir=ctor_t.srcdir)
+            rundir=rundir)
         
         self._task_m[task.name] = task
 
@@ -137,71 +136,45 @@ class TaskGraphBuilder(object):
         self._logger.debug("pkg_s: %d %s" % (
             len(self._pkg_s), (self._pkg_s[-1].name if len(self._pkg_s) else "<unknown>")))
 
+        # First, check the active pkg_def to see if any aliases
+        # Should be considered
+        pkg_name = spec.name
+        if pkg_def is not None:
+            # Look for an import alias
+            self._logger.debug("Search package %s for import alias %s" % (
+                pkg_def.name, pkg_spec.name))
+            for imp in pkg_def.imports:
+                self._logger.debug("imp: %s" % str(imp))
+                if imp.alias is not None and imp.alias == spec.name:
+                    # Found the alias name. Just need to get an instance of this package
+                    self._logger.debug("Found alias %s -> %s" % (imp.alias, imp.name))
+                    pkg_name = imp.name
+                    break
+
         # Note: _pkg_m needs to be context specific, such that imports from
         # one package don't end up visible in another
-        if len(self._pkg_s) and spec.name == self._pkg_s[-1].name:
-            pkg = self._pkg_s[-1]
+        spec.name = pkg_name
+
         if spec in self._pkg_m.keys():
+            self._logger.debug("Found cached package instance")
             pkg = self._pkg_m[spec]
+        elif self.pkg_rgy.hasPackage(spec.name):
+            self._logger.debug("Registry has a definition")
+            p_def =  self.pkg_rgy.getPackage(spec.name)
+
+            self._pkg_spec_s.append(p_def)
+            pkg = p_def.mkPackage(self)
+            self._pkg_spec_s.pop()
+            self._pkg_m[spec] = pkg
         else:
-            pkg = None
-
-            if pkg_def is not None:
-                # Look for an import alias
-                self._logger.debug("imports: %s" % str(pkg_def.imports))
-                for imp in pkg_def.imports:
-                    self._logger.debug("imp: %s" % str(imp))
-                    if imp.alias is not None and imp.alias == spec.name:
-                        # Found the alias name. Just need to get an instance of this package
-                        tgt_pkg_spec = PackageSpec(imp.name)
-                        if tgt_pkg_spec in self._pkg_m.keys():
-                            pkg = self._pkg_m[tgt_pkg_spec]
-                        elif self.pkg_rgy.hasPackage(tgt_pkg_spec.name):
-                            base = self.pkg_rgy.getPackage(tgt_pkg_spec.name)
-                            self._pkg_spec_s.append(base)
-                            pkg = base.mkPackage(self, spec.params)
-                            self._pkg_spec_s.pop()
-                        elif imp.path is not None:
-                            # See if we can load the package
-                            self._logger.critical("TODO: load referenced package")
-                        else:
-                            raise Exception("Failed to resolve target (%s) of import alias %s" % (
-                                imp.name,
-                                imp.alias))
-                        break
-                    else:
-                        # Need to compare the spec with the full import spec
-                        imp_spec = PackageSpec(imp.name)
-                        # TODO: set parameters
-                        if imp_spec == spec:
-                            base = self.pkg_rgy.getPackage(spec.name)
-                            if base is None:
-                                raise Exception("Failed to find imported package %s" % spec.name)
-                            self._pkg_spec_s.append(base)
-                            pkg = base.mkPackage(self, spec.params)
-                            self._pkg_m[spec] = pkg
-                            self._pkg_spec_s.pop()
-                            break
-            
-            if pkg is None:
-                self._logger.debug("Checking registry")
-                p_def =  self.pkg_rgy.getPackage(spec.name)
-
-                if p_def is not None:
-                    self._pkg_spec_s.append(p_def)
-                    pkg = p_def.mkPackage(self)
-                    self._pkg_spec_s.pop()
-                    self._pkg_m[spec] = pkg
-
-            if pkg is None:
-                raise Exception("Failed to find package %s from package %s" % (
-                    spec.name, (pkg_def.name if pkg_def is not None else "<null>")))
+            raise Exception("Failed to find definition of package %s" % spec.name)
 
         self._logger.debug("<-- getPackage: %s" % str(pkg))
 
         return pkg
         
     def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef = None) -> 'TaskCtor':
+        self._logger.debug("--> getTaskCtor %s" % spec.name)
         spec_e = spec.name.split(".")
         task_name = spec_e[-1]
 
@@ -219,4 +192,7 @@ class TaskGraphBuilder(object):
                 self._logger.critical("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
                 raise e
 
-        return pkg.getTaskCtor(task_name)
+        ctor = pkg.getTaskCtor(task_name)
+
+        self._logger.debug("--> getTaskCtor %s" % spec.name)
+        return ctor

dv_flow/mgr/task_graph_runner_local.py

@@ -23,6 +23,7 @@ import asyncio
 import os
 import yaml
 import dataclasses as dc
+import logging
 from toposort import toposort
 from typing import Any, Callable, ClassVar, Coroutine, Dict, List, Tuple, Union
 from .fragment_def import FragmentDef
@@ -47,6 +48,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
     # Search path for .dfs files
     create_subprocess : Callable = asyncio.create_subprocess_exec
     _root_dir : str = None
+    _log : ClassVar = logging.getLogger("TaskGraphRunnerLocal")
 
     def __post_init__(self):
         if self.nproc == -1:
@@ -71,11 +73,11 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
         for t in task:
             self._mkDeps(dep_m, task_m, t)
 
-        print("dep_m: %s" % str(dep_m))
+        self._log.debug("dep_m: %s" % str(dep_m))
 
         order = list(toposort(dep_m))
         
-        print("order: %s" % str(order))
+        self._log.debug("order: %s" % str(order))
 
         active_task_l : List[Tuple[Task,Coroutine]]= []
         # Now, iterate over the concurrent sets
@@ -96,8 +98,7 @@ class TaskGraphRunnerLocal(TaskGraphRunner):
                                 break
                 if t not in self.done_task_m.keys():
                     task_t = task_m[t]
-                    task_t.session = self
-                    coro = asyncio.Task(task_t.do_run())
+                    coro = asyncio.Task(task_t.do_run(self))
                     active_task_l.append((task_t, coro))
                
             # Now, wait for tasks to complete

{dv_flow_mgr-0.0.1.12968982426a1.dist-info → dv_flow_mgr-0.0.1.12971126211a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12968982426a1
+Version: 0.0.1.12971126211a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

dv_flow_mgr-0.0.1.12971126211a1.dist-info/RECORD

@@ -0,0 +1,28 @@
+dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
+dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
+dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
+dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
+dv_flow/mgr/package.py,sha256=AOLEEotVQF9VNMMl13uNQ7Na2TuHPXxEvybBOX7XIps,1615
+dv_flow/mgr/package_def.py,sha256=lokG420kcEPDNu11GYQSbCalX7gxeDoTJy8iMc1GmXM,13451
+dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
+dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
+dv_flow/mgr/task.py,sha256=7Nc7H2Wj9xWAHjspJ5TKw6DZ7g8DMisAqWuhjcy3MZU,9102
+dv_flow/mgr/task_data.py,sha256=XBPWwvuaQ3BZ94wknvv1bqLh98iTzAg5fskv7_kk6DQ,10524
+dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
+dv_flow/mgr/task_graph_builder.py,sha256=4dfy_T26MYI7ls2xiofr4V0ItZHDehU2iw6UBPt-SZQ,7074
+dv_flow/mgr/task_graph_runner.py,sha256=NwNYcOJ952lPMLwIIlYE9CoDdedqvcw2fWHYUsKFXuU,2164
+dv_flow/mgr/task_graph_runner_local.py,sha256=66JYJcu-W60LjfuT9UkbcKslNqDcD5q4UR7ZAWrSaGM,4707
+dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
+dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
+dv_flow/mgr/cmds/cmd_run.py,sha256=eths8kT7mBmpZqwOuMtpKAaux4rg-f7hPBxxTHbpKT4,2903
+dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow/mgr/std/fileset.py,sha256=Hn3_C1CczSRSaNYI3aDYbaaNdnKDlIqU16_GRIBP4PI,2461
+dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
+dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
+dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/METADATA,sha256=vFSzc2a0PhhsjazyaZhTv7BES6zdQ7V4bFD-6XkrUmc,13276
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.12971126211a1.dist-info/RECORD,,

dv_flow_mgr-0.0.1.12968982426a1.dist-info/RECORD

@@ -1,28 +0,0 @@
-dv_flow/mgr/__init__.py,sha256=IZA7I1u7RH34DrJXSkETxWFpK5Jn_A2zXfnCAzJ8XxA,289
-dv_flow/mgr/__main__.py,sha256=big-RSuqbx7P48_8rzaZKX5YW3B2US1i0mRH_TfoeIs,1340
-dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
-dv_flow/mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
-dv_flow/mgr/package.py,sha256=LAJNVOMlpWkez7eK7yfChoDKeIPakApUCpDh_No059g,1469
-dv_flow/mgr/package_def.py,sha256=ay4bYuzBEzrqqJsNr1ezZF9BxEAgWeS9xxIETKe-Bzs,11595
-dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
-dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
-dv_flow/mgr/task.py,sha256=ewJ7bCFWqwVuzHZZsX2LDZfzXWVFfFlH8yFyn-xxIVg,5043
-dv_flow/mgr/task_data.py,sha256=-6Dqa3oUI7RJc1Js2SRSnhxNTcASkamXFYMN6UiknZQ,10376
-dv_flow/mgr/task_def.py,sha256=96hSwqJo0MazJ1VcLhovYRmNCplsNLt47AumtyjSddU,1690
-dv_flow/mgr/task_graph_builder.py,sha256=ie93_dMwcNLvlAp_K0thDdTGZuWhQ4l1m2rVSfoVWDU,8599
-dv_flow/mgr/task_graph_runner.py,sha256=NwNYcOJ952lPMLwIIlYE9CoDdedqvcw2fWHYUsKFXuU,2164
-dv_flow/mgr/task_graph_runner_local.py,sha256=UTBV1AKanLns99CSYtEQQ3EEkgo1hM8XsHpc8E5jkIg,4646
-dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
-dv_flow/mgr/cmds/cmd_run.py,sha256=C1bd2yZxVU03hZizGLYoH1JfZgeq_G57Kajc279iZpo,2773
-dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
-dv_flow/mgr/std/fileset.py,sha256=0eBp0VIkxYbIYAt0YJg0m-6nUbPuzGkaVBtxpius7Cg,2354
-dv_flow/mgr/std/flow.dv,sha256=pSpzrPPEu_L8DHccGfArxsKYgUfyQidShZc0ShgGtsY,500
-dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
-dv_flow/mgr/std/task_null.py,sha256=UEJ3fIoIMYWVsagiQC7GHD23UES7WoH4wtq94b4tcs4,265
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/METADATA,sha256=LKGnpuX45HstDNEJ8pxN_eVTxzw0OqiW1WoZyVSGEE4,13276
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.12968982426a1.dist-info/RECORD,,