dv-flow-mgr 0.0.1.12968982426a1__tar.gz → 0.0.1.12981364130a1__tar.gz

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12968982426a1
+Version: 0.0.1.12981364130a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12968982426a1"
+version = "0.0.1.12981364130a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/__main__.py

@@ -20,10 +20,17 @@
 #*
 #****************************************************************************
 import argparse
+import logging
 from .cmds.cmd_run import CmdRun
 
 def get_parser():
     parser = argparse.ArgumentParser(description='dv_flow_mgr')
+    parser.add_argument("-d", "--debug", 
+                        help="Enable debug",
+                        action="store_true")
+    parser.add_argument("-v", "--verbose", 
+                        help="Enable verbose output",
+                        action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
     run_parser = subparsers.add_parser('run', help='run a flow')
@@ -35,6 +42,12 @@ def get_parser():
 def main():
     parser = get_parser()
     args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
     args.func(args)
 
 if __name__ == "__main__":

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/cmds/cmd_run.py

@@ -1,5 +1,7 @@
 import asyncio
 import os
+import logging
+from typing import ClassVar
 from ..task_graph_runner import TaskGraphRunner
 from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
@@ -8,6 +10,7 @@ from ..pkg_rgy import PkgRgy
 
 
 class CmdRun(object):
+    _log : ClassVar = logging.getLogger("CmdRun")
 
     def __call__(self, args):
 
@@ -17,7 +20,7 @@ class CmdRun(object):
         if pkg is None:
             raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
 
-        print("pkg: %s" % pkg.name)
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
 
         if len(args.tasks) > 0:
             pass

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/package.py

@@ -20,7 +20,8 @@
 #*
 #****************************************************************************
 import dataclasses as dc
-from typing import Any, Dict
+import logging
+from typing import Any, ClassVar, Dict
 from .task import TaskCtor
 
 @dc.dataclass
@@ -30,8 +31,10 @@ class Package(object):
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
     tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    _log : ClassVar = logging.getLogger("Package")
 
     def getTaskCtor(self, name : str) -> TaskCtor:
+        self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))
         return self.tasks[name]

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/package_def.py

@@ -32,7 +32,7 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-from .task import TaskCtor, TaskParams
+from .task import TaskCtor, TaskCtorProxy, TaskCtorCls, TaskCtorParam, TaskCtorParamCls
 from .task_def import TaskDef, TaskSpec
 from .std.task_null import TaskNull
 
@@ -90,53 +90,132 @@ class PackageDef(BaseModel):
         self._log.debug("<-- mkPackage %s" % self.name)
         return ret
     
-    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
-        ctor_t : TaskCtor = None
+    def getTaskCtor(self, session, task_name, tasks_m):
+        self._log.debug("--> getTaskCtor")
+        # Find package (not package_def) that implements this task
+        # Insert an indirect reference to that tasks's constructor
+        last_dot = task_name.rfind('.')
+
+        if last_dot != -1:
+            pkg_name = task_name[:last_dot]
+            task_name = task_name[last_dot+1:]
+        else:
+            pkg_name = None
+
+        if pkg_name is not None:
+            self._log.debug("Package-qualified 'uses'")
+            pkg = session.getPackage(PackageSpec(pkg_name))
+            if pkg is None:
+                raise Exception("Failed to find package %s" % pkg_name)
+            ctor_t = pkg.getTaskCtor(task_name)
+        else:
+            self._log.debug("Unqualified 'uses'")
+            if task_name not in tasks_m.keys():
+                raise Exception("Failed to find task %s" % task_name)
+            if len(tasks_m[task_name]) != 3:
+                raise Exception("Task %s not fully defined" % task_name)
 
-        if task.uses is not None:
-            # Find package (not package_def) that implements this task
-            # Insert an indirect reference to that tasks's constructor
-            last_dot = task.uses.rfind('.')
+            ctor_t = tasks_m[task_name][2]
+        return ctor_t
 
-            if last_dot != -1:
-                pkg_name = task.uses[:last_dot]
-                task_name = task.uses[last_dot+1:]
-            else:
-                pkg_name = None
-                task_name = task.uses
-
-            if pkg_name is not None:
-                pkg = session.getPackage(PackageSpec(pkg_name))
-                if pkg is None:
-                    raise Exception("Failed to find package %s" % pkg_name)
-                ctor_t = pkg.getTaskCtor(task_name)
-                ctor_t = ctor_t.copy()
-                ctor_t.srcdir = srcdir
-            else:
-                if task_name not in tasks_m.keys():
-                    raise Exception("Failed to find task %s" % task_name)
-                if len(tasks_m[task_name]) == 3:
-                    ctor_t = tasks_m[task_name][2].copy()
-                    ctor_t.srcdir = srcdir
-                else:
-                    task_i = tasks_m[task_name]
-                    ctor_t = self.mkTaskCtor(
-                        session, 
-                        task=task_i[0], 
-                        srcdir=srcdir,
-                        tasks_m=tasks_m)
-                    tasks_m[task_name] = ctor_t
-
-        if ctor_t is None:
-            # Provide a default implementation
-            ctor_t = TaskCtor(
-                task_ctor=TaskNull,
-                param_ctor=TaskParams,
-                srcdir=srcdir)
+    def handleParams(self, task, ctor_t):
+
+        if task.params is not None and len(task.params) > 0:
+            decl_params = False
+
+            # First, add in a parameter-setting stage 
+            ctor_t = TaskCtorParam(
+                name=ctor_t.name,
+                uses=ctor_t,
+                srcdir=ctor_t.srcdir)
+#            ctor_t.params.update(task.params)
+
+            for value in task.params.values():
+                if "type" in value:
+                    decl_params = True
+                    break
+
+            field_m = {}
+            # First, add parameters from the base class
+            base_o = ctor_t.mkParams()
+            for fname,info in base_o.model_fields.items():
+                self._log.debug("Field: %s (%s)" % (fname, info.default))
+                field_m[fname] = (info.annotation, info.default)
+
+            if decl_params:
+                # We need to combine base parameters with new parameters
+                ptype_m = {
+                    "str" : str,
+                    "int" : int,
+                    "float" : float,
+                    "bool" : bool,
+                    "list" : List
+                }
+                pdflt_m = {
+                    "str" : "",
+                    "int" : 0,
+                    "float" : 0.0,
+                    "bool" : False,
+                    "list" : []
+                }
+                for p in task.params.keys():
+                    param = task.params[p]
+                    if type(param) == dict and "type" in param.keys():
+                        ptype_s = param["type"]
+                        if ptype_s not in ptype_m.keys():
+                            raise Exception("Unknown type %s" % ptype_s)
+                        ptype = ptype_m[ptype_s]
+
+                        if p in field_m.keys():
+                            raise Exception("Duplicate field %s" % p)
+                        if "value" in param.keys():
+                            field_m[p] = (ptype, param["value"])
+                        else:
+                            field_m[p] = (ptype, pdflt_m[ptype_s])
+                    else:
+                        if p not in field_m.keys():
+                            raise Exception("Field %s not found" % p)
+                        if type(param) != dict:
+                            value = param
+                        elif "value" in param.keys():
+                            value = param["value"]
+                        else:
+                            raise Exception("No value specified for param %s: %s" % (
+                                p, str(param)))
+                        field_m[p] = (field_m[p][0], value)
+                self._log.debug("field_m: %s" % str(field_m))
+                param_t = pydantic.create_model(
+                    "Task%sParams" % task.name, **field_m)
+                ctor_t = TaskCtorParamCls(
+                    name=ctor_t.name,
+                    uses=ctor_t,
+                    params_ctor=param_t)
+            else: # no new parameters declared
+                for p in task.params.keys():
+                    param = task.params[p]
+                    if p not in field_m.keys():
+                        raise Exception("Field %s not found" % p)
+                    if type(param) != dict:
+                        value = param
+                    elif "value" in param.keys():
+                        value = param["value"]
+                    else:
+                        raise Exception("No value specified for param %s: %s" % (
+                            p, str(param)))
+                    field_m[p] = (field_m[p][0], value)
+                ctor_t.params[p] = value
 
+        return ctor_t
+
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+        self._log.debug("--> %s::mkTaskCtor %s" % (self.name, task.name))
+        ctor_t : TaskCtor = None
+
+        # Determine the implementation constructor first
         if task.pyclass is not None:
             # Built-in impl
             # Now, lookup the class
+            self._log.debug("Use PyClass implementation")
             last_dot = task.pyclass.rfind('.')
             clsname = task.pyclass[last_dot+1:]
             modname = task.pyclass[:last_dot]
@@ -154,72 +233,36 @@ class PackageDef(BaseModel):
                 
             if not hasattr(mod, clsname):
                 raise Exception("Class %s not found in module %s" % (clsname, modname))
-            ctor_t.task_ctor = getattr(mod, clsname)
-
-            if task.uses is None:
-                ctor_t.param_ctor = TaskParams
-
-        decl_params = False
-        for value in task.params.values():
-            if "type" in value:
-                decl_params = True
-                break
-        
-        if decl_params:
-            # We need to combine base parameters with new parameters
-            field_m = {}
-            # First, add parameters from the base class
-            for fname,info in ctor_t.param_ctor.model_fields.items():
-                self._log.debug("Field: %s (%s)" % (fname, info.default))
-                field_m[fname] = (info.annotation, info.default)
-            ptype_m = {
-                "str" : str,
-                "int" : int,
-                "float" : float,
-                "bool" : bool,
-                "list" : List
-            }
-            pdflt_m = {
-                "str" : "",
-                "int" : 0,
-                "float" : 0.0,
-                "bool" : False,
-                "list" : []
-            }
-            for p in task.params.keys():
-                param = task.params[p]
-                if type(param) == dict and "type" in param.keys():
-                    ptype_s = param["type"]
-                    if ptype_s not in ptype_m.keys():
-                        raise Exception("Unknown type %s" % ptype_s)
-                    ptype = ptype_m[ptype_s]
-
-                    if p in field_m.keys():
-                        raise Exception("Duplicate field %s" % p)
-                    if "value" in param.keys():
-                        field_m[p] = (ptype, param["value"])
-                    else:
-                        field_m[p] = (ptype, pdflt_m[ptype_s])
-                else:
-                    if p not in field_m.keys():
-                        raise Exception("Field %s not found" % p)
-                    if type(param) != dict:
-                        value = param
-                    elif "value" in param.keys():
-                        value = param["value"]
-                    else:
-                        raise Exception("No value specified for param %s: %s" % (
-                            p, str(param)))
-                    field_m[p] = (field_m[p][0], value)
-            self._log.debug("field_m: %s" % str(field_m))
-            ctor_t.param_ctor = pydantic.create_model(
-                "Task%sParams" % task.name, **field_m)
+            task_ctor = getattr(mod, clsname)
+
+            # Determine if we need to use a new 
+
+            if task.uses is not None:
+                uses = self.getTaskCtor(session, task.uses, tasks_m)
+            else:
+                uses = None
+
+            ctor_t = TaskCtorCls(
+                name=task.name,
+                uses=uses,
+                task_ctor=task_ctor,
+                srcdir=srcdir)
+        elif task.uses is not None:
+            # Use the existing (base) to create the implementation
+            ctor_t = TaskCtor(
+                name=task.name,
+                uses=self.getTaskCtor(session, task.uses, tasks_m),
+                srcdir=srcdir)
         else:
-            if len(task.params) > 0:
-                ctor_t.params = task.params
-            if len(task.depends) > 0:
-                ctor_t.depends.extend(task.depends)
+            self._log.debug("Use 'Null' as the class implementation")
+            ctor_t = TaskCtorCls(
+                name=task.name,
+                task_ctor=TaskNull,
+                srcdir=srcdir)
+
+        ctor_t = self.handleParams(task, ctor_t)
 
+        self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t
 
     @staticmethod

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/std/fileset.py

@@ -2,8 +2,9 @@
 import os
 import fnmatch
 import glob
+import logging
 import pydantic.dataclasses as dc
-from typing import List, Tuple
+from typing import ClassVar, List, Tuple
 from dv_flow.mgr import Task, TaskData, TaskMemento
 from dv_flow.mgr import FileSet as _FileSet
 
@@ -12,8 +13,10 @@ class TaskFileSetMemento(TaskMemento):
 
 class FileSet(Task):
 
+    _log : ClassVar = logging.getLogger("FileSet")
+
     async def run(self, input : TaskData) -> TaskData:
-        print("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
+        self._log.debug("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
             self.name,
             self.srcdir,
             self.params.base, self.params.type, str(self.params.include)
@@ -52,8 +55,8 @@ class FileSet(Task):
         if ex_memento is not None and not input.changed:
             ex_memento.files.sort(key=lambda x: x[0])
             memento.files.sort(key=lambda x: x[0])
-            print("ex_memento.files: %s" % str(ex_memento.files))
-            print("memento.files: %s" % str(memento.files))
+            self._log.debug("ex_memento.files: %s" % str(ex_memento.files))
+            self._log.debug("memento.files: %s" % str(memento.files))
             input.changed = ex_memento != memento
         else:
             input.changed = True

{dv_flow_mgr-0.0.1.12968982426a1 → dv_flow_mgr-0.0.1.12981364130a1}/src/dv_flow/mgr/std/task_null.py

@@ -6,5 +6,6 @@ class TaskNull(Task):
 
     async def run(self, input : TaskData) -> TaskData:
         # No memento ; data pass-through
+        self._log.debug("%s: TaskNull.run" % self.name)
         return input