dv-flow-mgr 0.0.1.12761553329a1__py3-none-any.whl → 0.0.1.12849118090a1__py3-none-any.whl

dv_flow_mgr/fileset.py

@@ -24,8 +24,9 @@ from pydantic import BaseModel
 from typing import Any, Dict, List, Tuple
 
 class FileSet(BaseModel):
-    src : str
     type : str
     basedir : str
+    name : str = ""
+    src : str = None
     files : List[str] = dc.Field(default_factory=list)
     params : Dict[str,str] = dc.Field(default_factory=dict)

dv_flow_mgr/package.py

@@ -54,26 +54,6 @@ class Package(object):
     def getTaskCtor(self, name : str) -> TaskCtor:
         return self.tasks[name]
             
-    def mkTaskParams(self, name : str) -> TaskParams:
-        if name not in self.tasks:
-            raise Exception("Task " + name + " not found")
-        return self.tasks[name].mkTaskParams()
-    
-    def setTaskParams(self, name : str, params : TaskParams, pvals : Dict[str,Any]):
-        if name not in self.tasks:
-            raise Exception("Task " + name + " not found")
-        self.tasks[name].setTaskParams(params, pvals)
-
-    def mkTask(self, 
-               name : str, 
-               task_id : int, 
-               session : 'Session',
-               params : TaskParams,
-               depends : List['Task']) -> 'Task':
-        # TODO: combine parameters to create the full taskname
-        task =  self.tasks[name].mkTask(name, task_id, session, params, depends)
-        return task
-
     def __hash__(self):
         return hash(self.fullname())
 

dv_flow_mgr/package_def.py

@@ -19,16 +19,20 @@
 #*     Author: 
 #*
 #****************************************************************************
-import pydantic.dataclasses as dc
 import json
+import importlib
+import sys
+import pydantic
+import pydantic.dataclasses as dc
 from pydantic import BaseModel
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Callable, Tuple
 from .flow import Flow
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
-from .task import TaskParamCtor
+from .task import TaskCtor, TaskParams
 from .task_def import TaskDef, TaskSpec
+from .tasklib.builtin_pkg import TaskNull
 
 
 class PackageDef(BaseModel):
@@ -53,46 +57,154 @@ class PackageDef(BaseModel):
     def mkPackage(self, session, params : Dict[str,Any] = None) -> 'Package':
         ret = Package(self.name)
 
+        session.push_package(ret)
+
+        tasks_m : Dict[str,str,TaskCtor]= {}
+
         for task in self.tasks:
-            if task.type is not None:
-                # Find package (not package_def) that implements this task
-                # Insert an indirect reference to that tasks's constructor
-
-                # Only call getTaskCtor if the task is in a different package
-                task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
-                ctor_t = session.getTaskCtor(task_t, self)
-
-                ctor_t = TaskParamCtor(
-                    base=ctor_t, 
-                    params=task.params, 
-                    basedir=self.basedir,
-                    depend_refs=task.depends)
-            else:
-                # We use the Null task from the std package
-                raise Exception("")
-            ret.tasks[task.name] = ctor_t
+            if task.name in tasks_m.keys():
+                raise Exception("Duplicate task %s" % task.name)
+            tasks_m[task.name] = (task, self.basedir, ) # We'll add a TaskCtor later
 
         for frag in self.fragment_l:
             for task in frag.tasks:
-                if task.type is not None:
-                    # Find package (not package_def) that implements this task
-                    # Insert an indirect reference to that tasks's constructor
-
-                    # Only call getTaskCtor if the task is in a different package
-                    task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
-                    ctor_t = session.getTaskCtor(task_t, self)
-
-                    ctor_t = TaskParamCtor(
-                        base=ctor_t, 
-                        params=task.params, 
-                        basedir=frag.basedir,
-                        depend_refs=task.depends)
-                else:
-                    # We use the Null task from the std package
-                    raise Exception("")
-                if task.name in ret.tasks:
-                    raise Exception("Task %s already defined" % task.name)
-                ret.tasks[task.name] = ctor_t
+                if task.name in tasks_m.keys():
+                    raise Exception("Duplicate task %s" % task.name)
+                tasks_m[task.name] = (task, frag.basedir, ) # We'll add a TaskCtor later
+
+        # Now we have a unified map of the tasks declared in this package
+        for name in list(tasks_m.keys()):
+            task_i = tasks_m[name]
+            if len(task_i) < 3:
+                # Need to create the task ctor
+                ctor_t = self.mkTaskCtor(session, task_i[0], task_i[1], tasks_m)
+                tasks_m[name] = (task_i[0], task_i[1], ctor_t)
+            ret.tasks[name] = tasks_m[name][2]
+
+        session.pop_package(ret)
 
         return ret
+    
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+        ctor_t : TaskCtor = None
+
+        if task.uses is not None:
+            # Find package (not package_def) that implements this task
+            # Insert an indirect reference to that tasks's constructor
+            last_dot = task.uses.rfind('.')
+
+            if last_dot != -1:
+                pkg_name = task.uses[:last_dot]
+                task_name = task.uses[last_dot+1:]
+            else:
+                pkg_name = None
+                task_name = task.uses
+
+            if pkg_name is not None:
+                pkg = session.getPackage(PackageSpec(pkg_name))
+                if pkg is None:
+                    raise Exception("Failed to find package %s" % pkg_name)
+                ctor_t = pkg.getTaskCtor(task_name)
+                ctor_t = ctor_t.copy()
+                ctor_t.srcdir = srcdir
+            else:
+                if task_name not in tasks_m.keys():
+                    raise Exception("Failed to find task %s" % task_name)
+                if len(tasks_m[task_name]) == 3:
+                    ctor_t = tasks_m[task_name][2].copy()
+                    ctor_t.srcdir = srcdir
+                else:
+                    task_i = tasks_m[task_name]
+                    ctor_t = self.mkTaskCtor(
+                        session, 
+                        task=task_i[0], 
+                        srcdir=srcdir,
+                        tasks_m=tasks_m)
+                    tasks_m[task_name] = ctor_t
+
+        if ctor_t is None:
+            # Provide a default implementation
+            ctor_t = TaskCtor(
+                task_ctor=TaskNull,
+                param_ctor=TaskParams,
+                srcdir=srcdir)
+
+        if task.pyclass is not None:
+            # Built-in impl
+            # Now, lookup the class
+            last_dot = task.pyclass.rfind('.')
+            clsname = task.pyclass[last_dot+1:]
+            modname = task.pyclass[:last_dot]
+
+            try:
+                if modname not in sys.modules:
+                    if self.basedir not in sys.path:
+                        sys.path.append(self.basedir)
+                    mod = importlib.import_module(modname)
+                else:
+                    mod = sys.modules[modname]
+            except ModuleNotFoundError as e:
+                raise Exception("Failed to import module %s" % modname)
+                
+            if not hasattr(mod, clsname):
+                raise Exception("Class %s not found in module %s" % (clsname, modname))
+            ctor_t.task_ctor = getattr(mod, clsname)
+
+            if task.uses is None:
+                ctor_t.param_ctor = TaskParams
+
+        decl_params = False
+        for value in task.params.values():
+            if "type" in value:
+                decl_params = True
+                break
+        
+        if decl_params:
+            # We need to combine base parameters with new parameters
+            field_m = {}
+            # First, add parameters from the base class
+            for fname,info in ctor_t.param_ctor.model_fields.items():
+                print("Field: %s (%s)" % (fname, info.default))
+                field_m[fname] = (info.annotation, info.default)
+            ptype_m = {
+                "str" : str,
+                "int" : int,
+                "float" : float,
+                "bool" : bool
+            }
+            for p in task.params.keys():
+                param = task.params[p]
+                if type(param) == dict and "type" in param.keys():
+                    ptype_s = param["type"]
+                    if ptype_s not in ptype_m.keys():
+                        raise Exception("Unknown type %s" % ptype_s)
+                    ptype = ptype_m[ptype_s]
+
+                    if p in field_m.keys():
+                        raise Exception("Duplicate field %s" % p)
+                    if "value" in param.keys():
+                        field_m[p] = (ptype, param["value"])
+                    else:
+                        field_m[p] = (ptype, )
+                else:
+                    if p not in field_m.keys():
+                        raise Exception("Field %s not found" % p)
+                    if type(param) != dict:
+                        value = param
+                    elif "value" in param.keys():
+                        value = param["value"]
+                    else:
+                        raise Exception("No value specified for param %s: %s" % (
+                            p, str(param)))
+                    field_m[p] = (field_m[p][0], value)
+            print("field_m: %s" % str(field_m))
+            ctor_t.param_ctor = pydantic.create_model(
+                "Task%sParams" % task.name, **field_m)
+        else:
+            if len(task.params) > 0:
+                ctor_t.params = task.params
+            if len(task.depends) > 0:
+                ctor_t.depends.extends(task.depends)
+
+        return ctor_t
 

dv_flow_mgr/session.py

@@ -27,7 +27,7 @@ from typing import Any, Callable, Dict, List
 from .fragment_def import FragmentDef
 from .package import Package
 from .package_def import PackageDef, PackageSpec
-from .task import Task,TaskSpec
+from .task import Task, TaskSpec, TaskCtor
 
 @dc.dataclass
 class Session(object):
@@ -45,29 +45,36 @@ class Session(object):
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
     _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
+    _pkg_file_m : Dict[str,str] = dc.field(default_factory=dict)
+    _pkg_path : List[str] = dc.field(default_factory=list)
     _task_list : List[Task] = dc.field(default_factory=list)
     _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
     _task_id : int = 0
 
     def __post_init__(self):
-        from .tasklib.std.pkg_std import PackageStd
-        from .tasklib.hdl.sim.vcs_pkg import VcsPackage
-        from .tasklib.hdl.sim.vlt_pkg import VltPackage
-        from .tasklib.hdl.sim.mti_pkg import MtiPackage
-        self._pkg_m[PackageSpec("std")] = PackageStd("std")
-        self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
-        self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
-        self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
-
-    def load(self, root : str):
-        if not os.path.isdir(root):
-            raise Exception("Root directory %s does not exist" % root)
-
-        if not os.path.isfile(os.path.join(root, "flow.yaml")):
+        # Add a reference to the built-in 'std' package
+        this_dir = os.path.dirname(os.path.abspath(__file__))
+        self._pkg_file_m["std"] = os.path.join(this_dir, "tasklib/std/flow.dv")
+
+        # from .tasklib.std.pkg_std import PackageStd
+        # from .tasklib.hdl.sim.vcs_pkg import VcsPackage
+        # from .tasklib.hdl.sim.vlt_pkg import VltPackage
+        # from .tasklib.hdl.sim.mti_pkg import MtiPackage
+        # self._pkg_m[PackageSpec("std")] = PackageStd("std")
+        # self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
+        # self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
+        # self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
+        pass
+
+    def load(self):
+        if not os.path.isdir(self.srcdir):
+            raise Exception("Root directory %s does not exist" % self.srcdir)
+
+        if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
             raise Exception("No root flow file")
 
-        self._root_dir = os.path.dirname(root)
-        self.package = self._load_package(os.path.join(root, "flow.yaml"), [])
+        self._root_dir = os.path.dirname(self.srcdir)
+        self.package = self._load_package(os.path.join(self.srcdir, "flow.dv"), [])
 
         return self.package
 
@@ -77,7 +84,7 @@ class Session(object):
 
         return self._mkTaskGraph(task, self.rundir)
         
-    def _mkTaskGraph(self, task : str, parent_rundir : str, params : dict = None) -> Task:
+    def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
 
         elems = task.split(".")
 
@@ -99,23 +106,19 @@ class Session(object):
         
         self._pkg_s.append(pkg)
 
-        #task_def = pkg.getTask(task_name)
+        ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
 
         depends = []
 
-        params = pkg.mkTaskParams(task_name)
-
-        task_id = self.mkTaskId(None)
-#        task_name = "%s.%s" % (pkg.name, task_def.name)
-
         # The returned task should have all param references resolved
-        task = pkg.mkTask(
-            task_name,
-            task_id,
-            self,
-            params,
-            depends)
-        task.rundir = rundir
+        print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
+        task = ctor_t.task_ctor(
+            name=task_name,
+            session=self,
+            params=ctor_t.mkParams(),
+            depends=depends,
+            rundir=rundir,
+            srcdir=ctor_t.srcdir)
         
         for i,d in enumerate(task.depend_refs):
             if d in self._task_m.keys():
@@ -130,6 +133,15 @@ class Session(object):
         self._pkg_spec_s.pop()
 
         return task
+
+    def push_package(self, pkg : Package):
+        self._pkg_s.append(pkg)
+
+    def pop_package(self, pkg : Package):
+        self._pkg_s.pop()
+
+    def package(self):
+        return self._pkg_s[-1]
     
     def mkTaskId(self, task : 'Task') -> int:
         self._task_id += 1
@@ -158,13 +170,13 @@ class Session(object):
 
         if not len(self._pkg_spec_s):
             self._pkg_spec_s.append(PackageSpec(pkg.name))
-            self._pkg_def_m[PackageSpec(pkg.name)] = pkg
         else:
-            if self._pkg_spec_s[0].name != pkg.name:
-                raise Exception("Package name mismatch: %s != %s" % (self._pkg_m[0].name, pkg.name))
+            if self._pkg_spec_s[-1].name != pkg.name:
+                raise Exception("Package name mismatch: %s != %s" % (self._pkg_spec_s[-1].name, pkg.name))
             else:
                 # TODO: merge content
                 self._pkg_spec_s.append(PackageSpec(pkg.name))
+        self._pkg_def_m[PackageSpec(pkg.name)] = pkg
 
         print("pkg: %s" % str(pkg))
 
@@ -219,9 +231,12 @@ class Session(object):
 
 
     def getPackage(self, spec : PackageSpec) -> Package:
+        # Obtain the active package definition
         pkg_spec = self._pkg_spec_s[-1]
         pkg_def = self._pkg_def_m[pkg_spec]
 
+        print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
+
         # Need a stack to track which package we are currently in
         # Need a map to get a concrete package from a name with parameterization
 
@@ -261,30 +276,49 @@ class Session(object):
                         pkg = base.mkPackage(self, spec.params)
                         self._pkg_m[spec] = pkg
                         break
+            
+            if pkg is None:
+                # Look in the set of registered packages
+                if spec.name in self._pkg_file_m.keys():
+                    # Load the package
+                    self._pkg_spec_s.append(spec)
+                    pkg_def = self._load_package(
+                        self._pkg_file_m[spec.name], 
+                        [])
+
+                    self._pkg_spec_s.pop()
+
+                    # The definition is now in the map, so recurse to create it
+                    pkg = self.getPackage(spec)
+                else:
+                    # Go search the package path
+                    pass
+                
+
 
             if pkg is None:
                 raise Exception("Failed to find package %s from package %s" % (
                     spec.name, pkg_def.name))
 
-#            base_spec = PackageSpec(spec.name)
-#            if not base_spec in self._pkg_def_m.keys():
-#                # Template is not present. Go find it...
-#
-#                # If not found...
-#                raise Exception("Package %s not found" % spec.name)
-
         return pkg
         
     def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef) -> 'TaskCtor':
         spec_e = spec.name.split(".")
         task_name = spec_e[-1]
-        pkg_name = ".".join(spec_e[0:-1])
 
-        try:
-            pkg = self.getPackage(PackageSpec(pkg_name))
-        except Exception as e:
-            print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
-            raise e
+        if len(spec_e) == 1:
+            # Just have a task name. Use the current package
+            if len(self._pkg_s) == 0:
+                raise Exception("No package context for task %s" % spec.name)
+            pkg = self._pkg_s[-1]
+        else:
+            pkg_name = ".".join(spec_e[0:-1])
+
+            try:
+                pkg = self.getPackage(PackageSpec(pkg_name))
+            except Exception as e:
+                print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
+                raise e
 
         return pkg.getTaskCtor(task_name)
 

dv_flow_mgr/task.py

@@ -24,7 +24,7 @@ import json
 import asyncio
 import dataclasses as dc
 from pydantic import BaseModel
-from typing import Any, Dict, List, Tuple
+from typing import Any, Callable, Dict, List, Tuple
 from .task_data import TaskData
 from .task_memento import TaskMemento
 
@@ -37,78 +37,32 @@ class TaskParams(BaseModel):
 
 @dc.dataclass
 class TaskCtor(object):
-    def mkTaskParams(self) -> TaskParams:
-        raise NotImplementedError()
-    
-    def setTaskParams(self, params : TaskParams, pvals : Dict[str,Any]):
-        for p in pvals.keys():
-            if not hasattr(params, p):
-                raise Exception("Unsupported parameter: " + p)
-            else:
-                setattr(params, p, pvals[p])
-
-    def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
-        raise NotImplementedError()
-
-@dc.dataclass
-class TaskCtorT(TaskCtor):
-    TaskParamsT : TaskParams
-    TaskT : 'Task'
-
-    def mkTaskParams(self) -> TaskParams:
-        return self.TaskParamsT()
-
-    def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
-        task = self.TaskT(
-            name=name, 
-            task_id=task_id, 
-            session=session, 
-            params=params, 
-            depends=depends)
-        task.depends.extend(depends)
-        return task
-
-@dc.dataclass
-class TaskParamCtor(object):
-    base : TaskCtor
-    params : Dict[str,Any]
-    basedir : str
-    depend_refs : List['TaskSpec']
-
-    def mkTaskParams(self) -> TaskParams:
-        params = self.base.mkTaskParams()
-        self.base.setTaskParams(params, self.params)
-        return params
-    
-    def setTaskParams(self, params : Dict, pvals : Dict[str,Any]):
-        pass
-
-    def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
-        task =  self.base.mkTask(
-            name=name, 
-            task_id=task_id, 
-            session=session, 
-            params=params, 
-            depends=depends)
-        task.basedir = self.basedir
-        task.depend_refs.extend(self.depend_refs)
-        return task
-
-@dc.dataclass
-class PackageTaskCtor(TaskCtor):
-    name : str
-    pkg : 'Package'
-
-    def mkTaskParams(self, params : Dict) -> Dict:
-        return self.pkg.mkTaskParams(self.name, params)
-    def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
-        return self.pkg.mkTask(self.name, task_id, session, params, depends)
+    task_ctor : Callable
+    param_ctor : Callable
+    params : Dict[str,Any] = None
+    srcdir : str = None
+    depends : List[TaskSpec] = dc.field(default_factory=list)
+
+    def copy(self):
+        return TaskCtor(
+            task_ctor=self.task_ctor,
+            param_ctor=self.param_ctor,
+            params=self.params,
+            srcdir=self.srcdir,
+            depends=self.depends.copy())
+
+    def mkParams(self):
+        print("mkParams: %s" % str(self.params))
+        ret = self.param_ctor()
+        if self.params is not None:
+            for k,v in self.params.items():
+                setattr(ret, k, v)
+        return ret
 
 @dc.dataclass
 class Task(object):
     """Executable view of a task"""
     name : str
-    task_id : int
     session : 'Session'
     params : TaskParams
     basedir : str
@@ -155,14 +109,6 @@ class Task(object):
             # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
-            # First, merge the dep maps of all the inputs
-            deps_m = self.depends[0].output.deps.copy()
-            for deps in map(lambda d: d.deps, self.depends[1:]):
-                for k,v in deps.items():
-                    if k in deps_m:
-                        deps_m[k].add(v)
-                    else:
-                        deps_m[k] = set(v)
 
             print("deps_m: %s" % str(deps_m))
 
@@ -182,8 +128,7 @@ class Task(object):
         # Mark the source of this data as being this task
         input.src = self.name
 
-        if not os.path.isdir(self.rundir):
-            os.makedirs(self.rundir)
+        self.init_rundir()
 
         result = await self.run(input)
 
@@ -202,9 +147,7 @@ class Task(object):
             result = self.getOutput()
 
         # Write-back the memento, if specified
-        if self.memento is not None:
-            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
-                fp.write(self.memento.model_dump_json(indent=2))
+        self.save_memento()
 
         self.running = False
 
@@ -214,6 +157,15 @@ class Task(object):
     async def run(self, input : TaskData) -> TaskData:
         raise NotImplementedError("TaskImpl.run() not implemented")
     
+    def init_rundir(self):
+        if not os.path.isdir(self.rundir):
+            os.makedirs(self.rundir)
+
+    def save_memento(self):
+        if self.memento is not None:
+            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
+                fp.write(self.memento.model_dump_json(indent=2))
+    
     def setOutput(self, output : TaskData):
         self.output_set = True
         output.src = self.name

dv_flow_mgr/task_data.py

@@ -24,6 +24,7 @@ import pydantic.dataclasses as dc
 from pydantic import BaseModel
 from typing import Any, Dict, Set, List, Tuple
 from .fileset import FileSet
+from toposort import toposort
 
 class TaskDataParamOpE(enum.Enum):
     Set = enum.auto()
@@ -32,18 +33,25 @@ class TaskDataParamOpE(enum.Enum):
     PathAppend = enum.auto()
     PathPrepend = enum.auto()
 
+class TaskDataParamKindE(enum.Enum):
+    String = enum.auto()
+    FilePath = enum.auto()
+    SearchPath = enum.auto()
+    List = enum.auto()
+
 class TaskDataParamOp(BaseModel):
     op : TaskDataParamOpE
     value : Any
 
 class TaskDataParam(BaseModel):
-    value : Any
+    kind : TaskDataParamKindE
     ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
 
 class TaskData(BaseModel):
     src : str = None
-    params : Dict[str,Any] = dc.Field(default_factory=dict)
+    params : Dict[str,TaskDataParam] = dc.Field(default_factory=dict)
     deps : Dict[str,Set[str]] = dc.Field(default_factory=dict)
+    filesets : List[FileSet] = dc.Field(default_factory=list)
     changed : bool = False
 
     def hasParam(self, name: str) -> bool:
@@ -56,16 +64,27 @@ class TaskData(BaseModel):
         self.params[name] = value
 
     def addFileSet(self, fs : FileSet):
-        fs.src = self.src
-        if "filesets" not in self.params:
-            self.params["filesets"] = []
-        self.params["filesets"].append(fs)
+        self.filesets.append(fs)
 
-    def getFileSets(self, type=None) -> List[FileSet]:
+    def getFileSets(self, type=None, order=True) -> List[FileSet]:
         ret = []
 
-        if "filesets" in self.params:
-            for fs in self.params["filesets"]:
+        if order:
+            # The deps map specifies task dependencies
+
+            candidate_fs = []
+            for fs in self.filesets:
+                if type is None or fs.type in type:
+                    candidate_fs.append(fs)
+
+            order = toposort(self.deps)
+
+            for order_s in order:
+                for fs in candidate_fs:
+                    if fs.src in order_s:
+                            ret.append(fs)
+        else:
+            for fs in self.filesets:
                 if type is None or fs.type in type:
                     ret.append(fs)
         
@@ -80,15 +99,171 @@ class TaskData(BaseModel):
         ret.changed = self.changed
         return ret
     
-    def merge(self, other):
-        for k,v in other.params.items():
-            if k not in self.params:
-                if hasattr(v, "copy"):
-                    self.params[k] = v.copy()
+    def setParamVal(self, name: str, kind : TaskDataParamKindE, value: Any):
+        if name not in self.params:
+            self.params[name] = TaskDataParam(kind=kind)
+        self.params[name].ops.append(TaskDataParamOp(op=TaskDataParamOpE.Set, value=value))
+    
+    def getParamVal(self, name: str) -> Any:
+        if name not in self.params.keys():
+            raise Exception("No such parameter: %s" % name)
+        param = self.params[name]
+        value = param.ops[0].value
+
+        if len(param.ops) > 1:
+            for op in param.ops[1:]:
+                if op.op == TaskDataParamOpE.Append:
+                    if isinstance(value, list):
+                        value.extend(op.value)
+                    else:
+                        value += op.value
+                elif op.op == TaskDataParamOpE.Prepend:
+                    if isinstance(value, list):
+                        for nv in op.value:
+                            value.insert(0, nv)
+                    else:
+                        value = op.value + value
+                elif op.op == TaskDataParamOpE.PathAppend:
+                    if isinstance(value, list):
+                        value = ":".join(value)
+                    value = value + ":" + op.value
+                elif op.op == TaskDataParamOpE.PathPrepend:
+                    if isinstance(value, list):
+                        value = ":".join(value)
+                    value = op.value + ":" + value
+
+        return value
+
+    @staticmethod    
+    def merge(incoming : List['TaskData'], local : 'TaskData' = None) -> 'TaskData':
+        """Merges incoming data with local settings and produces an output"""
+
+        # Deal with the dependency trees first
+        output = TaskData()
+
+        # First, merge the dep maps of all the inputs
+        output.deps = incoming[0].deps.copy()
+        for deps in map(lambda i: i.deps, incoming[1:]):
+            for k,v in deps.items():
+                if k not in output.deps:
+                    output.deps[k] = []
+                for vi in v:
+                    if vi not in output.deps[k]:
+                        output.deps[k].append(v)
+
+        # Process filesets
+        for inp in incoming:
+            for fs in inp.filesets:
+                exists = False
+                for fs_o in output.filesets:
+                    if fs_o.name == fs.name and fs_o.src == fs.src:
+                        exists = True
+                        break
+                if not exists:
+                    output.addFileSet(fs.model_copy())
+
+        # Now, deal with parameters
+        # Find collisions first
+        colliding_keys = set()
+        passthrough_keys = set()
+
+        for i in incoming:
+            for k in i.params.keys():
+                if k in passthrough_keys:
+                    colliding_keys.add(k)
+                else:
+                    passthrough_keys.add(k)
+
+        # Now, removes those that are locally set
+        local_set_params = set()
+        if local is not None:
+            for k,v in local.params.items():
+                if len(v.ops) == 1 and v.ops[0].op == TaskDataParamOpE.Set:
+                    local_set_params.add(k)
+                    # If are setting locally, it's not passthrough
+                    passthrough_keys.remove(k)
+                    if k in colliding_keys:
+                        colliding_keys.remove(k)
+        
+        # Construct the passthrough set by removing 
+        # colliding entries and those that we will set locally
+        for k in colliding_keys:
+            if k in passthrough_keys:
+                passthrough_keys.remove(k)
+
+        # For the remaining keys, check for conflicts by
+        # confirming that the last 'set' in each incoming parameter
+        # are equal
+        for k in colliding_keys:
+            value = None
+            for i,inp in enumerate(incoming):
+                value_i = None
+                param = inp.params[k]
+                if len(param.ops) == 1:
+                    value_i = param.ops[0].value
+                else:
+                    # Iterate in reverse over the operations
+                    for op in param.ops[::-1]:
+                        if op.op == TaskDataParamOpE.Set:
+                            value_i = op.value
+                            break
+                if not i:
+                    value = value_i
+                else:
+                    if value != value_i:
+                        raise Exception("Parameter %s has conflicting values (%s %s)" % (
+                            k,
+                            str(value),
+                            value(value_i)))
+
+
+        # Now, we need to construct the result
+        # - copy over passthrough parameters
+        # - add locally-set parameters
+        # - for others
+        #   - Apply full list for first input
+        #   - Apply all beyond the last 'set' operation for others
+        for k in passthrough_keys:
+            # Find an input that has the parameter
+            for inp in incoming:
+                if k in inp.params:
+                    break
+            # Find the value of the param
+            param = inp.params[k]
+
+            if len(param.ops) == 1:
+                output.params[k] = TaskDataParam(kind=param.kind)
+                output.params[k].ops.append(param.ops[0])
+            else:
+                for op in param.ops[::-1]:
+                    if op.op == TaskDataParamOpE.Set:
+                        output.params[k] = TaskDataParam(kind=param.kind)
+                        output.params[k].ops.append(op)
+                        break
+        for k in local_set_params:
+            output.params[k] = local.params[k].model_copy()
+
+        for k in colliding_keys:
+            value = None
+            for i,inp in enumerate(incoming):
+                # Find the last location that performs a 'set' 
+                last_set_i = -1
+                param = inp.params[k]
+                if len(param.ops) == 1:
+                    last_set_i = 0
+                else:
+                    # Iterate in reverse over the operations
+                    for j,op in enumerate(param.ops[::-1]):
+                        if op.op == TaskDataParamOpE.Set:
+                            last_set_i = j
+                            break
+                
+                if not i:
+                    # Copy the full list, including the last 'set'
+                    output.params[k].ops = param.param[last_set_i:].copy()
                 else:
-                    self.params[k] = v
-            elif hasattr(self.params[k], "merge"):
-                self.params[k].merge(v)
-            elif self.params[k] != v:
-                raise Exception("Parameter %s has conflicting values" % k)
+                    # append any additional directives
+                    if last_set_i+1 < len(param.ops):
+                        output.params[k].extend(param.ops[last_set_i+1:])
 
+        return output

dv_flow_mgr/task_def.py

@@ -28,11 +28,13 @@ from .task import Task
 class TaskSpec(object):
     name : str
 
-
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
     name : str
-    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
+#    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
+    uses : str = dc.Field(default=None)
+    pyclass : str = dc.Field(default=None)
+    doc : str = dc.Field(default=None)
     depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
 

dv_flow_mgr/tasklib/builtin_pkg.py

@@ -0,0 +1,62 @@
+# import os
+# import sys
+# import glob
+# import fnmatch
+# import importlib
+# import pydantic.dataclasses as dc
+# from ..package import TaskCtor
+from ..task import Task
+from ..task_data import TaskData
+# from ..task_memento import TaskMemento
+# from typing import List, Tuple
+# import dataclasses as dc
+# from ..package_def import Package
+
+# class TaskPyClass(Task):
+#     pyclass : str = None
+
+#     async def run(self, input : TaskData) -> TaskData:
+
+#         if self.srcdir not in sys.path:
+#             sys.path.insert(0, self.srcdir)
+
+#         print("sys.path: %s" % str(sys.path), flush=True)
+#         idx = self.params.pyclass.rfind('.')
+#         modname = self.params.pyclass[:idx]
+#         clsname = self.params.pyclass[idx+1:]
+
+#         if os.path.isfile(os.path.join(self.basedir, "my_module.py")):
+#             print("my_module.py exists", flush=True)
+#         else:
+#             print("my_module.py does not exist", flush=True)
+
+#         if modname in sys.modules.keys():
+#             module = sys.modules[modname]
+#         else:
+#             try:
+#                 print("modname=%s" % modname, flush=True)
+#                 module = importlib.import_module(modname)
+#             except ModuleNotFoundError as e:
+#                 print("Module not found: %s syspath=%s" % (str(e), str(sys.path)), flush=True)
+#                 raise e
+
+#         cls = getattr(module, clsname)
+
+#         obj = cls(self.name, self.task_id, self.session, self.basedir, srcdir=self.srcdir)
+
+#         return await obj.run(input)
+
+
+# class TaskPyClassMemento(TaskMemento):
+#     pass
+
+class TaskNull(Task):
+    async def run(self, input : TaskData) -> TaskData:
+        return input
+
+# @dc.dataclass
+# class PackageBuiltin(Package):
+
+#     def __post_init__(self):
+#         print("PackageBuiltin::__post_init__", flush=True)
+#         self.tasks["PyClass"] = TaskPyClass()

dv_flow_mgr/tasklib/std/fileset.py

@@ -1,5 +1,68 @@
 
-from dv_flow_mgr import Task
+import os
+import fnmatch
+import glob
+import pydantic.dataclasses as dc
+from typing import List, Tuple
+from dv_flow_mgr import Task, TaskData, TaskMemento
+from dv_flow_mgr import FileSet as _FileSet
+
+class TaskFileSetMemento(TaskMemento):
+    files : List[Tuple[str,float]] = dc.Field(default_factory=list)
 
 class FileSet(Task):
+
+    async def run(self, input : TaskData) -> TaskData:
+        print("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
+            self.name,
+            self.srcdir,
+            self.params.base, self.params.type, str(self.params.include)
+        ))
+
+
+        ex_memento = self.getMemento(TaskFileSetMemento)
+        memento = TaskFileSetMemento()
+
+        if self.params is not None:
+            glob_root = os.path.join(self.srcdir, self.params.base)
+
+            print("glob_root: %s" % glob_root)
+
+            fs = _FileSet(
+                src=self.name, 
+                type=self.params.type,
+                basedir=glob_root)
+            print("glob_root: %s" % glob_root)
+
+            if not isinstance(self.params.include, list):
+                self.params.include = [self.params.include]
+
+            included_files = []
+            for pattern in self.params.include:
+                print("pattern: %s" % pattern)
+                included_files.extend(glob.glob(os.path.join(glob_root, pattern), recursive=False))
+
+            for file in included_files:
+                if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in self.params.exclude):
+                    memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))
+                    fs.files.append(file[len(glob_root):])
+
+        # Check to see if the filelist or fileset have changed
+        # Only bother doing this if the upstream task data has not changed
+        if ex_memento is not None and not input.changed:
+            ex_memento.files.sort(key=lambda x: x[0])
+            memento.files.sort(key=lambda x: x[0])
+            print("ex_memento.files: %s" % str(ex_memento.files))
+            print("memento.files: %s" % str(memento.files))
+            input.changed = ex_memento != memento
+        else:
+            input.changed = True
+
+        self.setMemento(memento)
+
+        if fs is not None:
+            input.addFileSet(fs)
+
+        return input
+
     pass

dv_flow_mgr/tasklib/std/flow.dv

@@ -0,0 +1,26 @@
+
+package:
+  name: std
+
+  tasks:
+  - name: Message
+    pyclass: message.Message
+    with:
+      msg:
+        type: str
+        value: ""
+  - name: FileSet
+    pyclass: fileset.FileSet
+    with:
+      base:
+        type: str
+        value: ""
+      type:
+        type: str
+        value: ""
+      include:
+        type: str
+        value: ""
+      exclude:
+        type: str
+        value: ""

dv_flow_mgr/tasklib/std/message.py

@@ -0,0 +1,7 @@
+
+from dv_flow_mgr import Task, TaskData
+
+class Message(Task):
+    async def run(self, input : TaskData) -> TaskData:
+        print("%s: %s" % (self.name, self.params.msg))
+        return input

{dv_flow_mgr-0.0.1.12761553329a1.dist-info → dv_flow_mgr-0.0.1.12849118090a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12761553329a1
+Version: 0.0.1.12849118090a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12761553329a1.dist-info → dv_flow_mgr-0.0.1.12849118090a1.dist-info}/RECORD

@@ -1,19 +1,20 @@
 dv_flow_mgr/__init__.py,sha256=GLfYxZONxJNNUplfmlsXrcycJ2VhTc8gvXN6qdlH7kg,97
 dv_flow_mgr/__main__.py,sha256=4xXA4XCqLELlIak6bs-H6rRHp2URucipHPvCiVdR01c,513
-dv_flow_mgr/fileset.py,sha256=89knoTNsDF85QJwrctLiM6ryqHS1oxW25Gd5jgpX-IQ,1123
+dv_flow_mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow_mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
 dv_flow_mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
-dv_flow_mgr/package.py,sha256=jFhXgKDEYSRfs_VRUjKY4C7KScQfmrNLEXQgtWKaBfI,2767
-dv_flow_mgr/package_def.py,sha256=Js8bZj8bSmJQ6Wk9xAhIBJScNGJOHNbXNhw_zkDVU5w,3834
+dv_flow_mgr/package.py,sha256=k6gaDun9mJeGwGsFP5YOYOaFLmKb3KyPZy5wGRgJd_E,1965
+dv_flow_mgr/package_def.py,sha256=YZkXk6alkP6hjGT61EAglAHPEE-Rax8VCybIYbQ9NJs,8040
 dv_flow_mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
 dv_flow_mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
-dv_flow_mgr/session.py,sha256=sYRyTojsfPFZOGu4mz9yNTX6ESNye0i5szgj5QVE-Pk,10802
-dv_flow_mgr/task.py,sha256=ZTuSulc6lBLcTByD6gQjmbaJqqL1c1mlVtP84iLrLbE,7973
-dv_flow_mgr/task_data.py,sha256=32rgwPxSthIjNbzbXK2sZoPo6Jedio_L54Qibu65fYk,2962
-dv_flow_mgr/task_def.py,sha256=Yjw1Q-bnbi7Ct5X1PriMMNJ6i4TBVFWrO8BTnipz9hg,1532
+dv_flow_mgr/session.py,sha256=oj75khugWLdBe7ouAmQX9sVAMRZi6O4CdLquokIkXDY,12163
+dv_flow_mgr/task.py,sha256=AIz9YI-K3riv1EQ-hArL6eYsfbWjHMoN07Ao4mRdiW8,6146
+dv_flow_mgr/task_data.py,sha256=8AcFCxVmv9agzUDXnwUNJImpfp0N4jB1IHodIECf-0k,9735
+dv_flow_mgr/task_def.py,sha256=w0V5FSHi8D6PFyq_3j_smO3fmJp-cAMpigWT0yU4-GY,1654
 dv_flow_mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
 dv_flow_mgr/cmds/cmd_run.py,sha256=ii48_jdxCxh4MOJCeKKX66Cui4IvQQy_RrJEIixsph8,733
 dv_flow_mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow_mgr/tasklib/builtin_pkg.py,sha256=ps5SkO7s29xVzyeEw96qbMAg9Dk6UPErH2Lc87foIcs,1883
 dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py,sha256=l_jerOEQrVSmxT6qJx0PlsorZsK037Y-arQvWzdm9No,348
 dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py,sha256=U1XADvEGaH412n_JujS1Maz08GJ07UL6mRT4B4qihZQ,2229
 dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py,sha256=QkKYOs0OhQ42F5bUbfCsezE4L50TfiIm7lVflVY4On8,1290
@@ -26,14 +27,16 @@ dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py,sha256=QJP1gjclUSYfq-B9NIA46C1E
 dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py,sha256=Pu9mWAR93kiTNT8GZJrX-VDjMOfr9P5ddcSPt1BElQQ,413
 dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py,sha256=9RDJ3r37C4tA4PIa7tDUYYgRpIh19up-Y9HTOwKxr0I,1564
 dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py,sha256=RxPaE1RSRi2QdadrZp2J3lZqTt6EnLjCnk__uPAV6_o,1199
-dv_flow_mgr/tasklib/std/fileset.py,sha256=u88qjKQpIip_eETk0k6J1AGxk9blKlqAqYinHGKwco8,61
+dv_flow_mgr/tasklib/std/fileset.py,sha256=m7b85A81pNkkq_HM7XdDO5zdkoGvbbOCaTckEH666jg,2350
+dv_flow_mgr/tasklib/std/flow.dv,sha256=d3jFrZnTqdI2IZB-lSOs5tjA_beLtuLHgjusytf2_nA,391
+dv_flow_mgr/tasklib/std/message.py,sha256=O61dY00bQPCd7Ak5o-KGXh3C7a9wfZcbU_D2fiBudd8,193
 dv_flow_mgr/tasklib/std/pkg_std.py,sha256=ERDd515qtetQaldk9opUH19MyNwo8nP4-GLz-ke-5gE,370
 dv_flow_mgr/tasklib/std/std.dfs,sha256=An2NUvvWaR9xI4U-kRQ4NN6U-GhX6GQmd3mNyRkW4NQ,68
 dv_flow_mgr/tasklib/std/task_fileset.py,sha256=ZBG4e2gsTsRMsr_FhhzIoE3J7YwYQZ9HigpxpGePd74,3145
 dv_flow_mgr/tasklib/std/task_null.py,sha256=bQH1HN4r-TtdUaJxfnmvoqKZSY4MWO48eK7z_yngdNU,599
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/METADATA,sha256=vQplCReVKB6eEKc6Ys_XM6nEB_T1m7Mw35b8ZV_5FzM,13252
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/entry_points.txt,sha256=VHxIon5AeHCtK73fYb1veDImAGwv76hrtBiuZePmzhI,51
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/top_level.txt,sha256=FGuhGIxN1FgD_YBkPcDq05fTxZcwaX9G_O8FFBvrTk4,12
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/METADATA,sha256=Si9x7uqsXSWlCl6c6eQUJ8CpIi3duJ_4rBM2QkPTJ_Q,13252
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/entry_points.txt,sha256=VHxIon5AeHCtK73fYb1veDImAGwv76hrtBiuZePmzhI,51
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/top_level.txt,sha256=FGuhGIxN1FgD_YBkPcDq05fTxZcwaX9G_O8FFBvrTk4,12
+dv_flow_mgr-0.0.1.12849118090a1.dist-info/RECORD,,