dv-flow-mgr 0.0.1.12761553329a1__py3-none-any.whl → 0.0.1.12822558956a1__py3-none-any.whl

dv_flow_mgr/fileset.py

@@ -24,8 +24,9 @@ from pydantic import BaseModel
 from typing import Any, Dict, List, Tuple
 
 class FileSet(BaseModel):
-    src : str
     type : str
     basedir : str
+    name : str = ""
+    src : str = None
     files : List[str] = dc.Field(default_factory=list)
     params : Dict[str,str] = dc.Field(default_factory=dict)

dv_flow_mgr/task.py

@@ -155,14 +155,6 @@ class Task(object):
             # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
-            # First, merge the dep maps of all the inputs
-            deps_m = self.depends[0].output.deps.copy()
-            for deps in map(lambda d: d.deps, self.depends[1:]):
-                for k,v in deps.items():
-                    if k in deps_m:
-                        deps_m[k].add(v)
-                    else:
-                        deps_m[k] = set(v)
 
             print("deps_m: %s" % str(deps_m))
 
@@ -182,8 +174,7 @@ class Task(object):
         # Mark the source of this data as being this task
         input.src = self.name
 
-        if not os.path.isdir(self.rundir):
-            os.makedirs(self.rundir)
+        self.init_rundir()
 
         result = await self.run(input)
 
@@ -202,9 +193,7 @@ class Task(object):
             result = self.getOutput()
 
         # Write-back the memento, if specified
-        if self.memento is not None:
-            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
-                fp.write(self.memento.model_dump_json(indent=2))
+        self.save_memento()
 
         self.running = False
 
@@ -214,6 +203,15 @@ class Task(object):
     async def run(self, input : TaskData) -> TaskData:
         raise NotImplementedError("TaskImpl.run() not implemented")
     
+    def init_rundir(self):
+        if not os.path.isdir(self.rundir):
+            os.makedirs(self.rundir)
+
+    def save_memento(self):
+        if self.memento is not None:
+            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
+                fp.write(self.memento.model_dump_json(indent=2))
+    
     def setOutput(self, output : TaskData):
         self.output_set = True
         output.src = self.name

dv_flow_mgr/task_data.py

@@ -24,6 +24,7 @@ import pydantic.dataclasses as dc
 from pydantic import BaseModel
 from typing import Any, Dict, Set, List, Tuple
 from .fileset import FileSet
+from toposort import toposort
 
 class TaskDataParamOpE(enum.Enum):
     Set = enum.auto()
@@ -32,18 +33,25 @@ class TaskDataParamOpE(enum.Enum):
     PathAppend = enum.auto()
     PathPrepend = enum.auto()
 
+class TaskDataParamKindE(enum.Enum):
+    String = enum.auto()
+    FilePath = enum.auto()
+    SearchPath = enum.auto()
+    List = enum.auto()
+
 class TaskDataParamOp(BaseModel):
     op : TaskDataParamOpE
     value : Any
 
 class TaskDataParam(BaseModel):
-    value : Any
+    kind : TaskDataParamKindE
     ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
 
 class TaskData(BaseModel):
     src : str = None
-    params : Dict[str,Any] = dc.Field(default_factory=dict)
+    params : Dict[str,TaskDataParam] = dc.Field(default_factory=dict)
     deps : Dict[str,Set[str]] = dc.Field(default_factory=dict)
+    filesets : List[FileSet] = dc.Field(default_factory=list)
     changed : bool = False
 
     def hasParam(self, name: str) -> bool:
@@ -56,16 +64,27 @@ class TaskData(BaseModel):
         self.params[name] = value
 
     def addFileSet(self, fs : FileSet):
-        fs.src = self.src
-        if "filesets" not in self.params:
-            self.params["filesets"] = []
-        self.params["filesets"].append(fs)
+        self.filesets.append(fs)
 
-    def getFileSets(self, type=None) -> List[FileSet]:
+    def getFileSets(self, type=None, order=True) -> List[FileSet]:
         ret = []
 
-        if "filesets" in self.params:
-            for fs in self.params["filesets"]:
+        if order:
+            # The deps map specifies task dependencies
+
+            candidate_fs = []
+            for fs in self.filesets:
+                if type is None or fs.type in type:
+                    candidate_fs.append(fs)
+
+            order = toposort(self.deps)
+
+            for order_s in order:
+                for fs in candidate_fs:
+                    if fs.src in order_s:
+                            ret.append(fs)
+        else:
+            for fs in self.filesets:
                 if type is None or fs.type in type:
                     ret.append(fs)
         
@@ -80,15 +99,171 @@ class TaskData(BaseModel):
         ret.changed = self.changed
         return ret
     
-    def merge(self, other):
-        for k,v in other.params.items():
-            if k not in self.params:
-                if hasattr(v, "copy"):
-                    self.params[k] = v.copy()
+    def setParamVal(self, name: str, kind : TaskDataParamKindE, value: Any):
+        if name not in self.params:
+            self.params[name] = TaskDataParam(kind=kind)
+        self.params[name].ops.append(TaskDataParamOp(op=TaskDataParamOpE.Set, value=value))
+    
+    def getParamVal(self, name: str) -> Any:
+        if name not in self.params.keys():
+            raise Exception("No such parameter: %s" % name)
+        param = self.params[name]
+        value = param.ops[0].value
+
+        if len(param.ops) > 1:
+            for op in param.ops[1:]:
+                if op.op == TaskDataParamOpE.Append:
+                    if isinstance(value, list):
+                        value.extend(op.value)
+                    else:
+                        value += op.value
+                elif op.op == TaskDataParamOpE.Prepend:
+                    if isinstance(value, list):
+                        for nv in op.value:
+                            value.insert(0, nv)
+                    else:
+                        value = op.value + value
+                elif op.op == TaskDataParamOpE.PathAppend:
+                    if isinstance(value, list):
+                        value = ":".join(value)
+                    value = value + ":" + op.value
+                elif op.op == TaskDataParamOpE.PathPrepend:
+                    if isinstance(value, list):
+                        value = ":".join(value)
+                    value = op.value + ":" + value
+
+        return value
+
+    @staticmethod    
+    def merge(incoming : List['TaskData'], local : 'TaskData' = None) -> 'TaskData':
+        """Merges incoming data with local settings and produces an output"""
+
+        # Deal with the dependency trees first
+        output = TaskData()
+
+        # First, merge the dep maps of all the inputs
+        output.deps = incoming[0].deps.copy()
+        for deps in map(lambda i: i.deps, incoming[1:]):
+            for k,v in deps.items():
+                if k not in output.deps:
+                    output.deps[k] = []
+                for vi in v:
+                    if vi not in output.deps[k]:
+                        output.deps[k].append(v)
+
+        # Process filesets
+        for inp in incoming:
+            for fs in inp.filesets:
+                exists = False
+                for fs_o in output.filesets:
+                    if fs_o.name == fs.name and fs_o.src == fs.src:
+                        exists = True
+                        break
+                if not exists:
+                    output.addFileSet(fs.model_copy())
+
+        # Now, deal with parameters
+        # Find collisions first
+        colliding_keys = set()
+        passthrough_keys = set()
+
+        for i in incoming:
+            for k in i.params.keys():
+                if k in passthrough_keys:
+                    colliding_keys.add(k)
+                else:
+                    passthrough_keys.add(k)
+
+        # Now, removes those that are locally set
+        local_set_params = set()
+        if local is not None:
+            for k,v in local.params.items():
+                if len(v.ops) == 1 and v.ops[0].op == TaskDataParamOpE.Set:
+                    local_set_params.add(k)
+                    # If are setting locally, it's not passthrough
+                    passthrough_keys.remove(k)
+                    if k in colliding_keys:
+                        colliding_keys.remove(k)
+        
+        # Construct the passthrough set by removing 
+        # colliding entries and those that we will set locally
+        for k in colliding_keys:
+            if k in passthrough_keys:
+                passthrough_keys.remove(k)
+
+        # For the remaining keys, check for conflicts by
+        # confirming that the last 'set' in each incoming parameter
+        # are equal
+        for k in colliding_keys:
+            value = None
+            for i,inp in enumerate(incoming):
+                value_i = None
+                param = inp.params[k]
+                if len(param.ops) == 1:
+                    value_i = param.ops[0].value
+                else:
+                    # Iterate in reverse over the operations
+                    for op in param.ops[::-1]:
+                        if op.op == TaskDataParamOpE.Set:
+                            value_i = op.value
+                            break
+                if not i:
+                    value = value_i
+                else:
+                    if value != value_i:
+                        raise Exception("Parameter %s has conflicting values (%s %s)" % (
+                            k,
+                            str(value),
+                            value(value_i)))
+
+
+        # Now, we need to construct the result
+        # - copy over passthrough parameters
+        # - add locally-set parameters
+        # - for others
+        #   - Apply full list for first input
+        #   - Apply all beyond the last 'set' operation for others
+        for k in passthrough_keys:
+            # Find an input that has the parameter
+            for inp in incoming:
+                if k in inp.params:
+                    break
+            # Find the value of the param
+            param = inp.params[k]
+
+            if len(param.ops) == 1:
+                output.params[k] = TaskDataParam(kind=param.kind)
+                output.params[k].ops.append(param.ops[0])
+            else:
+                for op in param.ops[::-1]:
+                    if op.op == TaskDataParamOpE.Set:
+                        output.params[k] = TaskDataParam(kind=param.kind)
+                        output.params[k].ops.append(op)
+                        break
+        for k in local_set_params:
+            output.params[k] = local.params[k].model_copy()
+
+        for k in colliding_keys:
+            value = None
+            for i,inp in enumerate(incoming):
+                # Find the last location that performs a 'set' 
+                last_set_i = -1
+                param = inp.params[k]
+                if len(param.ops) == 1:
+                    last_set_i = 0
+                else:
+                    # Iterate in reverse over the operations
+                    for j,op in enumerate(param.ops[::-1]):
+                        if op.op == TaskDataParamOpE.Set:
+                            last_set_i = j
+                            break
+                
+                if not i:
+                    # Copy the full list, including the last 'set'
+                    output.params[k].ops = param.param[last_set_i:].copy()
                 else:
-                    self.params[k] = v
-            elif hasattr(self.params[k], "merge"):
-                self.params[k].merge(v)
-            elif self.params[k] != v:
-                raise Exception("Parameter %s has conflicting values" % k)
+                    # append any additional directives
+                    if last_set_i+1 < len(param.ops):
+                        output.params[k].extend(param.ops[last_set_i+1:])
 
+        return output

dv_flow_mgr/tasklib/builtin_pkg.py

@@ -0,0 +1,61 @@
+import os
+import sys
+import glob
+import fnmatch
+import importlib
+import pydantic.dataclasses as dc
+from ..package import TaskCtor
+from ..task import Task, TaskParams, TaskCtorT
+from ..task_data import TaskData
+from ..task_memento import TaskMemento
+from typing import List, Tuple
+import dataclasses as dc
+from ..package_def import Package
+
+class TaskPyClass(Task):
+
+    async def run(self, input : TaskData) -> TaskData:
+
+        if self.srcdir not in sys.path:
+            sys.path.insert(0, self.srcdir)
+
+        print("sys.path: %s" % str(sys.path), flush=True)
+        idx = self.params.pyclass.rfind('.')
+        modname = self.params.pyclass[:idx]
+        clsname = self.params.pyclass[idx+1:]
+
+        if os.path.isfile(os.path.join(self.basedir, "my_module.py")):
+            print("my_module.py exists", flush=True)
+        else:
+            print("my_module.py does not exist", flush=True)
+
+        try:
+            print("modname=%s" % modname, flush=True)
+            module = importlib.import_module(modname)
+        except ModuleNotFoundError as e:
+            print("Module not found: %s syspath=%s" % (str(e), str(sys.path)), flush=True)
+            raise e
+
+        cls = getattr(module, clsname)
+
+        obj = cls(self.name, self.task_id, self.session, self.basedir, srcdir=self.srcdir)
+
+        return await obj.run(input)
+
+
+class TaskPyClassParams(TaskParams):
+    pyclass : str
+
+class TaskPyClassMemento(TaskMemento):
+    pass
+
+class TaskPyClassCtor(TaskCtorT):
+    def __init__(self):
+        super().__init__(TaskPyClassParams, TaskPyClass)
+
+@dc.dataclass
+class PackageBuiltin(Package):
+
+    def __post_init__(self):
+        print("PackageBuiltin::__post_init__", flush=True)
+        self.tasks["PyClass"] = TaskPyClass()

dv_flow_mgr/tasklib/std/flow.dv

@@ -0,0 +1,12 @@
+
+package:
+  name: std
+
+  tasks:
+  - name: Exec
+    uses: builtin.PyClass
+    params:
+    - name: cmd
+      type: string
+    - name: pyclass
+      type: string

{dv_flow_mgr-0.0.1.12761553329a1.dist-info → dv_flow_mgr-0.0.1.12822558956a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12761553329a1
+Version: 0.0.1.12822558956a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12761553329a1.dist-info → dv_flow_mgr-0.0.1.12822558956a1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 dv_flow_mgr/__init__.py,sha256=GLfYxZONxJNNUplfmlsXrcycJ2VhTc8gvXN6qdlH7kg,97
 dv_flow_mgr/__main__.py,sha256=4xXA4XCqLELlIak6bs-H6rRHp2URucipHPvCiVdR01c,513
-dv_flow_mgr/fileset.py,sha256=89knoTNsDF85QJwrctLiM6ryqHS1oxW25Gd5jgpX-IQ,1123
+dv_flow_mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
 dv_flow_mgr/flow.py,sha256=UdgJOIqBS2wTRpO-sNWCCqO9oQFxDfGPGVD0r42aTrA,1562
 dv_flow_mgr/fragment_def.py,sha256=p5i6ONtBWlDHTBFsduu3Z36_76Bn8PCIylp_xoZ7jfQ,1552
 dv_flow_mgr/package.py,sha256=jFhXgKDEYSRfs_VRUjKY4C7KScQfmrNLEXQgtWKaBfI,2767
@@ -8,12 +8,13 @@ dv_flow_mgr/package_def.py,sha256=Js8bZj8bSmJQ6Wk9xAhIBJScNGJOHNbXNhw_zkDVU5w,38
 dv_flow_mgr/package_import_spec.py,sha256=bStPa727wAKMcixydVY1Ht6ylzXsSMy2K31HWPXhc9k,921
 dv_flow_mgr/parameters.py,sha256=kUjRss5VtMMz5eL3-Z_M6BS-wFs7MhQu3ANXO54UPo0,896
 dv_flow_mgr/session.py,sha256=sYRyTojsfPFZOGu4mz9yNTX6ESNye0i5szgj5QVE-Pk,10802
-dv_flow_mgr/task.py,sha256=ZTuSulc6lBLcTByD6gQjmbaJqqL1c1mlVtP84iLrLbE,7973
-dv_flow_mgr/task_data.py,sha256=32rgwPxSthIjNbzbXK2sZoPo6Jedio_L54Qibu65fYk,2962
+dv_flow_mgr/task.py,sha256=976-6lsx0qarXDgzOHvg8szS1dIjsp1VoQTrcOWUiQE,7723
+dv_flow_mgr/task_data.py,sha256=8AcFCxVmv9agzUDXnwUNJImpfp0N4jB1IHodIECf-0k,9735
 dv_flow_mgr/task_def.py,sha256=Yjw1Q-bnbi7Ct5X1PriMMNJ6i4TBVFWrO8BTnipz9hg,1532
 dv_flow_mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
 dv_flow_mgr/cmds/cmd_run.py,sha256=ii48_jdxCxh4MOJCeKKX66Cui4IvQQy_RrJEIixsph8,733
 dv_flow_mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow_mgr/tasklib/builtin_pkg.py,sha256=qdLUrhF7ZGPpT4EQmtSeOiB4q_tbdLmY6JLxhAYevEA,1746
 dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py,sha256=l_jerOEQrVSmxT6qJx0PlsorZsK037Y-arQvWzdm9No,348
 dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py,sha256=U1XADvEGaH412n_JujS1Maz08GJ07UL6mRT4B4qihZQ,2229
 dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py,sha256=QkKYOs0OhQ42F5bUbfCsezE4L50TfiIm7lVflVY4On8,1290
@@ -27,13 +28,14 @@ dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py,sha256=Pu9mWAR93kiTNT8GZJrX-VDjMOfr9P5ddc
 dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py,sha256=9RDJ3r37C4tA4PIa7tDUYYgRpIh19up-Y9HTOwKxr0I,1564
 dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py,sha256=RxPaE1RSRi2QdadrZp2J3lZqTt6EnLjCnk__uPAV6_o,1199
 dv_flow_mgr/tasklib/std/fileset.py,sha256=u88qjKQpIip_eETk0k6J1AGxk9blKlqAqYinHGKwco8,61
+dv_flow_mgr/tasklib/std/flow.dv,sha256=onqRPp-71uAvLlQdXivvnrIs-OANZC5WYJQMn427TrE,158
 dv_flow_mgr/tasklib/std/pkg_std.py,sha256=ERDd515qtetQaldk9opUH19MyNwo8nP4-GLz-ke-5gE,370
 dv_flow_mgr/tasklib/std/std.dfs,sha256=An2NUvvWaR9xI4U-kRQ4NN6U-GhX6GQmd3mNyRkW4NQ,68
 dv_flow_mgr/tasklib/std/task_fileset.py,sha256=ZBG4e2gsTsRMsr_FhhzIoE3J7YwYQZ9HigpxpGePd74,3145
 dv_flow_mgr/tasklib/std/task_null.py,sha256=bQH1HN4r-TtdUaJxfnmvoqKZSY4MWO48eK7z_yngdNU,599
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/METADATA,sha256=vQplCReVKB6eEKc6Ys_XM6nEB_T1m7Mw35b8ZV_5FzM,13252
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/entry_points.txt,sha256=VHxIon5AeHCtK73fYb1veDImAGwv76hrtBiuZePmzhI,51
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/top_level.txt,sha256=FGuhGIxN1FgD_YBkPcDq05fTxZcwaX9G_O8FFBvrTk4,12
-dv_flow_mgr-0.0.1.12761553329a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/METADATA,sha256=sN9jrnKJpUhpO6NILL6OZBr1jC38YhnbyDtlbt8nESk,13252
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/entry_points.txt,sha256=VHxIon5AeHCtK73fYb1veDImAGwv76hrtBiuZePmzhI,51
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/top_level.txt,sha256=FGuhGIxN1FgD_YBkPcDq05fTxZcwaX9G_O8FFBvrTk4,12
+dv_flow_mgr-0.0.1.12822558956a1.dist-info/RECORD,,