dv-flow-mgr 0.0.1.12664275031a1__py3-none-any.whl

dv_flow_mgr/share/flow.json

@@ -0,0 +1,172 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://fvutils.github.io/dv-flow-mgr/flow.json",
+    "title": "Flow-specification schema",
+    "description": "Flow-specification schema",
+    "type": "object",
+    "properties": {
+        "package": {
+            "$ref": "#/defs/package-def"
+        },
+        "fragment": {
+            "$ref": "#/defs/fragment-def"
+        },
+        "oneOf": [
+            {"required": ["package"]},
+            {"required": ["fragment"]}
+        ]
+    },
+    "defs": {
+
+        "package-def": {
+            "title": "Package Definition",
+            "$$target": "#/defs/package-def",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "imports": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/import-def"
+                    }
+                },
+                "with": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/param"
+                    }
+                },
+                "tasks": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/task-def"
+                    }
+                },
+                "fragments": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+
+        "import-def": {
+            "title": "Import Definition",
+            "$$target": "#/defs/import-def",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "as": {
+                    "type": "string"
+                },
+                "with": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/param"
+                    }
+                },
+                "from": {
+                    "type": "string"
+                }
+            }
+        },
+
+        "fragment-def": {
+            "title": "Fragment Definition",
+            "$$target": "#/defs/fragment-def",
+            "type": "object",
+            "properties": {
+                "imports": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "tasks": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/task-def"
+                    }
+                },
+                "fragments": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "param": {
+            "title": "Parameter Definition",
+            "$$target": "#/defs/param",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "type": {
+                    "type": "string"
+                },
+                "value": {
+                    "type": "string"
+                }
+            }
+        },
+        "task-dep": {
+            "title": "Task dependency specification",
+            "$$target": "#/defs/task-dep",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "with": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/param"
+                    }
+                }
+            }
+        },
+        "task-def": {
+            "title": "Task Definition",
+            "$$target": "#/defs/task-def",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string"
+                },
+                "type": {
+                    "type": "string"
+                },
+                "with": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/defs/param"
+                    }
+                },
+                "depends": {
+                    "oneOf": [
+                        {
+                            "type": "array",
+                            "items": {
+                                "$ref": "#/defs/task-dep"
+                            }
+                        }, {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        }
+                    ]
+                }
+            }
+        }
+    }
+}
+

dv_flow_mgr/task.py

@@ -0,0 +1,225 @@
+#****************************************************************************
+#* task.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import os
+import json
+import asyncio
+import dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List, Tuple
+from .task_data import TaskData
+from .task_memento import TaskMemento
+
+@dc.dataclass
+class TaskSpec(object):
+    name : str
+
+class TaskParams(BaseModel):
+    pass
+
+@dc.dataclass
+class TaskCtor(object):
+    def mkTaskParams(self) -> TaskParams:
+        raise NotImplementedError()
+    
+    def setTaskParams(self, params : TaskParams, pvals : Dict[str,Any]):
+        for p in pvals.keys():
+            if not hasattr(params, p):
+                raise Exception("Unsupported parameter: " + p)
+            else:
+                setattr(params, p, pvals[p])
+
+    def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
+        raise NotImplementedError()
+
+@dc.dataclass
+class TaskCtorT(TaskCtor):
+    TaskParamsT : TaskParams
+    TaskT : 'Task'
+
+    def mkTaskParams(self) -> TaskParams:
+        return self.TaskParamsT()
+
+    def mkTask(self, name : str, task_id : int, session : 'Session', params : TaskParams, depends : List['Task']) -> 'Task':
+        task = self.TaskT(
+            name=name, 
+            task_id=task_id, 
+            session=session, 
+            params=params, 
+            depends=depends)
+        task.depends.extend(depends)
+        return task
+
+@dc.dataclass
+class TaskParamCtor(object):
+    base : TaskCtor
+    params : Dict[str,Any]
+    basedir : str
+    depend_refs : List['TaskSpec']
+
+    def mkTaskParams(self) -> TaskParams:
+        params = self.base.mkTaskParams()
+        self.base.setTaskParams(params, self.params)
+        return params
+    
+    def setTaskParams(self, params : Dict, pvals : Dict[str,Any]):
+        pass
+
+    def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
+        task =  self.base.mkTask(
+            name=name, 
+            task_id=task_id, 
+            session=session, 
+            params=params, 
+            depends=depends)
+        task.basedir = self.basedir
+        task.depend_refs.extend(self.depend_refs)
+        return task
+
+@dc.dataclass
+class PackageTaskCtor(TaskCtor):
+    name : str
+    pkg : 'Package'
+
+    def mkTaskParams(self, params : Dict) -> Dict:
+        return self.pkg.mkTaskParams(self.name, params)
+    def mkTask(self, name : str, task_id : int, session : 'Session', params : Dict, depends : List['Task']) -> 'Task':
+        return self.pkg.mkTask(self.name, task_id, session, params, depends)
+
+@dc.dataclass
+class Task(object):
+    """Executable view of a task"""
+    name : str
+    task_id : int
+    session : 'Session'
+    params : TaskParams
+    basedir : str
+    srcdir : str = None
+    memento : TaskMemento = None
+    depend_refs : List['TaskSpec'] = dc.field(default_factory=list)
+    depends : List[int] = dc.field(default_factory=list)
+    running : bool = False
+    output_set : bool = False
+    output : Any = None
+    output_ev : Any = asyncio.Event()
+
+    # Implementation data below
+    basedir : str = dc.field(default=None)
+    rundir : str = dc.field(default=None)
+    impl : str = None
+    body: Dict[str,Any] = dc.field(default_factory=dict)
+    impl_t : Any = None
+
+    def getMemento(self, T) -> TaskMemento:
+        if os.path.isfile(os.path.join(self.rundir, "memento.json")):
+            with open(os.path.join(self.rundir, "memento.json"), "r") as fp:
+                try:
+                    data = json.load(fp)
+                    self.memento = T(**data)
+                except Exception as e:
+                    print("Failed to load memento %s: %s" % (
+                        os.path.join(self.rundir, "memento.json"), str(e)))
+                    os.unlink(os.path.join(self.rundir, "memento.json"))
+        return self.memento
+
+    def setMemento(self, memento : TaskMemento):
+        self.memento = memento
+
+    async def do_run(self) -> TaskData:
+        print("do_run: %s - %d depends" % (self.name, len(self.depends)))
+        if len(self.depends) > 0:
+            awaitables = [dep.waitOutput() for dep in self.depends]
+            deps_o = await asyncio.gather(*awaitables)
+
+            print("deps_o: %s" % str(deps_o))
+
+            # Merge the output of the dependencies into a single input data
+#            if len(self.depends) > 1:
+#                raise Exception("TODO: handle >1 inputs")
+
+            input = self.depends[0].output.copy()
+        else:
+            input = TaskData()
+
+        if not os.path.isdir(self.rundir):
+            os.makedirs(self.rundir)
+
+        result = await self.run(input)
+
+        if not self.output_set:
+            if result is None:
+                result = TaskData()
+
+            # We perform an auto-merge algorithm if the task 
+            # doesn't take control
+#            for dep_o in deps_o:
+#                result.deps.append(dep_o.clone())
+
+            self.setOutput(result)
+        else:
+            # The task has taken control of the output
+            result = self.getOutput()
+
+        # Write-back the memento, if specified
+        if self.memento is not None:
+            with open(os.path.join(self.rundir, "memento.json"), "w") as fp:
+                fp.write(self.memento.model_dump_json(indent=2))
+
+        self.running = False
+
+        # Combine data from the deps to produce a result
+        return result
+
+    async def run(self, input : TaskData) -> TaskData:
+        raise NotImplementedError("TaskImpl.run() not implemented")
+    
+    def setOutput(self, output : TaskData):
+        self.output_set = True
+        output.task_id = self.task_id
+        self.output = output
+        self.output_ev.set()
+
+    async def waitOutput(self) -> TaskData:
+        if not self.output_set:
+            if self.running:
+                # Task is already running
+                print("wait")
+                await self.output_ev.wait()
+            else:
+                self.running = True
+                print("start task")
+                await self.do_run()
+        return self.output
+    
+    def getOutput(self) -> TaskData:
+        return self.output
+
+    def getField(self, name : str) -> Any:
+        if name in self.__dict__.keys():
+            return self.__dict__[name]
+        elif name in self.__pydantic_extra__.keys():
+            return self.__pydantic_extra__[name]
+        else:
+            raise Exception("No such field %s" % name)
+        
+
+
+

dv_flow_mgr/task_data.py

@@ -0,0 +1,77 @@
+#****************************************************************************
+#* task_data.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, Set, List, Tuple
+from .fileset import FileSet
+
+class TaskData(BaseModel):
+    task_id : int = -1
+    params : Dict[str,Any] = dc.Field(default_factory=dict)
+    deps : List['TaskData'] = dc.Field(default_factory=list)
+    changed : bool = False
+
+    def hasParam(self, name: str) -> bool:
+        return name in self.params
+    
+    def getParam(self, name: str) -> Any:
+        return self.params[name]
+    
+    def setParam(self, name: str, value: Any):
+        self.params[name] = value
+
+    def addFileSet(self, fs : FileSet):
+        if "filesets" not in self.params:
+            self.params["filesets"] = []
+        self.params["filesets"].append(fs)
+
+    def getFileSets(self, type : (str|Set[str])=None) -> List[FileSet]:
+        ret = []
+
+        if "filesets" in self.params:
+            for fs in self.params["filesets"]:
+                if type is None or fs.type in type:
+                    ret.append(fs)
+        
+        return ret
+
+    def copy(self) -> 'TaskData':
+        ret = TaskData()
+        ret.task_id = self.task_id
+        ret.params = self.params.copy()
+        for d in self.deps:
+            ret.deps.append(d.clone())
+        ret.changed = self.changed
+        return ret
+    
+    def merge(self, other):
+        for k,v in other.params.items():
+            if k not in self.params:
+                if hasattr(v, "copy"):
+                    self.params[k] = v.copy()
+                else:
+                    self.params[k] = v
+            elif hasattr(self.params[k], "merge"):
+                self.params[k].merge(v)
+            elif self.params[k] != v:
+                raise Exception("Parameter %s has conflicting values" % k)
+

dv_flow_mgr/task_def.py

@@ -0,0 +1,45 @@
+#****************************************************************************
+#* task_def.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List, Tuple
+from .task import Task
+
+@dc.dataclass
+class TaskSpec(object):
+    name : str
+
+
+class TaskDef(BaseModel):
+    """Holds definition information (ie the YAML view) for a task"""
+    name : str
+    type : (str|TaskSpec) = dc.Field(default_factory=list)
+    depends : List[(str|TaskSpec)] = dc.Field(default_factory=list)
+    params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
+
+    def copy(self) -> 'TaskDef':
+        ret = TaskDef(
+            name=self.name,
+            type=self.type,
+            depends=self.depends.copy())
+        return ret  
+

dv_flow_mgr/task_memento.py

@@ -0,0 +1,34 @@
+#****************************************************************************
+#* task_memento.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List
+
+class TaskMemento(BaseModel):
+#    dep_ids : List[int] = dc.Field(default_factory=list)
+#    params : Dict[str,Any] = dc.Field(default_factory=dict)
+
+    pass
+
+
+
+

dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py

@@ -0,0 +1,11 @@
+import dataclasses as dc
+from .pkg_hdl_sim import PackageHdlSim
+from .mti_task_sim_image import TaskMtiSimImageCtor
+from .mti_task_sim_run import TaskMtiSimRunCtor
+
+@dc.dataclass
+class MtiPackage(PackageHdlSim):
+    def __post_init__(self):
+        self.tasks["SimImage"] = TaskMtiSimImageCtor()
+        self.tasks["SimRun"] = TaskMtiSimRunCtor()
+

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py

@@ -0,0 +1,69 @@
+import os
+import fnmatch
+import dataclasses as dc
+from ....fileset import FileSet
+from ....package import TaskCtor
+from ....task import Task, TaskParams, TaskCtorT
+from ....task_data import TaskData
+from ....task_memento import TaskMemento
+from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
+from typing import List, Tuple
+
+from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
+
+@dc.dataclass
+class TaskMtiSimImage(VlTaskSimImage):
+
+    def getRefTime(self):
+        if os.path.isdir(os.path.join(self.rundir, 'work.d')):
+            return os.path.getmtime(os.path.join(self.rundir, 'work.d'))
+        else:
+            raise Exception("work.d not found")
+    
+    async def build(self, files : List[str], incdirs : List[str]):
+        if not os.path.isdir(os.path.join(self.rundir, 'work')):
+            cmd = ['vlib', 'work']
+            proc = await self.session.create_subprocess(*cmd,
+                                                        cwd=self.rundir)
+            await proc.wait()
+            if proc.returncode != 0:
+                raise Exception("Questa vlib failed")
+
+        cmd = ['vlog', '-sv']
+
+        for incdir in incdirs:
+            cmd.append('+incdir+%s' % incdir)
+
+        cmd.extend(files)
+
+        proc = await self.session.create_subprocess(*cmd,
+                                                        cwd=self.rundir)
+        await proc.wait()
+        if proc.returncode != 0:
+            raise Exception("Questa compile failed")
+
+        cmd = ['vopt', '-o', 'simv_opt']
+
+        for top in self.params.top:
+            cmd.append(top)
+
+        proc = await self.session.create_subprocess(*cmd,
+                                                        cwd=self.rundir)
+
+        await proc.wait()
+
+        with open(os.path.join(self.rundir, 'work.d'), "w") as fp:
+            pass
+
+        if proc.returncode != 0:
+            raise Exception("Questa opt failed")
+
+class TaskMtiSimImageParams(VlTaskSimImageParams):
+    pass
+
+class TaskMtiSimImageMemento(VlTaskSimImageMemento):
+    pass
+
+class TaskMtiSimImageCtor(TaskCtorT):
+    def __init__(self):
+        super().__init__(TaskMtiSimImageParams, TaskMtiSimImage)

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py

@@ -0,0 +1,47 @@
+import os
+import fnmatch
+import pydantic.dataclasses as dc
+from ....fileset import FileSet
+from ....package import TaskCtor
+from ....task import Task, TaskParams, TaskCtorT
+from ....task_data import TaskData
+from ....task_memento import TaskMemento
+from typing import List, Tuple
+
+class TaskMtiSimRun(Task):
+
+    async def run(self, input : TaskData) -> TaskData:
+        vl_fileset = input.getFileSets("verilatorBinary")
+
+        build_dir = vl_fileset[0].basedir
+
+        cmd = [
+            'vsim', '-batch', '-do', 'run -all; quit -f',
+            '-work', os.path.join(build_dir, 'work'),
+            'simv_opt'
+        ]
+
+        fp = open(os.path.join(self.rundir, 'sim.log'), "w")
+        proc = await self.session.create_subprocess(*cmd,
+                                                    cwd=self.rundir,
+                                                    stdout=fp)
+
+        await proc.wait()
+
+        fp.close()
+
+        output = TaskData()
+        output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
+
+        return output
+
+class TaskMtiSimRunParams(TaskParams):
+    pass
+
+class TaskMtiSimRunMemento(TaskMemento):
+    pass
+
+class TaskMtiSimRunCtor(TaskCtorT):
+    def __init__(self):
+        super().__init__(TaskMtiSimRunParams, TaskMtiSimRun)
+

dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py

@@ -0,0 +1,8 @@
+import dataclasses as dc
+from ....package import Package
+
+@dc.dataclass
+class PackageHdlSim(Package):
+    # TODO: parameter to select proper implementation
+    pass
+