dv-flow-mgr 0.0.1.12822558956a1__py3-none-any.whl → 0.0.1.12911707440a1__py3-none-any.whl

dv_flow_mgr/package_def.py

@@ -1,98 +0,0 @@
-#****************************************************************************
-#* package_def.py
-#*
-#* Copyright 2023 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import pydantic.dataclasses as dc
-import json
-from pydantic import BaseModel
-from typing import Any, Dict, List
-from .flow import Flow
-from .fragment_def import FragmentDef
-from .package import Package
-from .package_import_spec import PackageImportSpec, PackageSpec
-from .task import TaskParamCtor
-from .task_def import TaskDef, TaskSpec
-
-
-class PackageDef(BaseModel):
-    name : str
-    params : Dict[str,Any] = dc.Field(default_factory=dict)
-    type : List[PackageSpec] = dc.Field(default_factory=list)
-    tasks : List[TaskDef] = dc.Field(default_factory=list)
-    imports : List[PackageImportSpec] = dc.Field(default_factory=list)
-    fragments: List[str] = dc.Field(default_factory=list)
-
-    fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
-
-#    import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
-
-    basedir : str = None
-
-    def getTask(self, name : str) -> 'TaskDef':
-        for t in self.tasks:
-            if t.name == name:
-                return t
-    
-    def mkPackage(self, session, params : Dict[str,Any] = None) -> 'Package':
-        ret = Package(self.name)
-
-        for task in self.tasks:
-            if task.type is not None:
-                # Find package (not package_def) that implements this task
-                # Insert an indirect reference to that tasks's constructor
-
-                # Only call getTaskCtor if the task is in a different package
-                task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
-                ctor_t = session.getTaskCtor(task_t, self)
-
-                ctor_t = TaskParamCtor(
-                    base=ctor_t, 
-                    params=task.params, 
-                    basedir=self.basedir,
-                    depend_refs=task.depends)
-            else:
-                # We use the Null task from the std package
-                raise Exception("")
-            ret.tasks[task.name] = ctor_t
-
-        for frag in self.fragment_l:
-            for task in frag.tasks:
-                if task.type is not None:
-                    # Find package (not package_def) that implements this task
-                    # Insert an indirect reference to that tasks's constructor
-
-                    # Only call getTaskCtor if the task is in a different package
-                    task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
-                    ctor_t = session.getTaskCtor(task_t, self)
-
-                    ctor_t = TaskParamCtor(
-                        base=ctor_t, 
-                        params=task.params, 
-                        basedir=frag.basedir,
-                        depend_refs=task.depends)
-                else:
-                    # We use the Null task from the std package
-                    raise Exception("")
-                if task.name in ret.tasks:
-                    raise Exception("Task %s already defined" % task.name)
-                ret.tasks[task.name] = ctor_t
-
-        return ret
-

dv_flow_mgr/session.py

@@ -1,290 +0,0 @@
-#****************************************************************************
-#* session.py
-#*
-#* Copyright 2023 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import asyncio
-import os
-import yaml
-import dataclasses as dc
-from typing import Any, Callable, Dict, List
-from .fragment_def import FragmentDef
-from .package import Package
-from .package_def import PackageDef, PackageSpec
-from .task import Task,TaskSpec
-
-@dc.dataclass
-class Session(object):
-    """Manages the running of a flow"""
-
-    srcdir : str
-    rundir : str
-
-    # Search path for .dfs files
-    package_path : List[str] = dc.field(default_factory=list)
-    package : PackageDef = None
-    create_subprocess : Callable = asyncio.create_subprocess_exec
-    _root_dir : str = None
-    _pkg_s : List[Package] = dc.field(default_factory=list)
-    _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
-    _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
-    _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
-    _task_list : List[Task] = dc.field(default_factory=list)
-    _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
-    _task_id : int = 0
-
-    def __post_init__(self):
-        from .tasklib.std.pkg_std import PackageStd
-        from .tasklib.hdl.sim.vcs_pkg import VcsPackage
-        from .tasklib.hdl.sim.vlt_pkg import VltPackage
-        from .tasklib.hdl.sim.mti_pkg import MtiPackage
-        self._pkg_m[PackageSpec("std")] = PackageStd("std")
-        self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
-        self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
-        self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
-
-    def load(self, root : str):
-        if not os.path.isdir(root):
-            raise Exception("Root directory %s does not exist" % root)
-
-        if not os.path.isfile(os.path.join(root, "flow.yaml")):
-            raise Exception("No root flow file")
-
-        self._root_dir = os.path.dirname(root)
-        self.package = self._load_package(os.path.join(root, "flow.yaml"), [])
-
-        return self.package
-
-    def mkTaskGraph(self, task : str) -> Task:
-        self._pkg_s.clear()
-        self._task_m.clear()
-
-        return self._mkTaskGraph(task, self.rundir)
-        
-    def _mkTaskGraph(self, task : str, parent_rundir : str, params : dict = None) -> Task:
-
-        elems = task.split(".")
-
-        pkg_name = ".".join(elems[0:-1])
-        task_name = elems[-1]
-
-        if pkg_name == "":
-            if len(self._pkg_spec_s) == 0:
-                raise Exception("No package context for %s" % task)
-            pkg_spec = self._pkg_spec_s[-1]
-            pkg_name = pkg_spec.name
-        else:
-            pkg_spec = PackageSpec(pkg_name)
-
-        rundir = os.path.join(parent_rundir, pkg_name, task_name)
-
-        self._pkg_spec_s.append(pkg_spec)
-        pkg = self.getPackage(pkg_spec)
-        
-        self._pkg_s.append(pkg)
-
-        #task_def = pkg.getTask(task_name)
-
-        depends = []
-
-        params = pkg.mkTaskParams(task_name)
-
-        task_id = self.mkTaskId(None)
-#        task_name = "%s.%s" % (pkg.name, task_def.name)
-
-        # The returned task should have all param references resolved
-        task = pkg.mkTask(
-            task_name,
-            task_id,
-            self,
-            params,
-            depends)
-        task.rundir = rundir
-        
-        for i,d in enumerate(task.depend_refs):
-            if d in self._task_m.keys():
-                task.depends.append(self._task_m[d])
-            else:
-                print("mkTaskGraph: %s" % d)
-                task.depends.append(self._mkTaskGraph(d, parent_rundir))
-
-        self._task_m[task.name] = task
-
-        self._pkg_s.pop()
-        self._pkg_spec_s.pop()
-
-        return task
-    
-    def mkTaskId(self, task : 'Task') -> int:
-        self._task_id += 1
-        # TODO: save task <-> id map for later?
-        return self._task_id
-
-    async def run(self, task : str) -> 'TaskData':
-        impl = self.mkTaskGraph(task)
-        return await impl.do_run()
-
-    def _load_package(self, root : str, file_s : List[str]) -> PackageDef:
-        if root in file_s:
-            raise Exception("Recursive file processing @ %s: %s" % (root, ",".join(self._file_s)))
-        file_s.append(root)
-        ret = None
-        with open(root, "r") as fp:
-            print("open %s" % root)
-            doc = yaml.load(fp, Loader=yaml.FullLoader)
-            if "package" not in doc.keys():
-                raise Exception("Missing 'package' key in %s" % root)
-            pkg = PackageDef(**(doc["package"]))
-            pkg.basedir = os.path.dirname(root)
-
-#            for t in pkg.tasks:
-#                t.basedir = os.path.dirname(root)
-
-        if not len(self._pkg_spec_s):
-            self._pkg_spec_s.append(PackageSpec(pkg.name))
-            self._pkg_def_m[PackageSpec(pkg.name)] = pkg
-        else:
-            if self._pkg_spec_s[0].name != pkg.name:
-                raise Exception("Package name mismatch: %s != %s" % (self._pkg_m[0].name, pkg.name))
-            else:
-                # TODO: merge content
-                self._pkg_spec_s.append(PackageSpec(pkg.name))
-
-        print("pkg: %s" % str(pkg))
-
-        print("fragments: %s" % str(pkg.fragments))
-        for spec in pkg.fragments:
-            self._load_fragment_spec(pkg, spec, file_s)
-
-        self._pkg_spec_s.pop()
-        file_s.pop()
-
-        return pkg
-    
-    def _load_fragment_spec(self, pkg : PackageDef, spec : str, file_s : List[str]):
-
-        # We're either going to have:
-        # - File path
-        # - Directory path
-
-        if os.path.isfile(os.path.join(pkg.basedir, spec)):
-            self._load_fragment_file(pkg, spec, file_s)
-        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
-            self._load_fragment_dir(pkg, os.path.join(pkg.basedir, spec), file_s)
-        else:
-            raise Exception("Fragment spec %s not found" % spec)
-        
-
-    def _load_fragment_dir(self, pkg : PackageDef, dir : str, file_s : List[str]):
-
-        for file in os.listdir(dir):
-            if os.path.isdir(os.path.join(dir, file)):
-                self._load_fragment_dir(pkg, os.path.join(dir, file), file_s)
-            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.yaml":
-                self._load_fragment_file(pkg, os.path.join(dir, file), file_s)
-
-    def _load_fragment_file(self, pkg : PackageDef, file : str, file_s : List[str]):
-
-        if file in file_s:
-            raise Exception("Recursive file processing @ %s: %s" % (file, ",".join(self._file_s)))
-        file_s.append(file)
-
-        with open(file, "r") as fp:
-            doc = yaml.load(fp, Loader=yaml.FullLoader)
-            print("doc: %s" % str(doc), flush=True)
-            if "fragment" in doc.keys():
-                # Merge the package definition
-                frag = FragmentDef(**(doc["fragment"]))
-                frag.basedir = os.path.dirname(file)
-                pkg.fragment_l.append(frag)
-            else:
-                print("Warning: file %s is not a fragment" % file)
-        
-
-
-    def getPackage(self, spec : PackageSpec) -> Package:
-        pkg_spec = self._pkg_spec_s[-1]
-        pkg_def = self._pkg_def_m[pkg_spec]
-
-        # Need a stack to track which package we are currently in
-        # Need a map to get a concrete package from a name with parameterization
-
-        # Note: _pkg_m needs to be context specific, such that imports from
-        # one package don't end up visible in another
-        if spec in self._pkg_m.keys():
-            pkg = self._pkg_m[spec]
-        elif spec in self._pkg_def_m.keys():
-            pkg = self._pkg_def_m[spec].mkPackage(self)
-            self._pkg_m[spec] = pkg
-        else:
-            pkg = None
-            print("imports: %s" % str(pkg_def.imports))
-            for imp in pkg_def.imports:
-                print("imp: %s" % str(imp))
-                if imp.alias is not None and imp.alias == spec.name:
-                    # Found the alias name. Just need to get an instance of this package
-                    tgt_pkg_spec = PackageSpec(imp.name)
-                    if tgt_pkg_spec in self._pkg_m.keys():
-                        pkg = self._pkg_m[tgt_pkg_spec]
-                    elif tgt_pkg_spec in self._pkg_def_m.keys():
-                        base = self._pkg_def_m[tgt_pkg_spec]
-                        pkg = base.mkPackage(self, spec.params)
-                        self._pkg_m[spec] = pkg
-                    elif imp.path is not None:
-                        # See if we can load the package
-                        print("TODO: load referenced package")
-                    else:
-                        raise Exception("Import alias %s not found" % imp.name)
-                    break
-                else:
-                    # Need to compare the spec with the full import spec
-                    imp_spec = PackageSpec(imp.name)
-                    # TODO: set parameters
-                    if imp_spec == spec:
-                        base = self._pkg_def_m[PackageSpec(spec.name)]
-                        pkg = base.mkPackage(self, spec.params)
-                        self._pkg_m[spec] = pkg
-                        break
-
-            if pkg is None:
-                raise Exception("Failed to find package %s from package %s" % (
-                    spec.name, pkg_def.name))
-
-#            base_spec = PackageSpec(spec.name)
-#            if not base_spec in self._pkg_def_m.keys():
-#                # Template is not present. Go find it...
-#
-#                # If not found...
-#                raise Exception("Package %s not found" % spec.name)
-
-        return pkg
-        
-    def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef) -> 'TaskCtor':
-        spec_e = spec.name.split(".")
-        task_name = spec_e[-1]
-        pkg_name = ".".join(spec_e[0:-1])
-
-        try:
-            pkg = self.getPackage(PackageSpec(pkg_name))
-        except Exception as e:
-            print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
-            raise e
-
-        return pkg.getTaskCtor(task_name)
-

dv_flow_mgr/tasklib/builtin_pkg.py

@@ -1,61 +0,0 @@
-import os
-import sys
-import glob
-import fnmatch
-import importlib
-import pydantic.dataclasses as dc
-from ..package import TaskCtor
-from ..task import Task, TaskParams, TaskCtorT
-from ..task_data import TaskData
-from ..task_memento import TaskMemento
-from typing import List, Tuple
-import dataclasses as dc
-from ..package_def import Package
-
-class TaskPyClass(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-
-        if self.srcdir not in sys.path:
-            sys.path.insert(0, self.srcdir)
-
-        print("sys.path: %s" % str(sys.path), flush=True)
-        idx = self.params.pyclass.rfind('.')
-        modname = self.params.pyclass[:idx]
-        clsname = self.params.pyclass[idx+1:]
-
-        if os.path.isfile(os.path.join(self.basedir, "my_module.py")):
-            print("my_module.py exists", flush=True)
-        else:
-            print("my_module.py does not exist", flush=True)
-
-        try:
-            print("modname=%s" % modname, flush=True)
-            module = importlib.import_module(modname)
-        except ModuleNotFoundError as e:
-            print("Module not found: %s syspath=%s" % (str(e), str(sys.path)), flush=True)
-            raise e
-
-        cls = getattr(module, clsname)
-
-        obj = cls(self.name, self.task_id, self.session, self.basedir, srcdir=self.srcdir)
-
-        return await obj.run(input)
-
-
-class TaskPyClassParams(TaskParams):
-    pyclass : str
-
-class TaskPyClassMemento(TaskMemento):
-    pass
-
-class TaskPyClassCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskPyClassParams, TaskPyClass)
-
-@dc.dataclass
-class PackageBuiltin(Package):
-
-    def __post_init__(self):
-        print("PackageBuiltin::__post_init__", flush=True)
-        self.tasks["PyClass"] = TaskPyClass()

dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py

@@ -1,11 +0,0 @@
-import dataclasses as dc
-from .pkg_hdl_sim import PackageHdlSim
-from .mti_task_sim_image import TaskMtiSimImageCtor
-from .mti_task_sim_run import TaskMtiSimRunCtor
-
-@dc.dataclass
-class MtiPackage(PackageHdlSim):
-    def __post_init__(self):
-        self.tasks["SimImage"] = TaskMtiSimImageCtor()
-        self.tasks["SimRun"] = TaskMtiSimRunCtor()
-

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py

@@ -1,69 +0,0 @@
-import os
-import fnmatch
-import dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
-from typing import List, Tuple
-
-from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
-
-@dc.dataclass
-class TaskMtiSimImage(VlTaskSimImage):
-
-    def getRefTime(self):
-        if os.path.isfile(os.path.join(self.rundir, 'work.d')):
-            return os.path.getmtime(os.path.join(self.rundir, 'work.d'))
-        else:
-            raise Exception("work.d not found (%s)")
-    
-    async def build(self, files : List[str], incdirs : List[str]):
-        if not os.path.isdir(os.path.join(self.rundir, 'work')):
-            cmd = ['vlib', 'work']
-            proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-            await proc.wait()
-            if proc.returncode != 0:
-                raise Exception("Questa vlib failed")
-
-        cmd = ['vlog', '-sv']
-
-        for incdir in incdirs:
-            cmd.append('+incdir+%s' % incdir)
-
-        cmd.extend(files)
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-        await proc.wait()
-        if proc.returncode != 0:
-            raise Exception("Questa compile failed")
-
-        cmd = ['vopt', '-o', 'simv_opt']
-
-        for top in self.params.top:
-            cmd.append(top)
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-
-        await proc.wait()
-
-        with open(os.path.join(self.rundir, 'work.d'), "w") as fp:
-            fp.write("\n")
-
-        if proc.returncode != 0:
-            raise Exception("Questa opt failed")
-
-class TaskMtiSimImageParams(VlTaskSimImageParams):
-    pass
-
-class TaskMtiSimImageMemento(VlTaskSimImageMemento):
-    pass
-
-class TaskMtiSimImageCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskMtiSimImageParams, TaskMtiSimImage)

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py

@@ -1,47 +0,0 @@
-import os
-import fnmatch
-import pydantic.dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from typing import List, Tuple
-
-class TaskMtiSimRun(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("verilatorBinary")
-
-        build_dir = vl_fileset[0].basedir
-
-        cmd = [
-            'vsim', '-batch', '-do', 'run -all; quit -f',
-            '-work', os.path.join(build_dir, 'work'),
-            'simv_opt'
-        ]
-
-        fp = open(os.path.join(self.rundir, 'sim.log'), "w")
-        proc = await self.session.create_subprocess(*cmd,
-                                                    cwd=self.rundir,
-                                                    stdout=fp)
-
-        await proc.wait()
-
-        fp.close()
-
-        output = TaskData()
-        output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
-
-        return output
-
-class TaskMtiSimRunParams(TaskParams):
-    pass
-
-class TaskMtiSimRunMemento(TaskMemento):
-    pass
-
-class TaskMtiSimRunCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskMtiSimRunParams, TaskMtiSimRun)
-

dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py

@@ -1,8 +0,0 @@
-import dataclasses as dc
-from ....package import Package
-
-@dc.dataclass
-class PackageHdlSim(Package):
-    # TODO: parameter to select proper implementation
-    pass
-

dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py

@@ -1,16 +0,0 @@
-import os
-import fnmatch
-import pydantic.dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from typing import List, Tuple
-
-class TaskSimImage(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-        return input
-
-

dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py

@@ -1,14 +0,0 @@
-import dataclasses as dc
-from .pkg_hdl_sim import PackageHdlSim
-from .vcs_task_sim_image import TaskVcsSimImageCtor
-from .vcs_task_sim_run import TaskVcsSimRunCtor
-
-@dc.dataclass
-class VcsPackage(PackageHdlSim):
-
-    def __post_init__(self):
-        print("PackageVcs::__post_init__", flush=True)
-        self.tasks["SimImage"] = TaskVcsSimImageCtor()
-        self.tasks["SimRun"] = TaskVcsSimRunCtor()
-    pass
-

dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py

@@ -1,49 +0,0 @@
-import os
-import fnmatch
-import dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
-from typing import List, Tuple
-
-from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
-
-@dc.dataclass
-class TaskVcsSimImage(VlTaskSimImage):
-
-    def getRefTime(self):
-        if os.path.isfile(os.path.join(self.rundir, 'simv')):
-            return os.path.getmtime(os.path.join(self.rundir, 'simv'))
-        else:
-            raise Exception
-    
-    async def build(self, files : List[str], incdirs : List[str]):
-        cmd = ['vcs', '-sverilog']
-
-        for incdir in incdirs:
-            cmd.append('+incdir+%s' % incdir)
-
-        cmd.extend(files)
-
-        if len(self.params.top):
-            cmd.extend(['-top', "+".join(self.params.top)])
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-        await proc.wait()
-
-        if proc.returncode != 0:
-            raise Exception("VCS simv failed")
-
-class TaskVcsSimImageParams(VlTaskSimImageParams):
-    pass
-
-class TaskVcsSimImageMemento(VlTaskSimImageMemento):
-    pass
-
-class TaskVcsSimImageCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskVcsSimImageParams, TaskVcsSimImage)