dv-flow-mgr 0.0.1.12849118090a1__py3-none-any.whl → 0.0.1.12911707440a1__py3-none-any.whl

dv_flow_mgr/session.py

@@ -1,324 +0,0 @@
-#****************************************************************************
-#* session.py
-#*
-#* Copyright 2023 Matthew Ballance and Contributors
-#*
-#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
-#* not use this file except in compliance with the License.  
-#* You may obtain a copy of the License at:
-#*
-#*   http://www.apache.org/licenses/LICENSE-2.0
-#*
-#* Unless required by applicable law or agreed to in writing, software 
-#* distributed under the License is distributed on an "AS IS" BASIS, 
-#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
-#* See the License for the specific language governing permissions and 
-#* limitations under the License.
-#*
-#* Created on:
-#*     Author: 
-#*
-#****************************************************************************
-import asyncio
-import os
-import yaml
-import dataclasses as dc
-from typing import Any, Callable, Dict, List
-from .fragment_def import FragmentDef
-from .package import Package
-from .package_def import PackageDef, PackageSpec
-from .task import Task, TaskSpec, TaskCtor
-
-@dc.dataclass
-class Session(object):
-    """Manages the running of a flow"""
-
-    srcdir : str
-    rundir : str
-
-    # Search path for .dfs files
-    package_path : List[str] = dc.field(default_factory=list)
-    package : PackageDef = None
-    create_subprocess : Callable = asyncio.create_subprocess_exec
-    _root_dir : str = None
-    _pkg_s : List[Package] = dc.field(default_factory=list)
-    _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
-    _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
-    _pkg_def_m : Dict[str,PackageDef] = dc.field(default_factory=dict)
-    _pkg_file_m : Dict[str,str] = dc.field(default_factory=dict)
-    _pkg_path : List[str] = dc.field(default_factory=list)
-    _task_list : List[Task] = dc.field(default_factory=list)
-    _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
-    _task_id : int = 0
-
-    def __post_init__(self):
-        # Add a reference to the built-in 'std' package
-        this_dir = os.path.dirname(os.path.abspath(__file__))
-        self._pkg_file_m["std"] = os.path.join(this_dir, "tasklib/std/flow.dv")
-
-        # from .tasklib.std.pkg_std import PackageStd
-        # from .tasklib.hdl.sim.vcs_pkg import VcsPackage
-        # from .tasklib.hdl.sim.vlt_pkg import VltPackage
-        # from .tasklib.hdl.sim.mti_pkg import MtiPackage
-        # self._pkg_m[PackageSpec("std")] = PackageStd("std")
-        # self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
-        # self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
-        # self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
-        pass
-
-    def load(self):
-        if not os.path.isdir(self.srcdir):
-            raise Exception("Root directory %s does not exist" % self.srcdir)
-
-        if not os.path.isfile(os.path.join(self.srcdir, "flow.dv")):
-            raise Exception("No root flow file")
-
-        self._root_dir = os.path.dirname(self.srcdir)
-        self.package = self._load_package(os.path.join(self.srcdir, "flow.dv"), [])
-
-        return self.package
-
-    def mkTaskGraph(self, task : str) -> Task:
-        self._pkg_s.clear()
-        self._task_m.clear()
-
-        return self._mkTaskGraph(task, self.rundir)
-        
-    def _mkTaskGraph(self, task : str, parent_rundir : str) -> Task:
-
-        elems = task.split(".")
-
-        pkg_name = ".".join(elems[0:-1])
-        task_name = elems[-1]
-
-        if pkg_name == "":
-            if len(self._pkg_spec_s) == 0:
-                raise Exception("No package context for %s" % task)
-            pkg_spec = self._pkg_spec_s[-1]
-            pkg_name = pkg_spec.name
-        else:
-            pkg_spec = PackageSpec(pkg_name)
-
-        rundir = os.path.join(parent_rundir, pkg_name, task_name)
-
-        self._pkg_spec_s.append(pkg_spec)
-        pkg = self.getPackage(pkg_spec)
-        
-        self._pkg_s.append(pkg)
-
-        ctor_t : TaskCtor = pkg.getTaskCtor(task_name)
-
-        depends = []
-
-        # The returned task should have all param references resolved
-        print("task_ctor=%s" % str(ctor_t.task_ctor), flush=True)
-        task = ctor_t.task_ctor(
-            name=task_name,
-            session=self,
-            params=ctor_t.mkParams(),
-            depends=depends,
-            rundir=rundir,
-            srcdir=ctor_t.srcdir)
-        
-        for i,d in enumerate(task.depend_refs):
-            if d in self._task_m.keys():
-                task.depends.append(self._task_m[d])
-            else:
-                print("mkTaskGraph: %s" % d)
-                task.depends.append(self._mkTaskGraph(d, parent_rundir))
-
-        self._task_m[task.name] = task
-
-        self._pkg_s.pop()
-        self._pkg_spec_s.pop()
-
-        return task
-
-    def push_package(self, pkg : Package):
-        self._pkg_s.append(pkg)
-
-    def pop_package(self, pkg : Package):
-        self._pkg_s.pop()
-
-    def package(self):
-        return self._pkg_s[-1]
-    
-    def mkTaskId(self, task : 'Task') -> int:
-        self._task_id += 1
-        # TODO: save task <-> id map for later?
-        return self._task_id
-
-    async def run(self, task : str) -> 'TaskData':
-        impl = self.mkTaskGraph(task)
-        return await impl.do_run()
-
-    def _load_package(self, root : str, file_s : List[str]) -> PackageDef:
-        if root in file_s:
-            raise Exception("Recursive file processing @ %s: %s" % (root, ",".join(self._file_s)))
-        file_s.append(root)
-        ret = None
-        with open(root, "r") as fp:
-            print("open %s" % root)
-            doc = yaml.load(fp, Loader=yaml.FullLoader)
-            if "package" not in doc.keys():
-                raise Exception("Missing 'package' key in %s" % root)
-            pkg = PackageDef(**(doc["package"]))
-            pkg.basedir = os.path.dirname(root)
-
-#            for t in pkg.tasks:
-#                t.basedir = os.path.dirname(root)
-
-        if not len(self._pkg_spec_s):
-            self._pkg_spec_s.append(PackageSpec(pkg.name))
-        else:
-            if self._pkg_spec_s[-1].name != pkg.name:
-                raise Exception("Package name mismatch: %s != %s" % (self._pkg_spec_s[-1].name, pkg.name))
-            else:
-                # TODO: merge content
-                self._pkg_spec_s.append(PackageSpec(pkg.name))
-        self._pkg_def_m[PackageSpec(pkg.name)] = pkg
-
-        print("pkg: %s" % str(pkg))
-
-        print("fragments: %s" % str(pkg.fragments))
-        for spec in pkg.fragments:
-            self._load_fragment_spec(pkg, spec, file_s)
-
-        self._pkg_spec_s.pop()
-        file_s.pop()
-
-        return pkg
-    
-    def _load_fragment_spec(self, pkg : PackageDef, spec : str, file_s : List[str]):
-
-        # We're either going to have:
-        # - File path
-        # - Directory path
-
-        if os.path.isfile(os.path.join(pkg.basedir, spec)):
-            self._load_fragment_file(pkg, spec, file_s)
-        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
-            self._load_fragment_dir(pkg, os.path.join(pkg.basedir, spec), file_s)
-        else:
-            raise Exception("Fragment spec %s not found" % spec)
-        
-
-    def _load_fragment_dir(self, pkg : PackageDef, dir : str, file_s : List[str]):
-
-        for file in os.listdir(dir):
-            if os.path.isdir(os.path.join(dir, file)):
-                self._load_fragment_dir(pkg, os.path.join(dir, file), file_s)
-            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.yaml":
-                self._load_fragment_file(pkg, os.path.join(dir, file), file_s)
-
-    def _load_fragment_file(self, pkg : PackageDef, file : str, file_s : List[str]):
-
-        if file in file_s:
-            raise Exception("Recursive file processing @ %s: %s" % (file, ",".join(self._file_s)))
-        file_s.append(file)
-
-        with open(file, "r") as fp:
-            doc = yaml.load(fp, Loader=yaml.FullLoader)
-            print("doc: %s" % str(doc), flush=True)
-            if "fragment" in doc.keys():
-                # Merge the package definition
-                frag = FragmentDef(**(doc["fragment"]))
-                frag.basedir = os.path.dirname(file)
-                pkg.fragment_l.append(frag)
-            else:
-                print("Warning: file %s is not a fragment" % file)
-        
-
-
-    def getPackage(self, spec : PackageSpec) -> Package:
-        # Obtain the active package definition
-        pkg_spec = self._pkg_spec_s[-1]
-        pkg_def = self._pkg_def_m[pkg_spec]
-
-        print("spec: %s ; _pkg_def_m: %s" % (str(spec), str(self._pkg_def_m.keys())))
-
-        # Need a stack to track which package we are currently in
-        # Need a map to get a concrete package from a name with parameterization
-
-        # Note: _pkg_m needs to be context specific, such that imports from
-        # one package don't end up visible in another
-        if spec in self._pkg_m.keys():
-            pkg = self._pkg_m[spec]
-        elif spec in self._pkg_def_m.keys():
-            pkg = self._pkg_def_m[spec].mkPackage(self)
-            self._pkg_m[spec] = pkg
-        else:
-            pkg = None
-            print("imports: %s" % str(pkg_def.imports))
-            for imp in pkg_def.imports:
-                print("imp: %s" % str(imp))
-                if imp.alias is not None and imp.alias == spec.name:
-                    # Found the alias name. Just need to get an instance of this package
-                    tgt_pkg_spec = PackageSpec(imp.name)
-                    if tgt_pkg_spec in self._pkg_m.keys():
-                        pkg = self._pkg_m[tgt_pkg_spec]
-                    elif tgt_pkg_spec in self._pkg_def_m.keys():
-                        base = self._pkg_def_m[tgt_pkg_spec]
-                        pkg = base.mkPackage(self, spec.params)
-                        self._pkg_m[spec] = pkg
-                    elif imp.path is not None:
-                        # See if we can load the package
-                        print("TODO: load referenced package")
-                    else:
-                        raise Exception("Import alias %s not found" % imp.name)
-                    break
-                else:
-                    # Need to compare the spec with the full import spec
-                    imp_spec = PackageSpec(imp.name)
-                    # TODO: set parameters
-                    if imp_spec == spec:
-                        base = self._pkg_def_m[PackageSpec(spec.name)]
-                        pkg = base.mkPackage(self, spec.params)
-                        self._pkg_m[spec] = pkg
-                        break
-            
-            if pkg is None:
-                # Look in the set of registered packages
-                if spec.name in self._pkg_file_m.keys():
-                    # Load the package
-                    self._pkg_spec_s.append(spec)
-                    pkg_def = self._load_package(
-                        self._pkg_file_m[spec.name], 
-                        [])
-
-                    self._pkg_spec_s.pop()
-
-                    # The definition is now in the map, so recurse to create it
-                    pkg = self.getPackage(spec)
-                else:
-                    # Go search the package path
-                    pass
-                
-
-
-            if pkg is None:
-                raise Exception("Failed to find package %s from package %s" % (
-                    spec.name, pkg_def.name))
-
-        return pkg
-        
-    def getTaskCtor(self, spec : TaskSpec, pkg : PackageDef) -> 'TaskCtor':
-        spec_e = spec.name.split(".")
-        task_name = spec_e[-1]
-
-        if len(spec_e) == 1:
-            # Just have a task name. Use the current package
-            if len(self._pkg_s) == 0:
-                raise Exception("No package context for task %s" % spec.name)
-            pkg = self._pkg_s[-1]
-        else:
-            pkg_name = ".".join(spec_e[0:-1])
-
-            try:
-                pkg = self.getPackage(PackageSpec(pkg_name))
-            except Exception as e:
-                print("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
-                raise e
-
-        return pkg.getTaskCtor(task_name)
-

dv_flow_mgr/tasklib/builtin_pkg.py

@@ -1,62 +0,0 @@
-# import os
-# import sys
-# import glob
-# import fnmatch
-# import importlib
-# import pydantic.dataclasses as dc
-# from ..package import TaskCtor
-from ..task import Task
-from ..task_data import TaskData
-# from ..task_memento import TaskMemento
-# from typing import List, Tuple
-# import dataclasses as dc
-# from ..package_def import Package
-
-# class TaskPyClass(Task):
-#     pyclass : str = None
-
-#     async def run(self, input : TaskData) -> TaskData:
-
-#         if self.srcdir not in sys.path:
-#             sys.path.insert(0, self.srcdir)
-
-#         print("sys.path: %s" % str(sys.path), flush=True)
-#         idx = self.params.pyclass.rfind('.')
-#         modname = self.params.pyclass[:idx]
-#         clsname = self.params.pyclass[idx+1:]
-
-#         if os.path.isfile(os.path.join(self.basedir, "my_module.py")):
-#             print("my_module.py exists", flush=True)
-#         else:
-#             print("my_module.py does not exist", flush=True)
-
-#         if modname in sys.modules.keys():
-#             module = sys.modules[modname]
-#         else:
-#             try:
-#                 print("modname=%s" % modname, flush=True)
-#                 module = importlib.import_module(modname)
-#             except ModuleNotFoundError as e:
-#                 print("Module not found: %s syspath=%s" % (str(e), str(sys.path)), flush=True)
-#                 raise e
-
-#         cls = getattr(module, clsname)
-
-#         obj = cls(self.name, self.task_id, self.session, self.basedir, srcdir=self.srcdir)
-
-#         return await obj.run(input)
-
-
-# class TaskPyClassMemento(TaskMemento):
-#     pass
-
-class TaskNull(Task):
-    async def run(self, input : TaskData) -> TaskData:
-        return input
-
-# @dc.dataclass
-# class PackageBuiltin(Package):
-
-#     def __post_init__(self):
-#         print("PackageBuiltin::__post_init__", flush=True)
-#         self.tasks["PyClass"] = TaskPyClass()

dv_flow_mgr/tasklib/hdl/sim/mti_pkg.py

@@ -1,11 +0,0 @@
-import dataclasses as dc
-from .pkg_hdl_sim import PackageHdlSim
-from .mti_task_sim_image import TaskMtiSimImageCtor
-from .mti_task_sim_run import TaskMtiSimRunCtor
-
-@dc.dataclass
-class MtiPackage(PackageHdlSim):
-    def __post_init__(self):
-        self.tasks["SimImage"] = TaskMtiSimImageCtor()
-        self.tasks["SimRun"] = TaskMtiSimRunCtor()
-

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py

@@ -1,69 +0,0 @@
-import os
-import fnmatch
-import dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
-from typing import List, Tuple
-
-from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
-
-@dc.dataclass
-class TaskMtiSimImage(VlTaskSimImage):
-
-    def getRefTime(self):
-        if os.path.isfile(os.path.join(self.rundir, 'work.d')):
-            return os.path.getmtime(os.path.join(self.rundir, 'work.d'))
-        else:
-            raise Exception("work.d not found (%s)")
-    
-    async def build(self, files : List[str], incdirs : List[str]):
-        if not os.path.isdir(os.path.join(self.rundir, 'work')):
-            cmd = ['vlib', 'work']
-            proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-            await proc.wait()
-            if proc.returncode != 0:
-                raise Exception("Questa vlib failed")
-
-        cmd = ['vlog', '-sv']
-
-        for incdir in incdirs:
-            cmd.append('+incdir+%s' % incdir)
-
-        cmd.extend(files)
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-        await proc.wait()
-        if proc.returncode != 0:
-            raise Exception("Questa compile failed")
-
-        cmd = ['vopt', '-o', 'simv_opt']
-
-        for top in self.params.top:
-            cmd.append(top)
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-
-        await proc.wait()
-
-        with open(os.path.join(self.rundir, 'work.d'), "w") as fp:
-            fp.write("\n")
-
-        if proc.returncode != 0:
-            raise Exception("Questa opt failed")
-
-class TaskMtiSimImageParams(VlTaskSimImageParams):
-    pass
-
-class TaskMtiSimImageMemento(VlTaskSimImageMemento):
-    pass
-
-class TaskMtiSimImageCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskMtiSimImageParams, TaskMtiSimImage)

dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py

@@ -1,47 +0,0 @@
-import os
-import fnmatch
-import pydantic.dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from typing import List, Tuple
-
-class TaskMtiSimRun(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("verilatorBinary")
-
-        build_dir = vl_fileset[0].basedir
-
-        cmd = [
-            'vsim', '-batch', '-do', 'run -all; quit -f',
-            '-work', os.path.join(build_dir, 'work'),
-            'simv_opt'
-        ]
-
-        fp = open(os.path.join(self.rundir, 'sim.log'), "w")
-        proc = await self.session.create_subprocess(*cmd,
-                                                    cwd=self.rundir,
-                                                    stdout=fp)
-
-        await proc.wait()
-
-        fp.close()
-
-        output = TaskData()
-        output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
-
-        return output
-
-class TaskMtiSimRunParams(TaskParams):
-    pass
-
-class TaskMtiSimRunMemento(TaskMemento):
-    pass
-
-class TaskMtiSimRunCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskMtiSimRunParams, TaskMtiSimRun)
-

dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py

@@ -1,8 +0,0 @@
-import dataclasses as dc
-from ....package import Package
-
-@dc.dataclass
-class PackageHdlSim(Package):
-    # TODO: parameter to select proper implementation
-    pass
-

dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py

@@ -1,16 +0,0 @@
-import os
-import fnmatch
-import pydantic.dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from typing import List, Tuple
-
-class TaskSimImage(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-        return input
-
-

dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py

@@ -1,14 +0,0 @@
-import dataclasses as dc
-from .pkg_hdl_sim import PackageHdlSim
-from .vcs_task_sim_image import TaskVcsSimImageCtor
-from .vcs_task_sim_run import TaskVcsSimRunCtor
-
-@dc.dataclass
-class VcsPackage(PackageHdlSim):
-
-    def __post_init__(self):
-        print("PackageVcs::__post_init__", flush=True)
-        self.tasks["SimImage"] = TaskVcsSimImageCtor()
-        self.tasks["SimRun"] = TaskVcsSimRunCtor()
-    pass
-

dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py

@@ -1,49 +0,0 @@
-import os
-import fnmatch
-import dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
-from typing import List, Tuple
-
-from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
-
-@dc.dataclass
-class TaskVcsSimImage(VlTaskSimImage):
-
-    def getRefTime(self):
-        if os.path.isfile(os.path.join(self.rundir, 'simv')):
-            return os.path.getmtime(os.path.join(self.rundir, 'simv'))
-        else:
-            raise Exception
-    
-    async def build(self, files : List[str], incdirs : List[str]):
-        cmd = ['vcs', '-sverilog']
-
-        for incdir in incdirs:
-            cmd.append('+incdir+%s' % incdir)
-
-        cmd.extend(files)
-
-        if len(self.params.top):
-            cmd.extend(['-top', "+".join(self.params.top)])
-
-        proc = await self.session.create_subprocess(*cmd,
-                                                        cwd=self.rundir)
-        await proc.wait()
-
-        if proc.returncode != 0:
-            raise Exception("VCS simv failed")
-
-class TaskVcsSimImageParams(VlTaskSimImageParams):
-    pass
-
-class TaskVcsSimImageMemento(VlTaskSimImageMemento):
-    pass
-
-class TaskVcsSimImageCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskVcsSimImageParams, TaskVcsSimImage)

dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py

@@ -1,45 +0,0 @@
-import os
-import fnmatch
-import pydantic.dataclasses as dc
-from ....fileset import FileSet
-from ....package import TaskCtor
-from ....task import Task, TaskParams, TaskCtorT
-from ....task_data import TaskData
-from ....task_memento import TaskMemento
-from typing import List, Tuple
-
-class TaskVcsSimRun(Task):
-
-    async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("simDir")
-
-        build_dir = vl_fileset[0].basedir
-
-        cmd = [
-            os.path.join(build_dir, 'simv'),
-        ]
-
-        fp = open(os.path.join(self.rundir, 'sim.log'), "w")
-        proc = await self.session.create_subprocess(*cmd,
-                                                    cwd=self.rundir,
-                                                    stdout=fp)
-
-        await proc.wait()
-
-        fp.close()
-
-        output = TaskData()
-        output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
-
-        return output
-
-class TaskVcsSimRunParams(TaskParams):
-    pass
-
-class TaskVcsSimRunMemento(TaskMemento):
-    pass
-
-class TaskVcsSimRunCtor(TaskCtorT):
-    def __init__(self):
-        super().__init__(TaskVcsSimRunParams, TaskVcsSimRun)
-