dv-flow-mgr 0.0.1.12672327205a1__tar.gz → 0.0.1.12750690879a1__tar.gz

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12672327205a1
+Version: 0.0.1.12750690879a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/Notes.md

@@ -160,4 +160,35 @@ package:
 # DV Flow Manager 1.0.0
 - Packages and Fragments
 - Package Import
-- 
+- Python implementation of YAML-defined tasks
+- Parameters and Properties
+- Modifying the value of input parameters
+- Strategy for tool-specific options
+- Tasks for reusable sim-image components (sim-image, library, ip)
+- Work-avoidance schemes for SV
+
+## Use Cases
+- Compile simulation image (debug, optimized)
+- Run user-specified simulations
+
+# DV Flow Manager 2.0.0
+- Package inheritance ("project templates")
+- VSCode log browser - visualize task graph ; navigate / view individual task logs
+  - See execution times
+  - See which tasks performed real work (output of 'changed')
+- Task/sub-graph execution delegation interface (implementation for LSF, SLURM, etc)
+- 
+
+# DV Flow Manager 3.0.0
+
+# DV Flow Manager 4.0.0
+- VSCode flow-file development support (error checking, completion, navigation)
+- VSCode run integration (launch run from VSCode ; monitor execution/completion)
+
+# DV Flow Manager 5.0.0
+- VSCode debug support (step through task graph ; inspect variable values)
+
+
+
+
+

dv_flow_mgr-0.0.1.12750690879a1/docs/Roadmap.md

@@ -0,0 +1,95 @@
+
+# Flow Specification
+The Flow Specification is comprised of the Flow YAML Schema and the 
+semantic definition of how task graphs defined using the flow specification
+are evaluated.
+
+## 1.0.0
+- Package definition
+- Package import
+- Task definition
+  - "with" variable usage
+  - Operations on input and output data
+  - Operations on task parameters
+- Package fragments
+
+## 2.0.0
+- Parameterized package definition and use
+- Package "uses" (type/inheritance)
+- Task "with"-data definition (tasks can add their own parameters)
+- Task Groups / Sub-DAG
+- Coarse/fine-grained dependency management
+  - Mark task dependency as having a "coarse" requirement. Causes 
+    the task to be run if it hasn't been run already. Doesn't perform
+    exhaustive analysis.
+  - Maybe allow subtree dependency analysis? Single analysis point to
+    determine up-to-date status on a whole collection of source
+
+## 3.0.0
+- JQ-based data extraction
+- YAML task templates / expansions
+- Support for annotating job requirements 
+
+# Library
+
+## 1.0.0
+- Std
+  - Null (combine dependencies, set variables). Implements tasks that do not specify 'uses'
+  - Exec
+  - FileSet
+  - PyClass - implement a task as a Python class (load from a module)
+
+- HdlSim
+  - Library  - creates a reusable simulator-specific library
+  - IP       - create a reusable single-module IP library
+  - SimImage - creates a simulation image 
+  - SimRun
+
+
+## 2.0.0
+- Std
+  - DefineData (Declares a data structure to pass)
+  - Export   - Captures the result of some task in a dedicated directory
+
+## 3.0.0
+- Std
+  - 
+
+# DV Flow Manager
+
+## 1.0.0
+- Simple 'max-jobs' process scheduler
+
+## 2.0.0
+- Progress meter and status line to monitor builds (non-verbose)
+- Multi-level mechanism for monitoring jobs
+  - High-level with a progress bar
+  - 
+- Log-creation feature that assembles a total log from per-task logs
+
+## 3.0.0
+- Provide link to source for error messages
+- Improve debug logging
+
+## 4.0.0
+- Improve status display by allowing actions to signal count-like information (+pass, +fail)
+- OpenTelemetry support
+
+
+## 5.0.0
+- Need some form of site settings
+
+
+# Core Principles
+- Relevant DAG can be constructed statically
+  - Tasks are not inferred based on dataflow content
+  - Tasks may be created based on statically-available data
+- Dataflow is dynamically typed (union of content)
+- Tasks pass-through filesets that are not relevant for them
+  - Enables injecting new data when convenient, even if it won't be used until later
+  - Can have 'filter' tasks if there is a need to clean-up
+- Dependency analysis is not delegated 
+  - Allows static DAG construction
+- Extensible -- by users and organizations
+- Implementation independent (not tied to a specific implementation language)
+

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/conf.py

@@ -14,7 +14,8 @@ author = 'Matthew Ballance'
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
-    'sphinx-jsonschema'
+    'sphinx-jsonschema',
+    'sphinxcontrib.mermaid'
 ]
 
 templates_path = ['_templates']

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/index.rst

@@ -10,6 +10,18 @@ DV Flow Manager helps users capture the tasks and dataflow in their
 design and verification (DV) flows. You can think of DV Flow Manager as a 
 sort of "make for silicon engineering".
 
+.. mermaid::
+
+    flowchart TD
+      A[IP Fileset] --> B[Testbench]
+      C[VIP Fileset] --> D[Precompile]
+      D --> B
+      B --> E[SimImage]
+      E --> F[Test1]
+      E --> G[Test2]
+      E --> H[Test3]
+
+
 
 .. toctree::
    :maxdepth: 2
@@ -17,4 +29,3 @@ sort of "make for silicon engineering".
 
    quickstart
    reference
-

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/quickstart.rst

@@ -60,16 +60,19 @@ simulate this module.
         tasks:
           - name: rtl
             type: std.FileSet
-            params:
+            with:
+              type: "systemVerilogSource"
               include: "*.sv"
 
           - name: sim-image
             type: hdl.sim.SimImage
-            depends: [rtl]
+            with:
+              - top: [top]
+            needs: [rtl]
 
           - name: sim-run
             type: hdl.sim.SimRun
-            depends: [sim-image]
+            needs: [sim-image]
 
 
 If we run the `dvfm run` command, DV Flow Manager will:
@@ -80,6 +83,52 @@ If we run the `dvfm run` command, DV Flow Manager will:
 
 .. code-block:: bash
 
-    % dvfm run
+    % dvfm run sim-run
 
-The command should complete successfully, and we should see the following
+This will compile the source, build a simulation image for module `top`,
+and run the resulting image. Not too bad for 20-odd lines of build specification.
+
+A Bit More Detail
+=================
+Let's break this down just a bit:
+
+.. code-block:: yaml
+
+    package:
+        name: my_design
+
+        imports:
+          - name: hdl.sim.vlt
+            as: hdl.sim
+
+DV Flow Manager views the world as a series of *packages* that reference each
+other and contain *tasks* to operate on sources within the *packages*.
+
+Here, we have declared a new package (my_design) and specified that it 
+references a built-in package named `hdl.sim.vlt`. This is a package that
+implements tasks for performing HDL simulation with the Verilator simulator.
+
+Note that we specify an alias (hdl.sim) for the package when importing it.
+This will allow us to easily swap in a different simulator without changing
+anything else within our package definition.
+
+.. code-block:: yaml
+    :highlight: 8,12
+
+    package:
+        name: my_design
+
+        imports:
+          - name: hdl.sim.vlt
+            as: hdl.sim
+
+        tasks:
+          - name: rtl
+            type: std.FileSet
+            with:
+              type: "systemVerilogSource"
+              include: "*.sv"
+
+Our first task is to specify the sources we want to process. This is done
+by specifying a `FileSet` task. The parameters of this task specify where
+the task should look for sources and which sources it should include

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/ivpm.yaml

@@ -21,11 +21,16 @@ package:
       src: pypi
     - name: svdep
       url: https://github.com/fvutils/svdep.git
+      anonymous: True
+    - name: ply 
+      src: pypi
     - name: pyyaml
       src: pypi
     - name: Sphinx
       src: pypi
     - name: sphinx-jsonschema
       src: pypi
+    - name: sphinxcontrib-mermaid
+      src: pypi
 
 

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12672327205a1"
+version = "0.0.1.12750690879a1"
 dependencies = [
     "pyyaml",
     "toposort",

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/fragment_def.py

@@ -22,16 +22,14 @@
 import pydantic.dataclasses as dc
 import json
 from pydantic import BaseModel
-from typing import Any, Dict, List
-from .flow import Flow
+from typing import Any, Dict, List, Union
 from .package import Package
-from .package_def import PackageImportSpec
-from .task import TaskParamCtor
-from .task_def import TaskDef, TaskSpec
+from .package_import_spec import PackageImportSpec
+from .task_def import TaskDef
 
 class FragmentDef(BaseModel):
     tasks : List[TaskDef] = dc.Field(default_factory=list)
-    imports : List[(str|PackageImportSpec)] = dc.Field(default_factory=list, alias="imports")
+    imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list, alias="imports")
     fragments: List[str] = dc.Field(default_factory=list)
 
     basedir : str = None

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/package_def.py

@@ -24,36 +24,12 @@ import json
 from pydantic import BaseModel
 from typing import Any, Dict, List
 from .flow import Flow
+from .fragment_def import FragmentDef
 from .package import Package
+from .package_import_spec import PackageImportSpec, PackageSpec
 from .task import TaskParamCtor
 from .task_def import TaskDef, TaskSpec
 
-@dc.dataclass
-class PackageSpec(object):
-    name : str
-    params : Dict[str,Any] = dc.Field(default_factory=dict)
-    _fullname : str = None
-
-    def get_fullname(self) -> str:
-        if self._fullname is None:
-            if len(self.params) != 0:
-                self._fullname = "%s%s}" % (
-                    self.name,
-                    json.dumps(self.params, separators=(',', ':')))
-            else:
-                self._fullname = self.name
-        return self._fullname    
-    
-    def __hash__(self):
-        return hash(self.get_fullname())
-
-    def __eq__(self, value):
-        return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
-
-@dc.dataclass
-class PackageImportSpec(PackageSpec):
-    path : str = dc.Field(default=None, alias="from")
-    alias : str = dc.Field(default=None, alias="as")
 
 class PackageDef(BaseModel):
     name : str
@@ -63,6 +39,8 @@ class PackageDef(BaseModel):
     imports : List[PackageImportSpec] = dc.Field(default_factory=list)
     fragments: List[str] = dc.Field(default_factory=list)
 
+    fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
+
 #    import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
 
     basedir : str = None
@@ -90,8 +68,31 @@ class PackageDef(BaseModel):
                     basedir=self.basedir,
                     depend_refs=task.depends)
             else:
+                # We use the Null task from the std package
                 raise Exception("")
             ret.tasks[task.name] = ctor_t
 
+        for frag in self.fragment_l:
+            for task in frag.tasks:
+                if task.type is not None:
+                    # Find package (not package_def) that implements this task
+                    # Insert an indirect reference to that tasks's constructor
+
+                    # Only call getTaskCtor if the task is in a different package
+                    task_t = task.type if isinstance(task.type, TaskSpec) else TaskSpec(task.type)
+                    ctor_t = session.getTaskCtor(task_t, self)
+
+                    ctor_t = TaskParamCtor(
+                        base=ctor_t, 
+                        params=task.params, 
+                        basedir=frag.basedir,
+                        depend_refs=task.depends)
+                else:
+                    # We use the Null task from the std package
+                    raise Exception("")
+                if task.name in ret.tasks:
+                    raise Exception("Task %s already defined" % task.name)
+                ret.tasks[task.name] = ctor_t
+
         return ret
 

dv_flow_mgr-0.0.1.12750690879a1/src/dv_flow_mgr/package_import_spec.py

@@ -0,0 +1,31 @@
+
+import pydantic.dataclasses as dc
+import json
+from typing import Dict, Any
+
+@dc.dataclass
+class PackageSpec(object):
+    name : str
+    params : Dict[str,Any] = dc.Field(default_factory=dict)
+    _fullname : str = None
+
+    def get_fullname(self) -> str:
+        if self._fullname is None:
+            if len(self.params) != 0:
+                self._fullname = "%s%s}" % (
+                    self.name,
+                    json.dumps(self.params, separators=(',', ':')))
+            else:
+                self._fullname = self.name
+        return self._fullname    
+    
+    def __hash__(self):
+        return hash(self.get_fullname())
+
+    def __eq__(self, value):
+        return isinstance(value, PackageSpec) and value.get_fullname() == self.get_fullname()
+
+@dc.dataclass
+class PackageImportSpec(PackageSpec):
+    path : str = dc.Field(default=None, alias="from")
+    alias : str = dc.Field(default=None, alias="as")

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/session.py

@@ -24,6 +24,7 @@ import os
 import yaml
 import dataclasses as dc
 from typing import Any, Callable, Dict, List
+from .fragment_def import FragmentDef
 from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .task import Task,TaskSpec
@@ -47,14 +48,15 @@ class Session(object):
     _task_list : List[Task] = dc.field(default_factory=list)
     _task_m : Dict[TaskSpec,Task] = dc.field(default_factory=dict)
     _task_id : int = 0
-    _rundir_s : List[str] = dc.field(default_factory=list)
 
     def __post_init__(self):
         from .tasklib.std.pkg_std import PackageStd
+        from .tasklib.hdl.sim.vcs_pkg import VcsPackage
         from .tasklib.hdl.sim.vlt_pkg import VltPackage
         from .tasklib.hdl.sim.mti_pkg import MtiPackage
         self._pkg_m[PackageSpec("std")] = PackageStd("std")
         self._pkg_m[PackageSpec("hdl.sim.mti")] = MtiPackage("hdl.sim.mti")
+        self._pkg_m[PackageSpec("hdl.sim.vcs")] = VcsPackage("hdl.sim.vcs")
         self._pkg_m[PackageSpec("hdl.sim.vlt")] = VltPackage("hdl.sim.vlt")
 
     def load(self, root : str):
@@ -73,27 +75,25 @@ class Session(object):
         self._pkg_s.clear()
         self._task_m.clear()
 
-        self._rundir_s.clear()
-        self._rundir_s.append(self.rundir)
-
-        return self._mkTaskGraph(task)
+        return self._mkTaskGraph(task, self.rundir)
         
-    def _mkTaskGraph(self, task : str, params : dict = None) -> Task:
+    def _mkTaskGraph(self, task : str, parent_rundir : str, params : dict = None) -> Task:
 
         elems = task.split(".")
 
         pkg_name = ".".join(elems[0:-1])
         task_name = elems[-1]
 
-        self._rundir_s.append(os.path.join(self._rundir_s[-1], pkg_name, task_name))
-
         if pkg_name == "":
             if len(self._pkg_spec_s) == 0:
                 raise Exception("No package context for %s" % task)
             pkg_spec = self._pkg_spec_s[-1]
+            pkg_name = pkg_spec.name
         else:
             pkg_spec = PackageSpec(pkg_name)
 
+        rundir = os.path.join(parent_rundir, pkg_name, task_name)
+
         self._pkg_spec_s.append(pkg_spec)
         pkg = self.getPackage(pkg_spec)
         
@@ -115,20 +115,19 @@ class Session(object):
             self,
             params,
             depends)
-        task.rundir = self._rundir_s[-1]
+        task.rundir = rundir
         
         for i,d in enumerate(task.depend_refs):
             if d in self._task_m.keys():
                 task.depends.append(self._task_m[d])
             else:
                 print("mkTaskGraph: %s" % d)
-                task.depends.append(self._mkTaskGraph(d))
+                task.depends.append(self._mkTaskGraph(d, parent_rundir))
 
         self._task_m[task.name] = task
 
         self._pkg_s.pop()
         self._pkg_spec_s.pop()
-        self._rundir_s.pop()
 
         return task
     
@@ -147,6 +146,7 @@ class Session(object):
         file_s.append(root)
         ret = None
         with open(root, "r") as fp:
+            print("open %s" % root)
             doc = yaml.load(fp, Loader=yaml.FullLoader)
             if "package" not in doc.keys():
                 raise Exception("Missing 'package' key in %s" % root)
@@ -167,13 +167,56 @@ class Session(object):
                 self._pkg_spec_s.append(PackageSpec(pkg.name))
 
         print("pkg: %s" % str(pkg))
-        
-        # TODO: read in fragments
+
+        print("fragments: %s" % str(pkg.fragments))
+        for spec in pkg.fragments:
+            self._load_fragment_spec(pkg, spec, file_s)
 
         self._pkg_spec_s.pop()
         file_s.pop()
 
         return pkg
+    
+    def _load_fragment_spec(self, pkg : PackageDef, spec : str, file_s : List[str]):
+
+        # We're either going to have:
+        # - File path
+        # - Directory path
+
+        if os.path.isfile(os.path.join(pkg.basedir, spec)):
+            self._load_fragment_file(pkg, spec, file_s)
+        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
+            self._load_fragment_dir(pkg, os.path.join(pkg.basedir, spec), file_s)
+        else:
+            raise Exception("Fragment spec %s not found" % spec)
+        
+
+    def _load_fragment_dir(self, pkg : PackageDef, dir : str, file_s : List[str]):
+
+        for file in os.listdir(dir):
+            if os.path.isdir(os.path.join(dir, file)):
+                self._load_fragment_dir(pkg, os.path.join(dir, file), file_s)
+            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.yaml":
+                self._load_fragment_file(pkg, os.path.join(dir, file), file_s)
+
+    def _load_fragment_file(self, pkg : PackageDef, file : str, file_s : List[str]):
+
+        if file in file_s:
+            raise Exception("Recursive file processing @ %s: %s" % (file, ",".join(self._file_s)))
+        file_s.append(file)
+
+        with open(file, "r") as fp:
+            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            print("doc: %s" % str(doc), flush=True)
+            if "fragment" in doc.keys():
+                # Merge the package definition
+                frag = FragmentDef(**(doc["fragment"]))
+                frag.basedir = os.path.dirname(file)
+                pkg.fragment_l.append(frag)
+            else:
+                print("Warning: file %s is not a fragment" % file)
+        
+
 
     def getPackage(self, spec : PackageSpec) -> Package:
         pkg_spec = self._pkg_spec_s[-1]

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/share/flow.json

@@ -23,7 +23,6 @@
         }
     ],
     "defs": {
-
         "package-def": {
             "title": "Package Definition",
             "$$target": "#/defs/package-def",
@@ -156,21 +155,23 @@
                         "$ref": "#/defs/param"
                     }
                 },
-                "depends": {
-                    "oneOf": [
-                        {
+                "oneOf": [
+                    {
+                        "needs": {
                             "type": "array",
                             "items": {
                                 "$ref": "#/defs/task-dep"
                             }
-                        }, {
+                        }
+                    }, {
+                        "needs": {
                             "type": "array",
                             "items": {
                                 "type": "string"
                             }
                         }
-                    ]
-                }
+                    }
+                ]
             }
         }
     }

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task.py

@@ -143,21 +143,44 @@ class Task(object):
     def setMemento(self, memento : TaskMemento):
         self.memento = memento
 
+    async def isUpToDate(self, memento) -> bool:
+        return False
+
     async def do_run(self) -> TaskData:
         print("do_run: %s - %d depends" % (self.name, len(self.depends)))
         if len(self.depends) > 0:
             awaitables = [dep.waitOutput() for dep in self.depends]
             deps_o = await asyncio.gather(*awaitables)
 
+            # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
+            # First, merge the dep maps of all the inputs
+            deps_m = self.depends[0].output.deps.copy()
+            for deps in map(lambda d: d.deps, self.depends[1:]):
+                for k,v in deps.items():
+                    if k in deps_m:
+                        deps_m[k].add(v)
+                    else:
+                        deps_m[k] = set(v)
+
+            print("deps_m: %s" % str(deps_m))
+
             # Merge the output of the dependencies into a single input data
 #            if len(self.depends) > 1:
 #                raise Exception("TODO: handle >1 inputs")
 
+            # Now that we have a clean input object, we need
+            # to build the dep map
+
             input = self.depends[0].output.copy()
         else:
             input = TaskData()
+        
+
+
+        # Mark the source of this data as being this task
+        input.src = self.name
 
         if not os.path.isdir(self.rundir):
             os.makedirs(self.rundir)
@@ -193,7 +216,7 @@ class Task(object):
     
     def setOutput(self, output : TaskData):
         self.output_set = True
-        output.task_id = self.task_id
+        output.src = self.name
         self.output = output
         self.output_ev.set()
 

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task_data.py

@@ -19,15 +19,31 @@
 #*     Author: 
 #*
 #****************************************************************************
+import enum
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
 from typing import Any, Dict, Set, List, Tuple
 from .fileset import FileSet
 
+class TaskDataParamOpE(enum.Enum):
+    Set = enum.auto()
+    Append = enum.auto()
+    Prepend = enum.auto()
+    PathAppend = enum.auto()
+    PathPrepend = enum.auto()
+
+class TaskDataParamOp(BaseModel):
+    op : TaskDataParamOpE
+    value : Any
+
+class TaskDataParam(BaseModel):
+    value : Any
+    ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
+
 class TaskData(BaseModel):
-    task_id : int = -1
+    src : str = None
     params : Dict[str,Any] = dc.Field(default_factory=dict)
-    deps : List['TaskData'] = dc.Field(default_factory=list)
+    deps : Dict[str,Set[str]] = dc.Field(default_factory=dict)
     changed : bool = False
 
     def hasParam(self, name: str) -> bool:
@@ -40,11 +56,12 @@ class TaskData(BaseModel):
         self.params[name] = value
 
     def addFileSet(self, fs : FileSet):
+        fs.src = self.src
         if "filesets" not in self.params:
             self.params["filesets"] = []
         self.params["filesets"].append(fs)
 
-    def getFileSets(self, type : (str|Set[str])=None) -> List[FileSet]:
+    def getFileSets(self, type=None) -> List[FileSet]:
         ret = []
 
         if "filesets" in self.params:
@@ -56,7 +73,7 @@ class TaskData(BaseModel):
 
     def copy(self) -> 'TaskData':
         ret = TaskData()
-        ret.task_id = self.task_id
+        ret.src = self.src
         ret.params = self.params.copy()
         for d in self.deps:
             ret.deps.append(d.clone())

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task_def.py

@@ -21,7 +21,7 @@
 #****************************************************************************
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Union, Tuple
 from .task import Task
 
 @dc.dataclass
@@ -32,8 +32,8 @@ class TaskSpec(object):
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
     name : str
-    type : (str|TaskSpec) = dc.Field(default_factory=list)
-    depends : List[(str|TaskSpec)] = dc.Field(default_factory=list)
+    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
+    depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
 
     def copy(self) -> 'TaskDef':

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py

@@ -15,10 +15,10 @@ from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
 class TaskMtiSimImage(VlTaskSimImage):
 
     def getRefTime(self):
-        if os.path.isdir(os.path.join(self.rundir, 'work.d')):
+        if os.path.isfile(os.path.join(self.rundir, 'work.d')):
             return os.path.getmtime(os.path.join(self.rundir, 'work.d'))
         else:
-            raise Exception("work.d not found")
+            raise Exception("work.d not found (%s)")
     
     async def build(self, files : List[str], incdirs : List[str]):
         if not os.path.isdir(os.path.join(self.rundir, 'work')):
@@ -53,7 +53,7 @@ class TaskMtiSimImage(VlTaskSimImage):
         await proc.wait()
 
         with open(os.path.join(self.rundir, 'work.d'), "w") as fp:
-            pass
+            fp.write("\n")
 
         if proc.returncode != 0:
             raise Exception("Questa opt failed")

dv_flow_mgr-0.0.1.12750690879a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py

@@ -0,0 +1,14 @@
+import dataclasses as dc
+from .pkg_hdl_sim import PackageHdlSim
+from .vcs_task_sim_image import TaskVcsSimImageCtor
+from .vcs_task_sim_run import TaskVcsSimRunCtor
+
+@dc.dataclass
+class VcsPackage(PackageHdlSim):
+
+    def __post_init__(self):
+        print("PackageVcs::__post_init__", flush=True)
+        self.tasks["SimImage"] = TaskVcsSimImageCtor()
+        self.tasks["SimRun"] = TaskVcsSimRunCtor()
+    pass
+

dv_flow_mgr-0.0.1.12750690879a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py

@@ -0,0 +1,49 @@
+import os
+import fnmatch
+import dataclasses as dc
+from ....fileset import FileSet
+from ....package import TaskCtor
+from ....task import Task, TaskParams, TaskCtorT
+from ....task_data import TaskData
+from ....task_memento import TaskMemento
+from .vl_task_sim_image import VlTaskSimImage, VlTaskSimImageParams, VlTaskSimImageMemento
+from typing import List, Tuple
+
+from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
+
+@dc.dataclass
+class TaskVcsSimImage(VlTaskSimImage):
+
+    def getRefTime(self):
+        if os.path.isfile(os.path.join(self.rundir, 'simv')):
+            return os.path.getmtime(os.path.join(self.rundir, 'simv'))
+        else:
+            raise Exception
+    
+    async def build(self, files : List[str], incdirs : List[str]):
+        cmd = ['vcs', '-sverilog']
+
+        for incdir in incdirs:
+            cmd.append('+incdir+%s' % incdir)
+
+        cmd.extend(files)
+
+        if len(self.params.top):
+            cmd.extend(['-top', "+".join(self.params.top)])
+
+        proc = await self.session.create_subprocess(*cmd,
+                                                        cwd=self.rundir)
+        await proc.wait()
+
+        if proc.returncode != 0:
+            raise Exception("VCS simv failed")
+
+class TaskVcsSimImageParams(VlTaskSimImageParams):
+    pass
+
+class TaskVcsSimImageMemento(VlTaskSimImageMemento):
+    pass
+
+class TaskVcsSimImageCtor(TaskCtorT):
+    def __init__(self):
+        super().__init__(TaskVcsSimImageParams, TaskVcsSimImage)

dv_flow_mgr-0.0.1.12750690879a1/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py

@@ -0,0 +1,45 @@
+import os
+import fnmatch
+import pydantic.dataclasses as dc
+from ....fileset import FileSet
+from ....package import TaskCtor
+from ....task import Task, TaskParams, TaskCtorT
+from ....task_data import TaskData
+from ....task_memento import TaskMemento
+from typing import List, Tuple
+
+class TaskVcsSimRun(Task):
+
+    async def run(self, input : TaskData) -> TaskData:
+        vl_fileset = input.getFileSets("simDir")
+
+        build_dir = vl_fileset[0].basedir
+
+        cmd = [
+            os.path.join(build_dir, 'simv'),
+        ]
+
+        fp = open(os.path.join(self.rundir, 'sim.log'), "w")
+        proc = await self.session.create_subprocess(*cmd,
+                                                    cwd=self.rundir,
+                                                    stdout=fp)
+
+        await proc.wait()
+
+        fp.close()
+
+        output = TaskData()
+        output.addFileSet(FileSet(src=self.name, type="simRunDir", basedir=self.rundir))
+
+        return output
+
+class TaskVcsSimRunParams(TaskParams):
+    pass
+
+class TaskVcsSimRunMemento(TaskMemento):
+    pass
+
+class TaskVcsSimRunCtor(TaskCtorT):
+    def __init__(self):
+        super().__init__(TaskVcsSimRunParams, TaskVcsSimRun)
+

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py

@@ -3,6 +3,7 @@ import fnmatch
 import dataclasses
 import shutil
 import pydantic.dataclasses as dc
+from toposort import toposort
 from ....fileset import FileSet
 from ....package import TaskCtor
 from ....task import Task, TaskParams, TaskCtorT
@@ -31,9 +32,8 @@ class VlTaskSimImage(Task):
         incdirs = []
         memento = ex_memento
 
-        vl_filesets = input.getFileSets(("verilogSource", "systemVerilogSource"))
-        for file in vl_filesets[0].files:
-            files.append(os.path.join(vl_filesets[0].basedir, file))
+
+        self._gatherSvSources(files, incdirs, input)
 
         if not in_changed:
             try:
@@ -41,7 +41,7 @@ class VlTaskSimImage(Task):
                 info = FileCollection.from_dict(ex_memento.svdeps)
                 in_changed = not TaskCheckUpToDate(files, incdirs).check(info, ref_mtime)
             except Exception as e:
-                print("Unexpected output-directory format. Rebuilding")
+                print("Unexpected output-directory format (%s). Rebuilding" % str(e))
                 shutil.rmtree(self.rundir)
                 os.makedirs(self.rundir)
                 in_changed = True
@@ -56,11 +56,35 @@ class VlTaskSimImage(Task):
             await self.build(files, incdirs) 
 
         output = TaskData()
-        output.addFileSet(FileSet(src=self.name, type="verilatorBinary", basedir=self.rundir))
+        output.addFileSet(FileSet(src=self.name, type="simDir", basedir=self.rundir))
         output.changed = in_changed
 
         self.setMemento(memento)
         return output
+    
+    def _gatherSvSources(self, files, incdirs, input):
+        # input must represent dependencies for all tasks related to filesets
+        # references must support transitivity
+
+        vl_filesets = input.getFileSets(("verilogSource", "systemVerilogSource"))
+        fs_tasks = [fs.src for fs in vl_filesets]
+
+        # Want dependencies just for the filesets
+        # - key is the task associated with a filelist
+        # - deps is the dep-set of the on the incoming
+        #
+        # -> Send output set of dependencies
+        # - Task -> deps map
+        #     "task" : ["dep1", "dep2", ...],
+        #     "task2" : 
+        # - All tasks are represented in the map
+        # -> Assume projects will often flatten before exporting
+
+        # Sort the deps
+        order = list(toposort(input.deps))
+
+        print("order: %s" % str(order))
+
 
 
 class VlTaskSimImageParams(TaskParams):

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py

@@ -11,7 +11,7 @@ from typing import List, Tuple
 class TaskVltSimRun(Task):
 
     async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("verilatorBinary")
+        vl_fileset = input.getFileSets("simDir")
 
         build_dir = vl_fileset[0].basedir
 

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/std/task_fileset.py

@@ -12,8 +12,9 @@ from typing import List, Tuple
 class TaskFileSet(Task):
 
     async def run(self, input : TaskData) -> TaskData:
-        print("run: %s: base=%s type=%s include=%s" % (
+        print("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
             self.name,
+            self.basedir,
             self.params.base, self.params.type, str(self.params.include)
         ))
 

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12672327205a1
+Version: 0.0.1.12750690879a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12672327205a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ pyproject.toml
 .vscode/settings.json
 docs/Makefile
 docs/Notes.md
+docs/Roadmap.md
 docs/Stages.md
 docs/TypesAndDefs.md
 docs/conf.py
@@ -20,6 +21,7 @@ src/dv_flow_mgr/flow.py
 src/dv_flow_mgr/fragment_def.py
 src/dv_flow_mgr/package.py
 src/dv_flow_mgr/package_def.py
+src/dv_flow_mgr/package_import_spec.py
 src/dv_flow_mgr/parameters.py
 src/dv_flow_mgr/session.py
 src/dv_flow_mgr/task.py
@@ -39,6 +41,9 @@ src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_image.py
 src/dv_flow_mgr/tasklib/hdl/sim/mti_task_sim_run.py
 src/dv_flow_mgr/tasklib/hdl/sim/pkg_hdl_sim.py
 src/dv_flow_mgr/tasklib/hdl/sim/task_sim_image.py
+src/dv_flow_mgr/tasklib/hdl/sim/vcs_pkg.py
+src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py
+src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py
 src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py
 src/dv_flow_mgr/tasklib/hdl/sim/vlt_pkg.py
 src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_image.py