dv-flow-mgr 0.0.1.12703462647a1__tar.gz → 0.0.1.12750690879a1__tar.gz

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12703462647a1
+Version: 0.0.1.12750690879a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/Roadmap.md

@@ -18,6 +18,12 @@ are evaluated.
 - Package "uses" (type/inheritance)
 - Task "with"-data definition (tasks can add their own parameters)
 - Task Groups / Sub-DAG
+- Coarse/fine-grained dependency management
+  - Mark task dependency as having a "coarse" requirement. Causes 
+    the task to be run if it hasn't been run already. Doesn't perform
+    exhaustive analysis.
+  - Maybe allow subtree dependency analysis? Single analysis point to
+    determine up-to-date status on a whole collection of source
 
 ## 3.0.0
 - JQ-based data extraction
@@ -73,3 +79,17 @@ are evaluated.
 ## 5.0.0
 - Need some form of site settings
 
+
+# Core Principles
+- Relevant DAG can be constructed statically
+  - Tasks are not inferred based on dataflow content
+  - Tasks may be created based on statically-available data
+- Dataflow is dynamically typed (union of content)
+- Tasks pass-through filesets that are not relevant for them
+  - Enables injecting new data when convenient, even if it won't be used until later
+  - Can have 'filter' tasks if there is a need to clean-up
+- Dependency analysis is not delegated 
+  - Allows static DAG construction
+- Extensible -- by users and organizations
+- Implementation independent (not tied to a specific implementation language)
+

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/conf.py

@@ -14,7 +14,8 @@ author = 'Matthew Ballance'
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
-    'sphinx-jsonschema'
+    'sphinx-jsonschema',
+    'sphinxcontrib.mermaid'
 ]
 
 templates_path = ['_templates']

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/docs/index.rst

@@ -10,6 +10,18 @@ DV Flow Manager helps users capture the tasks and dataflow in their
 design and verification (DV) flows. You can think of DV Flow Manager as a 
 sort of "make for silicon engineering".
 
+.. mermaid::
+
+    flowchart TD
+      A[IP Fileset] --> B[Testbench]
+      C[VIP Fileset] --> D[Precompile]
+      D --> B
+      B --> E[SimImage]
+      E --> F[Test1]
+      E --> G[Test2]
+      E --> H[Test3]
+
+
 
 .. toctree::
    :maxdepth: 2
@@ -17,4 +29,3 @@ sort of "make for silicon engineering".
 
    quickstart
    reference
-

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/ivpm.yaml

@@ -21,11 +21,16 @@ package:
       src: pypi
     - name: svdep
       url: https://github.com/fvutils/svdep.git
+      anonymous: True
+    - name: ply 
+      src: pypi
     - name: pyyaml
       src: pypi
     - name: Sphinx
       src: pypi
     - name: sphinx-jsonschema
       src: pypi
+    - name: sphinxcontrib-mermaid
+      src: pypi
 
 

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12703462647a1"
+version = "0.0.1.12750690879a1"
 dependencies = [
     "pyyaml",
     "toposort",

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/fragment_def.py

@@ -22,14 +22,14 @@
 import pydantic.dataclasses as dc
 import json
 from pydantic import BaseModel
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 from .package import Package
 from .package_import_spec import PackageImportSpec
 from .task_def import TaskDef
 
 class FragmentDef(BaseModel):
     tasks : List[TaskDef] = dc.Field(default_factory=list)
-    imports : List[(str|PackageImportSpec)] = dc.Field(default_factory=list, alias="imports")
+    imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list, alias="imports")
     fragments: List[str] = dc.Field(default_factory=list)
 
     basedir : str = None

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/session.py

@@ -196,7 +196,7 @@ class Session(object):
         for file in os.listdir(dir):
             if os.path.isdir(os.path.join(dir, file)):
                 self._load_fragment_dir(pkg, os.path.join(dir, file), file_s)
-            elif os.path.isfile(os.path.join(dir, file)):
+            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.yaml":
                 self._load_fragment_file(pkg, os.path.join(dir, file), file_s)
 
     def _load_fragment_file(self, pkg : PackageDef, file : str, file_s : List[str]):
@@ -207,6 +207,7 @@ class Session(object):
 
         with open(file, "r") as fp:
             doc = yaml.load(fp, Loader=yaml.FullLoader)
+            print("doc: %s" % str(doc), flush=True)
             if "fragment" in doc.keys():
                 # Merge the package definition
                 frag = FragmentDef(**(doc["fragment"]))

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task.py

@@ -152,15 +152,35 @@ class Task(object):
             awaitables = [dep.waitOutput() for dep in self.depends]
             deps_o = await asyncio.gather(*awaitables)
 
+            # Merge filesets. A fileset with the same
             print("deps_o: %s" % str(deps_o))
 
+            # First, merge the dep maps of all the inputs
+            deps_m = self.depends[0].output.deps.copy()
+            for deps in map(lambda d: d.deps, self.depends[1:]):
+                for k,v in deps.items():
+                    if k in deps_m:
+                        deps_m[k].add(v)
+                    else:
+                        deps_m[k] = set(v)
+
+            print("deps_m: %s" % str(deps_m))
+
             # Merge the output of the dependencies into a single input data
 #            if len(self.depends) > 1:
 #                raise Exception("TODO: handle >1 inputs")
 
+            # Now that we have a clean input object, we need
+            # to build the dep map
+
             input = self.depends[0].output.copy()
         else:
             input = TaskData()
+        
+
+
+        # Mark the source of this data as being this task
+        input.src = self.name
 
         if not os.path.isdir(self.rundir):
             os.makedirs(self.rundir)
@@ -196,7 +216,7 @@ class Task(object):
     
     def setOutput(self, output : TaskData):
         self.output_set = True
-        output.task_id = self.task_id
+        output.src = self.name
         self.output = output
         self.output_ev.set()
 

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task_data.py

@@ -19,15 +19,31 @@
 #*     Author: 
 #*
 #****************************************************************************
+import enum
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
 from typing import Any, Dict, Set, List, Tuple
 from .fileset import FileSet
 
+class TaskDataParamOpE(enum.Enum):
+    Set = enum.auto()
+    Append = enum.auto()
+    Prepend = enum.auto()
+    PathAppend = enum.auto()
+    PathPrepend = enum.auto()
+
+class TaskDataParamOp(BaseModel):
+    op : TaskDataParamOpE
+    value : Any
+
+class TaskDataParam(BaseModel):
+    value : Any
+    ops : List[TaskDataParamOp] = dc.Field(default_factory=list)
+
 class TaskData(BaseModel):
-    task_id : int = -1
+    src : str = None
     params : Dict[str,Any] = dc.Field(default_factory=dict)
-    deps : List['TaskData'] = dc.Field(default_factory=list)
+    deps : Dict[str,Set[str]] = dc.Field(default_factory=dict)
     changed : bool = False
 
     def hasParam(self, name: str) -> bool:
@@ -40,11 +56,12 @@ class TaskData(BaseModel):
         self.params[name] = value
 
     def addFileSet(self, fs : FileSet):
+        fs.src = self.src
         if "filesets" not in self.params:
             self.params["filesets"] = []
         self.params["filesets"].append(fs)
 
-    def getFileSets(self, type : (str|Set[str])=None) -> List[FileSet]:
+    def getFileSets(self, type=None) -> List[FileSet]:
         ret = []
 
         if "filesets" in self.params:
@@ -56,7 +73,7 @@ class TaskData(BaseModel):
 
     def copy(self) -> 'TaskData':
         ret = TaskData()
-        ret.task_id = self.task_id
+        ret.src = self.src
         ret.params = self.params.copy()
         for d in self.deps:
             ret.deps.append(d.clone())

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/task_def.py

@@ -21,7 +21,7 @@
 #****************************************************************************
 import pydantic.dataclasses as dc
 from pydantic import BaseModel
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Union, Tuple
 from .task import Task
 
 @dc.dataclass
@@ -32,8 +32,8 @@ class TaskSpec(object):
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
     name : str
-    type : (str|TaskSpec) = dc.Field(default_factory=list)
-    depends : List[(str|TaskSpec)] = dc.Field(default_factory=list, alias="needs")
+    type : Union[str,TaskSpec] = dc.Field(default_factory=list)
+    depends : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
     params: Dict[str,Any] = dc.Field(default_factory=dict, alias="with")
 
     def copy(self) -> 'TaskDef':

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_image.py

@@ -15,7 +15,7 @@ from svdep import FileCollection, TaskCheckUpToDate, TaskBuildFileCollection
 class TaskVcsSimImage(VlTaskSimImage):
 
     def getRefTime(self):
-        if os.path.isdir(os.path.join(self.rundir, 'simv')):
+        if os.path.isfile(os.path.join(self.rundir, 'simv')):
             return os.path.getmtime(os.path.join(self.rundir, 'simv'))
         else:
             raise Exception

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vcs_task_sim_run.py

@@ -11,7 +11,7 @@ from typing import List, Tuple
 class TaskVcsSimRun(Task):
 
     async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("simBinary")
+        vl_fileset = input.getFileSets("simDir")
 
         build_dir = vl_fileset[0].basedir
 

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vl_task_sim_image.py

@@ -3,6 +3,7 @@ import fnmatch
 import dataclasses
 import shutil
 import pydantic.dataclasses as dc
+from toposort import toposort
 from ....fileset import FileSet
 from ....package import TaskCtor
 from ....task import Task, TaskParams, TaskCtorT
@@ -31,9 +32,8 @@ class VlTaskSimImage(Task):
         incdirs = []
         memento = ex_memento
 
-        vl_filesets = input.getFileSets(("verilogSource", "systemVerilogSource"))
-        for file in vl_filesets[0].files:
-            files.append(os.path.join(vl_filesets[0].basedir, file))
+
+        self._gatherSvSources(files, incdirs, input)
 
         if not in_changed:
             try:
@@ -61,6 +61,30 @@ class VlTaskSimImage(Task):
 
         self.setMemento(memento)
         return output
+    
+    def _gatherSvSources(self, files, incdirs, input):
+        # input must represent dependencies for all tasks related to filesets
+        # references must support transitivity
+
+        vl_filesets = input.getFileSets(("verilogSource", "systemVerilogSource"))
+        fs_tasks = [fs.src for fs in vl_filesets]
+
+        # Want dependencies just for the filesets
+        # - key is the task associated with a filelist
+        # - deps is the dep-set of the on the incoming
+        #
+        # -> Send output set of dependencies
+        # - Task -> deps map
+        #     "task" : ["dep1", "dep2", ...],
+        #     "task2" : 
+        # - All tasks are represented in the map
+        # -> Assume projects will often flatten before exporting
+
+        # Sort the deps
+        order = list(toposort(input.deps))
+
+        print("order: %s" % str(order))
+
 
 
 class VlTaskSimImageParams(TaskParams):

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/hdl/sim/vlt_task_sim_run.py

@@ -11,7 +11,7 @@ from typing import List, Tuple
 class TaskVltSimRun(Task):
 
     async def run(self, input : TaskData) -> TaskData:
-        vl_fileset = input.getFileSets("simBinary")
+        vl_fileset = input.getFileSets("simDir")
 
         build_dir = vl_fileset[0].basedir
 

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr/tasklib/std/task_fileset.py

@@ -12,8 +12,9 @@ from typing import List, Tuple
 class TaskFileSet(Task):
 
     async def run(self, input : TaskData) -> TaskData:
-        print("run: %s: base=%s type=%s include=%s" % (
+        print("TaskFileSet run: %s: basedir=%s, base=%s type=%s include=%s" % (
             self.name,
+            self.basedir,
             self.params.base, self.params.type, str(self.params.include)
         ))
 

{dv_flow_mgr-0.0.1.12703462647a1 → dv_flow_mgr-0.0.1.12750690879a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12703462647a1
+Version: 0.0.1.12750690879a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License