dv-flow-mgr 0.0.1.12822558956a1__tar.gz → 0.0.1.12911707440a1__tar.gz

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
       run: |
         cd docs
         make html
-        cp ../src/dv_flow_mgr/share/flow.json _build/html/
+        cp ../src/dv_flow/mgr/share/flow.json _build/html/
         touch _build/html/.nojekyll
 
     - name: Publish to PyPi

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12822558956a1
+Version: 0.0.1.12911707440a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/docs/Roadmap.md

@@ -12,6 +12,18 @@ are evaluated.
   - Operations on input and output data
   - Operations on task parameters
 - Package fragments
+- Define task status. Tasks can have at least two types of failures
+  - Pass/Fail: Fail halts successors
+    - Fail must come with a message and extra info
+  - Status markers/counters
+    - errors / warnings / notes 
+  - Want known fileset to capture logfiles and related info
+  - Central params and datasets?
+    - Datasets preserve dependency relationships
+    - Datasets are the best way to aggregate settings
+  - Typed parameter sets
+  - Dependencies provide order in which to evaluate
+  - Operations on variables
 
 ## 2.0.0
 - Parameterized package definition and use
@@ -29,11 +41,13 @@ are evaluated.
 - JQ-based data extraction
 - YAML task templates / expansions
 - Support for annotating job requirements 
+- Support capturing schema for structured task data
 - Mark tasks as producing and accepting certain data
   - FileSet task `produces` fileset of `type`
   - SimImage task `accepts` systemVerilogSource, verilogSource, verilogPreCompLib, etc
   => Mostly useful for checking and suggestion
   => As more are marked, can treat as more binding
+- 
 
 # Library
 

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/docs/reference.rst

@@ -10,7 +10,7 @@ Each package is defined by the content in its root `flow.yaml` file
 and that in any `fragment` files that are specified in the root 
 package file or its fragments.
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/package-def
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/package-def
 
 .. code-block:: yaml
 
@@ -24,7 +24,7 @@ package file or its fragments.
             - src/verif
 
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/fragment-def
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/fragment-def
 
 A fragment has similar content to a root-package file.
 
@@ -42,12 +42,12 @@ Remember that all fragments referenced by a given package contribute to
 the same package namespace. It would be illegal for another flow file
 to also define a task named `rtl`.
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/import-def
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/import-def
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/param
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/param
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/task-def
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/task-def
 
-.. jsonschema:: ../src/dv_flow_mgr/share/flow.json#/defs/task-dep
+.. jsonschema:: ../src/dv_flow/mgr/share/flow.json#/defs/task-dep
 
 And, now, after

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/ivpm.yaml

@@ -1,10 +1,6 @@
 
 package:
-  name: pytest-fv
-
-  #  setup-deps:
-  #- ninja
-  #- cython
+  name: dv-flow-mgr
 
   dep-sets:
 
@@ -22,6 +18,8 @@ package:
     - name: svdep
       url: https://github.com/fvutils/svdep.git
       anonymous: True
+    - name: dv-flow-lib-hdlsim
+      url: http://github.com/dv-flow/dv-flow-lib-hdlsim.git
     - name: ply 
       src: pypi
     - name: pyyaml

{dv_flow_mgr-0.0.1.12822558956a1 → dv_flow_mgr-0.0.1.12911707440a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12822558956a1"
+version = "0.0.1.12911707440a1"
 dependencies = [
     "pyyaml",
     "toposort",
@@ -18,8 +18,8 @@ description = "DV Flow Manager is a build system for silicon design"
 license = {file = "LICENSE" }
 
 [project.scripts]
-dvfm = "dv_flow_mgr.__main__:main"
+dfm = "dv_flow.mgr.__main__:main"
 
 [tool.setuptools.package-data]
-dv_Flow_mgr = ['share/*']
+"dv_flow.mgr" = ['share/*']
 

{dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr → dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr}/__init__.py

@@ -1,6 +1,6 @@
 
 from .package_def import *
-from .session import *
+from .task_graph_runner import *
 from .task import *
 from .task_data import *
 

{dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr → dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr}/__main__.py

@@ -7,7 +7,7 @@ def get_parser():
     subparsers = parser.add_subparsers(required=True)
 
     run_parser = subparsers.add_parser('run', help='run a flow')
-    run_parser.add_argument("tasks", nargs='+', help="tasks to run")
+    run_parser.add_argument("tasks", nargs='*', help="tasks to run")
     run_parser.set_defaults(func=CmdRun())
 
     return parser

dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/cmds/cmd_run.py

@@ -0,0 +1,90 @@
+import asyncio
+import os
+from ..task_graph_runner import TaskGraphRunner
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_graph_runner_local import TaskGraphRunnerLocal
+from ..pkg_rgy import PkgRgy
+
+
+class CmdRun(object):
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        print("pkg: %s" % pkg.name)
+
+        if len(args.tasks) > 0:
+            pass
+        else:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg.fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+
+            pass
+
+        # Create a session around <pkg>
+        # Need to select a backend
+        # Need somewhere to store project config data
+        # Maybe separate into a task-graph builder and a task-graph runner
+
+        # TODO: allow user to specify run root -- maybe relative to some fixed directory?
+        rundir = os.path.join(pkg.basedir, "rundir")
+
+        builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+        runner = TaskGraphRunnerLocal(rundir)
+
+        tasks = []
+
+        for spec in args.tasks:
+            task = builder.mkTaskGraph(spec)
+            tasks.append(task)
+
+        asyncio.run(runner.run(tasks))
+
+#        rgy = PkgRgy.inst()
+#        rgy.registerPackage(pkg)
+
+
+        # srcdir = os.getcwd()
+
+        # session = Session(srcdir, rundir)
+
+        # package = session.load(srcdir)
+
+        # graphs = []
+        # for task in args.tasks:
+        #     if task.find(".") == -1:
+        #         task = package.name + "." + task
+        #     subgraph = session.mkTaskGraph(task)
+        #     graphs.append(subgraph)
+
+        # awaitables = [subgraph.do_run() for subgraph in graphs]
+        # print("%d awaitables" % len(awaitables))
+
+        # out = asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
+
+        # print("out: %s" % str(out))
+

{dv_flow_mgr-0.0.1.12822558956a1/src/dv_flow_mgr → dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr}/package.py

@@ -54,26 +54,6 @@ class Package(object):
     def getTaskCtor(self, name : str) -> TaskCtor:
         return self.tasks[name]
             
-    def mkTaskParams(self, name : str) -> TaskParams:
-        if name not in self.tasks:
-            raise Exception("Task " + name + " not found")
-        return self.tasks[name].mkTaskParams()
-    
-    def setTaskParams(self, name : str, params : TaskParams, pvals : Dict[str,Any]):
-        if name not in self.tasks:
-            raise Exception("Task " + name + " not found")
-        self.tasks[name].setTaskParams(params, pvals)
-
-    def mkTask(self, 
-               name : str, 
-               task_id : int, 
-               session : 'Session',
-               params : TaskParams,
-               depends : List['Task']) -> 'Task':
-        # TODO: combine parameters to create the full taskname
-        task =  self.tasks[name].mkTask(name, task_id, session, params, depends)
-        return task
-
     def __hash__(self):
         return hash(self.fullname())
 

dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/package_def.py

@@ -0,0 +1,302 @@
+#****************************************************************************
+#* package_def.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import os
+import json
+import yaml
+import importlib
+import sys
+import pydantic
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import Any, Dict, List, Callable, Tuple
+from .flow import Flow
+from .fragment_def import FragmentDef
+from .package import Package
+from .package_import_spec import PackageImportSpec, PackageSpec
+from .task import TaskCtor, TaskParams
+from .task_def import TaskDef, TaskSpec
+from .std.task_null import TaskNull
+
+
+class PackageDef(BaseModel):
+    name : str
+    params : Dict[str,Any] = dc.Field(default_factory=dict)
+    type : List[PackageSpec] = dc.Field(default_factory=list)
+    tasks : List[TaskDef] = dc.Field(default_factory=list)
+    imports : List[PackageImportSpec] = dc.Field(default_factory=list)
+    fragments: List[str] = dc.Field(default_factory=list)
+
+    fragment_l : List['FragmentDef'] = dc.Field(default_factory=list, exclude=True)
+
+#    import_m : Dict['PackageSpec','Package'] = dc.Field(default_factory=dict)
+
+    basedir : str = None
+
+    def getTask(self, name : str) -> 'TaskDef':
+        for t in self.tasks:
+            if t.name == name:
+                return t
+    
+    def mkPackage(self, session, params : Dict[str,Any] = None) -> 'Package':
+        ret = Package(self.name)
+
+        session.push_package(ret)
+
+        tasks_m : Dict[str,str,TaskCtor]= {}
+
+        for task in self.tasks:
+            if task.name in tasks_m.keys():
+                raise Exception("Duplicate task %s" % task.name)
+            tasks_m[task.name] = (task, self.basedir, ) # We'll add a TaskCtor later
+
+        for frag in self.fragment_l:
+            for task in frag.tasks:
+                if task.name in tasks_m.keys():
+                    raise Exception("Duplicate task %s" % task.name)
+                tasks_m[task.name] = (task, frag.basedir, ) # We'll add a TaskCtor later
+
+        # Now we have a unified map of the tasks declared in this package
+        for name in list(tasks_m.keys()):
+            task_i = tasks_m[name]
+            if len(task_i) < 3:
+                # Need to create the task ctor
+                ctor_t = self.mkTaskCtor(session, task_i[0], task_i[1], tasks_m)
+                tasks_m[name] = (task_i[0], task_i[1], ctor_t)
+            ret.tasks[name] = tasks_m[name][2]
+
+        session.pop_package(ret)
+
+        return ret
+    
+    def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
+        ctor_t : TaskCtor = None
+
+        if task.uses is not None:
+            # Find package (not package_def) that implements this task
+            # Insert an indirect reference to that tasks's constructor
+            last_dot = task.uses.rfind('.')
+
+            if last_dot != -1:
+                pkg_name = task.uses[:last_dot]
+                task_name = task.uses[last_dot+1:]
+            else:
+                pkg_name = None
+                task_name = task.uses
+
+            if pkg_name is not None:
+                pkg = session.getPackage(PackageSpec(pkg_name))
+                if pkg is None:
+                    raise Exception("Failed to find package %s" % pkg_name)
+                ctor_t = pkg.getTaskCtor(task_name)
+                ctor_t = ctor_t.copy()
+                ctor_t.srcdir = srcdir
+            else:
+                if task_name not in tasks_m.keys():
+                    raise Exception("Failed to find task %s" % task_name)
+                if len(tasks_m[task_name]) == 3:
+                    ctor_t = tasks_m[task_name][2].copy()
+                    ctor_t.srcdir = srcdir
+                else:
+                    task_i = tasks_m[task_name]
+                    ctor_t = self.mkTaskCtor(
+                        session, 
+                        task=task_i[0], 
+                        srcdir=srcdir,
+                        tasks_m=tasks_m)
+                    tasks_m[task_name] = ctor_t
+
+        if ctor_t is None:
+            # Provide a default implementation
+            ctor_t = TaskCtor(
+                task_ctor=TaskNull,
+                param_ctor=TaskParams,
+                srcdir=srcdir)
+
+        if task.pyclass is not None:
+            # Built-in impl
+            # Now, lookup the class
+            last_dot = task.pyclass.rfind('.')
+            clsname = task.pyclass[last_dot+1:]
+            modname = task.pyclass[:last_dot]
+
+            try:
+                if modname not in sys.modules:
+                    if self.basedir not in sys.path:
+                        sys.path.append(self.basedir)
+                    mod = importlib.import_module(modname)
+                else:
+                    mod = sys.modules[modname]
+            except ModuleNotFoundError as e:
+                raise Exception("Failed to import module %s (basedir=%s): %s" % (
+                    modname, self.basedir, str(e)))
+                
+            if not hasattr(mod, clsname):
+                raise Exception("Class %s not found in module %s" % (clsname, modname))
+            ctor_t.task_ctor = getattr(mod, clsname)
+
+            if task.uses is None:
+                ctor_t.param_ctor = TaskParams
+
+        decl_params = False
+        for value in task.params.values():
+            if "type" in value:
+                decl_params = True
+                break
+        
+        if decl_params:
+            # We need to combine base parameters with new parameters
+            field_m = {}
+            # First, add parameters from the base class
+            for fname,info in ctor_t.param_ctor.model_fields.items():
+                print("Field: %s (%s)" % (fname, info.default))
+                field_m[fname] = (info.annotation, info.default)
+            ptype_m = {
+                "str" : str,
+                "int" : int,
+                "float" : float,
+                "bool" : bool,
+                "list" : List
+            }
+            pdflt_m = {
+                "str" : "",
+                "int" : 0,
+                "float" : 0.0,
+                "bool" : False,
+                "list" : []
+            }
+            for p in task.params.keys():
+                param = task.params[p]
+                if type(param) == dict and "type" in param.keys():
+                    ptype_s = param["type"]
+                    if ptype_s not in ptype_m.keys():
+                        raise Exception("Unknown type %s" % ptype_s)
+                    ptype = ptype_m[ptype_s]
+
+                    if p in field_m.keys():
+                        raise Exception("Duplicate field %s" % p)
+                    if "value" in param.keys():
+                        field_m[p] = (ptype, param["value"])
+                    else:
+                        field_m[p] = (ptype, pdflt_m[ptype_s])
+                else:
+                    if p not in field_m.keys():
+                        raise Exception("Field %s not found" % p)
+                    if type(param) != dict:
+                        value = param
+                    elif "value" in param.keys():
+                        value = param["value"]
+                    else:
+                        raise Exception("No value specified for param %s: %s" % (
+                            p, str(param)))
+                    field_m[p] = (field_m[p][0], value)
+            print("field_m: %s" % str(field_m))
+            ctor_t.param_ctor = pydantic.create_model(
+                "Task%sParams" % task.name, **field_m)
+        else:
+            if len(task.params) > 0:
+                ctor_t.params = task.params
+            if len(task.depends) > 0:
+                ctor_t.depends.extends(task.depends)
+
+        return ctor_t
+
+    @staticmethod
+    def load(path, exp_pkg_name=None):
+        return PackageDef._loadPkgDef(path, exp_pkg_name, [])
+        pass
+
+    @staticmethod
+    def _loadPkgDef(root, exp_pkg_name, file_s):
+        if root in file_s:
+            raise Exception("Recursive file processing @ %s: %s" % (root, ",".join(file_s)))
+        file_s.append(root)
+        ret = None
+        with open(root, "r") as fp:
+            print("open %s" % root)
+            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            if "package" not in doc.keys():
+                raise Exception("Missing 'package' key in %s" % root)
+            pkg = PackageDef(**(doc["package"]))
+            pkg.basedir = os.path.dirname(root)
+
+#            for t in pkg.tasks:
+#                t.basedir = os.path.dirname(root)
+
+        if exp_pkg_name is not None:
+            if exp_pkg_name != pkg.name:
+                raise Exception("Package name mismatch: %s != %s" % (exp_pkg_name, pkg.name))
+            # else:
+            #     self._pkg_m[exp_pkg_name] = [PackageSpec(pkg.name)
+            #     self._pkg_spec_s.append(PackageSpec(pkg.name))
+
+        # if not len(self._pkg_spec_s):
+        #     self._pkg_spec_s.append(PackageSpec(pkg.name))
+        # else:
+        # self._pkg_def_m[PackageSpec(pkg.name)] = pkg
+
+        print("pkg: %s" % str(pkg))
+
+        print("fragments: %s" % str(pkg.fragments))
+        for spec in pkg.fragments:
+            PackageDef._loadFragmentSpec(pkg, spec, file_s)
+
+        file_s.pop()
+
+        return pkg
+    
+    @staticmethod
+    def _loadFragmentSpec(pkg, spec, file_s):
+        # We're either going to have:
+        # - File path
+        # - Directory path
+
+        if os.path.isfile(os.path.join(pkg.basedir, spec)):
+            PackageDef._loadFragmentFile(pkg, spec, file_s)
+        elif os.path.isdir(os.path.join(pkg.basedir, spec)):
+            PackageDef._loadFragmentDir(pkg, os.path.join(pkg.basedir, spec), file_s)
+        else:
+            raise Exception("Fragment spec %s not found" % spec)
+
+    @staticmethod
+    def _loadFragmentDir(pkg, dir, file_s):
+        for file in os.listdir(dir):
+            if os.path.isdir(os.path.join(dir, file)):
+                PackageDef._loadFragmentDir(pkg, os.path.join(dir, file), file_s)
+            elif os.path.isfile(os.path.join(dir, file)) and file == "flow.dv":
+                PackageDef._loadFragmentFile(pkg, os.path.join(dir, file), file_s)
+
+    @staticmethod
+    def _loadFragmentFile(pkg, file, file_s):
+        if file in file_s:
+            raise Exception("Recursive file processing @ %s: %s" % (file, ", ".join(file_s)))
+        file_s.append(file)
+
+        with open(file, "r") as fp:
+            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            print("doc: %s" % str(doc), flush=True)
+            if "fragment" in doc.keys():
+                # Merge the package definition
+                frag = FragmentDef(**(doc["fragment"]))
+                frag.basedir = os.path.dirname(file)
+                pkg.fragment_l.append(frag)
+            else:
+                print("Warning: file %s is not a fragment" % file)

dv_flow_mgr-0.0.1.12911707440a1/src/dv_flow/mgr/pkg_rgy.py

@@ -0,0 +1,78 @@
+import os
+import sys
+from typing import Dict, Tuple
+from .package_def import PackageDef
+
+class PkgRgy(object):
+    _inst = None
+
+    def __init__(self):
+        self._pkgpath = []
+        self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
+
+    def hasPackage(self, name, search_path=False):
+        if name in self._pkg_m.keys():
+            return True
+        elif search_path:
+            for p in self._pkgpath:
+                if os.path.exists(os.path.join(p, name)):
+                    return True
+        else:
+            return False
+    
+    def getPackage(self, name):
+        if name in self._pkg_m.keys():
+            if self._pkg_m[name][1] is None:
+                pkg_def = PackageDef.load(self._pkg_m[name][0])
+                # Load the package
+                self._pkg_m[name] = (
+                    self._pkg_m[name][0],
+                    pkg_def
+                )
+                pass
+            return self._pkg_m[name][1]
+        else:
+            # Go search the package path
+            return None
+
+    def registerPackage(self, pkg_def):
+        if pkg_def.name in self._pkg_m.keys():
+            raise Exception("Duplicate package %s" % pkg_def.name)
+        self._pkg_m[pkg_def.name] = pkg_def
+
+    def _discover_plugins(self):
+        # Register built-in package
+        self._pkg_m["std"] = (os.path.join(os.path.dirname(__file__), "std/flow.dv"), None)
+
+        if sys.version_info < (3,10):
+            from importlib_metadata import entry_points
+        else:
+            from importlib.metadata import entry_points
+
+        discovered_plugins = entry_points(group='dv_flow.mgr')
+        for p in discovered_plugins:
+            try:
+                mod = p.load()
+
+                if hasattr(mod, "dvfm_packages"):
+                    pkg_m = mod.dvfm_packages()
+                    
+                    for name,path in pkg_m.items():
+                        if name in self._pkg_m.keys():
+                            raise Exception("Package %s already registered using path %s. Conflicting path: %s" % (
+                                name, self._pkg_m[name][0], path))
+                        self._pkg_m[name] = (path, None)
+            except Exception as e:
+                print("Error loading plugin %s: %s" % (p.name, str(e)))
+                raise e
+
+        # self._pkgs = {}
+        # for pkg in self._load_pkg_list():
+        #     self._pkgs[pkg.name] = pkg
+
+    @classmethod
+    def inst(cls):
+        if cls._inst is None:
+            cls._inst = cls()
+            cls._inst._discover_plugins()
+        return cls._inst