dv-flow-mgr 0.0.1.13979842530a1__tar.gz → 0.0.1.14097297609a1__tar.gz

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.1.13979842530a1
+Version: 0.0.1.14097297609a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/docs/Makefile

@@ -1,4 +1,5 @@
 DOCS_DIR:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+PROJ_DIR:=$(shell cd $(DOCS_DIR)/.. ; pwd)
 # Minimal makefile for Sphinx documentation
 #
 
@@ -18,4 +19,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@PYTHONPATH=$(PROJ_DIR)/src:$(PYTHONPATH) $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/docs/Roadmap.md

@@ -129,3 +129,5 @@ are evaluated.
   - Enables users to easily click on first N warnings/errors in each step
     without needing to find and open a logfile
 - Allow files to be attached as part of the result
+
+- Allow need relationships to be conditional

dv_flow_mgr-0.0.1.14097297609a1/docs/cmdref.rst

@@ -0,0 +1,9 @@
+#################
+Command reference
+#################
+
+.. argparse::
+    :module: dv_flow.mgr.__main__
+    :func: get_parser 
+    :prog: dfm
+

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/docs/conf.py

@@ -14,6 +14,7 @@ author = 'Matthew Ballance'
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
+    'sphinxarg.ext',
     'sphinx-jsonschema',
     'sphinxcontrib.mermaid',
     'sphinxcontrib-dv-flow'

dv_flow_mgr-0.0.1.14097297609a1/docs/index.rst

@@ -0,0 +1,21 @@
+.. DV Flow Manager documentation master file, created by
+   sphinx-quickstart on Tue Jan  7 02:06:13 2025.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+DV Flow Manager
+===============
+
+DV Flow Manager is an execution engine for the DV Flow build 
+specification. This YAML-based format captures design and verification 
+(DV) tasks and dataflow in a way that enables concurrent execution and 
+efficient avoidance of redundant work.
+You can learn more about the input specification <here>.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   quickstart
+   cmdref
+   pytask_api

dv_flow_mgr-0.0.1.14097297609a1/docs/pytask_api.rst

@@ -0,0 +1,61 @@
+===============
+Python Task API 
+===============
+
+The core implementation for tasks is provided by a Python `async` method. 
+This method is passed two parameters:
+
+* `runner` - Services that the task runner provides for the use of tasks
+* `input` - The input data for the task
+
+The method must return a `TaskDataResult` object with the execution 
+status of the task, result data, markers, and memento data.
+
+TaskDataInput
+=============
+
+An object of type `TaskDataInput` is passed as the `input` parameter 
+of the runner method. 
+
+.. autoclass:: dv_flow.mgr.TaskDataInput
+   :members:
+   :exclude-members: model_config
+
+TaskDataItem
+=============
+
+Data is passed between tasks via `TaskDataItem`-derived objects. 
+Each task may produce 0 or more `TaskDataItem` objects as output. 
+A task receives all the `TaskDataItem` objects produced by its
+dependencies. 
+
+.. autoclass:: dv_flow.mgr.TaskDataItem
+   :members:
+   :exclude-members: model_config
+
+
+TaskDataResult
+==============
+Task implementation methods must return an object of type 
+`TaskDataResult`. This object contains key data about 
+task execution.
+
+.. autoclass:: dv_flow.mgr.TaskDataResult
+   :members:
+   :exclude-members: model_config
+
+TaskMarker
+==========
+Tasks may produce markers to highlight key information to the
+user. A marker is typically a pointer to a file location with
+an associated severity level (error, warning, info).
+
+
+.. autoclass:: dv_flow.mgr.TaskMarker
+   :members:
+   :exclude-members: model_config
+
+.. autoclass:: dv_flow.mgr.TaskMarkerLoc
+   :members:
+   :exclude-members: model_config
+

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/ivpm.yaml

@@ -21,6 +21,8 @@ package:
       src: pypi
     - name: Sphinx
       src: pypi
+    - name: sphinx-argparse
+      src: pypi
     - name: sphinx-jsonschema
 #      src: pypi
       url: https://github.com/lnoor/sphinx-jsonschema.git

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13979842530a1"
+version = "0.0.1.14097297609a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/__init__.py

@@ -21,10 +21,10 @@
 #****************************************************************************
 from .package_def import *
 from .pkg_rgy import PkgRgy
-from .task import *
 from .task_data import *
 from .task_graph_builder import TaskGraphBuilder
-from .task_node import task
+from .task_runner import TaskRunner
+from .task_node_ctor_wrapper import task
 from .task_runner import TaskSetRunner
 from .task_listener_log import TaskListenerLog
 

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/__main__.py

@@ -21,10 +21,14 @@
 #****************************************************************************
 import argparse
 import logging
+from .cmds.cmd_graph import CmdGraph
 from .cmds.cmd_run import CmdRun
+from .cmds.cmd_show import CmdShow
 
 def get_parser():
-    parser = argparse.ArgumentParser(description='dv_flow_mgr')
+    parser = argparse.ArgumentParser(
+        description='dv_flow_mgr',
+        prog='dfm')
     # parser.add_argument("-d", "--debug", 
     #                     help="Enable debug",
     #                     action="store_true")
@@ -36,6 +40,16 @@ def get_parser():
     #                     action="store_true")
     subparsers = parser.add_subparsers(required=True)
 
+    graph_parser = subparsers.add_parser('graph', 
+                                         help='Generates the graph of a task')
+    graph_parser.add_argument("task", nargs="?", help="task to graph")
+    graph_parser.add_argument("-f", "--format", help="Specifies the output format",
+                              default="dot")
+    graph_parser.add_argument("-o", "--output", 
+                              help="Specifies the output file",
+                              default="-")
+    graph_parser.set_defaults(func=CmdGraph())
+
     run_parser = subparsers.add_parser('run', help='run a flow')
     run_parser.add_argument("tasks", nargs='*', help="tasks to run")
     run_parser.add_argument("-j",
@@ -43,6 +57,17 @@ def get_parser():
                         type=int, default=-1)
     run_parser.set_defaults(func=CmdRun())
 
+    show_parser = subparsers.add_parser('show', 
+                                        help='Display information about a task or tasks')
+    show_parser.add_argument("task", nargs='?', help="task to show")
+    show_parser.add_argument("-a", "--all",
+                        action="store_true",
+                        help="Shows all tasks required for the subject to run")
+    show_parser.add_argument("-v", "--verbose",
+                        action="store_true",
+                        help="Shows additional information about tasks")
+    show_parser.set_defaults(func=CmdShow())
+
     return parser
 
 def main():

dv_flow_mgr-0.0.1.14097297609a1/src/dv_flow/mgr/cmds/cmd_graph.py

@@ -0,0 +1,82 @@
+#****************************************************************************
+#* cmd_graph.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdGraph(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+            t = builder.mkTaskGraph(pkg.name + "." + args.task)
+
+            TaskGraphDotWriter().write(
+                t,
+                "-"
+            )
+
+        return 0
+
+

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/cmds/cmd_run.py

@@ -50,7 +50,7 @@ class CmdRun(object):
             tasks = []
             for task in pkg.tasks:
                 tasks.append(task)
-            for frag in pkg.fragment_l:
+            for frag in pkg._fragment_l:
                 for task in frag.tasks:
                     tasks.append(task)
             tasks.sort(key=lambda x: x.name)
@@ -75,7 +75,7 @@ class CmdRun(object):
         # Maybe separate into a task-graph builder and a task-graph runner
 
         # TODO: allow user to specify run root -- maybe relative to some fixed directory?
-        rundir = os.path.join(pkg.basedir, "rundir")
+        rundir = os.path.join(pkg._basedir, "rundir")
 
         builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
         runner = TaskSetRunner(rundir)

dv_flow_mgr-0.0.1.14097297609a1/src/dv_flow/mgr/cmds/cmd_show.py

@@ -0,0 +1,107 @@
+#****************************************************************************
+#* cmd_show.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import asyncio
+import os
+import logging
+import toposort
+from typing import ClassVar
+from ..util import loadProjPkgDef
+from ..task_graph_builder import TaskGraphBuilder
+from ..task_runner import TaskSetRunner
+from ..task_listener_log import TaskListenerLog
+from ..pkg_rgy import PkgRgy
+from ..task_graph_dot_writer import TaskGraphDotWriter
+
+
+class CmdShow(object):
+    _log : ClassVar = logging.getLogger("CmdGraph")
+
+    def __call__(self, args):
+
+        # First, find the project we're working with
+        pkg = loadProjPkgDef(os.getcwd())
+
+        if pkg is None:
+            raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
+
+        self._log.debug("Root flow file defines package: %s" % pkg.name)
+
+        if args.task is None:
+            # Print out available tasks
+            tasks = []
+            for task in pkg.tasks:
+                tasks.append(task)
+            for frag in pkg._fragment_l:
+                for task in frag.tasks:
+                    tasks.append(task)
+            tasks.sort(key=lambda x: x.name)
+
+            max_name_len = 0
+            for t in tasks:
+                if len(t.name) > max_name_len:
+                    max_name_len = len(t.name)
+
+            print("No task specified. Available Tasks:")
+            for t in tasks:
+                desc = t.desc
+                if desc is None or t.desc == "":
+                    "<no descripion>"
+                print("%s - %s" % (t.name.ljust(max_name_len), desc))
+        else:
+            rundir = os.path.join(pkg._basedir, "rundir")
+
+            builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
+
+
+            if args.all:
+                t = builder.mkTaskGraph(pkg.name + "." + args.task)
+                dep_m = TaskSetRunner(rundir=None).buildDepMap(t)
+
+                order = list(toposort.toposort(dep_m))
+
+                print("Task: %s" % t.name)
+
+                last_s = set()
+                for i,s in enumerate(order):
+                    print("-- TaskSet %d --" % (i+1))
+                    for t in s:
+                        print("  - %s" % t.name)
+                        if args.verbose and len(t.params.model_fields.items()):
+                            print("    params:")
+                            for name,field in t.params.model_fields.items():
+                                print("    - %s: %s" % (name, getattr(t.params, name)))
+                        if len(t.needs):
+                            print("    needs:")
+                            for n in t.needs:
+                                print("    - %s%s" % (
+                                    n[0].name,
+                                    ("*" if n[0] in last_s else "")))
+                    last_s = s
+                        
+                pass
+            else:
+                # Show show info about the current task
+                pass
+
+        return 0
+
+

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/fileset.py

@@ -29,6 +29,7 @@ class FileSet(BaseModel):
     basedir : str
     name : str = ""
     src : str = None
+    seq : int = -1
     files : List[str] = dc.Field(default_factory=list)
     incdirs: List[str] = dc.Field(default_factory=list)
     params : Dict[str,str] = dc.Field(default_factory=dict)

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/fragment_def.py

@@ -34,7 +34,7 @@ class FragmentDef(BaseModel):
     fragments: List[str] = dc.Field(default_factory=list)
     types : List[TypeDef] = dc.Field(default_factory=list)
 
-    basedir : str = None
+    _basedir : str = None
 
     def getTask(self, name : str) -> 'TaskDef':
         for t in self.tasks:

{dv_flow_mgr-0.0.1.13979842530a1 → dv_flow_mgr-0.0.1.14097297609a1}/src/dv_flow/mgr/package.py

@@ -22,7 +22,7 @@
 import dataclasses as dc
 import logging
 from typing import Any, ClassVar, Dict
-from .task_ctor import TaskCtor
+from .task_node_ctor import TaskNodeCtor
 
 @dc.dataclass
 class Package(object):
@@ -30,11 +30,11 @@ class Package(object):
     params : Dict[str,Any] = dc.field(default_factory=dict)
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
-    tasks : Dict[str,TaskCtor] = dc.field(default_factory=dict)
+    tasks : Dict[str,TaskNodeCtor] = dc.field(default_factory=dict)
     types : Dict[str,Any] = dc.field(default_factory=dict)
     _log : ClassVar = logging.getLogger("Package")
 
-    def getTaskCtor(self, name : str) -> TaskCtor:
+    def getTaskCtor(self, name : str) -> TaskNodeCtor:
         self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))