dv-flow-mgr 0.0.2.14182043984a1__tar.gz → 1.0.0.14370600369a1__tar.gz

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.2.14182043984a1
+Version: 1.0.0.14370600369a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/docs/Roadmap.md

@@ -5,24 +5,24 @@ semantic definition of how task graphs defined using the flow specification
 are evaluated.
 
 ## 1.0.0
-- Package definition
-- Package import
-- Task definition
-  - "with" variable usage
-  - Operations on input and output data
-  - Operations on task parameters
-- Package fragments
-- Define task status. Tasks can have at least two types of failures
-  - Pass/Fail: Fail halts successors
+* Package definition
+* Package import
+* Task definition
+  * "with" variable usage
+  * Operations on input and output data
+  * Operations on task parameters
+* Package fragments
+* Define task status. Tasks can have at least two types of failures
+  * Pass/Fail: Fail halts successors
     - Fail must come with a message and extra info
-  - Status markers/counters
+  * Status markers/counters
     - errors / warnings / notes 
   - Want known fileset to capture logfiles and related info
   - Central params and datasets?
     - Datasets preserve dependency relationships
     - Datasets are the best way to aggregate settings
-  - Typed parameter sets
-  - Dependencies provide order in which to evaluate
+  * Typed parameter sets
+  * Dependencies provide order in which to evaluate
   - Operations on variables
 
 ## 2.0.0
@@ -38,11 +38,11 @@ are evaluated.
     determine up-to-date status on a whole collection of source
 
 ## 3.0.0
-- JQ-based data extraction
+* JQ-based data extraction
 - YAML task templates / expansions
 - Support for annotating job requirements 
 - Support capturing schema for structured task data
-- Mark tasks as producing and accepting certain data
+~ Mark tasks as producing and accepting certain data
   - FileSet task `produces` fileset of `type`
   - SimImage task `accepts` systemVerilogSource, verilogSource, verilogPreCompLib, etc
   => Mostly useful for checking and suggestion
@@ -131,3 +131,17 @@ are evaluated.
 - Allow files to be attached as part of the result
 
 - Allow need relationships to be conditional
+
+
+# Apr-May
+- Configuration
+  - Conditional needs
+  - package-level parameters
+  - 
+- Subgraph builders
+  - Python
+  - Matrix strategy
+
+- Identify export tasks
+
+- Interop w/external formats

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/docs/Stages.md

@@ -179,3 +179,23 @@ Key: all
   - end-to-end system check
   - avoid recompile when primary sources do not change
 
+# Load/Elab/Build Refactor
+- Separated loading from graph building
+  - Now, package and task information is produced with source info
+- Working on recreating the task-node builder
+  - Consider getting rid of the Ctor infrastructure
+    - Have parameter-type info
+    - Have inheritance info
+    - Have fully-qualified names for all dependencies
+  - In this mode, graph builder would:
+    - Find named 'task' object
+    - Determine how to create a node and parameters from it
+      - Task should save statically-specified parameter values
+      - GraphBuilder can apply programmatically-specified parameters
+    - What is a programmatically-specified task?
+      - Prototype for a task node
+      - Not necessarily registered with the package loader
+      - Should have similar characteristics to a task (?)
+      - Maybe just a direct way to create a task node
+      - Don't think we expect the system to create is programmatically...
+

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.2.14182043984a1"
+version = "1.0.0.14370600369a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/__init__.py

@@ -20,7 +20,8 @@
 #*
 #****************************************************************************
 from .package_def import *
-from .pkg_rgy import PkgRgy
+from .package_loader import PackageLoader
+from .ext_rgy import ExtRgy
 from .task_data import *
 from .task_def import *
 from .task_graph_builder import TaskGraphBuilder

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/cmds/cmd_graph.py

@@ -27,7 +27,6 @@ from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
 from ..task_runner import TaskSetRunner
 from ..task_listener_log import TaskListenerLog
-from ..pkg_rgy import PkgRgy
 from ..task_graph_dot_writer import TaskGraphDotWriter
 
 
@@ -66,11 +65,11 @@ class CmdGraph(object):
                     "<no descripion>"
                 print("%s - %s" % (t.name.ljust(max_name_len), desc))
         else:
-            rundir = os.path.join(pkg._basedir, "rundir")
+            rundir = os.path.join(pkg.basedir, "rundir")
 
             builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
 
-            t = builder.mkTaskGraph(pkg.name + "." + args.task)
+            t = builder.mkTaskNode(pkg.name + "." + args.task)
 
             TaskGraphDotWriter().write(
                 t,

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/cmds/cmd_run.py

@@ -27,7 +27,6 @@ from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
 from ..task_runner import TaskSetRunner
 from ..task_listener_log import TaskListenerLog
-from ..pkg_rgy import PkgRgy
 
 
 class CmdRun(object):
@@ -36,7 +35,8 @@ class CmdRun(object):
     def __call__(self, args):
 
         # First, find the project we're working with
-        pkg = loadProjPkgDef(os.getcwd())
+        listener = TaskListenerLog()
+        pkg = loadProjPkgDef(os.getcwd(), listener=listener)
 
         if pkg is None:
             raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())
@@ -48,11 +48,8 @@ class CmdRun(object):
         else:
             # Print out available tasks
             tasks = []
-            for task in pkg.tasks:
+            for task in pkg.task_m.values():
                 tasks.append(task)
-            for frag in pkg._fragment_l:
-                for task in frag.tasks:
-                    tasks.append(task)
             tasks.sort(key=lambda x: x.name)
 
             max_name_len = 0
@@ -75,7 +72,8 @@ class CmdRun(object):
         # Maybe separate into a task-graph builder and a task-graph runner
 
         # TODO: allow user to specify run root -- maybe relative to some fixed directory?
-        rundir = os.path.join(pkg._basedir, "rundir")
+        rundir = os.path.join(pkg.basedir, "rundir")
+
 
         if args.clean:
             print("Note: Cleaning rundir %s" % rundir)
@@ -89,14 +87,14 @@ class CmdRun(object):
         if args.j != -1:
             runner.nproc = int(args.j)
 
-        runner.add_listener(TaskListenerLog().event)
+        runner.add_listener(listener.event)
 
         tasks = []
 
         for spec in args.tasks:
             if spec.find('.') == -1:
                 spec = pkg.name + "." + spec
-            task = builder.mkTaskGraph(spec, rundir=spec)
+            task = builder.mkTaskNode(spec)
             tasks.append(task)
 
         asyncio.run(runner.run(tasks))

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/cmds/cmd_show.py

@@ -28,7 +28,6 @@ from ..util import loadProjPkgDef
 from ..task_graph_builder import TaskGraphBuilder
 from ..task_runner import TaskSetRunner
 from ..task_listener_log import TaskListenerLog
-from ..pkg_rgy import PkgRgy
 from ..task_graph_dot_writer import TaskGraphDotWriter
 
 
@@ -73,7 +72,7 @@ class CmdShow(object):
 
 
             if args.all:
-                t = builder.mkTaskGraph(pkg.name + "." + args.task)
+                t = builder.mkTaskNode(pkg.name + "." + args.task)
                 dep_m = TaskSetRunner(rundir=None).buildDepMap(t)
 
                 order = list(toposort.toposort(dep_m))

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/cond_def.py

@@ -0,0 +1,16 @@
+
+import pydantic.dataclasses as pdc
+from pydantic import BaseModel
+
+#
+# - task:
+#   needs: 
+#   - if: abc
+#       then: abc
+#       else: def 
+#     
+class CondDef(BaseModel):
+    if_then : str = pdc.Field(
+        description="Condition to evaluate")
+    then : str = pdc.Field(
+        description="Task to execute if the condition is true")

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/config.py

@@ -0,0 +1,7 @@
+
+import dataclasses as dc
+from typing import List
+
+@dc.dataclass
+class Config(object):
+    overrides : List

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/config_def.py

@@ -0,0 +1,33 @@
+
+import pydantic.dataclasses as pdc
+from pydantic import BaseModel
+from typing import List, Union, Any
+from .extend_def import ExtendDef
+from .param_def import ParamDef
+
+class OverrideDef(BaseModel):
+    """Override definition"""
+    package : Union[str, None] = pdc.Field(
+        description="Package to override")
+    task : Union[str, None] = pdc.Field(
+        description="Task to override")
+    value : str = pdc.Field(
+        description="Override to use",
+        alias="with")
+
+class ConfigDef(BaseModel):
+    name : str = pdc.Field(
+        description="Name of the configuration")
+    params : List[ParamDef] = pdc.Field(
+        default_factory=list,
+        description="List of configuration parameters",
+        alias="with")
+    uses : str = pdc.Field(
+        default=None,
+        description="Name of the configuration to use as a base")
+    overrides : List[OverrideDef] = pdc.Field(
+        default_factory=list,
+        description="List of package and task overrides")
+    extensions : List[ExtendDef] = pdc.Field(
+        default_factory=list,
+        description="List of extensions to apply")

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/exec_callable.py

@@ -0,0 +1,88 @@
+import dataclasses as dc
+import importlib
+import importlib.util
+import logging
+import os
+import sys
+from typing import ClassVar, List
+from .task_data import TaskDataResult
+
+@dc.dataclass
+class ExecCallable(object):
+    body : str
+    _log : ClassVar = logging.getLogger("ExecCallable")
+
+    async def __call__(self, ctxt, input):
+        self._log.debug("--> ExecCallable")
+        self._log.debug("Body:\n%s" % self.body)
+
+        # If it is a single line, then we have a spec to load
+        # Otherwise, we have an inline task
+
+        if self.body.find("\n") == -1:
+            # Two forms:
+            # <path>::method
+            # <method-path>
+
+            ci = self.body.find("::")
+            if ci != -1:
+                # have a file to load
+                file = self.body[:ci]
+                method = self.body[ci+2:]
+                spec = importlib.util.spec_from_file_location(input.name, file)
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[input.name] = module
+                spec.loader.exec_module(module)
+
+                callable = getattr(module, method)
+                pass
+            else:
+            #     self._log.debug("Use PyTask implementation")
+                last_dot = self.body.rfind('.')
+                clsname = self.body[last_dot+1:]
+                modname = self.body[:last_dot]
+
+                try:
+                    if modname not in sys.modules:
+                        if input.srcdir not in sys.path:
+                           sys.path.append(input.srcdir)
+                        mod = importlib.import_module(modname)
+                    else:
+                        mod = sys.modules[modname]
+                except ModuleNotFoundError as e:
+                    raise Exception("Failed to import module %s: %s" % (modname, str(e)))
+                
+                if not hasattr(mod, clsname):
+                    raise Exception("Method %s not found in module %s" % (clsname, modname))
+                callable = getattr(mod, clsname)
+        else:
+            text_lines = self.body.splitlines()
+
+            least_whitespace = 2^32
+            have_content = False
+            for line in text_lines:
+                line_no_leading_ws = line.lstrip()
+                if line_no_leading_ws != "":
+                    have_content = True
+                    leading_ws = len(line) - len(line_no_leading_ws)
+                    if leading_ws < least_whitespace:
+                        least_whitespace = leading_ws
+            # Remove leading whitespace
+            if have_content:
+                for i,line in enumerate(text_lines):
+                    if len(line) >= least_whitespace:
+                        text_lines[i] = line[least_whitespace:]
+
+            method = "async def pytask(ctxt, input):\n" + "\n".join(["    %s" % l for l in text_lines])
+
+            exec(method)
+
+            callable = locals()['pytask']
+
+        result = await callable(ctxt, input)
+
+        if result is None:
+            result = TaskDataResult()
+
+        self._log.debug("<-- ExecCallable")
+        return result

dv_flow_mgr-0.0.2.14182043984a1/src/dv_flow/mgr/pkg_rgy.py → dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/ext_rgy.py

@@ -1,5 +1,5 @@
 #****************************************************************************
-#* pkg_rgy.py
+#* ext_rgy.py
 #*
 #* Copyright 2023-2025 Matthew Ballance and Contributors
 #*
@@ -19,18 +19,23 @@
 #*     Author: 
 #*
 #****************************************************************************
+import dataclasses as dc
 import os
 import logging
 import sys
-from typing import Dict, Tuple
-from .package_def import PackageDef
+from typing import Callable, ClassVar, Dict, Tuple
+from .exec_callable import ExecCallable
+from .pytask_callable import PytaskCallable
+from .shell_callable import ShellCallable
 
-class PkgRgy(object):
-    _inst = None
+@dc.dataclass
+class ExtRgy(object):
+    _inst : ClassVar = None
 
     def __init__(self):
         self._pkgpath = []
-        self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
+        self._pkg_m : Dict[str, str] = {}
+        self._shell_m : Dict[str, Callable] = {}
         self._log = logging.getLogger(type(self).__name__)
         self._override_m : Dict[str,str] = {}
 
@@ -47,21 +52,19 @@ class PkgRgy(object):
             return True
         else:
             return False
-    
-    def getPackage(self, name):
-        self._log.debug("--> getPackage(%s)" % name)
+        
+    def findShell(self, name) -> Callable:
+        if name in self._shell_m.keys():
+            return self._shell_m[name]
+        
+    def findPackagePath(self, name) -> str:
+        ret = None
+        self._log.debug("--> findPackagePath(%s)" % name)
         if name in self._pkg_m.keys():
-            if self._pkg_m[name][1] is None:
-                pkg_def = PackageDef.load(self._pkg_m[name][0])
-                # Load the package
-                self._pkg_m[name] = (
-                    self._pkg_m[name][0],
-                    pkg_def
-                )
-            ret = self._pkg_m[name][1]
+            ret = self._pkg_m[name]
         else:
             ret = self._findOnPath(name)
-        self._log.debug("<-- getPackage(%s)" % name)
+        self._log.debug("<-- findPackagePath(%s)" % name)
         return ret
         
     def _findOnPath(self, name):
@@ -72,33 +75,31 @@ class PkgRgy(object):
         else:
             name_pref = None
 
-        pkg = None
+        ret = None
 
         for path in self._pkgpath:
             if os.path.isfile(os.path.join(path, name_dir, "flow.dv")):
-                pkg = PackageDef.load(os.path.join(path, name_dir, "flow.dv"))
+                ret = os.path.join(path, name_dir, "flow.dv")
             elif name_pref is not None and os.path.isfile(os.path.join(path, name_pref, name_s[-1] + ".dv")):
-                pkg = PackageDef.load(os.path.join(path, name_pref, name_s[-1] + ".dv"))
+                ret = os.path.join(path, name_pref, name_s[-1] + ".dv")
             elif os.path.isfile(os.path.join(path, name + ".dv")):
-                pkg = PackageDef.load(os.path.join(path, name + ".dv"))
+                ret = os.path.join(path, name + ".dv")
 
-            if pkg is not None:
-                self._pkg_m[name] = (pkg.name, pkg)
+            if ret is not None:
+                self._pkg_m[name] = ret
                 break
 
-        return pkg
-
-    def registerPackage(self, pkg_def):
-        self._log.debug("--> registerPackage %s" % pkg_def.name)
-        if pkg_def.name in self._pkg_m.keys():
-            raise Exception("Duplicate package %s" % pkg_def.name)
-        self._pkg_m[pkg_def.name] = (pkg_def.basedir, pkg_def)
-        self._log.debug("<-- registerPackage %s" % pkg_def.name)
+        return ret
 
     def _discover_plugins(self):
         self._log.debug("--> discover_plugins")
         # Register built-in package
-        self._pkg_m["std"] = (os.path.join(os.path.dirname(__file__), "std/flow.dv"), None)
+        self._pkg_m["std"] = os.path.join(os.path.dirname(__file__), "std/flow.dv")
+
+        # Register built-in shells
+        self._shell_m["shell"] = ShellCallable
+        self._shell_m["pytask"] = ExecCallable
+
 
         if "DV_FLOW_PATH" in os.environ.keys() and os.environ["DV_FLOW_PATH"] != "":
             paths = os.environ["DV_FLOW_PATH"].split(':')
@@ -124,7 +125,15 @@ class PkgRgy(object):
                             self._log.debug("Package %s already registered using path %s. Conflicting path: %s" % (
                                 name, self._pkg_m[name][0], path))
                         else:
-                            self._pkg_m[name] = (path, None)
+                            self._pkg_m[name] = path
+                if hasattr(mod, "dvfm_shells"):
+                    shell_m = mod.dvfm_shells()
+                    for name, shell in shell_m.items():
+                        self._log.debug("Registering shell %s" % name)
+                        if name in self._shell_m.keys():
+                            self._log.debug("Shell %s already registered" % name)
+                        else:
+                            self._shell_m[name] = shell
             except Exception as e:
                 self._log.critical("Error loading plugin %s: %s" % (p.name, str(e)))
                 raise e
@@ -135,7 +144,7 @@ class PkgRgy(object):
         self._log.debug("<-- discover_plugins")
 
     def copy(self):
-        ret = PkgRgy()
+        ret = ExtRgy()
         ret._pkgpath = self._pkgpath.copy()
         ret._pkg_m = self._pkg_m.copy()
         return ret

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/extend_def.py

@@ -0,0 +1,21 @@
+
+import pydantic.dataclasses as pdc
+from pydantic import BaseModel
+from typing import List, Union
+from .param_def import ParamDef
+
+class ExtendDef(BaseModel):
+    """Extension definition"""
+    task : str = pdc.Field(
+        description="Name of the task to extend")
+    params : List[ParamDef] = pdc.Field(
+        default_factory=list,
+        description="Parameter extensions to apply to the task",
+        alias="with")
+    uses : str = pdc.Field(
+        default=None,
+        description="Name of the extension to use as a base")
+    needs: List[str] = pdc.Field(
+        default_factory=list,
+        description="List of tasks to depend on")
+

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/fragment_def.py

@@ -23,8 +23,8 @@ import pydantic.dataclasses as dc
 import json
 from pydantic import BaseModel
 from typing import Any, Dict, List, Union
-from .package import Package
 from .package_import_spec import PackageImportSpec
+from .srcinfo import SrcInfo
 from .task_def import TaskDef
 from .type_def import TypeDef
 
@@ -33,6 +33,7 @@ class FragmentDef(BaseModel):
     imports : List[Union[str,PackageImportSpec]] = dc.Field(default_factory=list, alias="imports")
     fragments: List[str] = dc.Field(default_factory=list)
     types : List[TypeDef] = dc.Field(default_factory=list)
+    srcinfo : SrcInfo = dc.Field(default=None)
 
     _basedir : str = None
 
@@ -41,6 +42,6 @@ class FragmentDef(BaseModel):
             if t.name == name:
                 return t
             
-    def apply(self, session, pkg : Package):
-        pass
+#    def apply(self, session, pkg : Package):
+#        pass
             

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/need_def.py

@@ -0,0 +1,6 @@
+import pydantic.dataclasses as pdc
+from pydantic import BaseModel
+
+class NeedDef(BaseModel):
+    task : str = pdc.Field(
+        description="Name of the task to depend on")

dv_flow_mgr-1.0.0.14370600369a1/src/dv_flow/mgr/null_callable.py

@@ -0,0 +1,10 @@
+import dataclasses as dc
+from .task_data import TaskDataResult
+
+@dc.dataclass
+class NullCallable(object):
+    body : str
+
+    async def __call__(self, ctxt, input):
+        """Null callable that does nothing"""
+        return TaskDataResult()

{dv_flow_mgr-0.0.2.14182043984a1 → dv_flow_mgr-1.0.0.14370600369a1}/src/dv_flow/mgr/package.py

@@ -21,25 +21,49 @@
 #****************************************************************************
 import dataclasses as dc
 import logging
-from typing import Any, ClassVar, Dict
-from .task_node_ctor import TaskNodeCtor
+from typing import Any, ClassVar, Dict, List
+from .fragment_def import FragmentDef
+from .package_def import PackageDef
+from .task import Task
 
 @dc.dataclass
 class Package(object):
-    name : str
+    pkg_def : PackageDef
+    basedir : str = None
     params : Dict[str,Any] = dc.field(default_factory=dict)
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
-    tasks : Dict[str,TaskNodeCtor] = dc.field(default_factory=dict)
+    task_m : Dict[str,Task] = dc.field(default_factory=dict)
     types : Dict[str,Any] = dc.field(default_factory=dict)
+    fragment_def_l : List[FragmentDef] = dc.field(default_factory=list)
+    pkg_m : Dict[str, 'Package'] = dc.field(default_factory=dict)
     _log : ClassVar = logging.getLogger("Package")
 
-    def getTaskCtor(self, name : str) -> TaskNodeCtor:
+    @property
+    def name(self):
+        return self.pkg_def.name
+
+    def getTaskCtor(self, name : str) -> Task:
         self._log.debug("-- %s::getTaskCtor: %s" % (self.name, name))
         if name not in self.tasks.keys():
             raise Exception("Task %s not present in package %s" % (name, self.name))
         return self.tasks[name]
+    
+    def dump(self):
+        tasks = {}
+        for k, v in self.task_m.items():
+            tasks[k] = v.dump()
+
+        pkg = {
+            "name": self.name,
+            "basedir": self.basedir,
+            "params": self.params,
+            "tasks": tasks,
+            "fragments": [f.dump() for f in self.fragment_def_l]
+        }
+
+        return pkg
             
     def __hash__(self):
-        return hash(self.fullname())
+        return id(self)