dv-flow-mgr 0.0.1.13824531439a1__py3-none-any.whl → 0.0.1.13862522550a1__py3-none-any.whl

dv_flow/mgr/pkg_rgy.py

@@ -32,6 +32,13 @@ class PkgRgy(object):
         self._pkgpath = []
         self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
         self._log = logging.getLogger(type(self).__name__)
+        self._override_m : Dict[str,str] = {}
+
+    def addOverride(self, key, value):
+        self._override_m[key] = value
+
+    def getOverrides(self):
+        return self._override_m
 
     def hasPackage(self, name, search_path=True):
         if name in self._pkg_m.keys():

dv_flow/mgr/std/create_file.py

@@ -0,0 +1,61 @@
+import os
+import hashlib
+import logging
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import ClassVar, List, Tuple
+from dv_flow.mgr import TaskDataResult
+from dv_flow.mgr import FileSet as _FileSet
+
+class TaskCreateFileMemento(BaseModel):
+    name : str = ""
+    hash : str = ""
+
+_log = logging.getLogger("CreateFile")
+
+async def CreateFile(runner, input) -> TaskDataResult:
+    _log.debug("CreateFile run: %s: rundir=%s, type=%s filename=%s content=%s" % (
+        input.name,
+        input.rundir,
+        input.params.type, input.params.filename, input.params.content))
+
+    # 
+    try:
+        ex_memento = TaskCreateFileMemento(**input.memento) if input.memento is not None else None
+    except Exception as e:
+        _log.error("Failed to load memento: %s" % str(e))
+        ex_memento = None 
+    memento = TaskCreateFileMemento()
+
+    _log.debug("ex_memento: %s" % str(ex_memento))
+    _log.debug("params: %s" % str(input.params))
+
+    changed = (input.changed or ex_memento is None)
+
+    if not changed and ex_memento is not None:
+        if ex_memento.name != input.params.filename:
+            changed = True
+        else:
+            new_hash = hashlib.md5(input.params.content.encode()).hexdigest()
+            changed = ex_memento.hash != new_hash
+    
+    if changed:
+        filename = os.path.join(input.rundir, input.params.filename)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        with open(filename, "w") as fp:
+            fp.write(input.params.content)
+
+    fs = _FileSet(
+                filetype=input.params.type,
+                src=input.name, 
+                basedir=input.rundir,
+                files=[input.params.filename])
+
+    _log.debug("<-- FileSet(%s) changed=%s" % (input.name, changed))
+
+    return TaskDataResult(
+        memento=memento,
+        changed=changed,
+        output=[fs]
+    )

dv_flow/mgr/std/flow.dv

@@ -29,6 +29,22 @@ package:
       exclude:
         type: str
         value: ""
+  - name: CreateFile
+    pytask: dv_flow.mgr.std.create_file.CreateFile
+    passthrough: true
+    doc: |
+      Creates one or more files in the run directory from 
+      literal content in the .dv file. Outputs a fileset 
+      referencing all the created files.
+    with:
+      type:
+        desc: Content-type to use for the fileset
+        type: str
+        value: ""
+      filename:
+        type: str
+      content:
+        type: str
   - name: Exec
     pytask: dv_flow.mgr.std.exec.Exec
     with: 

dv_flow/mgr/task_graph_builder.py

@@ -39,16 +39,20 @@ class TaskGraphBuilder(object):
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
     _task_m : Dict['TaskSpec',Task] = dc.field(default_factory=dict)
+    _override_m : Dict[str,str] = dc.field(default_factory=dict)
     _logger : logging.Logger = None
 
     def __post_init__(self):
         if self.pkg_rgy is None:
             self.pkg_rgy = PkgRgy.inst().copy()
 
+        # Initialize the overrides from the global registry
+        self._override_m.update(self.pkg_rgy.getOverrides())
+
         self._logger = logging.getLogger(type(self).__name__)
-        self._logger.debug("TaskGraphBuilder: root_pkg: %s" % str(self.root_pkg))
 
         if self.root_pkg is not None:
+            self._logger.debug("TaskGraphBuilder: root_pkg: %s" % str(self.root_pkg))
 
             # Register package definitions found during loading
             visited = set()
@@ -61,6 +65,14 @@ class TaskGraphBuilder(object):
             # Allows us to find ourselves
             self._pkg_m[PackageSpec(self.root_pkg.name)] = pkg
 
+    def loadPkg(self, pkgfile : str):
+        pkg = PackageDef.load(pkgfile)
+        visited = set()
+        self._registerPackages(pkg, visited)
+
+    def addOverride(self, key : str, val : str):
+        self._override_m[key] = val
+
     def _registerPackages(self, pkg : PackageDef, visited):
         self._logger.debug("Packages: %s" % str(pkg))
         if pkg.name not in visited:
@@ -210,6 +222,21 @@ class TaskGraphBuilder(object):
     
     def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
         self._logger.debug("--> mkTaskNode: %s" % task_t)
+
+        if task_t in self._override_m.keys():
+            self._logger.debug("Overriding task %s with %s" % (task_t, self._override_m[task_t]))
+            task_t = self._override_m[task_t]
+        else:
+            dot_idx = task_t.rfind(".")
+            if dot_idx != -1:
+                pkg = task_t[0:dot_idx]
+                tname = task_t[dot_idx+1:]
+
+                if pkg in self._override_m.keys():
+                    self._logger.debug("Overriding package %s with %s" % (pkg, self._override_m[pkg]))
+                    task_t = self._override_m[pkg] + "." + tname
+
+
         ctor = self.getTaskCtor(task_t)
         self._logger.debug("ctor: %s" % ctor.name)
         params = ctor.mkTaskParams(kwargs)

dv_flow/mgr/task_node.py

@@ -58,8 +58,8 @@ class TaskNode(object):
                 if subdep not in dep_m.keys():
                     dep_m[subdep] = []
                 dep_m[subdep].extend(need.output.dep_m[subdep])
+        self._log.debug("input dep_m: %s %s" % (self.name, str(dep_m)))
 
-        self._log.debug("input dep_m: %s" % str(dep_m))
         sorted = toposort.toposort(dep_m)
 
         in_params_m = {}
@@ -127,10 +127,12 @@ class TaskNode(object):
 
         self._log.debug("output[1]: %s" % str(output))
 
+        # Pass-through all dependencies
+        # Add an entry for ourselves
+        dep_m[self.name] = list(need.name for need in self.needs)
+
         if self.passthrough:
             self._log.debug("passthrough: %s" % self.name)
-            # Add an entry for ourselves
-            dep_m[self.name] = list(need.name for need in self.needs)
 
             if self.consumes is None and len(self.consumes):
                 self._log.debug("Propagating all input parameters to output")
@@ -167,11 +169,11 @@ class TaskNode(object):
         else:
             self._log.debug("non-passthrough: %s (only local outputs propagated)" % self.name)
             # empty dependency map
-            dep_m = {
-                self.name : []
-            }
+#            dep_m = {
+#                self.name : []
+#            }
 
-        self._log.debug("output dep_m: %s" % str(dep_m))
+        self._log.debug("output dep_m: %s %s" % (self.name, str(dep_m)))
         self._log.debug("output[2]: %s" % str(output))
 
         # Store the result
@@ -187,6 +189,30 @@ class TaskNode(object):
 
     def __hash__(self):
         return id(self)
+
+    def _matches(self, params, consumes):
+        """Determines if a parameter set matches a set of consumed parameters"""
+        self._log.debug("--> _matches: %s params=%s consumes=%s" % (
+            self.name, str(params), str(consumes)))
+        consumed = False
+        for c in consumes:
+            # All matching attribute keys must have same value
+            match = False
+            for k,v in c.items():
+                self._log.debug("k,v: %s,%s - hasattr=%s" % (k,v, hasattr(params, k)))
+                if hasattr(params, k):
+                    self._log.debug("getattr=%s v=%s" % (getattr(params, k), v))
+                    if getattr(params, k) == v:
+                        match = True
+                    else:
+                        match = False
+                        break
+            if match:
+                consumed = True
+                break
+        self._log.debug("<-- _matches: %s %s" % (self.name, consumed))
+        return consumed
+
     
 
 @dc.dataclass
@@ -353,24 +379,6 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                     setattr(obj, key, value)
         return obj
     
-    def _matches(self, params, consumes):
-        """Determines if a parameter set matches a set of consumed parameters"""
-        consumed = False
-        for c in consumes:
-            # All matching attribute keys must have same value
-            match = False
-            for k,v in c.items():
-                if hasattr(params, k):
-                    if getattr(params, k) != v:
-                        match = True
-                    else:
-                        match = False
-                        break
-            if match:
-                consumed = True
-                break
-        return consumed
-
 def task(paramT,passthrough=False,consumes=None):
     """Decorator to wrap a task method as a TaskNodeCtor"""
     def wrapper(T):

dv_flow/mgr/task_runner.py

@@ -38,13 +38,17 @@ class TaskRunner(object):
 
 @dc.dataclass
 class TaskSetRunner(TaskRunner):
-    nproc : int = 8
+    nproc : int = -1
     status : int = 0
 
     _anon_tid : int = 1
 
     _log : ClassVar = logging.getLogger("TaskSetRunner")
 
+    def __post_init__(self):
+        if self.nproc == -1:
+            self.nproc = os.cpu_count()
+
     async def run(self, task : Union[TaskNode,List[TaskNode]]):
         # Ensure that the rundir exists or can be created
 

{dv_flow_mgr-0.0.1.13824531439a1.dist-info → dv_flow_mgr-0.0.1.13862522550a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13824531439a1
+Version: 0.0.1.13862522550a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13824531439a1.dist-info → dv_flow_mgr-0.0.1.13862522550a1.dist-info}/RECORD

@@ -13,35 +13,36 @@ dv_flow/mgr/param.py,sha256=3BY-ucig6JRw73FhjyJQL-vpd57qhAzixgZ8I5FoUpw,553
 dv_flow/mgr/param_def.py,sha256=gLua-EQiY8V2CFX-2svLRIlrs8PEeGh4-EPtn4a2Mng,712
 dv_flow/mgr/param_ref_eval.py,sha256=U8QhDf1n_9bLnExdc1oQamq5-pOUXrFalOX4oyR9UoM,1138
 dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
-dv_flow/mgr/pkg_rgy.py,sha256=46XMWint3JpP2nx-OYTpzpHZgOkrvTseUU4Y87KusC8,5147
+dv_flow/mgr/pkg_rgy.py,sha256=UGQvBYUbAiabiBk8lKxknpk-yAOVfgeQzTB180iG9h4,5332
 dv_flow/mgr/task.py,sha256=kLQSvnVwj9ROIDtxq8lLu-4mJizTxOqvUeogmgN6QAA,5976
 dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
 dv_flow/mgr/task_data.py,sha256=m3W7k-xv09ZyhpP2gg33CLOhBLyQDJS2mlG35yOZjZQ,11432
 dv_flow/mgr/task_def.py,sha256=Wmh78mQLAiHXlpD2KCvFxTWtnjjwgct8fEm8gelCg1c,1987
 dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
-dv_flow/mgr/task_graph_builder.py,sha256=AOltPyzd2Q9B67m0MyL59RPu3i2d5QrX3K-0gKWLoY8,9097
+dv_flow/mgr/task_graph_builder.py,sha256=a9g_2WYlxqD_3R9wa7QDKxA5lqQozGskb8afg8ArP1Y,10101
 dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
 dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
 dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
 dv_flow/mgr/task_listener_log.py,sha256=d4GQcT3n_0Dq-XWQP-WNJIhuxvw948RbFPSiJ6X1_eI,2683
 dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
-dv_flow/mgr/task_node.py,sha256=oj9vZ9XxuqavBXY8uXKY_jE6y8jPoIhN1qC9b188SbY,13799
+dv_flow/mgr/task_node.py,sha256=I1wbq-EpuZZ94R42rtBSQRnk-2Yf6ZQhctsxjn2Vbus,14228
 dv_flow/mgr/task_output.py,sha256=dkJhhNRFGFQSkVsw_bGirK0_0ghTxBYq1LjRMZCVWnA,245
 dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
-dv_flow/mgr/task_runner.py,sha256=Z081AjdUQ6kK8bPgKU8m8rFRHbX8ce4lOI9kLQ5WF_w,7822
+dv_flow/mgr/task_runner.py,sha256=NFGAAlOKMeXkWQtSX3YcZ6RMPn5xlZ92R6q57xPpueg,7922
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=KdhuNlfw-NKU-4VZFCnMPyj775yEB7cpr5tz73a9yuQ,259
 dv_flow/mgr/util.py,sha256=06eVyURF4ga-s8C9Sd3ZSDebwO4QS0XXaB8xADVbWRc,1437
 dv_flow/mgr/cmds/cmd_run.py,sha256=w1KdnS5Skg4RH1xTB1RZP7gAElZMNcRuOhz3G98n9uk,2467
 dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
+dv_flow/mgr/std/create_file.py,sha256=gPNasXwjg3NSIc2QhMp-UlBW1r5dRDLt6pluFTJhfek,1886
 dv_flow/mgr/std/exec.py,sha256=ETx9xSxhdCD_iw6pcmhrafDCJ-41AneyEAPwQf3q-3w,452
 dv_flow/mgr/std/fileset.py,sha256=nW5Cg4IyCs-ccKKsIxA5W5aZvaluFk6LXDyFh9Jixcs,3076
-dv_flow/mgr/std/flow.dv,sha256=DpDTrRBfbIt4cJKLHqk_l6_nPkeFDKLVa5Ic4FZodNM,1587
+dv_flow/mgr/std/flow.dv,sha256=CcQOvSfT0JgsaNf7dH8gfbG8lPvQQXw9W9gqrT52Pw0,2025
 dv_flow/mgr/std/message.py,sha256=T8Ye44784rD6CT2qIAP5SD7sKPdAlGMFimc4IaPCxxE,200
 dv_flow/mgr/std/task_null.py,sha256=UKwUnqwFPBY8BO44ZAPcgehQB59kHZFa1qyZc1TwUqE,196
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/METADATA,sha256=K7yb-cW4SPXTT8QSoTTmPugmfmO7Ck5lX1SovLT_oMk,13314
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.13824531439a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/METADATA,sha256=B8h8L4Nr9ywSDxAGO4ElhZg4cIHndSyJCmNRMJGP4d4,13314
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.13862522550a1.dist-info/RECORD,,