dv-flow-mgr 0.0.1.13824498694a1__tar.gz → 0.0.1.13849962851a1__tar.gz

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13824498694a1
+Version: 0.0.1.13849962851a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.13824498694a1"
+version = "0.0.1.13849962851a1"
 dependencies = [
     'jq',
     'pydantic',

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/__init__.py

@@ -4,8 +4,6 @@ from .pkg_rgy import PkgRgy
 from .task_graph_runner import *
 from .task import *
 from .task_data import *
-from .task_graph_runner import TaskGraphRunner
-from .task_graph_runner_local import TaskGraphRunnerLocal
 from .task_graph_builder import TaskGraphBuilder
 from .task_node import task
 from .task_runner import TaskSetRunner

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/pkg_rgy.py

@@ -32,6 +32,13 @@ class PkgRgy(object):
         self._pkgpath = []
         self._pkg_m : Dict[str, Tuple[str,PackageDef]] = {}
         self._log = logging.getLogger(type(self).__name__)
+        self._override_m : Dict[str,str] = {}
+
+    def addOverride(self, key, value):
+        self._override_m[key] = value
+
+    def getOverrides(self):
+        return self._override_m
 
     def hasPackage(self, name, search_path=True):
         if name in self._pkg_m.keys():

dv_flow_mgr-0.0.1.13849962851a1/src/dv_flow/mgr/std/create_file.py

@@ -0,0 +1,61 @@
+import os
+import hashlib
+import logging
+import pydantic.dataclasses as dc
+from pydantic import BaseModel
+from typing import ClassVar, List, Tuple
+from dv_flow.mgr import TaskDataResult
+from dv_flow.mgr import FileSet as _FileSet
+
+class TaskCreateFileMemento(BaseModel):
+    name : str = ""
+    hash : str = ""
+
+_log = logging.getLogger("CreateFile")
+
+async def CreateFile(runner, input) -> TaskDataResult:
+    _log.debug("CreateFile run: %s: rundir=%s, type=%s filename=%s content=%s" % (
+        input.name,
+        input.rundir,
+        input.params.type, input.params.filename, input.params.content))
+
+    # 
+    try:
+        ex_memento = TaskCreateFileMemento(**input.memento) if input.memento is not None else None
+    except Exception as e:
+        _log.error("Failed to load memento: %s" % str(e))
+        ex_memento = None 
+    memento = TaskCreateFileMemento()
+
+    _log.debug("ex_memento: %s" % str(ex_memento))
+    _log.debug("params: %s" % str(input.params))
+
+    changed = (input.changed or ex_memento is None)
+
+    if not changed and ex_memento is not None:
+        if ex_memento.name != input.params.filename:
+            changed = True
+        else:
+            new_hash = hashlib.md5(input.params.content.encode()).hexdigest()
+            changed = ex_memento.hash != new_hash
+    
+    if changed:
+        filename = os.path.join(input.rundir, input.params.filename)
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        with open(filename, "w") as fp:
+            fp.write(input.params.content)
+
+    fs = _FileSet(
+                filetype=input.params.type,
+                src=input.name, 
+                basedir=input.rundir,
+                files=[input.params.filename])
+
+    _log.debug("<-- FileSet(%s) changed=%s" % (input.name, changed))
+
+    return TaskDataResult(
+        memento=memento,
+        changed=changed,
+        output=[fs]
+    )

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/std/flow.dv

@@ -29,6 +29,22 @@ package:
       exclude:
         type: str
         value: ""
+  - name: CreateFile
+    pytask: dv_flow.mgr.std.create_file.CreateFile
+    passthrough: true
+    doc: |
+      Creates one or more files in the run directory from 
+      literal content in the .dv file. Outputs a fileset 
+      referencing all the created files.
+    with:
+      type:
+        desc: Content-type to use for the fileset
+        type: str
+        value: ""
+      filename:
+        type: str
+      content:
+        type: str
   - name: Exec
     pytask: dv_flow.mgr.std.exec.Exec
     with: 

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/task_graph_builder.py

@@ -39,16 +39,20 @@ class TaskGraphBuilder(object):
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
     _task_m : Dict['TaskSpec',Task] = dc.field(default_factory=dict)
+    _override_m : Dict[str,str] = dc.field(default_factory=dict)
     _logger : logging.Logger = None
 
     def __post_init__(self):
         if self.pkg_rgy is None:
             self.pkg_rgy = PkgRgy.inst().copy()
 
+        # Initialize the overrides from the global registry
+        self._override_m.update(self.pkg_rgy.getOverrides())
+
         self._logger = logging.getLogger(type(self).__name__)
-        self._logger.debug("TaskGraphBuilder: root_pkg: %s" % str(self.root_pkg))
 
         if self.root_pkg is not None:
+            self._logger.debug("TaskGraphBuilder: root_pkg: %s" % str(self.root_pkg))
 
             # Register package definitions found during loading
             visited = set()
@@ -61,6 +65,14 @@ class TaskGraphBuilder(object):
             # Allows us to find ourselves
             self._pkg_m[PackageSpec(self.root_pkg.name)] = pkg
 
+    def loadPkg(self, pkgfile : str):
+        pkg = PackageDef.load(pkgfile)
+        visited = set()
+        self._registerPackages(pkg, visited)
+
+    def addOverride(self, key : str, val : str):
+        self._override_m[key] = val
+
     def _registerPackages(self, pkg : PackageDef, visited):
         self._logger.debug("Packages: %s" % str(pkg))
         if pkg.name not in visited:
@@ -210,6 +222,21 @@ class TaskGraphBuilder(object):
     
     def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
         self._logger.debug("--> mkTaskNode: %s" % task_t)
+
+        if task_t in self._override_m.keys():
+            self._logger.debug("Overriding task %s with %s" % (task_t, self._override_m[task_t]))
+            task_t = self._override_m[task_t]
+        else:
+            dot_idx = task_t.rfind(".")
+            if dot_idx != -1:
+                pkg = task_t[0:dot_idx]
+                tname = task_t[dot_idx+1:]
+
+                if pkg in self._override_m.keys():
+                    self._logger.debug("Overriding package %s with %s" % (pkg, self._override_m[pkg]))
+                    task_t = self._override_m[pkg] + "." + tname
+
+
         ctor = self.getTaskCtor(task_t)
         self._logger.debug("ctor: %s" % ctor.name)
         params = ctor.mkTaskParams(kwargs)

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/task_node.py

@@ -187,6 +187,30 @@ class TaskNode(object):
 
     def __hash__(self):
         return id(self)
+
+    def _matches(self, params, consumes):
+        """Determines if a parameter set matches a set of consumed parameters"""
+        self._log.debug("--> _matches: %s params=%s consumes=%s" % (
+            self.name, str(params), str(consumes)))
+        consumed = False
+        for c in consumes:
+            # All matching attribute keys must have same value
+            match = False
+            for k,v in c.items():
+                self._log.debug("k,v: %s,%s - hasattr=%s" % (k,v, hasattr(params, k)))
+                if hasattr(params, k):
+                    self._log.debug("getattr=%s v=%s" % (getattr(params, k), v))
+                    if getattr(params, k) == v:
+                        match = True
+                    else:
+                        match = False
+                        break
+            if match:
+                consumed = True
+                break
+        self._log.debug("<-- _matches: %s %s" % (self.name, consumed))
+        return consumed
+
     
 
 @dc.dataclass
@@ -353,24 +377,6 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
                     setattr(obj, key, value)
         return obj
     
-    def _matches(self, params, consumes):
-        """Determines if a parameter set matches a set of consumed parameters"""
-        consumed = False
-        for c in consumes:
-            # All matching attribute keys must have same value
-            match = False
-            for k,v in c.items():
-                if hasattr(params, k):
-                    if getattr(params, k) != v:
-                        match = True
-                    else:
-                        match = False
-                        break
-            if match:
-                consumed = True
-                break
-        return consumed
-
 def task(paramT,passthrough=False,consumes=None):
     """Decorator to wrap a task method as a TaskNodeCtor"""
     def wrapper(T):

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow/mgr/task_runner.py

@@ -38,13 +38,17 @@ class TaskRunner(object):
 
 @dc.dataclass
 class TaskSetRunner(TaskRunner):
-    nproc : int = 8
+    nproc : int = -1
     status : int = 0
 
     _anon_tid : int = 1
 
     _log : ClassVar = logging.getLogger("TaskSetRunner")
 
+    def __post_init__(self):
+        if self.nproc == -1:
+            self.nproc = os.cpu_count()
+
     async def run(self, task : Union[TaskNode,List[TaskNode]]):
         # Ensure that the rundir exists or can be created
 

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.13824498694a1
+Version: 0.0.1.13849962851a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/src/dv_flow_mgr.egg-info/SOURCES.txt

@@ -57,6 +57,7 @@ src/dv_flow/mgr/type_def.py
 src/dv_flow/mgr/util.py
 src/dv_flow/mgr/cmds/cmd_run.py
 src/dv_flow/mgr/share/flow.json
+src/dv_flow/mgr/std/create_file.py
 src/dv_flow/mgr/std/exec.py
 src/dv_flow/mgr/std/fileset.py
 src/dv_flow/mgr/std/flow.dv

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/tests/unit/test_imports.py

@@ -8,7 +8,7 @@ import pytest
 from typing import Any, List, Union
 import yaml
 from dv_flow.mgr import PackageDef, TaskGraphBuilder, TaskSetRunner, task, TaskDataResult
-from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
+from dv_flow.mgr import TaskGraphBuilder, PackageDef
 from dv_flow.mgr.task_runner import TaskSetRunner
 from dv_flow.mgr.task_listener_log import TaskListenerLog
 from dv_flow.mgr.fileset import FileSet

{dv_flow_mgr-0.0.1.13824498694a1 → dv_flow_mgr-0.0.1.13849962851a1}/tests/unit/test_pyclass.py

@@ -2,7 +2,7 @@
 import os
 import asyncio
 import pytest
-from dv_flow.mgr import TaskGraphBuilder, TaskGraphRunnerLocal, PackageDef
+from dv_flow.mgr import TaskGraphBuilder, PackageDef
 from dv_flow.mgr.task_runner import TaskSetRunner
 from dv_flow.mgr.task_listener_log import TaskListenerLog
 #from dv_flow_mgr.tasklib.builtin_pkg import TaskPyClass, TaskPyClassParams