dv-flow-mgr 0.0.1.12971126211a1__tar.gz → 0.0.1.12992722023a1__tar.gz

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12971126211a1
+Version: 0.0.1.12992722023a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/docs/Roadmap.md

@@ -53,9 +53,10 @@ are evaluated.
 
 ## 1.0.0
 - Std
-  - Null (combine dependencies, set variables). Implements tasks that do not specify 'uses'
+  * Null (combine dependencies, set variables). Implements tasks that do not specify 'uses'
   - Exec
-  - FileSet
+  - Make
+  * FileSet
   - PyClass - implement a task as a Python class (load from a module)
 
 - HdlSim
@@ -112,3 +113,14 @@ are evaluated.
 - Extensible -- by users and organizations
 - Implementation independent (not tied to a specific implementation language)
 
+
+# TODO
+- Need some way to signal rough core consumption. Challenge is to complete
+  build as quickly as possible via coarse and fine-grained parallelism
+  - Parallelism tends to follow exponential curve. High parallelism early ; Low later
+  - Reasoning about curves may guide resource over-subscription
+- Each task needs memory hints / requirements as well
+- Dependency strategy for Exec/Make
+  - Specify dependency file created by task
+  - When the timestamp is updated, task is known to have been rebuilt
+- 

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dv-flow-mgr"
-version = "0.0.1.12971126211a1"
+version = "0.0.1.12992722023a1"
 dependencies = [
     'pydantic',
     'pyyaml',

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/src/dv_flow/mgr/package_def.py

@@ -119,6 +119,7 @@ class PackageDef(BaseModel):
         return ctor_t
 
     def handleParams(self, task, ctor_t):
+        self._log.debug("--> handleParams %s params=%s" % (task.name, str(task.params)))
 
         if task.params is not None and len(task.params) > 0:
             decl_params = False
@@ -131,18 +132,21 @@ class PackageDef(BaseModel):
 #            ctor_t.params.update(task.params)
 
             for value in task.params.values():
-                if "type" in value:
+                self._log.debug("value: %s" % str(value))
+                if type(value) == dict and "type" in value.keys():
                     decl_params = True
                     break
 
+            field_m = {}
+            # First, add parameters from the base class
+            base_o = ctor_t.mkParams()
+            for fname,info in base_o.model_fields.items():
+                self._log.debug("Field: %s (%s)" % (fname, info.default))
+                field_m[fname] = (info.annotation, info.default)
+
             if decl_params:
+                self._log.debug("Type declares new parameters")
                 # We need to combine base parameters with new parameters
-                field_m = {}
-                # First, add parameters from the base class
-                base_o = ctor_t.mkParams()
-                for fname,info in base_o.model_fields.items():
-                    self._log.debug("Field: %s (%s)" % (fname, info.default))
-                    field_m[fname] = (info.annotation, info.default)
                 ptype_m = {
                     "str" : str,
                     "int" : int,
@@ -159,6 +163,7 @@ class PackageDef(BaseModel):
                 }
                 for p in task.params.keys():
                     param = task.params[p]
+                    self._log.debug("param: %s" % str(param))
                     if type(param) == dict and "type" in param.keys():
                         ptype_s = param["type"]
                         if ptype_s not in ptype_m.keys():
@@ -190,6 +195,7 @@ class PackageDef(BaseModel):
                     uses=ctor_t,
                     params_ctor=param_t)
             else: # no new parameters declared
+                self._log.debug("Type only overrides existing parameters")
                 for p in task.params.keys():
                     param = task.params[p]
                     if p not in field_m.keys():
@@ -202,12 +208,15 @@ class PackageDef(BaseModel):
                         raise Exception("No value specified for param %s: %s" % (
                             p, str(param)))
                     field_m[p] = (field_m[p][0], value)
-                ctor_t.params[p] = value
+                    self._log.debug("Set param=%s to %s" % (p, str(value)))
+                    ctor_t.params[p] = value
+
+        self._log.debug("<-- handleParams %s" % task.name)
 
         return ctor_t
 
     def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
-        self._log.debug("--> %s::mkTaskCtor %s" % (self.name, task.name))
+        self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
         ctor_t : TaskCtor = None
 
         # Determine the implementation constructor first
@@ -260,6 +269,7 @@ class PackageDef(BaseModel):
                 srcdir=srcdir)
 
         ctor_t = self.handleParams(task, ctor_t)
+        ctor_t.depends.extend(task.depends)
 
         self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
         return ctor_t

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/src/dv_flow/mgr/std/fileset.py

@@ -26,6 +26,9 @@ class FileSet(Task):
         ex_memento = self.getMemento(TaskFileSetMemento)
         memento = TaskFileSetMemento()
 
+        self._log.debug("ex_memento: %s" % str(ex_memento))
+        self._log.debug("params: %s" % str(self.params))
+
         if self.params is not None:
             glob_root = os.path.join(self.srcdir, self.params.base)
             glob_root = glob_root.strip()
@@ -33,6 +36,8 @@ class FileSet(Task):
             if glob_root[-1] == '/' or glob_root == '\\':
                 glob_root = glob_root[:-1]
 
+            self._log.debug("glob_root: %s" % glob_root)
+
             fs = _FileSet(
                 src=self.name, 
                 type=self.params.type,
@@ -45,6 +50,8 @@ class FileSet(Task):
             for pattern in self.params.include:
                 included_files.extend(glob.glob(os.path.join(glob_root, pattern), recursive=False))
 
+            self._log.debug("included_files: %s" % str(included_files))
+
             for file in included_files:
                 if not any(glob.fnmatch.fnmatch(file, os.path.join(glob_root, pattern)) for pattern in self.params.exclude):
                     memento.files.append((file, os.path.getmtime(os.path.join(glob_root, file))))

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/src/dv_flow/mgr/task.py

@@ -123,7 +123,7 @@ class TaskCtorCls(TaskCtor):
     _log : ClassVar = logging.getLogger("TaskCtorCls")
 
     def mkTask(self, name : str, depends, rundir, srcdir=None, params=None):
-        self._log.debug("--> %s::mkTask (%s)" % (self.name, str(self.task_ctor)))
+        self._log.debug("--> %s::mkTask (%s) srcdir=%s" % (self.name, str(self.task_ctor), srcdir))
 
         if srcdir is None:
             srcdir = self.srcdir
@@ -137,7 +137,6 @@ class TaskCtorCls(TaskCtor):
             rundir=rundir, 
             srcdir=srcdir, 
             params=params)
-        ret.srcdir = self.srcdir
 
         # Update parameters on the way back
         self.applyParams(ret.params)

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/src/dv_flow_mgr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dv-flow-mgr
-Version: 0.0.1.12971126211a1
+Version: 0.0.1.12992722023a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/tests/system/test_depends.py

@@ -12,13 +12,13 @@ package:
 
   tasks:
   - name: files
-    uses: std.FileSet
+    uses: std.Message
     with:
-      type: textFiles
-      include: "*.txt"
+      msg: "Running Files"
 
   - name: print
     uses: std.Message
+    needs: [files]
     with:
       msg: "Running Print"
 """
@@ -44,6 +44,9 @@ package:
 
     output = output.decode()
 
+    print("output: %s" % output)
+
+    assert output.find("Running Files") != -1
     assert output.find("Running Print") != -1
 
 def test_seq_2(tmpdir):

{dv_flow_mgr-0.0.1.12971126211a1 → dv_flow_mgr-0.0.1.12992722023a1}/tests/unit/test_fileset.py

@@ -39,7 +39,7 @@ def test_fileset_1(tmpdir):
     out = asyncio.run(runner.run(task))
     assert out.changed == False
 
-    # Now, add a files
+    # Now, add a file
     with open(os.path.join(tmpdir, "test1", "files1", "file1_3.sv"), "w") as f:
         f.write("// file1_3.sv\n")