dv-flow-mgr 0.0.1.14110240305a1__py3-none-any.whl → 0.0.1.14116344455a1__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -22,6 +22,7 @@
 from .package_def import *
 from .pkg_rgy import PkgRgy
 from .task_data import *
+from .task_def import *
 from .task_graph_builder import TaskGraphBuilder
 from .task_runner import TaskRunner
 from .task_node_ctor_wrapper import task

dv_flow/mgr/package_def.py

@@ -33,7 +33,7 @@ from .fragment_def import FragmentDef
 from .package import Package
 from .package_import_spec import PackageImportSpec, PackageSpec
 from .param_def import ParamDef
-from .task_def import TaskDef
+from .task_def import TaskDef, PassthroughE, ConsumesE
 from .task_node_ctor import TaskNodeCtor
 from .task_node_ctor_proxy import TaskNodeCtorProxy
 from .task_node_ctor_task import TaskNodeCtorTask
@@ -183,7 +183,7 @@ class PackageDef(BaseModel):
         base_params : BaseModel = None
         callable = None
         passthrough = task.passthrough
-        consumes = [] if task.consumes is None else task.consumes.copy()
+        consumes = task.consumes.copy() if isinstance(task.consumes, list) else task.consumes
         needs = [] if task.needs is None else task.needs.copy()
         fullname = self.name + "." + task.name
 
@@ -192,15 +192,21 @@ class PackageDef(BaseModel):
             base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
             base_params = base_ctor_t.mkTaskParams()
 
-            # Once we have passthrough, we can't turn it off
-            passthrough |= base_ctor_t.passthrough
-            consumes.extend(base_ctor_t.consumes)
+            if passthrough is None:
+                passthrough = base_ctor_t.passthrough
+            if consumes is None:
+                consumes = base_ctor_t.consumes
 
             if base_ctor_t is None:
                 self._log.error("Failed to load task ctor %s" % task.uses)
         else:
             self._log.debug("No 'uses' specified")
 
+        if passthrough is None:
+            passthrough = PassthroughE.No
+        if consumes is None:
+            consumes = ConsumesE.All
+
         # Determine the implementation constructor first
         if task.pytask is not None:
             # Built-in impl

dv_flow/mgr/std/flow.dv

@@ -37,7 +37,8 @@ package:
     doc: |
       Creates a fileset from a list of files or glob patterns
     pytask: dv_flow.mgr.std.fileset.FileSet
-    passthrough: true
+    passthrough: all
+    consumes: none
     with:
       base:
         doc: |
@@ -61,7 +62,8 @@ package:
         value: ""
   - name: CreateFile
     pytask: dv_flow.mgr.std.create_file.CreateFile
-    passthrough: true
+    passthrough: all
+    consumes: none
     doc: |
       Creates one or more files in the run directory from 
       literal content in the .dv file. Outputs a fileset 

dv_flow/mgr/task_def.py

@@ -39,8 +39,12 @@ class RundirE(enum.Enum):
     Unique = "unique"
     Inherit = "inherit"
 
+class ConsumesE(enum.Enum):
+    No = "none"
+    All = "all"
+
 class PassthroughE(enum.Enum):
-    No = "no"
+    No = "none"
     All = "all"
     Unused = "unused"
 
@@ -89,11 +93,11 @@ class TaskDef(BaseModel):
     rundir : RundirE = dc.Field(
         default=RundirE.Unique,
         description="Specifies handling of this tasks's run directory")
-    passthrough: Union[bool, PassthroughE] = dc.Field(
-        default=PassthroughE.No,
+    passthrough: Union[PassthroughE, List[Any], None] = dc.Field(
+        default=None,
         description="Specifies whether this task should pass its inputs to its output")
-    consumes : List[Any] = dc.Field(
-        default_factory=list,
+    consumes : Union[ConsumesE, List[Any], None] = dc.Field(
+        default=None,
         description="Specifies matching patterns for parameter sets that this task consumes")
 
 #    out: List[TaskOutput] = dc.Field(default_factory=list)

dv_flow/mgr/task_node.py

@@ -86,7 +86,7 @@ class TaskNode(object):
             for k,v in c.items():
                 self._log.debug("k,v: %s,%s - hasattr=%s" % (k,v, hasattr(params, k)))
                 if hasattr(params, k):
-                    self._log.debug("getattr=%s v=%s" % (getattr(params, k), v))
+                    self._log.debug("getattr=\"%s\" v=\"%s\"" % (getattr(params, k), v))
                     if getattr(params, k) == v:
                         match = True
                     else:

dv_flow/mgr/task_node_leaf.py

@@ -7,6 +7,7 @@ import logging
 import toposort
 from typing import Any, Callable, ClassVar, Dict, List, Tuple
 from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_def import ConsumesE, PassthroughE
 from .task_node import TaskNode
 from .task_params_ctor import TaskParamsCtor
 from .param_ref_eval import ParamRefEval
@@ -83,10 +84,12 @@ class TaskNodeLeaf(TaskNode):
 
         # Default inputs is the list of parameter sets that match 'consumes'
         inputs = []
-        if self.consumes is not None and len(self.consumes):
+        if isinstance(self.consumes, list) and len(self.consumes):
             for in_p in in_params:
                 if self._matches(in_p, self.consumes):
                     inputs.append(in_p)
+        elif self.consumes == ConsumesE.All:
+            inputs = in_params.copy()
 
         for name,field in self.params.model_fields.items():
             value = getattr(self.params, name)
@@ -125,10 +128,17 @@ class TaskNodeLeaf(TaskNode):
         # Add an entry for ourselves
         dep_m[self.name] = list(need.name for need,_ in self.needs)
 
-        if self.passthrough:
+        if isinstance(self.passthrough, list):
+            self._log.warning("List-based passthrough not yet supported")
+        elif self.passthrough == PassthroughE.All:
+            self._log.debug("Propagating all input parameters to output")
+            for need,block in self.needs:
+                if not block:
+                    output.extend(need.output.output)
+        elif self.passthrough == PassthroughE.Unused:
             self._log.debug("passthrough: %s" % self.name)
 
-            if self.consumes is None or len(self.consumes) == 0:
+            if self.consumes == ConsumesE.No or (isinstance(self.consumes, list) and len(self.consumes) == 0):
                 self._log.debug("Propagating all input parameters to output")
                 for need,block in self.needs:
                     if not block:
@@ -164,6 +174,11 @@ class TaskNodeLeaf(TaskNode):
         # TODO: 
         self._log.debug("<-- do_run: %s" % self.name)
 
+        if self.result is None:
+            raise Exception("Task %s did not produce a result" % self.name)
+
+        if self.output is None:
+            raise Exception("Task %s did not produce a result" % self.name)
         return self.result
 
     def __hash__(self):

dv_flow/mgr/task_runner.py

@@ -183,10 +183,18 @@ class TaskSetRunner(TaskRunner):
         with open(os.path.join(self.rundir, "cache", "mementos.json"), "w") as f:
             json.dump(dst_memento, f)
 
-        if isinstance(task, list):
-            return list(t.output for t in task)
+        if self.status == 0:
+            if isinstance(task, list):
+                for t in task:
+                    if t.output is None:
+                        raise Exception("Task %s did not produce output" % t.name)
+                return list(t.output for t in task)
+            else:
+                if task.output is None:
+                    raise Exception("Task %s did not produce output" % task.name)
+                return task.output
         else:
-            return task.output
+            return None
         
     def buildDepMap(self, task : Union[TaskNode, List[TaskNode]]) -> Dict[TaskNode, Set[TaskNode]]:
         tasks = task if isinstance(task, list) else [task]

{dv_flow_mgr-0.0.1.14110240305a1.dist-info → dv_flow_mgr-0.0.1.14116344455a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.1.14110240305a1
+Version: 0.0.1.14116344455a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.1.14110240305a1.dist-info → dv_flow_mgr-0.0.1.14116344455a1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dv_flow/mgr/__init__.py,sha256=EL7krYGBmN53z3HKvdrIiyHTD4TRx94LegCgx8whUAo,1130
+dv_flow/mgr/__init__.py,sha256=Zv-hbAhsxnQ49zy6_tKlwqgu5q12Yr7dCySyJMDtOO0,1154
 dv_flow/mgr/__main__.py,sha256=ExLZCOjDZtxqPk_SSF5SZKkbYUjMmVca4sT2OeePv10,3437
 dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
 dv_flow/mgr/expr_eval.py,sha256=N_8hRIgzJK9JVqhRt8F9rc4S7AAdKHMMltafqk6KhJs,3113
@@ -7,7 +7,7 @@ dv_flow/mgr/fileset.py,sha256=4izm4-qldWLiZxWDUa5Em0iFizzS-S-O0YU6zvtbUUs,1264
 dv_flow/mgr/fragment_def.py,sha256=AHx3ITbiNbGCzlDk5FM8Iimm0GJnWXDDRAoRC1b41jM,1647
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=9FhOu1hKYKzlD88zehwrI6KAcs1E7TbXDnq4v54dqDQ,1701
-dv_flow/mgr/package_def.py,sha256=L6KzBLSgArDgKPdjRAkkkq1c771BwV0NU-fMZRMMwws,20581
+dv_flow/mgr/package_def.py,sha256=WdVVpWVe4Pip5mR717iz2UWoZPkIfKrH8KzPIK-t73Y,20777
 dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
 dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
 dv_flow/mgr/param_def.py,sha256=mkAw3DanIxcVWRYeh9lUAfUMkpqDFly_Ku_iobr4ix8,1745
@@ -15,11 +15,11 @@ dv_flow/mgr/param_ref_eval.py,sha256=5yH37oIX6f2qmk7GfRgNT5qZx0jm3CJFgB9lLDZZ1yQ
 dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
 dv_flow/mgr/pkg_rgy.py,sha256=d1nIjRm3ymMNJT-yiMDxCS6bFisTPvLMqh5VrfsHVKM,5404
 dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
-dv_flow/mgr/task_def.py,sha256=H_8PUHbve7qBB_6ZEjI3kzIxOlXOor_rZ5yl1ZT-GUQ,3590
+dv_flow/mgr/task_def.py,sha256=9_VJfY8hfZI7x9LFqDxALurg6oWMQQw6rdsppcjTweM,3669
 dv_flow/mgr/task_graph_builder.py,sha256=3jDlC6UinOHNZO1scO4g6bdDdLZKJEjtexxV9RXy0_4,14371
 dv_flow/mgr/task_graph_dot_writer.py,sha256=GxqiYwQJbFgUQdnPCS9vpIYmYFbSXwnXCSbGmjbxC3M,2418
 dv_flow/mgr/task_listener_log.py,sha256=hrJEjSRXXoovDTcD1Cmhi3Spzw7uEJ-WP6tv6JUaa6s,4060
-dv_flow/mgr/task_node.py,sha256=ftAB3-j7ZeWV8Y8R1OwaqL8h2Xdc5tC77jLaL2Q390k,3532
+dv_flow/mgr/task_node.py,sha256=xkcdFZyEIIO6Natk-v4-5eyLZGI5kV0516kfjgqzQ34,3540
 dv_flow/mgr/task_node_compound.py,sha256=y8_jzsfT9Vnm-tCWc_-hFHFoSat95zBXFibySu38YRA,1696
 dv_flow/mgr/task_node_ctor.py,sha256=COFGvm5PR2B92H3uW1yhDIUCmppo9U4IfOcv_Jrsreo,3952
 dv_flow/mgr/task_node_ctor_compound.py,sha256=inJeqOtlkVa7QRCzktqvVkkZ0vZALHFadIGlRZyDTD8,4006
@@ -28,10 +28,10 @@ dv_flow/mgr/task_node_ctor_def_base.py,sha256=C2EeKgjeHB9xdC7nkLMBOcektwDoLhdP-4
 dv_flow/mgr/task_node_ctor_proxy.py,sha256=e1pfLBnJHAgQvkyNcB8sKw5DJdAPRphQz1-DT9Oheho,2017
 dv_flow/mgr/task_node_ctor_task.py,sha256=9HT39F3KcEpnOBd0SbUqE6FaeSY5fFp_tN1bzGNfFKk,2237
 dv_flow/mgr/task_node_ctor_wrapper.py,sha256=fRwO2RRJbt7IB1Ziwobmg0-nU5vFYsXoNV4IXfsS7mw,3435
-dv_flow/mgr/task_node_leaf.py,sha256=52qv1hwp5FLG9PuPvYVPLlTdzi1TcG4OTqMZC54iIwc,6481
+dv_flow/mgr/task_node_leaf.py,sha256=YofovCK8ozKOXyAAnoRq9fKcNh2kGGWxsQXh6l9K8t0,7276
 dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
 dv_flow/mgr/task_params_ctor.py,sha256=BPkbnoCtzhCxc1g8CJ6VimCcm5UAu92PXeDMhQ4lYsQ,1957
-dv_flow/mgr/task_runner.py,sha256=6Tq-ADmCjWzDmUktHs0yHsmJovUXAxcbZdOuJovdR2M,9117
+dv_flow/mgr/task_runner.py,sha256=Ho-u4xhiTVrOhSgjtcv-6pA0euwI3JFNNAHRdmSVBaA,9477
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=NDeyplKrPnWwEv4yHkhXEMK9d9j39b5MOeLB-1Mchqo,1095
 dv_flow/mgr/cmds/cmd_graph.py,sha256=Ykw__EdwamDBZZKxQZVbtMtFl7koq5dJMShTBXSM2pk,2672
@@ -41,7 +41,7 @@ dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5
 dv_flow/mgr/std/create_file.py,sha256=wmn5N_mObx_wr2LPdKOVbNmdM71hhs3UXOv6Ap9l3Ts,2726
 dv_flow/mgr/std/exec.py,sha256=UChqa_tAjvdB1NjqTsvlPgFomP8hMsX2rAOPyUonNpk,3896
 dv_flow/mgr/std/fileset.py,sha256=5IxS6T-x0wzA6fdEQcfHZ9kNP8IpH9hfJ3UMhL4A6Iw,3911
-dv_flow/mgr/std/flow.dv,sha256=rl0VZt_H-TyIJq0iERai07aDDZHI86mNozs4gUxidDA,3759
+dv_flow/mgr/std/flow.dv,sha256=t8m-urWYhKEAOFhG4wlrMqGHwiWDyO8H-b6-5vNQkBk,3795
 dv_flow/mgr/std/message.py,sha256=0JHLErg8whqMLAasG1fumZ2O7R7WNWeNQ9ibJaLDpVY,1029
 dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
 dv_flow/mgr/util/__init__.py,sha256=6uuA6z5cKS2hcjJw6YyEM2M79g6OpXb6tZF_Gku-AGU,22
@@ -49,9 +49,9 @@ dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,
 dv_flow/mgr/util/util.py,sha256=yg9oTPRiO87mkCSOQpOtlG9vyKPQzY3qp4OJkEMbWyY,1443
 dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/util/cmds/cmd_schema.py,sha256=lrEI-Jwb8j4I4yYOn9hq7_7NYbK8leVxLesrHyEWm-E,1879
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/METADATA,sha256=giD0HNKpfr3oaDAgHqo6nT2uUNm5_0H3DHiTMzv3itA,13336
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.1.14110240305a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/METADATA,sha256=eCspmsiQjw9KIDGVt0zXfKIQufdktI71Mn-LkQcL6Vc,13336
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.1.14116344455a1.dist-info/RECORD,,