dv-flow-mgr 0.0.2.14150065664a1__py3-none-any.whl → 0.0.2.14180123344a1__py3-none-any.whl

dv_flow/mgr/__main__.py

@@ -55,6 +55,9 @@ def get_parser():
     run_parser.add_argument("-j",
                         help="Specifies degree of parallelism. Uses all cores by default",
                         type=int, default=-1)
+    run_parser.add_argument("-c", "--clean",
+                            action="store_true",
+                            help="Cleans the rundir before running")
     run_parser.set_defaults(func=CmdRun())
 
     show_parser = subparsers.add_parser('show', 

dv_flow/mgr/cmds/cmd_run.py

@@ -77,6 +77,12 @@ class CmdRun(object):
         # TODO: allow user to specify run root -- maybe relative to some fixed directory?
         rundir = os.path.join(pkg._basedir, "rundir")
 
+        if args.clean:
+            print("Note: Cleaning rundir %s" % rundir)
+            if os.path.exists(rundir):
+                os.rmdir(rundir)
+            os.makedirs(rundir)
+
         builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
         runner = TaskSetRunner(rundir)
 

dv_flow/mgr/listener_list.py

@@ -0,0 +1,10 @@
+import dataclasses as dc
+from typing import List, Callable
+
+@dc.dataclass
+class ListenerList(object):
+    listeners : List[Callable] = dc.field(default_factory=list)
+
+    def __call__(self, *args, **kwargs):
+        for l in self.listeners:
+            l(*args, **kwargs)

dv_flow/mgr/package_def.py

@@ -64,6 +64,7 @@ class PackageDef(BaseModel):
     params : List[ParamDef] = dc.Field(
         default_factory=list, alias="with",
         description="List of package parameters to set")
+    srcinfo : Any = dc.Field(default=None)
 
     _fragment_l : List['FragmentDef'] = []
     _subpkg_m : Dict[str,'PackageDef'] = {}
@@ -410,13 +411,23 @@ class PackageDef(BaseModel):
 
     @classmethod
     def _loadPkgDef(cls, root, exp_pkg_name, file_s):
+        from yaml.loader import SafeLoader
+
+        class SafeLineLoader(SafeLoader):
+            def construct_mapping(self, node, deep=False):
+#                print("construct")
+                mapping = super(SafeLineLoader, self).construct_mapping(node, deep=deep)
+                # Add 1 so line numbering starts at 1
+#                mapping['_srcinfo'] = node.start_mark.line + 1
+                return mapping
+
         if root in file_s:
             raise Exception("Recursive file processing @ %s: %s" % (root, ",".join(file_s)))
         file_s.append(root)
         ret = None
         with open(root, "r") as fp:
             PackageDef._log.debug("open %s" % root)
-            doc = yaml.load(fp, Loader=yaml.FullLoader)
+            doc = yaml.load(fp, Loader=SafeLineLoader)
             if "package" not in doc.keys():
                 raise Exception("Missing 'package' key in %s" % root)
             try:

dv_flow/mgr/task_def.py

@@ -19,6 +19,7 @@
 #*     Author: 
 #*
 #****************************************************************************
+import pydantic
 import pydantic.dataclasses as dc
 import enum
 from pydantic import BaseModel
@@ -48,13 +49,31 @@ class PassthroughE(enum.Enum):
     All = "all"
     Unused = "unused"
 
-
 class StrategyDef(BaseModel):
     chain: Union[bool, None] = dc.Field(default=None)
     matrix : Union[Dict[str,List[Any]],None] = dc.Field(
         default=None,
         description="Matrix of parameter values to explore")
 
+class TaskExecDef(BaseModel):
+    pytask : Union[str, None] = dc.Field(
+        default=None,
+        description="Python method to execute to implement this task")
+    pydep  : Union[str, None] = dc.Field(
+        default=None,
+        description="Python method to check up-to-date status for this task")
+
+class TasksBuilder(BaseModel):
+    # TODO: control how much data this task is provided?
+    pydef : Union[str, None] = dc.Field(
+        default=None,
+        description="Python method to build the subgraph")
+
+class Tasks(BaseModel):
+    tasks: Union[List['TaskDef'], TasksBuilder] = dc.Field(
+        default_factory=list,
+        description="Sub-tasks")
+
 class TaskDef(BaseModel):
     """Holds definition information (ie the YAML view) for a task"""
     name : str = dc.Field(
@@ -72,7 +91,7 @@ class TaskDef(BaseModel):
         description="Python method to execute to implement this task")
     strategy : StrategyDef = dc.Field(
         default=None)
-    tasks: List['TaskDef'] = dc.Field(
+    tasks: Union[List['TaskDef'], TasksBuilder] = dc.Field(
         default_factory=list,
         description="Sub-tasks")
     desc : str = dc.Field(
@@ -100,9 +119,12 @@ class TaskDef(BaseModel):
     consumes : Union[ConsumesE, List[Any], None] = dc.Field(
         default=None,
         description="Specifies matching patterns for parameter sets that this task consumes")
-
-#    out: List[TaskOutput] = dc.Field(default_factory=list)
-
+    
+    def __init__(self, **data):
+#        print("--> task_def %s" % str(data))
+        super().__init__(**data)
+#        print("<-- task_def %s" % str(data))
+    
     def copy(self) -> 'TaskDef':
         ret = TaskDef(
             name=self.name,

dv_flow/mgr/task_graph_builder.py

@@ -22,6 +22,7 @@
 import os
 import dataclasses as dc
 import logging
+from typing import Callable
 from .package import Package
 from .package_def import PackageDef, PackageSpec
 from .pkg_rgy import PkgRgy
@@ -48,6 +49,7 @@ class TaskGraphBuilder(object):
     root_pkg : PackageDef
     rundir : str
     pkg_rgy : PkgRgy = None
+    marker_l : Callable = lambda *args, **kwargs: None
     _pkg_s : List[Package] = dc.field(default_factory=list)
     _pkg_m : Dict[PackageSpec,Package] = dc.field(default_factory=dict)
     _pkg_spec_s : List[PackageDef] = dc.field(default_factory=list)
@@ -330,6 +332,7 @@ class TaskGraphBuilder(object):
             self._pkg_spec_s.pop()
             self._pkg_m[spec] = pkg
         else:
+            self.error("Failed to find package %s" % spec.name)
             raise Exception("Failed to find definition of package %s" % spec.name)
 
         self._logger.debug("<-- getPackage: %s" % str(pkg))

dv_flow/mgr/task_node_ctor_compound.py

@@ -73,13 +73,23 @@ class TaskNodeCtorCompound(TaskNodeCtor):
             # Need to get the parent name
             needs = []
             for n in t.needs:
-                need_name = "%s.%s" % (builder.package().name, n)
-                task = builder.findTask(n)
-                if task is None:
+                # 'n' is the dependency as specified by the user
+                # Need to perform a search
+                # - Look locally inside the compound task (pkg.compound.name)
+                # - Look for the fully-qualified task name
+                # - Look for the task name in the package
+
+                names = []
+                for pref in (builder.get_name_prefix((), "", builder.package().name)):
+                    need_name = n if pref == "" else ("%s.%s" % (pref, n))
+                    names.append(need_name)
                     task = builder.findTask(need_name)
 
+                    if task is not None:
+                        break
+
                 if task is None:
-                    raise Exception("Failed to find task %s (%s)" % (n, need_name))
+                    raise Exception("Failed to find task %s (searched %s)" % (n, str(names)))
                 self._log.debug("Add %s as dependency of %s" % (
                     task.name, t.name
                 ))

{dv_flow_mgr-0.0.2.14150065664a1.dist-info → dv_flow_mgr-0.0.2.14180123344a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dv-flow-mgr
-Version: 0.0.2.14150065664a1
+Version: 0.0.2.14180123344a1
 Summary: DV Flow Manager is a build system for silicon design
 Author-email: Matthew Ballance <matt.ballance@gmail.com>
 License:                                  Apache License

{dv_flow_mgr-0.0.2.14150065664a1.dist-info → dv_flow_mgr-0.0.2.14180123344a1.dist-info}/RECORD

@@ -1,13 +1,14 @@
 dv_flow/mgr/__init__.py,sha256=hzwGMkN3t6RkfZSCHi_6mvEkCzEOd91EZBv9F2ZCa-8,1193
-dv_flow/mgr/__main__.py,sha256=ExLZCOjDZtxqPk_SSF5SZKkbYUjMmVca4sT2OeePv10,3437
+dv_flow/mgr/__main__.py,sha256=4_MJIH-FxIFXrk5pvic9bBereeZxQsrxnESA4SZ5q9U,3600
 dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
 dv_flow/mgr/expr_eval.py,sha256=N_8hRIgzJK9JVqhRt8F9rc4S7AAdKHMMltafqk6KhJs,3113
 dv_flow/mgr/expr_parser.py,sha256=P6u2FdSXeZbdamC3zpEnYKLcK2RULQJfSoev2Ol75fE,6543
 dv_flow/mgr/fileset.py,sha256=4izm4-qldWLiZxWDUa5Em0iFizzS-S-O0YU6zvtbUUs,1264
 dv_flow/mgr/fragment_def.py,sha256=AHx3ITbiNbGCzlDk5FM8Iimm0GJnWXDDRAoRC1b41jM,1647
+dv_flow/mgr/listener_list.py,sha256=BfqvEO2AyJvyc4ClU-hPgDPqWSlqvSdG-yaFEHvUrMc,272
 dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
 dv_flow/mgr/package.py,sha256=9FhOu1hKYKzlD88zehwrI6KAcs1E7TbXDnq4v54dqDQ,1701
-dv_flow/mgr/package_def.py,sha256=iilJPa1zJoF5OkjdtBVro_fFImEbiJqPijFkYltut-o,20881
+dv_flow/mgr/package_def.py,sha256=18y29oi_vQc4Jzl7v7iHjRo22eUQbndPxa1bucrZX8E,21343
 dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
 dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
 dv_flow/mgr/param_def.py,sha256=mkAw3DanIxcVWRYeh9lUAfUMkpqDFly_Ku_iobr4ix8,1745
@@ -15,14 +16,14 @@ dv_flow/mgr/param_ref_eval.py,sha256=5yH37oIX6f2qmk7GfRgNT5qZx0jm3CJFgB9lLDZZ1yQ
 dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
 dv_flow/mgr/pkg_rgy.py,sha256=d1nIjRm3ymMNJT-yiMDxCS6bFisTPvLMqh5VrfsHVKM,5404
 dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
-dv_flow/mgr/task_def.py,sha256=Yf9bkWqSETeFUSbddbpvl2Cu3pyssvhdz0X2jxabniA,3751
-dv_flow/mgr/task_graph_builder.py,sha256=Z14ySv1SFanFovR8HePwKglyFSzc6JsS8Z4RLmggM6c,15316
+dv_flow/mgr/task_def.py,sha256=ClCPFLlpT70Nxcr4xxBJBtdRxwFHq5YUtEDNVCiqFtE,4559
+dv_flow/mgr/task_graph_builder.py,sha256=ST8EseeO8JZUTUSU44txxaUVKlXEiTOPbvB_UVrlraU,15463
 dv_flow/mgr/task_graph_dot_writer.py,sha256=GxqiYwQJbFgUQdnPCS9vpIYmYFbSXwnXCSbGmjbxC3M,2418
 dv_flow/mgr/task_listener_log.py,sha256=hrJEjSRXXoovDTcD1Cmhi3Spzw7uEJ-WP6tv6JUaa6s,4060
 dv_flow/mgr/task_node.py,sha256=JXq2QimCZKxfhhkdhM-HKk4JbxpIlpW65xUJ0hVlQc0,4981
 dv_flow/mgr/task_node_compound.py,sha256=mNu4nf9hVqu2698ue5fpE3FeAOkvJH0Ke2W9V0G3-As,2975
 dv_flow/mgr/task_node_ctor.py,sha256=COFGvm5PR2B92H3uW1yhDIUCmppo9U4IfOcv_Jrsreo,3952
-dv_flow/mgr/task_node_ctor_compound.py,sha256=n2Ad1jtX147cb99HTlLrx_21XUn109bJ18zDXr0pn2Q,4259
+dv_flow/mgr/task_node_ctor_compound.py,sha256=HI-KDmBoRlvwPQpl3WSZit8jqhJ7Zbv8NRvh45Kp6PA,4736
 dv_flow/mgr/task_node_ctor_compound_proxy.py,sha256=D8x54nD8Pd-2-_mr1syhqVeSFfIVf100ldi3bdzmSfI,2073
 dv_flow/mgr/task_node_ctor_def_base.py,sha256=_8QQHKDkONio_ve0Z409yxC0AMO8ocNBPDjRiNED1FI,1503
 dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v2PA6VA,1906
@@ -36,7 +37,7 @@ dv_flow/mgr/task_runner.py,sha256=-919VntXAe2XSuFW2dFpgvUre-NkILBnDBbAKBZYn5w,95
 dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/type_def.py,sha256=NDeyplKrPnWwEv4yHkhXEMK9d9j39b5MOeLB-1Mchqo,1095
 dv_flow/mgr/cmds/cmd_graph.py,sha256=Ykw__EdwamDBZZKxQZVbtMtFl7koq5dJMShTBXSM2pk,2672
-dv_flow/mgr/cmds/cmd_run.py,sha256=f0zt2OxGZC50aaDR7wGDH8YBk3rMQoBl8YeRHWcJUD0,3211
+dv_flow/mgr/cmds/cmd_run.py,sha256=ADFBetIVk_CwMQXO3oSNdtUgrxx8FyQj9LYQ2cA8nuo,3394
 dv_flow/mgr/cmds/cmd_show.py,sha256=CZlgwB3Hcu-6HS-diqnWjCcPPpO-kjIIXU3DgWXvsf4,3773
 dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
 dv_flow/mgr/std/create_file.py,sha256=TAUhpXlTmUDUYw4Dw0cI9FPuYI84yCVkoadnWZxi_8U,2888
@@ -50,9 +51,9 @@ dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,
 dv_flow/mgr/util/util.py,sha256=yg9oTPRiO87mkCSOQpOtlG9vyKPQzY3qp4OJkEMbWyY,1443
 dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dv_flow/mgr/util/cmds/cmd_schema.py,sha256=lrEI-Jwb8j4I4yYOn9hq7_7NYbK8leVxLesrHyEWm-E,1879
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/METADATA,sha256=9jJBGIp_ylYV8SgeRvZh0P1EcYH4cQHSLaHEF2Mr_QA,13336
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
-dv_flow_mgr-0.0.2.14150065664a1.dist-info/RECORD,,
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/METADATA,sha256=uCchOggxk9U54veU7UCjQOyn65EEfR22ABTcYJd8P-4,13336
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
+dv_flow_mgr-0.0.2.14180123344a1.dist-info/RECORD,,