dv-flow-mgr 1.0.0.14528489065a1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dv_flow/mgr/__init__.py

@@ -31,3 +31,7 @@ from .task_node_ctor_wrapper import task
 from .task_runner import TaskSetRunner
 from .task_listener_log import TaskListenerLog
 
+VERSION="1.5.0"
+SUFFIX="14762702318rc0"
+__version__="%s%s" % (BASE, SUFFIX)
+

dv_flow/mgr/cmds/cmd_run.py

@@ -24,6 +24,7 @@ import os
 import logging
 from typing import ClassVar
 from ..util import loadProjPkgDef
+from ..task_data import SeverityE
 from ..task_graph_builder import TaskGraphBuilder
 from ..task_runner import TaskSetRunner
 from ..task_listener_log import TaskListenerLog
@@ -42,7 +43,11 @@ class CmdRun(object):
 
         # First, find the project we're working with
         listener = TaskListenerLog()
-        pkg = loadProjPkgDef(rootdir, listener=listener)
+        pkg = loadProjPkgDef(rootdir, listener=listener.marker)
+
+        if listener.has_severity[SeverityE.Error] > 0:
+            print("Error(s) encountered while loading package definition")
+            return 1
 
         if pkg is None:
             raise Exception("Failed to find a 'flow.dv' file that defines a package in %s or its parent directories" % os.getcwd())

dv_flow/mgr/cmds/cmd_show.py

@@ -47,9 +47,9 @@ class CmdShow(object):
         if args.task is None:
             # Print out available tasks
             tasks = []
-            for task in pkg.tasks:
+            for task in pkg.task_m.values():
                 tasks.append(task)
-            for frag in pkg._fragment_l:
+            for frag in pkg.fragment_def_l:
                 for task in frag.tasks:
                     tasks.append(task)
             tasks.sort(key=lambda x: x.name)
@@ -66,7 +66,7 @@ class CmdShow(object):
                     "<no descripion>"
                 print("%s - %s" % (t.name.ljust(max_name_len), desc))
         else:
-            rundir = os.path.join(pkg._basedir, "rundir")
+            rundir = os.path.join(pkg.basedir, "rundir")
 
             builder = TaskGraphBuilder(root_pkg=pkg, rundir=rundir)
 

dv_flow/mgr/config_def.py

@@ -27,7 +27,7 @@ class ConfigDef(BaseModel):
         description="Name of the configuration to use as a base")
     overrides : List[OverrideDef] = pdc.Field(
         default_factory=list,
-        description="List of package and task overrides")
+        description="List of package overrides")
     extensions : List[ExtendDef] = pdc.Field(
         default_factory=list,
         description="List of extensions to apply")

dv_flow/mgr/expr_eval.py

@@ -22,7 +22,8 @@
 import dataclasses as dc
 import json
 from typing import Any, Callable, Dict, List
-from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp, ExprCall, ExprId, ExprString, ExprInt
+from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp
+from .expr_parser import ExprCall, ExprHId, ExprId, ExprString, ExprInt
 
 @dc.dataclass
 class ExprEval(ExprVisitor):
@@ -30,6 +31,9 @@ class ExprEval(ExprVisitor):
     variables : Dict[str, object] = dc.field(default_factory=dict)
     value : Any = None
 
+    def set(self, name : str, value : object):
+        self.variables[name] = value
+
     def eval(self, e : Expr) -> str:
         self.value = None
         e.accept(self)
@@ -59,6 +63,25 @@ class ExprEval(ExprVisitor):
 
         return rval
 
+    def visitExprHId(self, e : ExprHId):
+        print("Hid: %s" % ".".join(e.id))
+        if e.id[0] in self.variables:
+            # Always represent data as a JSON object
+            root = self.variables[e.id[0]]
+            for i in range(1, len(e.id)):
+                if isinstance(root, dict):
+                    if e.id[i] in root.keys():
+                        root = root[e.id[i]]
+                    else:
+                        raise Exception("Sub-element '%s' not found in '%s'" % (e.id[i], ".".join(e.id)))
+                elif hasattr(root, e.id[i]):
+                    root = getattr(root, e.id[i])
+                else:
+                    raise Exception("Sub-element '%s' not found in '%s'" % (e.id[i], ".".join(e.id)))
+            self.value = root
+        else:
+            raise Exception("Variable '%s' not found" % e.id[0])
+
     def visitExprId(self, e : ExprId):
         if e.id in self.variables:
             # Always represent data as a JSON object

dv_flow/mgr/expr_parser.py

@@ -37,6 +37,13 @@ class ExprId(Expr):
     def accept(self, v):
         v.visitExprId(self)
 
+@dc.dataclass
+class ExprHId(Expr):
+    id : List[str] = dc.field(default_factory=list)
+
+    def accept(self, v):
+        v.visitExprHId(self)
+
 class ExprBinOp(enum.Enum):
     Pipe = enum.auto()
     Plus = enum.auto()
@@ -76,6 +83,9 @@ class ExprInt(Expr):
         v.visitExprInt(self)
 
 class ExprVisitor(object):
+    def visitExprHId(self, e : ExprId):
+        pass
+
     def visitExprId(self, e : ExprId):
         pass
 
@@ -148,7 +158,7 @@ class ExprParser(object):
         return cls._inst
 
     tokens = (
-        'ID','NUMBER','COMMA',
+        'ID', 'DOT', 'NUMBER','COMMA',
         'PLUS','MINUS','TIMES','DIVIDE',
         'LPAREN','RPAREN','PIPE','STRING1','STRING2'
         )
@@ -163,6 +173,7 @@ class ExprParser(object):
     t_LPAREN  = r'\('
     t_RPAREN  = r'\)'
     t_ID      = r'[a-zA-Z_][a-zA-Z0-9_]*'
+    t_DOT     = r'\.'
     t_PIPE    = r'\|'
     
     def t_NUMBER(self, t):
@@ -244,6 +255,20 @@ class ExprParser(object):
         'expression : ID'
         t[0] = ExprId(t[1])
 
+    def p_expression_hid(self, t):
+        'expression : hier_id'
+        t[0] = t[1]
+
+    def p_hier_id(self, t):
+        '''hier_id : ID DOT hier_id 
+                   | ID'''
+        if len(t) == 2:
+            t[0] = ExprHId()
+            t[0].id.append(t[1])
+        else:
+            t[3].id.insert(0, t[1])
+            t[0] = t[3]
+
     def p_expression_string1(self, t):
         'expression : STRING1'
         t[0] = ExprString(t[1])

dv_flow/mgr/package.py

@@ -26,16 +26,17 @@ from .fragment_def import FragmentDef
 from .package_def import PackageDef
 from .srcinfo import SrcInfo
 from .task import Task
+from .type import Type
 
 @dc.dataclass
 class Package(object):
     pkg_def : PackageDef
     basedir : str = None
-    params : Dict[str,Any] = dc.field(default_factory=dict)
+    paramT : Any = None
     # Package holds constructors for tasks
     # - Dict holds the default parameters for the task
     task_m : Dict[str,Task] = dc.field(default_factory=dict)
-    types : Dict[str,Any] = dc.field(default_factory=dict)
+    type_m : Dict[str,Type] = dc.field(default_factory=dict)
     fragment_def_l : List[FragmentDef] = dc.field(default_factory=list)
     pkg_m : Dict[str, 'Package'] = dc.field(default_factory=dict)
     srcinfo : SrcInfo = None
@@ -53,17 +54,23 @@ class Package(object):
     
     def dump(self):
         tasks = {}
+        types = {}
         for k, v in self.task_m.items():
             tasks[k] = v.dump()
+        for k, v in self.type_m.items():
+            types[k] = v.dump()
 
         pkg = {
             "name": self.name,
             "basedir": self.basedir,
-            "params": self.params,
             "tasks": tasks,
+            "types": types,
             "fragments": [f.dump() for f in self.fragment_def_l]
         }
 
+        if self.paramT is not None:
+            pkg["paramT"] = self.paramT().model_dump()
+
         return pkg
             
     def __hash__(self):
@@ -79,11 +86,11 @@ class Package(object):
             dict: Dictionary containing required package data and markers
         """
         # Collect all imported packages recursively
-        imports = set()
+        imports = {}
         def collect_imports(pkg):
             for name, p in pkg.pkg_m.items():
-                if name not in imports:
-                    imports.add(name)
+                if name not in imports.keys():
+                    imports[name] = p.basedir
                     collect_imports(p)
         collect_imports(self)
         

dv_flow/mgr/package_def.py

@@ -43,7 +43,6 @@ from .type_def import TypeDef
 class PackageDef(BaseModel):
     name : str = dc.Field(
         description="Name of the package")
-    params : Dict[str,Any] = dc.Field(default_factory=dict)
     type : List[PackageSpec] = dc.Field(default_factory=list)
     tasks : List[TaskDef] = dc.Field(
         default_factory=list,
@@ -58,9 +57,9 @@ class PackageDef(BaseModel):
     uses : str = dc.Field(
         default=None,
         description="Name of the package to use as a base")
-    params : List[ParamDef] = dc.Field(
-        default_factory=list, alias="with",
-        description="List of package parameters to set")
+    params : Dict[str,Union[str,list,ParamDef]] = dc.Field(
+        default_factory=dict, alias="with",
+        description="Package parameters")
     configs : List[ConfigDef] = dc.Field(
         default_factory=list,
         description="List of package configurations")