dv-flow-mgr 1.6.14849587166rc0__py3-none-any.whl → 1.6.14872991111rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/__init__.py +1 -1
- dv_flow/mgr/config.py +2 -1
- dv_flow/mgr/config_def.py +2 -4
- dv_flow/mgr/data_callable.py +26 -0
- dv_flow/mgr/expr_eval.py +60 -37
- dv_flow/mgr/name_resolution.py +73 -0
- dv_flow/mgr/package_loader.py +74 -34
- dv_flow/mgr/package_node.py +10 -0
- dv_flow/mgr/param_ref_eval.py +5 -2
- dv_flow/mgr/std/create_file.py +3 -0
- dv_flow/mgr/std/data_item.py +17 -0
- dv_flow/mgr/std/flow.dv +15 -0
- dv_flow/mgr/task.py +1 -0
- dv_flow/mgr/task_graph_builder.py +136 -65
- dv_flow/mgr/task_run_ctxt.py +16 -0
- dv_flow/mgr/task_runner.py +6 -0
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/METADATA +1 -1
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/RECORD +22 -18
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/WHEEL +0 -0
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/licenses/LICENSE +0 -0
- {dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/top_level.txt +0 -0
dv_flow/mgr/__init__.py
CHANGED
dv_flow/mgr/config.py
CHANGED
dv_flow/mgr/config_def.py
CHANGED
@@ -7,10 +7,8 @@ from .param_def import ParamDef
|
|
7
7
|
|
8
8
|
class OverrideDef(BaseModel):
|
9
9
|
"""Override definition"""
|
10
|
-
|
11
|
-
description="
|
12
|
-
task : Union[str, None] = pdc.Field(
|
13
|
-
description="Task to override")
|
10
|
+
override : Union[str, None] = pdc.Field(
|
11
|
+
description="Task or package to override")
|
14
12
|
value : str = pdc.Field(
|
15
13
|
description="Override to use",
|
16
14
|
alias="with")
|
@@ -0,0 +1,26 @@
|
|
1
|
+
import dataclasses as dc
|
2
|
+
from typing import Any
|
3
|
+
from .task_data import TaskDataResult, TaskDataInput
|
4
|
+
from .task_run_ctxt import TaskRunCtxt
|
5
|
+
|
6
|
+
@dc.dataclass
|
7
|
+
class DataCallable(object):
|
8
|
+
paramT : Any
|
9
|
+
|
10
|
+
async def __call__(self, ctxt : TaskRunCtxt, input : TaskDataInput) -> TaskDataResult:
|
11
|
+
output = []
|
12
|
+
status = 0
|
13
|
+
|
14
|
+
data = self.paramT()
|
15
|
+
|
16
|
+
# Now, copy in the parameter values
|
17
|
+
for key in self.paramT.model_fields.keys():
|
18
|
+
if hasattr(input.params, key):
|
19
|
+
setattr(data, key, getattr(input.params, key))
|
20
|
+
|
21
|
+
output.append(data)
|
22
|
+
|
23
|
+
return TaskDataResult(
|
24
|
+
status=status,
|
25
|
+
output=output
|
26
|
+
)
|
dv_flow/mgr/expr_eval.py
CHANGED
@@ -21,26 +21,40 @@
|
|
21
21
|
#****************************************************************************
|
22
22
|
import dataclasses as dc
|
23
23
|
import json
|
24
|
-
from typing import Any, Callable, Dict, List
|
25
|
-
from .expr_parser import ExprVisitor, Expr, ExprBin, ExprBinOp
|
24
|
+
from typing import Any, Callable, Dict, List, Optional
|
25
|
+
from .expr_parser import ExprParser, ExprVisitor, Expr, ExprBin, ExprBinOp
|
26
26
|
from .expr_parser import ExprCall, ExprHId, ExprId, ExprString, ExprInt
|
27
|
+
from .name_resolution import NameResolutionContext
|
27
28
|
|
28
29
|
@dc.dataclass
|
29
30
|
class ExprEval(ExprVisitor):
|
30
|
-
methods
|
31
|
-
|
32
|
-
|
31
|
+
methods: Dict[str, Callable] = dc.field(default_factory=dict)
|
32
|
+
name_resolution: Optional[NameResolutionContext] = None
|
33
|
+
variables: Dict[str, object] = dc.field(default_factory=dict)
|
34
|
+
value: Any = None
|
33
35
|
|
34
|
-
def set(self, name
|
36
|
+
def set(self, name: str, value: object):
|
35
37
|
self.variables[name] = value
|
36
38
|
|
37
|
-
def
|
38
|
-
self.
|
39
|
-
e.accept(self)
|
39
|
+
def set_name_resolution(self, ctx: NameResolutionContext):
|
40
|
+
self.name_resolution = ctx
|
40
41
|
|
41
|
-
|
42
|
+
def eval(self, expr_s: str) -> str:
|
43
|
+
if expr_s is None:
|
44
|
+
return None
|
45
|
+
elif isinstance(expr_s, Expr):
|
46
|
+
expr_s.accept(self)
|
47
|
+
return self._toString(self.value)
|
48
|
+
elif isinstance(expr_s, bool):
|
49
|
+
return expr_s
|
50
|
+
else:
|
51
|
+
parser = ExprParser()
|
52
|
+
ast = parser.parse(expr_s)
|
42
53
|
|
43
|
-
|
54
|
+
self.value = None
|
55
|
+
ast.accept(self)
|
56
|
+
val = self._toString(self.value)
|
57
|
+
return val
|
44
58
|
|
45
59
|
def _toString(self, val):
|
46
60
|
rval = val
|
@@ -48,11 +62,6 @@ class ExprEval(ExprVisitor):
|
|
48
62
|
obj = self._toObject(val)
|
49
63
|
rval = json.dumps(obj)
|
50
64
|
return rval
|
51
|
-
# if isinstance(val, list):
|
52
|
-
# val = '[' + ",".join(self._toString(v) for v in val) + ']'
|
53
|
-
# elif hasattr(val, "model_dump_json"):
|
54
|
-
# val = val.model_dump_json()
|
55
|
-
# return val
|
56
65
|
|
57
66
|
def _toObject(self, val):
|
58
67
|
rval = val
|
@@ -63,33 +72,47 @@ class ExprEval(ExprVisitor):
|
|
63
72
|
|
64
73
|
return rval
|
65
74
|
|
66
|
-
def visitExprHId(self, e
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
75
|
+
def visitExprHId(self, e: ExprHId):
|
76
|
+
# First try to resolve using name resolution context
|
77
|
+
value = None
|
78
|
+
|
79
|
+
if self.name_resolution:
|
80
|
+
value = self.name_resolution.resolve_variable(e.id[0])
|
81
|
+
|
82
|
+
# Fall back to variables dict
|
83
|
+
if value is None and e.id[0] in self.variables:
|
84
|
+
value = self.variables[e.id[0]]
|
85
|
+
|
86
|
+
if value is None:
|
87
|
+
raise Exception("Variable '%s' not found" % e.id[0])
|
88
|
+
|
89
|
+
for i in range(1, len(e.id)):
|
90
|
+
if isinstance(value, dict):
|
91
|
+
if e.id[i] in value.keys():
|
92
|
+
value = value[e.id[i]]
|
79
93
|
else:
|
80
94
|
raise Exception("Sub-element '%s' not found in '%s'" % (e.id[i], ".".join(e.id)))
|
81
|
-
|
82
|
-
|
83
|
-
|
95
|
+
elif hasattr(value, e.id[i]):
|
96
|
+
value = getattr(value, e.id[i])
|
97
|
+
else:
|
98
|
+
raise Exception("Sub-element '%s' not found in '%s' (%s)" % (e.id[i], ".".join(e.id), value))
|
99
|
+
self.value = value
|
100
|
+
|
101
|
+
def visitExprId(self, e: ExprId):
|
102
|
+
# First try to resolve using name resolution context
|
103
|
+
if self.name_resolution:
|
104
|
+
resolved = self.name_resolution.resolve_variable(e.id)
|
105
|
+
if resolved is not None:
|
106
|
+
self.value = resolved
|
107
|
+
return
|
84
108
|
|
85
|
-
|
109
|
+
# Fall back to variables dict
|
86
110
|
if e.id in self.variables:
|
87
|
-
# Always represent data as a JSON object
|
88
111
|
self.value = self._toObject(self.variables[e.id])
|
89
112
|
else:
|
90
113
|
raise Exception("Variable '%s' not found" % e.id)
|
91
114
|
|
92
|
-
def visitExprString(self, e
|
115
|
+
def visitExprString(self, e: ExprString):
|
93
116
|
self.value = e.value
|
94
117
|
|
95
118
|
def visitExprBin(self, e):
|
@@ -101,7 +124,7 @@ class ExprEval(ExprVisitor):
|
|
101
124
|
elif e.op == ExprBinOp.Plus:
|
102
125
|
pass
|
103
126
|
|
104
|
-
def visitExprCall(self, e
|
127
|
+
def visitExprCall(self, e: ExprCall):
|
105
128
|
if e.id in self.methods:
|
106
129
|
# Need to gather up argument values
|
107
130
|
in_value = self.value
|
@@ -115,5 +138,5 @@ class ExprEval(ExprVisitor):
|
|
115
138
|
else:
|
116
139
|
raise Exception("Method %s not found" % e.id)
|
117
140
|
|
118
|
-
def visitExprInt(self, e
|
141
|
+
def visitExprInt(self, e: ExprInt):
|
119
142
|
self.value = e.value
|
@@ -0,0 +1,73 @@
|
|
1
|
+
#****************************************************************************
|
2
|
+
#* name_resolution.py
|
3
|
+
#*
|
4
|
+
#* Copyright 2023-2025 Matthew Ballance and Contributors
|
5
|
+
#*
|
6
|
+
#* Licensed under the Apache License, Version 2.0 (the "License"); you may
|
7
|
+
#* not use this file except in compliance with the License.
|
8
|
+
#* You may obtain a copy of the License at:
|
9
|
+
#*
|
10
|
+
#* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#*
|
12
|
+
#* Unless required by applicable law or agreed to in writing, software
|
13
|
+
#* distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
#* See the License for the specific language governing permissions and
|
16
|
+
#* limitations under the License.
|
17
|
+
#*
|
18
|
+
#****************************************************************************
|
19
|
+
|
20
|
+
import dataclasses as dc
|
21
|
+
from typing import Any, Dict, List
|
22
|
+
from .package import Package
|
23
|
+
from .task_node import TaskNode
|
24
|
+
|
25
|
+
@dc.dataclass
|
26
|
+
class TaskNameResolutionScope:
|
27
|
+
"""Represents a single task scope in the name resolution stack"""
|
28
|
+
task: TaskNode
|
29
|
+
variables: Dict[str, Any] = dc.field(default_factory=dict)
|
30
|
+
|
31
|
+
@dc.dataclass
|
32
|
+
class NameResolutionContext:
|
33
|
+
"""Represents a complete name resolution context"""
|
34
|
+
builder: 'TaskGraphBuilder' # Forward reference to avoid circular import
|
35
|
+
package: Package
|
36
|
+
task_scopes: List[TaskNameResolutionScope] = dc.field(default_factory=list)
|
37
|
+
|
38
|
+
def resolve_variable(self, name: str) -> Any:
|
39
|
+
"""
|
40
|
+
Resolve a variable name following scoping rules:
|
41
|
+
1. Current task scope
|
42
|
+
2. Enclosing compound-task scopes
|
43
|
+
3. Current package variables
|
44
|
+
4. Package-qualified variables
|
45
|
+
"""
|
46
|
+
# Check if this is a package-qualified reference
|
47
|
+
ret = None
|
48
|
+
|
49
|
+
if name in self.builder._pkg_params_m.keys():
|
50
|
+
# Check if this is a package-qualified reference
|
51
|
+
ret = self.builder._pkg_params_m[name]
|
52
|
+
|
53
|
+
# Check task scopes from innermost to outermost
|
54
|
+
if ret is None:
|
55
|
+
for scope in reversed(self.task_scopes):
|
56
|
+
if name in scope.variables.keys():
|
57
|
+
ret = scope.variables[name]
|
58
|
+
break
|
59
|
+
elif hasattr(scope.task.params, name):
|
60
|
+
# Check task parameters
|
61
|
+
ret = getattr(scope.task.params, name)
|
62
|
+
break
|
63
|
+
|
64
|
+
# Check package variables
|
65
|
+
if ret is None and hasattr(self.package.paramT, name):
|
66
|
+
ret = getattr(self.package.paramT, name)
|
67
|
+
|
68
|
+
# if ret is None and name in self.builder._pkg_m.keys():
|
69
|
+
# pkg = self.builder._pkg_m[name]
|
70
|
+
# ret = pkg.paramT
|
71
|
+
|
72
|
+
print("<-- resolve: %s" % type(ret))
|
73
|
+
return ret
|
dv_flow/mgr/package_loader.py
CHANGED
@@ -6,7 +6,7 @@ import pydantic
|
|
6
6
|
import sys
|
7
7
|
import yaml
|
8
8
|
from pydantic import BaseModel
|
9
|
-
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
9
|
+
from typing import Any, Callable, ClassVar, Dict, List, Tuple, Union
|
10
10
|
from .fragment_def import FragmentDef
|
11
11
|
from .package_def import PackageDef
|
12
12
|
from .package import Package
|
@@ -93,8 +93,6 @@ class LoaderScope(SymbolScope):
|
|
93
93
|
pkg = find_pkg(pkg_name)
|
94
94
|
if pkg is not None:
|
95
95
|
break;
|
96
|
-
else:
|
97
|
-
raise Exception("Task name %s is not fully qualified" % name)
|
98
96
|
|
99
97
|
if pkg is not None and name in pkg.task_m.keys():
|
100
98
|
ret = pkg.task_m[name]
|
@@ -106,6 +104,7 @@ class LoaderScope(SymbolScope):
|
|
106
104
|
def findType(self, name) -> Type:
|
107
105
|
self._log.debug("--> findType: %s" % name)
|
108
106
|
ret = None
|
107
|
+
pkg = None
|
109
108
|
last_dot = name.rfind('.')
|
110
109
|
if last_dot != -1:
|
111
110
|
pkg_name = name[:last_dot]
|
@@ -410,31 +409,21 @@ class PackageLoader(object):
|
|
410
409
|
self._log.info("Loading imported package %s" % imp_path)
|
411
410
|
|
412
411
|
if not os.path.isabs(imp_path):
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
# Search down the tree looking for a flow.dv file
|
417
|
-
if os.path.isdir(imp_path):
|
418
|
-
path = imp_path
|
412
|
+
for root in (basedir, os.path.dirname(self._file_s[0])):
|
413
|
+
self._log.debug("Search basedir: %s ; imp_path: %s" % (root, imp_path))
|
419
414
|
|
420
|
-
|
421
|
-
# Look one directory down
|
422
|
-
next_dir = None
|
423
|
-
for dir in os.listdir(path):
|
424
|
-
if os.path.isdir(os.path.join(path, dir)):
|
425
|
-
if next_dir is None:
|
426
|
-
next_dir = dir
|
427
|
-
else:
|
428
|
-
path = None
|
429
|
-
break
|
430
|
-
if path is not None:
|
431
|
-
path = next_dir
|
415
|
+
resolved_path = self._findFlowDvInDir(root, imp_path)
|
432
416
|
|
433
|
-
|
434
|
-
|
417
|
+
if resolved_path is not None and os.path.isfile(resolved_path):
|
418
|
+
self._log.debug("Found root file: %s" % resolved_path)
|
419
|
+
imp_path = resolved_path
|
420
|
+
break
|
435
421
|
|
436
422
|
if not os.path.isfile(imp_path):
|
437
|
-
|
423
|
+
self.error("Import file %s not found" % imp_path, pkg.srcinfo)
|
424
|
+
# Don't want to error out
|
425
|
+
return
|
426
|
+
# raise Exception("Import file %s not found" % imp_path)
|
438
427
|
|
439
428
|
if imp_path in self._pkg_path_m.keys():
|
440
429
|
sub_pkg = self._pkg_path_m[imp_path]
|
@@ -446,7 +435,34 @@ class PackageLoader(object):
|
|
446
435
|
|
447
436
|
pkg.pkg_m[sub_pkg.name] = sub_pkg
|
448
437
|
self._log.debug("<-- _loadPackageImport %s" % str(imp))
|
449
|
-
|
438
|
+
|
439
|
+
def _findFlowDvInDir(self, base, leaf):
|
440
|
+
"""Search down the tree looking for a flow.dv file"""
|
441
|
+
self._log.debug("--> _findFlowDvInDir (%s, %s)" % (base, leaf))
|
442
|
+
imp_path = None
|
443
|
+
if os.path.isfile(os.path.join(base, leaf)):
|
444
|
+
imp_path = os.path.join(base, leaf)
|
445
|
+
self._log.debug("Found: %s" % imp_path)
|
446
|
+
elif os.path.isdir(os.path.join(base, leaf)):
|
447
|
+
path = os.path.join(base, leaf)
|
448
|
+
|
449
|
+
while path is not None and os.path.isdir(path) and not os.path.isfile(os.path.join(path, "flow.dv")):
|
450
|
+
# Look one directory down
|
451
|
+
next_dir = None
|
452
|
+
for dir in os.listdir(path):
|
453
|
+
if os.path.isdir(os.path.join(path, dir)):
|
454
|
+
if next_dir is None:
|
455
|
+
next_dir = dir
|
456
|
+
else:
|
457
|
+
path = None
|
458
|
+
break
|
459
|
+
if path is not None:
|
460
|
+
path = next_dir
|
461
|
+
if path is not None and os.path.isfile(os.path.join(path, "flow.dv")):
|
462
|
+
imp_path = os.path.join(path, "flow.dv")
|
463
|
+
self._log.debug("Found: %s" % imp_path)
|
464
|
+
self._log.debug("<-- _findFlowDvInDir %s" % imp_path)
|
465
|
+
return imp_path
|
450
466
|
|
451
467
|
def _loadFragments(self, pkg, fragments, basedir, taskdefs, typedefs):
|
452
468
|
for spec in fragments:
|
@@ -549,6 +565,7 @@ class PackageLoader(object):
|
|
549
565
|
name=self._getScopeFullname(taskdef.name),
|
550
566
|
desc=desc,
|
551
567
|
doc=doc,
|
568
|
+
package=pkg,
|
552
569
|
srcinfo=taskdef.srcinfo)
|
553
570
|
|
554
571
|
if taskdef.iff is not None:
|
@@ -562,7 +579,7 @@ class PackageLoader(object):
|
|
562
579
|
for taskdef, task in tasks:
|
563
580
|
|
564
581
|
if taskdef.uses is not None:
|
565
|
-
task.uses = self.
|
582
|
+
task.uses = self._findTaskOrType(taskdef.uses)
|
566
583
|
|
567
584
|
if task.uses is None:
|
568
585
|
self.error("failed to resolve task-uses %s" % taskdef.uses, taskdef.srcinfo)
|
@@ -592,6 +609,7 @@ class PackageLoader(object):
|
|
592
609
|
raise Exception("Failed to find task %s" % need)
|
593
610
|
task.needs.append(nt)
|
594
611
|
|
612
|
+
# Determine how to implement this task
|
595
613
|
if taskdef.body is not None and len(taskdef.body) > 0:
|
596
614
|
self._mkTaskBody(task, taskdef)
|
597
615
|
elif taskdef.run is not None:
|
@@ -601,7 +619,7 @@ class PackageLoader(object):
|
|
601
619
|
elif taskdef.pytask is not None: # Deprecated case
|
602
620
|
task.run = taskdef.pytask
|
603
621
|
task.shell = "pytask"
|
604
|
-
elif task.uses is not None and task.uses.run is not None:
|
622
|
+
elif task.uses is not None and isinstance(task.uses, Task) and task.uses.run is not None:
|
605
623
|
task.run = task.uses.run
|
606
624
|
task.shell = task.uses.shell
|
607
625
|
|
@@ -628,12 +646,14 @@ class PackageLoader(object):
|
|
628
646
|
tt.paramT = self._getParamT(
|
629
647
|
td,
|
630
648
|
tt.uses.paramT if tt.uses is not None else None,
|
631
|
-
typename=tt.name
|
649
|
+
typename=tt.name,
|
650
|
+
is_type=True)
|
632
651
|
self._log.debug("<-- _loadTypes")
|
633
652
|
pass
|
634
653
|
|
635
654
|
def _mkTaskBody(self, task, taskdef):
|
636
655
|
self._pkg_s[-1].push_scope(TaskScope(name=taskdef.name))
|
656
|
+
pkg = self.package_scope()
|
637
657
|
|
638
658
|
# Need to add subtasks from 'uses' scope?
|
639
659
|
if task.uses is not None:
|
@@ -653,6 +673,7 @@ class PackageLoader(object):
|
|
653
673
|
name=self._getScopeFullname(td.name),
|
654
674
|
desc=desc,
|
655
675
|
doc=doc,
|
676
|
+
package=pkg.pkg,
|
656
677
|
srcinfo=td.srcinfo)
|
657
678
|
|
658
679
|
if td.iff is not None:
|
@@ -726,6 +747,16 @@ class PackageLoader(object):
|
|
726
747
|
return self._pkg_s[-1].findTask(name)
|
727
748
|
else:
|
728
749
|
return self._loader_scope.findTask(name)
|
750
|
+
|
751
|
+
def _findTaskOrType(self, name):
|
752
|
+
self._log.debug("--> _findTaskOrType %s" % name)
|
753
|
+
uses = self._findTask(name)
|
754
|
+
|
755
|
+
if uses is None:
|
756
|
+
uses = self._findType(name)
|
757
|
+
|
758
|
+
self._log.debug("<-- _findTaskOrType %s (%s)" % (name, ("found" if uses is not None else "not found")))
|
759
|
+
return uses
|
729
760
|
|
730
761
|
|
731
762
|
def _getScopeFullname(self, leaf=None):
|
@@ -735,14 +766,14 @@ class PackageLoader(object):
|
|
735
766
|
# Determine
|
736
767
|
pass
|
737
768
|
|
738
|
-
def _getPTConsumesRundir(self, taskdef : TaskDef, base_t : Task):
|
769
|
+
def _getPTConsumesRundir(self, taskdef : TaskDef, base_t : Union[Task,Type]):
|
739
770
|
self._log.debug("_getPTConsumesRundir %s" % taskdef.name)
|
740
771
|
passthrough = taskdef.passthrough
|
741
772
|
consumes = taskdef.consumes.copy() if isinstance(taskdef.consumes, list) else taskdef.consumes
|
742
773
|
rundir = taskdef.rundir
|
743
774
|
# needs = [] if task.needs is None else task.needs.copy()
|
744
775
|
|
745
|
-
if base_t is not None:
|
776
|
+
if base_t is not None and isinstance(base_t, Task):
|
746
777
|
if passthrough is None:
|
747
778
|
passthrough = base_t.passthrough
|
748
779
|
if consumes is None:
|
@@ -758,7 +789,12 @@ class PackageLoader(object):
|
|
758
789
|
|
759
790
|
return (passthrough, consumes, rundir)
|
760
791
|
|
761
|
-
def _getParamT(
|
792
|
+
def _getParamT(
|
793
|
+
self,
|
794
|
+
taskdef,
|
795
|
+
base_t : BaseModel,
|
796
|
+
typename=None,
|
797
|
+
is_type=False):
|
762
798
|
self._log.debug("--> _getParamT %s" % taskdef.name)
|
763
799
|
# Get the base parameter type (if available)
|
764
800
|
# We will build a new type with updated fields
|
@@ -768,14 +804,16 @@ class PackageLoader(object):
|
|
768
804
|
"int" : int,
|
769
805
|
"float" : float,
|
770
806
|
"bool" : bool,
|
771
|
-
"list" : List
|
807
|
+
"list" : List,
|
808
|
+
"map" : Dict
|
772
809
|
}
|
773
810
|
pdflt_m = {
|
774
811
|
"str" : "",
|
775
812
|
"int" : 0,
|
776
813
|
"float" : 0.0,
|
777
814
|
"bool" : False,
|
778
|
-
"list" : []
|
815
|
+
"list" : [],
|
816
|
+
"map" : {}
|
779
817
|
}
|
780
818
|
|
781
819
|
fields = []
|
@@ -795,6 +833,9 @@ class PackageLoader(object):
|
|
795
833
|
field_m[name] = (f.annotation, getattr(base_o, name))
|
796
834
|
else:
|
797
835
|
self._log.debug("No base type")
|
836
|
+
if is_type:
|
837
|
+
field_m["src"] = (str, "")
|
838
|
+
field_m["seq"] = (int, "")
|
798
839
|
|
799
840
|
for p in taskdef.params.keys():
|
800
841
|
param = taskdef.params[p]
|
@@ -860,6 +901,5 @@ class PackageLoader(object):
|
|
860
901
|
self.marker(marker)
|
861
902
|
|
862
903
|
def marker(self, marker):
|
863
|
-
print("listeners: %d" % len(self.marker_listeners))
|
864
904
|
for l in self.marker_listeners:
|
865
905
|
l(marker)
|
dv_flow/mgr/param_ref_eval.py
CHANGED
@@ -37,6 +37,9 @@ class ParamRefEval(object):
|
|
37
37
|
def set(self, name : str, value : object):
|
38
38
|
self.expr_eval.set(name, value)
|
39
39
|
|
40
|
+
def set_name_resolution(self, ctx: 'NameResolutionContext'):
|
41
|
+
self.expr_eval.set_name_resolution(ctx)
|
42
|
+
|
40
43
|
def eval(self, val : str) -> str:
|
41
44
|
idx = 0
|
42
45
|
|
@@ -51,8 +54,8 @@ class ParamRefEval(object):
|
|
51
54
|
|
52
55
|
ref = val[idx+3:eidx].strip()
|
53
56
|
|
54
|
-
expr_ast = self.parser.parse(ref)
|
55
|
-
exp_val = self.expr_eval.eval(
|
57
|
+
# expr_ast = self.parser.parse(ref)
|
58
|
+
exp_val = self.expr_eval.eval(ref)
|
56
59
|
|
57
60
|
# Replacing [idx..eidx+2] with len(exp_val)
|
58
61
|
val = val[:idx] + exp_val + val[eidx+2:]
|
dv_flow/mgr/std/create_file.py
CHANGED
@@ -76,6 +76,9 @@ async def CreateFile(runner, input) -> TaskDataResult:
|
|
76
76
|
src=input.name,
|
77
77
|
basedir=input.rundir,
|
78
78
|
files=[input.params.filename])
|
79
|
+
|
80
|
+
if input.params.incdir:
|
81
|
+
fs.incdirs = [input.rundir]
|
79
82
|
|
80
83
|
_log.debug("<-- FileSet(%s) changed=%s" % (input.name, changed))
|
81
84
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
from ..task_data import TaskDataInput, TaskDataResult
|
2
|
+
from ..task_run_ctxt import TaskRunCtxt
|
3
|
+
|
4
|
+
async def DataItem(ctxt : TaskRunCtxt, input : TaskDataInput) -> TaskDataResult:
|
5
|
+
status = 0
|
6
|
+
output = []
|
7
|
+
|
8
|
+
print("DataItem: %s" % input.params)
|
9
|
+
|
10
|
+
with_c = getattr(input.params, "with")
|
11
|
+
item = ctxt.mkDataItem(type=input.params.type, **with_c)
|
12
|
+
|
13
|
+
return TaskDataResult(
|
14
|
+
status=status,
|
15
|
+
output=output
|
16
|
+
)
|
17
|
+
|
dv_flow/mgr/std/flow.dv
CHANGED
@@ -82,6 +82,9 @@ package:
|
|
82
82
|
type: str
|
83
83
|
content:
|
84
84
|
type: str
|
85
|
+
incdir:
|
86
|
+
type: bool
|
87
|
+
value: false
|
85
88
|
- name: IncDirs
|
86
89
|
shell: pytask
|
87
90
|
run: dv_flow.mgr.std.incdirs.IncDirs
|
@@ -116,6 +119,18 @@ package:
|
|
116
119
|
doc: |
|
117
120
|
Optional timestamp file to determine if running
|
118
121
|
the command changed the output
|
122
|
+
|
123
|
+
- name: DataItem
|
124
|
+
shell: pytask
|
125
|
+
run: dv_flow.mgr.std.data_item.DataItem
|
126
|
+
doc: |
|
127
|
+
Creates and outputs a data item
|
128
|
+
with:
|
129
|
+
type:
|
130
|
+
type: str
|
131
|
+
value: ""
|
132
|
+
with:
|
133
|
+
type: map
|
119
134
|
types:
|
120
135
|
- name: DataItem
|
121
136
|
with:
|
dv_flow/mgr/task.py
CHANGED
@@ -23,6 +23,7 @@ class Task(object):
|
|
23
23
|
doc : str = ""
|
24
24
|
paramT : Any = None
|
25
25
|
uses : 'Task' = None
|
26
|
+
package : 'Package' = None
|
26
27
|
iff : str = None
|
27
28
|
needs : List[str] = dc.field(default_factory=list)
|
28
29
|
consumes : Union[ConsumesE, List[Dict[str, Any]]] = dc.field(default=None)
|
@@ -28,22 +28,19 @@ from .package import Package
|
|
28
28
|
from .package_def import PackageDef, PackageSpec
|
29
29
|
from .package_loader import PackageLoader
|
30
30
|
from .param_ref_eval import ParamRefEval
|
31
|
+
from .name_resolution import NameResolutionContext, TaskNameResolutionScope
|
31
32
|
from .ext_rgy import ExtRgy
|
32
33
|
from .task import Task
|
33
34
|
from .task_def import RundirE
|
34
35
|
from .task_data import TaskMarker, TaskMarkerLoc, SeverityE
|
35
36
|
from .task_node import TaskNode
|
36
37
|
from .task_node_ctor import TaskNodeCtor
|
37
|
-
from .task_node_ctor_compound import TaskNodeCtorCompound
|
38
|
-
from .task_node_ctor_compound_proxy import TaskNodeCtorCompoundProxy
|
39
|
-
from .task_node_ctor_proxy import TaskNodeCtorProxy
|
40
|
-
from .task_node_ctor_task import TaskNodeCtorTask
|
41
|
-
from .task_node_ctor_wrapper import TaskNodeCtorWrapper
|
42
38
|
from .task_node_compound import TaskNodeCompound
|
43
39
|
from .task_node_ctxt import TaskNodeCtxt
|
44
40
|
from .task_node_leaf import TaskNodeLeaf
|
45
41
|
from .type import Type
|
46
42
|
from .std.task_null import TaskNull
|
43
|
+
from .data_callable import DataCallable
|
47
44
|
from .exec_callable import ExecCallable
|
48
45
|
from .null_callable import NullCallable
|
49
46
|
from .shell_callable import ShellCallable
|
@@ -80,6 +77,7 @@ class TaskGraphBuilder(object):
|
|
80
77
|
_ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
|
81
78
|
_compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
|
82
79
|
_task_rundir_s : List[List[str]] = dc.field(default_factory=list)
|
80
|
+
_name_resolution_stack : List[NameResolutionContext] = dc.field(default_factory=list)
|
83
81
|
_task_node_s : List[TaskNode] = dc.field(default_factory=list)
|
84
82
|
_eval : ParamRefEval = dc.field(default_factory=ParamRefEval)
|
85
83
|
_ctxt : TaskNodeCtxt = None
|
@@ -136,6 +134,8 @@ class TaskGraphBuilder(object):
|
|
136
134
|
|
137
135
|
def _addPackageTasks(self, pkg, pkg_s):
|
138
136
|
|
137
|
+
self._pkg_m[pkg.name] = pkg
|
138
|
+
|
139
139
|
# Build out the package parameters
|
140
140
|
params = pkg.paramT()
|
141
141
|
self._expandParams(params, self._eval)
|
@@ -265,40 +265,81 @@ class TaskGraphBuilder(object):
|
|
265
265
|
def mkTaskGraph(self, task : str, rundir=None) -> TaskNode:
|
266
266
|
return self.mkTaskNode(task, rundir=rundir)
|
267
267
|
|
268
|
+
def push_name_resolution_context(self, pkg: Package):
|
269
|
+
"""Create and push a new name resolution context"""
|
270
|
+
ctx = NameResolutionContext(
|
271
|
+
builder=self,
|
272
|
+
package=pkg)
|
273
|
+
self._name_resolution_stack.append(ctx)
|
274
|
+
|
275
|
+
def pop_name_resolution_context(self):
|
276
|
+
"""Pop the current name resolution context"""
|
277
|
+
if self._name_resolution_stack:
|
278
|
+
self._name_resolution_stack.pop()
|
279
|
+
|
280
|
+
def push_task_scope(self, task: TaskNode):
|
281
|
+
"""Push a new task scope onto the current context"""
|
282
|
+
if self._name_resolution_stack:
|
283
|
+
scope = TaskNameResolutionScope(task=task)
|
284
|
+
# Add task parameters as 'this' in the scope's variables
|
285
|
+
if isinstance(task, TaskNodeCompound):
|
286
|
+
scope.variables['this'] = task.params
|
287
|
+
self._name_resolution_stack[-1].task_scopes.append(scope)
|
288
|
+
|
289
|
+
def task_scope(self):
|
290
|
+
"""Get the current task scope"""
|
291
|
+
if self._name_resolution_stack and self._name_resolution_stack[-1].task_scopes:
|
292
|
+
return self._name_resolution_stack[-1].task_scopes[-1]
|
293
|
+
return None
|
294
|
+
|
295
|
+
def pop_task_scope(self):
|
296
|
+
"""Pop the current task scope"""
|
297
|
+
if self._name_resolution_stack and self._name_resolution_stack[-1].task_scopes:
|
298
|
+
self._name_resolution_stack[-1].task_scopes.pop()
|
299
|
+
|
300
|
+
def resolve_variable(self, name: str) -> Any:
|
301
|
+
"""Resolve a variable using the current name resolution context"""
|
302
|
+
if self._name_resolution_stack:
|
303
|
+
ret = self._name_resolution_stack[-1].resolve_variable(name)
|
304
|
+
return ret
|
305
|
+
|
268
306
|
def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
|
269
307
|
self._log.debug("--> mkTaskNode: %s" % task_t)
|
270
308
|
|
271
309
|
if task_t in self._task_m.keys():
|
272
310
|
task = self._task_m[task_t]
|
311
|
+
# Get the package for this task
|
312
|
+
self.push_name_resolution_context(task.package)
|
273
313
|
elif self.loader is not None:
|
274
314
|
task = self.loader.getTask(task_t)
|
275
|
-
|
276
315
|
if task is None:
|
277
316
|
raise Exception("task_t (%s) not present" % str(task_t))
|
317
|
+
self.push_name_resolution_context(task.package)
|
278
318
|
else:
|
279
319
|
raise Exception("task_t (%s) not present" % str(task_t))
|
280
|
-
|
281
|
-
# TODO: need to obtain the package
|
282
320
|
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
321
|
+
try:
|
322
|
+
ret = self._mkTaskNode(
|
323
|
+
task,
|
324
|
+
name=name,
|
325
|
+
srcdir=srcdir,
|
326
|
+
eval=self._eval)
|
289
327
|
|
290
|
-
|
291
|
-
|
292
|
-
|
328
|
+
if needs is not None:
|
329
|
+
for need in needs:
|
330
|
+
ret.needs.append((need, False))
|
293
331
|
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
332
|
+
for k,v in kwargs.items():
|
333
|
+
if hasattr(ret.params, k):
|
334
|
+
setattr(ret.params, k, v)
|
335
|
+
else:
|
336
|
+
raise Exception("Task %s parameters do not include %s" % (task.name, k))
|
299
337
|
|
300
|
-
|
301
|
-
|
338
|
+
self._log.debug("<-- mkTaskNode: %s (%d needs)" % (task_t, len(ret.needs)))
|
339
|
+
return ret
|
340
|
+
finally:
|
341
|
+
# Clean up package context if we created one
|
342
|
+
self.pop_name_resolution_context()
|
302
343
|
|
303
344
|
def mkDataItem(self, name, **kwargs):
|
304
345
|
self._log.debug("--> mkDataItem: %s" % name)
|
@@ -448,10 +489,13 @@ class TaskGraphBuilder(object):
|
|
448
489
|
return ret
|
449
490
|
|
450
491
|
def _isCompound(self, task):
|
451
|
-
if task
|
452
|
-
|
453
|
-
|
454
|
-
|
492
|
+
if isinstance(task, Task):
|
493
|
+
if task.subtasks is not None and len(task.subtasks):
|
494
|
+
return True
|
495
|
+
elif task.uses is not None:
|
496
|
+
return self._isCompound(task.uses)
|
497
|
+
else:
|
498
|
+
return False
|
455
499
|
|
456
500
|
def _getTaskNode(self, name):
|
457
501
|
if name in self._task_node_m.keys():
|
@@ -460,7 +504,8 @@ class TaskGraphBuilder(object):
|
|
460
504
|
return self.mkTaskNode(name)
|
461
505
|
|
462
506
|
def _mkTaskLeafNode(self,
|
463
|
-
task : Task,
|
507
|
+
task : Task,
|
508
|
+
name=None,
|
464
509
|
srcdir=None,
|
465
510
|
params=None,
|
466
511
|
hierarchical=False,
|
@@ -476,19 +521,22 @@ class TaskGraphBuilder(object):
|
|
476
521
|
if params is None:
|
477
522
|
params = task.paramT()
|
478
523
|
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
#
|
488
|
-
|
524
|
+
# Create and push task scope for parameter resolution
|
525
|
+
node = TaskNodeLeaf(
|
526
|
+
name=name,
|
527
|
+
srcdir=srcdir,
|
528
|
+
params=params,
|
529
|
+
ctxt=self._ctxt,
|
530
|
+
passthrough=task.passthrough,
|
531
|
+
consumes=task.consumes,
|
532
|
+
task=None) # We'll set this later
|
533
|
+
|
534
|
+
self.push_task_scope(node)
|
489
535
|
|
490
|
-
|
491
|
-
|
536
|
+
self.task_scope().variables["rundir"] = "/".join([str(e) for e in self.get_rundir()])
|
537
|
+
|
538
|
+
# Now expand parameters in the scope context
|
539
|
+
self._expandParams(params, eval)
|
492
540
|
|
493
541
|
|
494
542
|
if task.rundir == RundirE.Unique:
|
@@ -496,26 +544,29 @@ class TaskGraphBuilder(object):
|
|
496
544
|
|
497
545
|
# TODO: handle callable in light of overrides
|
498
546
|
|
499
|
-
|
500
547
|
callable = None
|
548
|
+
|
501
549
|
if task.run is not None:
|
502
550
|
shell = task.shell if task.shell is not None else "shell"
|
503
551
|
if shell in self._shell_m.keys():
|
504
552
|
self._log.debug("Use shell implementation")
|
505
|
-
callable = self._shell_m[shell]
|
553
|
+
callable = self._shell_m[shell](task.run)
|
506
554
|
else:
|
507
555
|
raise Exception("Shell %s not found" % shell)
|
508
|
-
|
509
|
-
|
556
|
+
|
557
|
+
# Setup the callable
|
558
|
+
if callable is None and task.uses is not None:
|
559
|
+
if isinstance(task.uses, Type):
|
560
|
+
callable = DataCallable(task.uses.paramT)
|
561
|
+
else:
|
562
|
+
uses = self._getTaskNode(task.uses.name)
|
563
|
+
callable = uses.task
|
564
|
+
|
565
|
+
if callable is None:
|
566
|
+
callable = NullCallable(task.run)
|
567
|
+
|
568
|
+
node.task = callable
|
510
569
|
|
511
|
-
node = TaskNodeLeaf(
|
512
|
-
name=name,
|
513
|
-
srcdir=srcdir,
|
514
|
-
params=params,
|
515
|
-
ctxt=self._ctxt,
|
516
|
-
passthrough=task.passthrough,
|
517
|
-
consumes=task.consumes,
|
518
|
-
task=callable(task.run))
|
519
570
|
self._task_node_m[name] = node
|
520
571
|
node.rundir = self.get_rundir()
|
521
572
|
|
@@ -530,6 +581,9 @@ class TaskGraphBuilder(object):
|
|
530
581
|
if task.rundir == RundirE.Unique:
|
531
582
|
self.leave_rundir()
|
532
583
|
|
584
|
+
# Clean up
|
585
|
+
self.pop_task_scope()
|
586
|
+
|
533
587
|
self._log.debug("<-- _mkTaskLeafNode %s" % task.name)
|
534
588
|
return node
|
535
589
|
|
@@ -554,6 +608,13 @@ class TaskGraphBuilder(object):
|
|
554
608
|
# expand any variable references
|
555
609
|
self._expandParams(params, eval)
|
556
610
|
|
611
|
+
# Create a new task scope for this compound task
|
612
|
+
node = TaskNodeCompound(
|
613
|
+
name=name,
|
614
|
+
srcdir=srcdir,
|
615
|
+
params=params,
|
616
|
+
ctxt=self._ctxt)
|
617
|
+
|
557
618
|
if task.rundir == RundirE.Unique:
|
558
619
|
self.enter_rundir(name)
|
559
620
|
|
@@ -566,8 +627,9 @@ class TaskGraphBuilder(object):
|
|
566
627
|
# Determine whether this task is overridden
|
567
628
|
task_uses = self._findOverride(task_uses)
|
568
629
|
|
630
|
+
self.push_task_scope(node) # Push scope before enter_uses
|
569
631
|
self.enter_uses()
|
570
|
-
|
632
|
+
uses_node = self._mkTaskNode(
|
571
633
|
task_uses,
|
572
634
|
name=name,
|
573
635
|
srcdir=srcdir,
|
@@ -576,16 +638,18 @@ class TaskGraphBuilder(object):
|
|
576
638
|
eval=eval)
|
577
639
|
self.leave_uses()
|
578
640
|
|
579
|
-
if not isinstance(
|
641
|
+
if not isinstance(uses_node, TaskNodeCompound):
|
580
642
|
# TODO: need to enclose the leaf node in a compound wrapper
|
581
643
|
raise Exception("Task %s is not compound" % task_uses)
|
644
|
+
|
645
|
+
# Copy properties from uses_node to our node
|
646
|
+
node.tasks = uses_node.tasks
|
647
|
+
node.input = uses_node.input
|
648
|
+
node.needs = uses_node.needs
|
649
|
+
self.pop_task_scope() # Pop the scope
|
582
650
|
else:
|
583
651
|
# Node represents the terminal node of the sub-DAG
|
584
|
-
node
|
585
|
-
name=name,
|
586
|
-
srcdir=srcdir,
|
587
|
-
params=params,
|
588
|
-
ctxt=self._ctxt)
|
652
|
+
self.push_task_scope(node) # Push scope for non-uses compound task
|
589
653
|
|
590
654
|
if len(self._task_node_s):
|
591
655
|
node.parent = self._task_node_s[-1]
|
@@ -617,10 +681,10 @@ class TaskGraphBuilder(object):
|
|
617
681
|
# Need a local map of name -> task
|
618
682
|
# For now, build out local tasks and link up the needs
|
619
683
|
tasks = []
|
684
|
+
|
620
685
|
for t in task.subtasks:
|
621
686
|
nn = self._mkTaskNode(t, hierarchical=True, eval=eval)
|
622
687
|
node.tasks.append(nn)
|
623
|
-
# tasks.append((t, self._getTaskNode(t.name)))
|
624
688
|
tasks.append((t, nn))
|
625
689
|
|
626
690
|
# Pop the node stack, since we're done constructing the body
|
@@ -669,6 +733,10 @@ class TaskGraphBuilder(object):
|
|
669
733
|
if task.rundir == RundirE.Unique:
|
670
734
|
self.leave_rundir()
|
671
735
|
|
736
|
+
# Clean up task scope if we created one for a non-uses compound task
|
737
|
+
if not task.uses:
|
738
|
+
self.pop_task_scope()
|
739
|
+
|
672
740
|
return node
|
673
741
|
|
674
742
|
def _expandParams(self, params, eval):
|
@@ -681,21 +749,25 @@ class TaskGraphBuilder(object):
|
|
681
749
|
new_val = value
|
682
750
|
if type(value) == str:
|
683
751
|
if value.find("${{") != -1:
|
684
|
-
|
752
|
+
if len(self._name_resolution_stack) > 0:
|
753
|
+
eval.set_name_resolution(self._name_resolution_stack[-1])
|
685
754
|
new_val = eval.eval(value)
|
686
755
|
self._log.debug("Param: Evaluate expression \"%s\" => \"%s\"" % (value, new_val))
|
687
756
|
elif isinstance(value, list):
|
688
757
|
new_val = []
|
689
758
|
for i,elem in enumerate(value):
|
690
759
|
if elem.find("${{") != -1:
|
691
|
-
|
760
|
+
if len(self._name_resolution_stack) > 0:
|
761
|
+
eval.set_name_resolution(self._name_resolution_stack[-1])
|
762
|
+
resolved = eval.eval(elem)
|
763
|
+
new_val.append(resolved)
|
692
764
|
else:
|
693
765
|
new_val.append(elem)
|
694
766
|
return new_val
|
695
767
|
|
696
768
|
def _gatherNeeds(self, task_t, node):
|
697
769
|
self._log.debug("--> _gatherNeeds %s (%d)" % (task_t.name, len(task_t.needs)))
|
698
|
-
if task_t.uses is not None:
|
770
|
+
if task_t.uses is not None and isinstance(task_t.uses, Task):
|
699
771
|
self._gatherNeeds(task_t.uses, node)
|
700
772
|
|
701
773
|
for need in task_t.needs:
|
@@ -717,4 +789,3 @@ class TaskGraphBuilder(object):
|
|
717
789
|
|
718
790
|
def _findOverride(self, task):
|
719
791
|
return task
|
720
|
-
|
dv_flow/mgr/task_run_ctxt.py
CHANGED
@@ -27,6 +27,22 @@ class TaskRunCtxt(object):
|
|
27
27
|
@property
|
28
28
|
def root_rundir(self):
|
29
29
|
return self.ctxt.root_rundir
|
30
|
+
|
31
|
+
def mkDataItem(self, type, **kwargs):
|
32
|
+
"""
|
33
|
+
Create a data item in the task's rundir. The data item will be
|
34
|
+
created in the task's rundir, and will be available to the
|
35
|
+
task's implementation.
|
36
|
+
"""
|
37
|
+
try:
|
38
|
+
item = self.runner.mkDataItem(
|
39
|
+
type=type,
|
40
|
+
rundir=self.rundir,
|
41
|
+
**kwargs)
|
42
|
+
except Exception as e:
|
43
|
+
self.error("Failed to create data item: %s" % str(e))
|
44
|
+
raise e
|
45
|
+
|
30
46
|
|
31
47
|
async def exec(self,
|
32
48
|
cmd : List[str],
|
dv_flow/mgr/task_runner.py
CHANGED
@@ -59,6 +59,7 @@ class TaskRunner(object):
|
|
59
59
|
|
60
60
|
@dc.dataclass
|
61
61
|
class TaskSetRunner(TaskRunner):
|
62
|
+
builder : 'TaskGraphBuilder' = None
|
62
63
|
nproc : int = -1
|
63
64
|
status : int = 0
|
64
65
|
|
@@ -186,6 +187,11 @@ class TaskSetRunner(TaskRunner):
|
|
186
187
|
else:
|
187
188
|
return None
|
188
189
|
|
190
|
+
def mkDataItem(self, name, **kwargs):
|
191
|
+
if self.builder is None:
|
192
|
+
raise Exception("TaskSetRunner.mkDataItem() requires a builder")
|
193
|
+
return self.builder.mkDataItem(name, **kwargs)
|
194
|
+
|
189
195
|
def _completeTasks(self, active_task_l, done_task_s, done_l, dst_memento):
|
190
196
|
for d in done_l:
|
191
197
|
for i in range(len(active_task_l)):
|
{dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/RECORD
RENAMED
@@ -1,37 +1,40 @@
|
|
1
|
-
dv_flow/mgr/__init__.py,sha256=
|
1
|
+
dv_flow/mgr/__init__.py,sha256=0V0QkBX5fatUjS-uCcdDJ2UmB1zKt0se6Kpki-C__Ko,1315
|
2
2
|
dv_flow/mgr/__main__.py,sha256=BogNdBkXhgg05E8_IumNkVoag6WwvfbpiI8346oOtPo,3844
|
3
3
|
dv_flow/mgr/cond_def.py,sha256=2ZkzPusqVkN1fFMTvkDl9O_OJLPdD_cK3xzX9J75RMw,343
|
4
|
-
dv_flow/mgr/config.py,sha256=
|
5
|
-
dv_flow/mgr/config_def.py,sha256=
|
4
|
+
dv_flow/mgr/config.py,sha256=b2MVlVVNB0psk8x4bQRAYshkpNJrtyMtV1Ymhmx9AfM,137
|
5
|
+
dv_flow/mgr/config_def.py,sha256=N5nBc_2VPrx0pCDyIe-VPhQ7ItnbyJqrak3ZiPEAbtI,1040
|
6
|
+
dv_flow/mgr/data_callable.py,sha256=-GeMgBUdUEcdAkWsqvUYcpKqUd5BQsFgi6ARflnp2B8,687
|
6
7
|
dv_flow/mgr/eval_jq.py,sha256=bRsHtaN51qIOiZK1VJV52W0-vj5VH0nQ7XIFebZi5kI,1129
|
7
8
|
dv_flow/mgr/exec_callable.py,sha256=NbhTVqvrTd-vsTi3cJ3iGrpVe8tfAqxFsSopaNXApRE,3117
|
8
|
-
dv_flow/mgr/expr_eval.py,sha256=
|
9
|
+
dv_flow/mgr/expr_eval.py,sha256=ojHo3U_QGqm0CWVSEGtFJRo3jrjiTKhy_B58yRNxN8g,4814
|
9
10
|
dv_flow/mgr/expr_parser.py,sha256=LMLe4RTbgcQVIpuZIi1vowHsKRgRmzKzomGvlJL8kJI,7110
|
10
11
|
dv_flow/mgr/ext_rgy.py,sha256=BeHbTjB6KFcI5xhbl_O00YQW7MQ8efCFTD_Y8xf7I44,5651
|
11
12
|
dv_flow/mgr/extend_def.py,sha256=Ndh4z4XqqimtoS8-6ZNMDOvkVpVmEaWZsNEPM26PDZ0,653
|
12
13
|
dv_flow/mgr/fileset.py,sha256=kSjz-RiuQdKGMVjmbGK5YlwY-vJKSHmgggu-1_H0DHU,1320
|
13
14
|
dv_flow/mgr/fragment_def.py,sha256=2qx6KWiHCDvnb9JXnqIOb4n6kemAClAYuBoM6ex6xJE,1696
|
14
15
|
dv_flow/mgr/listener_list.py,sha256=BfqvEO2AyJvyc4ClU-hPgDPqWSlqvSdG-yaFEHvUrMc,272
|
16
|
+
dv_flow/mgr/name_resolution.py,sha256=yeOQuPRWaZRWbEc-82W7qO854RY1qicqC4IYxk8jpvQ,2727
|
15
17
|
dv_flow/mgr/need_def.py,sha256=X52FJnhukwFsGCkIM-W6apZEdxqS5Gmm-rRO-TS83aU,173
|
16
18
|
dv_flow/mgr/null_callable.py,sha256=x6hlJ9EL6xFLSGFgd14eXbxSWxsUGF56jpbLhERjPtg,243
|
17
19
|
dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
|
18
20
|
dv_flow/mgr/package.py,sha256=L1ba5xguOZWuyg9s4MfafKgSoIka3SgAQzYAmcV7XgQ,4430
|
19
21
|
dv_flow/mgr/package_def.py,sha256=1O1SBJATnw-zeYKRNENgw-RWxZK0hxNQhnncfE_NR-U,5674
|
20
22
|
dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
|
21
|
-
dv_flow/mgr/package_loader.py,sha256=
|
23
|
+
dv_flow/mgr/package_loader.py,sha256=HBLhrMgqaZ9xxA9SmQEnNMlo5PK3T8hN3TGvqkRNwYA,33457
|
24
|
+
dv_flow/mgr/package_node.py,sha256=CccD2ECiIXy9JBkRR3c7qDxeBiLiPbRT9CqR-Al_niI,214
|
22
25
|
dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
|
23
26
|
dv_flow/mgr/param_def.py,sha256=9uO-7UI4OotYixQBo8Nwb2o8FQRFxxJylYCZm9rjt48,1917
|
24
|
-
dv_flow/mgr/param_ref_eval.py,sha256=
|
27
|
+
dv_flow/mgr/param_ref_eval.py,sha256=x15lGEJNkWthXXfX3rRTh6tP7Y7A7cmNdo-iOHeOPKw,2179
|
25
28
|
dv_flow/mgr/parser.out,sha256=mWx6hgqORUFhM2OlxV2gp43Y7w3sX2u4ua4t-5wqmck,27566
|
26
29
|
dv_flow/mgr/parsetab.py,sha256=S2o2FfobhhVWKwZg_GA7zFg62VwT7tiRxxNZJIyqP98,4351
|
27
30
|
dv_flow/mgr/pytask_callable.py,sha256=JiFN2vq6qanI6FXJBeidwiBSn6t0oC0izPJ01c5QJTE,688
|
28
31
|
dv_flow/mgr/root_package.py,sha256=nYLOpZVQEDbY0Ui5X5UNeUHXOvqRENew7B2zpC2V2gQ,136
|
29
32
|
dv_flow/mgr/shell_callable.py,sha256=IWb0C4L-v_hfnvnhS3g2rhXqutURex_U-dBJgdNxP7I,302
|
30
33
|
dv_flow/mgr/srcinfo.py,sha256=xrfp-relVr7hYNbxOjY5jqX4H0nNMm9HK5AAM6_Piyk,366
|
31
|
-
dv_flow/mgr/task.py,sha256=
|
34
|
+
dv_flow/mgr/task.py,sha256=n3v9A9HvgMWNeQhXVB_L2qRjgjounXMs9-bbxtyEau8,2088
|
32
35
|
dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
|
33
36
|
dv_flow/mgr/task_def.py,sha256=8NPwtTROfWDkMqcO9mKXV4dw0sC4mCMmnsNuv8uTdTY,5094
|
34
|
-
dv_flow/mgr/task_graph_builder.py,sha256=
|
37
|
+
dv_flow/mgr/task_graph_builder.py,sha256=w3el95hS9vUOb8xKb4lHDdpaP1Z4V0_i3_wYBdybN0c,28171
|
35
38
|
dv_flow/mgr/task_graph_dot_writer.py,sha256=qK4Imy9o2_F1aKoU1tJ-qoBHslq2BhSMbdjAUPfpN7I,6009
|
36
39
|
dv_flow/mgr/task_listener_log.py,sha256=Ai-6X5BOoGsaNTgnlXEW0-czrjJm7__ShNK501CUmko,4337
|
37
40
|
dv_flow/mgr/task_node.py,sha256=3L1qqwisV_qRdbUdRG25uzcsO57T2Z7fbvXYF87Ta1U,5191
|
@@ -47,8 +50,8 @@ dv_flow/mgr/task_node_ctxt.py,sha256=dQp68ppFENK0zGo7TKaFXHInPCCsFai1yZwoBKDwJ20
|
|
47
50
|
dv_flow/mgr/task_node_leaf.py,sha256=od6INBSqMkKZTbCAGJFXzZ3_eD-g7_rt_v3HNa61cQ8,8926
|
48
51
|
dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
|
49
52
|
dv_flow/mgr/task_params_ctor.py,sha256=qlrzibGAFHmbqOu88jEoh1wOSFHu68Gwdgc259-50e8,1915
|
50
|
-
dv_flow/mgr/task_run_ctxt.py,sha256=
|
51
|
-
dv_flow/mgr/task_runner.py,sha256=
|
53
|
+
dv_flow/mgr/task_run_ctxt.py,sha256=Nm3QSB9iZtxwFaaQQ3h2ffU5aStDlO1p1vVKmvL6i8I,3817
|
54
|
+
dv_flow/mgr/task_runner.py,sha256=P94Fn3exSRX6McnSkh-6GpgiWrjjVjJhbe_BpTAL7ws,10240
|
52
55
|
dv_flow/mgr/type.py,sha256=hoJTatlPC0yOazKSWduK-5CfY38RPkc6qXFzOCcVSdM,723
|
53
56
|
dv_flow/mgr/type_def.py,sha256=4sge3PibO1jDnS0cXdX0PiurcKbDA3kT6rb4DGIKwEM,1176
|
54
57
|
dv_flow/mgr/yaml_srcinfo_loader.py,sha256=29BNRiB8Hj1FepkrLtdjHSv5U_85Q432gBeeK80nKEA,1606
|
@@ -56,10 +59,11 @@ dv_flow/mgr/cmds/cmd_graph.py,sha256=yg5KbNrGvm3dySiI0Qei_pMWZkOE9GtEyaQ4IdNVmoE
|
|
56
59
|
dv_flow/mgr/cmds/cmd_run.py,sha256=vjlgpoMPkSCrJErlGSC9DPRCrRmVuUqyLXNKrb8BKW0,3690
|
57
60
|
dv_flow/mgr/cmds/cmd_show.py,sha256=JOIFaVXt8YCf9bKXfq6qzV5wQdxdLHqE-yCj0ecGREs,3755
|
58
61
|
dv_flow/mgr/share/flow.json,sha256=lNmZex9NXkYbyb2aZseQfUOkV9CMyfH0iLODEI7EPBw,5096
|
59
|
-
dv_flow/mgr/std/create_file.py,sha256=
|
62
|
+
dv_flow/mgr/std/create_file.py,sha256=SEpKTQdiY32002C7b4kYfAiK9v_xajixOJU5WftW75I,2957
|
63
|
+
dv_flow/mgr/std/data_item.py,sha256=olKtspEVFOvr92reUl4QaQV27UVyjSGIfIWpsy0Myvo,434
|
60
64
|
dv_flow/mgr/std/exec.py,sha256=UChqa_tAjvdB1NjqTsvlPgFomP8hMsX2rAOPyUonNpk,3896
|
61
65
|
dv_flow/mgr/std/fileset.py,sha256=_9IUjZ2_doygc29Qg1hJPaFPzlaIaTdvHnYTai8mE3A,4183
|
62
|
-
dv_flow/mgr/std/flow.dv,sha256=
|
66
|
+
dv_flow/mgr/std/flow.dv,sha256=vizP3u3yU5iyCmsAeVFeIKy2Ikvhol-_NjCYWOcJCOE,4003
|
63
67
|
dv_flow/mgr/std/incdirs.py,sha256=F0wraOxkRJRkrhsyk9v2YvpquFtcK4ZsMO1G9sgmhmA,588
|
64
68
|
dv_flow/mgr/std/message.py,sha256=0JHLErg8whqMLAasG1fumZ2O7R7WNWeNQ9ibJaLDpVY,1029
|
65
69
|
dv_flow/mgr/std/task_null.py,sha256=dw6LXBXVwth6gLPeduDvlz5znAhcVpDH8r1DticD-0w,1041
|
@@ -69,9 +73,9 @@ dv_flow/mgr/util/util.py,sha256=BO7iqP_c9ttmXkojq7nKDN-g8wl1_Pco9k-KnrXxjwE,1889
|
|
69
73
|
dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
70
74
|
dv_flow/mgr/util/cmds/cmd_schema.py,sha256=IJzZdxCSEgIQ79LpYiM7UqJ9RJ-7yraqmBN2XVgAgXA,1752
|
71
75
|
dv_flow/mgr/util/cmds/cmd_workspace.py,sha256=egmaIXpe5L-TePwmcfisfrG6tdiTUWSjqa9Za5WChVs,890
|
72
|
-
dv_flow_mgr-1.6.
|
73
|
-
dv_flow_mgr-1.6.
|
74
|
-
dv_flow_mgr-1.6.
|
75
|
-
dv_flow_mgr-1.6.
|
76
|
-
dv_flow_mgr-1.6.
|
77
|
-
dv_flow_mgr-1.6.
|
76
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
77
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/METADATA,sha256=qCLOakAO4rA0KSzWyRKZPPqsJD3GPzt4UhRwsNEEBKs,13335
|
78
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
|
79
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
80
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
81
|
+
dv_flow_mgr-1.6.14872991111rc0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
{dv_flow_mgr-1.6.14849587166rc0.dist-info → dv_flow_mgr-1.6.14872991111rc0.dist-info}/top_level.txt
RENAMED
File without changes
|