dv-flow-mgr 0.0.1.13551739897a1__py3-none-any.whl → 0.0.1.13644197178a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/__init__.py +1 -0
- dv_flow/mgr/eval_jq.py +3 -2
- dv_flow/mgr/expr_eval.py +5 -2
- dv_flow/mgr/package_def.py +99 -38
- dv_flow/mgr/param.py +20 -0
- dv_flow/mgr/param_def.py +0 -1
- dv_flow/mgr/param_ref_eval.py +2 -5
- dv_flow/mgr/task_def.py +3 -3
- dv_flow/mgr/task_graph_builder.py +16 -10
- dv_flow/mgr/task_node.py +132 -29
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/METADATA +1 -1
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/RECORD +16 -15
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/WHEEL +1 -1
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/top_level.txt +0 -0
dv_flow/mgr/__init__.py
CHANGED
dv_flow/mgr/eval_jq.py
CHANGED
@@ -8,8 +8,9 @@ def eval_jq(input, args):
|
|
8
8
|
filt = jq.compile(args[0])
|
9
9
|
|
10
10
|
if type(input) == str:
|
11
|
-
ret = filt.input_text(input).
|
11
|
+
ret = filt.input_text(input).text()
|
12
12
|
else:
|
13
|
-
ret = filt.input_value(input).
|
13
|
+
ret = filt.input_value(input).text()
|
14
|
+
|
14
15
|
|
15
16
|
return ret
|
dv_flow/mgr/expr_eval.py
CHANGED
@@ -19,8 +19,11 @@ class ExprEval(ExprVisitor):
|
|
19
19
|
return val
|
20
20
|
|
21
21
|
def _toString(self, val):
|
22
|
-
|
23
|
-
|
22
|
+
rval = val
|
23
|
+
if type(val) != str:
|
24
|
+
obj = self._toObject(val)
|
25
|
+
rval = json.dumps(obj)
|
26
|
+
return rval
|
24
27
|
# if isinstance(val, list):
|
25
28
|
# val = '[' + ",".join(self._toString(v) for v in val) + ']'
|
26
29
|
# elif hasattr(val, "model_dump_json"):
|
dv_flow/mgr/package_def.py
CHANGED
@@ -33,7 +33,7 @@ from typing import Any, Dict, List, Callable, Tuple, ClassVar
|
|
33
33
|
from .fragment_def import FragmentDef
|
34
34
|
from .package import Package
|
35
35
|
from .package_import_spec import PackageImportSpec, PackageSpec
|
36
|
-
|
36
|
+
from .task_node import TaskNodeCtor, TaskNodeCtorProxy, TaskNodeCtorTask
|
37
37
|
from .task_ctor import TaskCtor
|
38
38
|
from .task_def import TaskDef, TaskSpec
|
39
39
|
from .std.task_null import TaskNull
|
@@ -151,20 +151,7 @@ class PackageDef(BaseModel):
|
|
151
151
|
if decl_params:
|
152
152
|
self._log.debug("Type declares new parameters")
|
153
153
|
# We need to combine base parameters with new parameters
|
154
|
-
|
155
|
-
"str" : str,
|
156
|
-
"int" : int,
|
157
|
-
"float" : float,
|
158
|
-
"bool" : bool,
|
159
|
-
"list" : List
|
160
|
-
}
|
161
|
-
pdflt_m = {
|
162
|
-
"str" : "",
|
163
|
-
"int" : 0,
|
164
|
-
"float" : 0.0,
|
165
|
-
"bool" : False,
|
166
|
-
"list" : []
|
167
|
-
}
|
154
|
+
|
168
155
|
for p in task.params.keys():
|
169
156
|
param = task.params[p]
|
170
157
|
self._log.debug("param: %s" % str(param))
|
@@ -221,16 +208,24 @@ class PackageDef(BaseModel):
|
|
221
208
|
|
222
209
|
def mkTaskCtor(self, session, task, srcdir, tasks_m) -> TaskCtor:
|
223
210
|
self._log.debug("--> %s::mkTaskCtor %s (srcdir: %s)" % (self.name, task.name, srcdir))
|
211
|
+
base_ctor_t : TaskCtor = None
|
224
212
|
ctor_t : TaskCtor = None
|
213
|
+
base_params : BaseModel = None
|
214
|
+
callable = None
|
215
|
+
needs = [] if task.needs is None else task.needs.copy()
|
216
|
+
|
217
|
+
if task.uses is not None:
|
218
|
+
base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
|
219
|
+
base_params = base_ctor_t.mkTaskParams()
|
225
220
|
|
226
221
|
# Determine the implementation constructor first
|
227
|
-
if task.
|
222
|
+
if task.pytask is not None:
|
228
223
|
# Built-in impl
|
229
224
|
# Now, lookup the class
|
230
|
-
self._log.debug("Use
|
231
|
-
last_dot = task.
|
232
|
-
clsname = task.
|
233
|
-
modname = task.
|
225
|
+
self._log.debug("Use PyTask implementation")
|
226
|
+
last_dot = task.pytask.rfind('.')
|
227
|
+
clsname = task.pytask[last_dot+1:]
|
228
|
+
modname = task.pytask[:last_dot]
|
234
229
|
|
235
230
|
try:
|
236
231
|
if modname not in sys.modules:
|
@@ -245,26 +240,26 @@ class PackageDef(BaseModel):
|
|
245
240
|
|
246
241
|
if not hasattr(mod, clsname):
|
247
242
|
raise Exception("Class %s not found in module %s" % (clsname, modname))
|
248
|
-
|
249
|
-
|
250
|
-
# Determine if we need to use a new
|
251
|
-
|
252
|
-
if task.uses is not None:
|
253
|
-
uses = self.getTaskCtor(session, task.uses, tasks_m)
|
254
|
-
else:
|
255
|
-
uses = None
|
243
|
+
callable = getattr(mod, clsname)
|
256
244
|
|
257
|
-
|
245
|
+
# Determine if we need to use a new
|
246
|
+
paramT = self._getParamT(task, base_params)
|
247
|
+
|
248
|
+
if callable is not None:
|
249
|
+
ctor_t = TaskNodeCtorTask(
|
258
250
|
name=task.name,
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
251
|
+
srcdir=srcdir,
|
252
|
+
paramT=paramT, # TODO: need to determine the parameter type
|
253
|
+
needs=needs, # TODO: need to determine the needs
|
254
|
+
task=callable)
|
255
|
+
elif base_ctor_t is not None:
|
263
256
|
# Use the existing (base) to create the implementation
|
264
|
-
ctor_t =
|
257
|
+
ctor_t = TaskNodeCtorProxy(
|
265
258
|
name=task.name,
|
266
|
-
|
267
|
-
|
259
|
+
srcdir=srcdir,
|
260
|
+
paramT=paramT, # TODO: need to determine the parameter type
|
261
|
+
needs=needs,
|
262
|
+
uses=base_ctor_t)
|
268
263
|
else:
|
269
264
|
self._log.debug("Use 'Null' as the class implementation")
|
270
265
|
ctor_t = TaskCtorCls(
|
@@ -272,11 +267,77 @@ class PackageDef(BaseModel):
|
|
272
267
|
task_ctor=TaskNull,
|
273
268
|
srcdir=srcdir)
|
274
269
|
|
275
|
-
|
276
|
-
ctor_t.
|
270
|
+
# TODO:
|
271
|
+
# ctor_t = self.handleParams(task, ctor_t)
|
272
|
+
# ctor_t.depends.extend(task.depends)
|
277
273
|
|
278
274
|
self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
|
279
275
|
return ctor_t
|
276
|
+
|
277
|
+
def _getParamT(self, task, base_t : BaseModel):
|
278
|
+
# Get the base parameter type (if available)
|
279
|
+
# We will build a new type with updated fields
|
280
|
+
|
281
|
+
ptype_m = {
|
282
|
+
"str" : str,
|
283
|
+
"int" : int,
|
284
|
+
"float" : float,
|
285
|
+
"bool" : bool,
|
286
|
+
"list" : List
|
287
|
+
}
|
288
|
+
pdflt_m = {
|
289
|
+
"str" : "",
|
290
|
+
"int" : 0,
|
291
|
+
"float" : 0.0,
|
292
|
+
"bool" : False,
|
293
|
+
"list" : []
|
294
|
+
}
|
295
|
+
|
296
|
+
fields = []
|
297
|
+
field_m : Dict[str,int] = {}
|
298
|
+
|
299
|
+
# First, pull out existing fields (if there's a base type)
|
300
|
+
if base_t is not None:
|
301
|
+
self._log.debug("Base type: %s" % str(base_t))
|
302
|
+
for name,f in base_t.model_fields.items():
|
303
|
+
ff : dc.Field = f
|
304
|
+
fields.append(f)
|
305
|
+
field_m[name] = (f.annotation, getattr(base_t, name))
|
306
|
+
else:
|
307
|
+
self._log.debug("No base type")
|
308
|
+
|
309
|
+
for p in task.params.keys():
|
310
|
+
param = task.params[p]
|
311
|
+
self._log.debug("param: %s %s (%s)" % (p, str(param), str(type(param))))
|
312
|
+
if hasattr(param, "type") and param.type is not None:
|
313
|
+
ptype_s = param.type
|
314
|
+
if ptype_s not in ptype_m.keys():
|
315
|
+
raise Exception("Unknown type %s" % ptype_s)
|
316
|
+
ptype = ptype_m[ptype_s]
|
317
|
+
|
318
|
+
if p in field_m.keys():
|
319
|
+
raise Exception("Duplicate field %s" % p)
|
320
|
+
if param.value is not None:
|
321
|
+
field_m[p] = (ptype, param.value)
|
322
|
+
else:
|
323
|
+
field_m[p] = (ptype, pdflt_m[ptype_s])
|
324
|
+
self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
|
325
|
+
else:
|
326
|
+
if p not in field_m.keys():
|
327
|
+
raise Exception("Field %s not found" % p)
|
328
|
+
if type(param) != dict:
|
329
|
+
value = param
|
330
|
+
elif "value" in param.keys():
|
331
|
+
value = param["value"]
|
332
|
+
else:
|
333
|
+
raise Exception("No value specified for param %s: %s" % (
|
334
|
+
p, str(param)))
|
335
|
+
field_m[p] = (field_m[p][0], value)
|
336
|
+
self._log.debug("Set param=%s to %s" % (p, str(field_m[p][1])))
|
337
|
+
|
338
|
+
params_t = pydantic.create_model("Task%sParams" % task.name, **field_m)
|
339
|
+
|
340
|
+
return params_t
|
280
341
|
|
281
342
|
@staticmethod
|
282
343
|
def load(path, exp_pkg_name=None):
|
dv_flow/mgr/param.py
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
|
2
|
+
from pydantic import BaseModel
|
3
|
+
import pydantic.dataclasses as pdc
|
4
|
+
from typing import Any, List, Union
|
5
|
+
|
6
|
+
class ParamMeta(type):
|
7
|
+
def __getitem__(self, T):
|
8
|
+
ret = Union[T, Param]
|
9
|
+
return ret
|
10
|
+
|
11
|
+
class ParamT(metaclass=ParamMeta):
|
12
|
+
pass
|
13
|
+
|
14
|
+
class Param(BaseModel):
|
15
|
+
append : Union[Any,List] = pdc.Field(default=None)
|
16
|
+
prepend : Union[Any,List] = pdc.Field(default=None)
|
17
|
+
append_path : Union[Any,List] = pdc.Field(default=None, alias="append-path")
|
18
|
+
prepend_path : Union[Any,List] = pdc.Field(default=None, alias="prepend-path")
|
19
|
+
|
20
|
+
|
dv_flow/mgr/param_def.py
CHANGED
dv_flow/mgr/param_ref_eval.py
CHANGED
@@ -7,8 +7,8 @@ from .eval_jq import eval_jq
|
|
7
7
|
@dc.dataclass
|
8
8
|
class ParamRefEval(object):
|
9
9
|
|
10
|
-
parser : ExprParser = ExprParser
|
11
|
-
expr_eval : ExprEval = ExprEval
|
10
|
+
parser : ExprParser = dc.field(default_factory=ExprParser)
|
11
|
+
expr_eval : ExprEval = dc.field(default_factory=ExprEval)
|
12
12
|
|
13
13
|
def __post_init__(self):
|
14
14
|
self.expr_eval.methods["jq"] = eval_jq
|
@@ -26,12 +26,9 @@ class ParamRefEval(object):
|
|
26
26
|
raise Exception("unterminated variable ref")
|
27
27
|
|
28
28
|
ref = val[idx+3:eidx].strip()
|
29
|
-
print("ref: %s" % ref)
|
30
29
|
|
31
30
|
expr_ast = self.parser.parse(ref)
|
32
|
-
print("expr_ast: %s" % str(expr_ast))
|
33
31
|
exp_val = self.expr_eval.eval(expr_ast)
|
34
|
-
print("exp_val: %s" % str(exp_val))
|
35
32
|
|
36
33
|
# Replacing [idx..eidx+2] with len(exp_val)
|
37
34
|
val = val[:idx] + exp_val + val[eidx+2:]
|
dv_flow/mgr/task_def.py
CHANGED
@@ -35,11 +35,11 @@ class TaskDef(BaseModel):
|
|
35
35
|
name : str
|
36
36
|
# type : Union[str,TaskSpec] = dc.Field(default_factory=list)
|
37
37
|
uses : str = dc.Field(default=None)
|
38
|
-
|
38
|
+
pytask : str = dc.Field(default=None)
|
39
39
|
desc : str = dc.Field(default="")
|
40
40
|
doc : str = dc.Field(default="")
|
41
|
-
|
42
|
-
params:
|
41
|
+
needs : List[Union[str,TaskSpec]] = dc.Field(default_factory=list, alias="needs")
|
42
|
+
params: Dict[str,Union[str,ParamDef]] = dc.Field(default_factory=dict, alias="with")
|
43
43
|
# out: List[TaskOutput] = dc.Field(default_factory=list)
|
44
44
|
|
45
45
|
def copy(self) -> 'TaskDef':
|
@@ -96,20 +96,26 @@ class TaskGraphBuilder(object):
|
|
96
96
|
|
97
97
|
self._logger.debug("ctor_t: %s" % ctor_t.name)
|
98
98
|
|
99
|
-
|
99
|
+
needs = []
|
100
100
|
|
101
|
-
for dep in ctor_t.depends:
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
101
|
+
# for dep in ctor_t.depends:
|
102
|
+
# if not dep in self._task_m.keys():
|
103
|
+
# task = self._mkTaskGraph(dep, rundir)
|
104
|
+
# self._task_m[dep] = task
|
105
|
+
# pass
|
106
|
+
# depends.append(self._task_m[dep])
|
107
107
|
|
108
108
|
# The returned task should have all param references resolved
|
109
|
-
|
109
|
+
params = ctor_t.mkTaskParams()
|
110
|
+
|
111
|
+
if params is None:
|
112
|
+
raise Exception("ctor %s returned None for params" % str(ctor_t))
|
113
|
+
|
114
|
+
task = ctor_t.mkTaskNode(
|
115
|
+
params=params,
|
110
116
|
name=task_name,
|
111
|
-
|
112
|
-
|
117
|
+
needs=needs)
|
118
|
+
task.rundir = rundir
|
113
119
|
|
114
120
|
self._task_m[task.name] = task
|
115
121
|
|
dv_flow/mgr/task_node.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
import os
|
2
|
+
import sys
|
2
3
|
import dataclasses as dc
|
3
4
|
import pydantic.dataclasses as pdc
|
4
5
|
import logging
|
@@ -6,6 +7,7 @@ from typing import Any, Callable, ClassVar, Dict, List
|
|
6
7
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
7
8
|
from .task_params_ctor import TaskParamsCtor
|
8
9
|
from .param_ref_eval import ParamRefEval
|
10
|
+
from .param import Param
|
9
11
|
|
10
12
|
@dc.dataclass
|
11
13
|
class TaskNode(object):
|
@@ -26,6 +28,10 @@ class TaskNode(object):
|
|
26
28
|
|
27
29
|
_log : ClassVar = logging.getLogger("TaskNode")
|
28
30
|
|
31
|
+
def __post_init__(self):
|
32
|
+
if self.needs is None:
|
33
|
+
self.needs = []
|
34
|
+
|
29
35
|
async def do_run(self,
|
30
36
|
runner,
|
31
37
|
rundir,
|
@@ -45,16 +51,19 @@ class TaskNode(object):
|
|
45
51
|
|
46
52
|
eval.setVar("in", in_params)
|
47
53
|
|
48
|
-
# for attr in dir(self.params):
|
49
|
-
# if not attr.startswith("_"):
|
50
|
-
# print("Attr: %s" % attr)
|
51
54
|
for name,field in self.params.model_fields.items():
|
52
55
|
value = getattr(self.params, name)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
56
|
+
if type(value) == str:
|
57
|
+
if value.find("${{") != -1:
|
58
|
+
new_val = eval.eval(value)
|
59
|
+
setattr(self.params, name, new_val)
|
60
|
+
elif isinstance(value, list):
|
61
|
+
for i,elem in enumerate(value):
|
62
|
+
if elem.find("${{") != -1:
|
63
|
+
new_val = eval.eval(elem)
|
64
|
+
value[i] = new_val
|
65
|
+
else:
|
66
|
+
raise Exception("Unhandled param type: %s" % str(value))
|
58
67
|
|
59
68
|
input = TaskDataInput(
|
60
69
|
changed=changed,
|
@@ -83,12 +92,6 @@ class TaskNode(object):
|
|
83
92
|
def __hash__(self):
|
84
93
|
return id(self)
|
85
94
|
|
86
|
-
@staticmethod
|
87
|
-
def task(paramT):
|
88
|
-
def wrapper(T):
|
89
|
-
ctor = TaskNodeCtorWrapper(T.__name__, T, paramT)
|
90
|
-
return ctor
|
91
|
-
return wrapper
|
92
95
|
|
93
96
|
@dc.dataclass
|
94
97
|
class TaskNodeCtor(object):
|
@@ -98,39 +101,110 @@ class TaskNodeCtor(object):
|
|
98
101
|
- Produces a TaskNode
|
99
102
|
"""
|
100
103
|
name : str
|
104
|
+
srcdir : str
|
105
|
+
paramT : Any
|
101
106
|
|
107
|
+
def getNeeds(self) -> List[str]:
|
108
|
+
return []
|
102
109
|
|
103
|
-
def mkTaskNode(self,
|
110
|
+
def mkTaskNode(self,
|
111
|
+
params,
|
112
|
+
srcdir=None,
|
113
|
+
name=None,
|
114
|
+
needs=None) -> TaskNode:
|
104
115
|
raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
|
105
116
|
|
106
|
-
def mkTaskParams(self, params : Dict) -> Any:
|
107
|
-
|
117
|
+
def mkTaskParams(self, params : Dict = None) -> Any:
|
118
|
+
obj = self.paramT()
|
119
|
+
|
120
|
+
# Apply user-specified params
|
121
|
+
if params is not None:
|
122
|
+
for key,value in params.items():
|
123
|
+
if not hasattr(obj, key):
|
124
|
+
raise Exception("Parameters class %s does not contain field %s" % (
|
125
|
+
str(type(obj)),
|
126
|
+
key))
|
127
|
+
else:
|
128
|
+
if isinstance(value, Param):
|
129
|
+
if value.append is not None:
|
130
|
+
ex_value = getattr(obj, key, [])
|
131
|
+
ex_value.extend(value.append)
|
132
|
+
setattr(obj, key, ex_value)
|
133
|
+
elif value.prepend is not None:
|
134
|
+
ex_value = getattr(obj, key, [])
|
135
|
+
value = value.copy()
|
136
|
+
value.extend(ex_value)
|
137
|
+
setattr(obj, key, value)
|
138
|
+
pass
|
139
|
+
else:
|
140
|
+
raise Exception("Unhandled value spec: %s" % str(value))
|
141
|
+
else:
|
142
|
+
setattr(obj, key, value)
|
143
|
+
return obj
|
144
|
+
|
145
|
+
@dc.dataclass
|
146
|
+
class TaskNodeCtorDefBase(TaskNodeCtor):
|
147
|
+
"""Task defines its own needs, that will need to be filled in"""
|
148
|
+
needs : List['str']
|
149
|
+
|
150
|
+
def __post_init__(self):
|
151
|
+
if self.needs is None:
|
152
|
+
self.needs = []
|
153
|
+
|
154
|
+
def getNeeds(self) -> List[str]:
|
155
|
+
return self.needs
|
156
|
+
|
157
|
+
@dc.dataclass
|
158
|
+
class TaskNodeCtorProxy(TaskNodeCtorDefBase):
|
159
|
+
"""Task has a 'uses' clause, so we delegate creation of the node"""
|
160
|
+
uses : TaskNodeCtor
|
161
|
+
|
162
|
+
def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
|
163
|
+
if srcdir is None:
|
164
|
+
srcdir = self.srcdir
|
165
|
+
node = self.uses.mkTaskNode(params=params, srcdir=srcdir, name=name, needs=needs)
|
166
|
+
return node
|
167
|
+
|
168
|
+
@dc.dataclass
|
169
|
+
class TaskNodeCtorTask(TaskNodeCtorDefBase):
|
170
|
+
task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
|
171
|
+
|
172
|
+
def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
|
173
|
+
if srcdir is None:
|
174
|
+
srcdir = self.srcdir
|
175
|
+
|
176
|
+
node = TaskNode(name, srcdir, params, self.task, needs=needs)
|
177
|
+
node.task = self.task
|
178
|
+
|
179
|
+
return node
|
108
180
|
|
109
181
|
@dc.dataclass
|
110
182
|
class TaskNodeCtorWrapper(TaskNodeCtor):
|
111
183
|
T : Any
|
112
|
-
paramT : Any
|
113
184
|
|
114
185
|
def __call__(self,
|
115
|
-
|
116
|
-
|
117
|
-
params=None,
|
186
|
+
name=None,
|
187
|
+
srcdir=None,
|
188
|
+
params=None,
|
118
189
|
needs=None,
|
119
190
|
**kwargs):
|
120
191
|
"""Convenience method for direct creation of tasks"""
|
121
192
|
if params is None:
|
122
193
|
params = self.mkTaskParams(kwargs)
|
123
194
|
|
124
|
-
node = self.mkTaskNode(
|
125
|
-
|
126
|
-
|
195
|
+
node = self.mkTaskNode(
|
196
|
+
srcdir=srcdir,
|
197
|
+
params=params,
|
198
|
+
name=name,
|
199
|
+
needs=needs)
|
200
|
+
|
127
201
|
return node
|
128
202
|
|
129
|
-
def mkTaskNode(self,
|
130
|
-
node = TaskNode(name, srcdir, params, self.T)
|
203
|
+
def mkTaskNode(self, params, srcdir=None, name=None, needs=None) -> TaskNode:
|
204
|
+
node = TaskNode(name, srcdir, params, self.T, needs=needs)
|
131
205
|
return node
|
132
206
|
|
133
|
-
def mkTaskParams(self, params : Dict) -> Any:
|
207
|
+
def mkTaskParams(self, params : Dict = None) -> Any:
|
134
208
|
obj = self.paramT()
|
135
209
|
|
136
210
|
# Apply user-specified params
|
@@ -140,5 +214,34 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
|
|
140
214
|
str(type(obj)),
|
141
215
|
key))
|
142
216
|
else:
|
143
|
-
|
217
|
+
if isinstance(value, Param):
|
218
|
+
if value.append is not None:
|
219
|
+
ex_value = getattr(obj, key, [])
|
220
|
+
ex_value.extend(value.append)
|
221
|
+
setattr(obj, key, ex_value)
|
222
|
+
elif value.prepend is not None:
|
223
|
+
ex_value = getattr(obj, key, [])
|
224
|
+
value = value.copy()
|
225
|
+
value.extend(ex_value)
|
226
|
+
setattr(obj, key, value)
|
227
|
+
pass
|
228
|
+
else:
|
229
|
+
raise Exception("Unhandled value spec: %s" % str(value))
|
230
|
+
else:
|
231
|
+
setattr(obj, key, value)
|
144
232
|
return obj
|
233
|
+
|
234
|
+
def task(paramT):
|
235
|
+
"""Decorator to wrap a task method as a TaskNodeCtor"""
|
236
|
+
def wrapper(T):
|
237
|
+
task_mname = T.__module__
|
238
|
+
task_module = sys.modules[task_mname]
|
239
|
+
ctor = TaskNodeCtorWrapper(
|
240
|
+
T.__name__,
|
241
|
+
os.path.dirname(os.path.abspath(task_module.__file__)),
|
242
|
+
paramT,
|
243
|
+
T)
|
244
|
+
return ctor
|
245
|
+
return wrapper
|
246
|
+
|
247
|
+
|
{dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/RECORD
RENAMED
@@ -1,29 +1,30 @@
|
|
1
|
-
dv_flow/mgr/__init__.py,sha256=
|
1
|
+
dv_flow/mgr/__init__.py,sha256=_IH_cJws79HRCdn0-Q11WHFuMQYA6oPCZIufeXT-BOc,317
|
2
2
|
dv_flow/mgr/__main__.py,sha256=uik6gDAvtZNG0yyDKYc4FTl6R1QHAR543jNG2lCAa6E,1771
|
3
|
-
dv_flow/mgr/eval_jq.py,sha256=
|
4
|
-
dv_flow/mgr/expr_eval.py,sha256=
|
3
|
+
dv_flow/mgr/eval_jq.py,sha256=Ue-qkyNW6uOu2Wy8u6nrTbPPY7ujaHd0-4iki0eV3Ec,294
|
4
|
+
dv_flow/mgr/expr_eval.py,sha256=Mp0TvrV6gZWcj1uYwLfPNI-ARwwdAfwntE5byLBfBuY,2276
|
5
5
|
dv_flow/mgr/expr_parser.py,sha256=aR8n6hgk5xmFknIaSVC02K-So-eU4pkkFtwb7zWsSPs,5474
|
6
6
|
dv_flow/mgr/fileset.py,sha256=FNvC5sU2ArxJ0OO3v8dXTv8zX-bZ5t0a0ljne0fQQ1o,1150
|
7
7
|
dv_flow/mgr/fragment_def.py,sha256=cyzp1XeWtNOaagScmeS-BPsoXj9j2LTBbKq5ZUioz8I,1641
|
8
8
|
dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
|
9
9
|
dv_flow/mgr/package.py,sha256=878twhPD-E1pFlDNUtuyeFEgJ_Y89b560og4St-Iwrs,1679
|
10
|
-
dv_flow/mgr/package_def.py,sha256=
|
10
|
+
dv_flow/mgr/package_def.py,sha256=m3vnoEcV4jEhOg_ijwYPTplrF4YN5dVJYJhVBTL5AgM,17408
|
11
11
|
dv_flow/mgr/package_import_spec.py,sha256=ah3r15v5Jdub2poc3sgi6Uar1L3oGoYsCPPNiOHV-a4,1760
|
12
|
-
dv_flow/mgr/
|
13
|
-
dv_flow/mgr/
|
12
|
+
dv_flow/mgr/param.py,sha256=3BY-ucig6JRw73FhjyJQL-vpd57qhAzixgZ8I5FoUpw,553
|
13
|
+
dv_flow/mgr/param_def.py,sha256=gLua-EQiY8V2CFX-2svLRIlrs8PEeGh4-EPtn4a2Mng,712
|
14
|
+
dv_flow/mgr/param_ref_eval.py,sha256=U8QhDf1n_9bLnExdc1oQamq5-pOUXrFalOX4oyR9UoM,1138
|
14
15
|
dv_flow/mgr/parsetab.py,sha256=enSOnMQ-woIsMEzHyeYiefvhAl8enxfX9Ct_o8-jkqs,3780
|
15
16
|
dv_flow/mgr/pkg_rgy.py,sha256=2R_EaeBDJn5qUq9DzSnLc37wUP36MWSv-p0LgUjJRAg,4471
|
16
17
|
dv_flow/mgr/task.py,sha256=kLQSvnVwj9ROIDtxq8lLu-4mJizTxOqvUeogmgN6QAA,5976
|
17
18
|
dv_flow/mgr/task_ctor.py,sha256=hlfl-UVvyjzLFN6D0Oel9eBs0xUQPqCX7gQ0uEHoL7o,1382
|
18
19
|
dv_flow/mgr/task_data.py,sha256=9c5NrVZWECHFXrCPOpwZBQXRhr1O3ni5aFeP2I1P5Rw,11398
|
19
|
-
dv_flow/mgr/task_def.py,sha256=
|
20
|
+
dv_flow/mgr/task_def.py,sha256=gOqh-sUE7MKC1fsobFxcO2jlF-jtjBJRFzWB2iDYlZY,1831
|
20
21
|
dv_flow/mgr/task_exec_data.py,sha256=aT__kmVmNxqnS_GbTRS1vrYgKiv8cH-HUSmRb6YYKsI,640
|
21
|
-
dv_flow/mgr/task_graph_builder.py,sha256
|
22
|
+
dv_flow/mgr/task_graph_builder.py,sha256=iprAMw0LVROOxWa563OHg8KTEm4IOS9hZVymjDeCnEo,7243
|
22
23
|
dv_flow/mgr/task_graph_runner.py,sha256=jUGI49QvxUCfQoKQDDk2psbeapIcCg72qNOW1JipHzM,2182
|
23
24
|
dv_flow/mgr/task_graph_runner_local.py,sha256=OrydPwtQ8E7hYWvSXx0h7lI3nfUNFyklULhsyMwz9dA,4687
|
24
25
|
dv_flow/mgr/task_impl_data.py,sha256=bFPijoKrh9x7fZN2DsvRJp0UHo-gGM0VjtDQISyfhFk,321
|
25
26
|
dv_flow/mgr/task_memento.py,sha256=C7VTQpBhDEoYuDmE6YTM-6TLMLnqHp6Y0Vat1aTgtCs,1096
|
26
|
-
dv_flow/mgr/task_node.py,sha256=
|
27
|
+
dv_flow/mgr/task_node.py,sha256=J_i53KntyljY7gwSqBq-lK_Gbu0jryNp4Xq_jUsa0PA,7903
|
27
28
|
dv_flow/mgr/task_output.py,sha256=l-W-FvVo6YDah1RQS-I9N0KUtB3vp-kl7lxIdmNz0l4,178
|
28
29
|
dv_flow/mgr/task_params_ctor.py,sha256=aXgB8o9xFPjaEjGW_xYkEC0N0apzGzGUPDj7g2ZLvus,1112
|
29
30
|
dv_flow/mgr/task_runner.py,sha256=JmerE6CyXiJJK7-n6P65OzwW9UTYFQqV8Cck7eHe0B8,3882
|
@@ -36,9 +37,9 @@ dv_flow/mgr/std/fileset.py,sha256=uP7bGntRq-Tn5_GEFnt0_J_OAmfvep3GlCwCuE8by4o,27
|
|
36
37
|
dv_flow/mgr/std/flow.dv,sha256=j9wLrF3Ghh1ZLbJxmk7WiNiRYUYEer-8CCUA5hsgtfk,1409
|
37
38
|
dv_flow/mgr/std/message.py,sha256=BPTHnEMD4tBufQ9LvsS9Sa_0xjaJATbBpwqosWslvVA,193
|
38
39
|
dv_flow/mgr/std/task_null.py,sha256=KObmjG_4D08GJ1k6neqKIQrFY72Sj0jLnwXxEkq5HA0,321
|
39
|
-
dv_flow_mgr-0.0.1.
|
40
|
-
dv_flow_mgr-0.0.1.
|
41
|
-
dv_flow_mgr-0.0.1.
|
42
|
-
dv_flow_mgr-0.0.1.
|
43
|
-
dv_flow_mgr-0.0.1.
|
44
|
-
dv_flow_mgr-0.0.1.
|
40
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
41
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/METADATA,sha256=yOdqkLx8LGFA6PcCcaQ3eMIwC3_lM0iup5Tx0iEra4U,13276
|
42
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
|
43
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
44
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
45
|
+
dv_flow_mgr-0.0.1.13644197178a1.dist-info/RECORD,,
|
{dv_flow_mgr-0.0.1.13551739897a1.dist-info → dv_flow_mgr-0.0.1.13644197178a1.dist-info}/LICENSE
RENAMED
File without changes
|
File without changes
|
File without changes
|