dv-flow-mgr 0.0.1.14116344455a1__py3-none-any.whl → 0.0.1.14147547123a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/package_def.py +29 -20
- dv_flow/mgr/task_graph_builder.py +27 -5
- dv_flow/mgr/task_node.py +5 -3
- dv_flow/mgr/task_node_compound.py +47 -4
- dv_flow/mgr/task_node_ctor_compound.py +7 -3
- dv_flow/mgr/task_node_ctor_compound_proxy.py +3 -0
- dv_flow/mgr/task_node_ctor_def_base.py +2 -3
- dv_flow/mgr/task_node_ctor_proxy.py +0 -3
- dv_flow/mgr/task_node_ctor_task.py +5 -4
- dv_flow/mgr/task_node_ctor_wrapper.py +12 -3
- dv_flow/mgr/task_node_leaf.py +3 -1
- dv_flow/mgr/task_runner.py +39 -34
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/METADATA +1 -1
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/RECORD +18 -18
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/WHEEL +0 -0
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/licenses/LICENSE +0 -0
- {dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/top_level.txt +0 -0
dv_flow/mgr/package_def.py
CHANGED
@@ -33,7 +33,7 @@ from .fragment_def import FragmentDef
|
|
33
33
|
from .package import Package
|
34
34
|
from .package_import_spec import PackageImportSpec, PackageSpec
|
35
35
|
from .param_def import ParamDef
|
36
|
-
from .task_def import TaskDef, PassthroughE, ConsumesE
|
36
|
+
from .task_def import TaskDef, PassthroughE, ConsumesE, RundirE
|
37
37
|
from .task_node_ctor import TaskNodeCtor
|
38
38
|
from .task_node_ctor_proxy import TaskNodeCtorProxy
|
39
39
|
from .task_node_ctor_task import TaskNodeCtorTask
|
@@ -182,30 +182,21 @@ class PackageDef(BaseModel):
|
|
182
182
|
ctor_t : TaskNodeCtor = None
|
183
183
|
base_params : BaseModel = None
|
184
184
|
callable = None
|
185
|
-
passthrough = task.passthrough
|
186
|
-
consumes = task.consumes.copy() if isinstance(task.consumes, list) else task.consumes
|
187
|
-
needs = [] if task.needs is None else task.needs.copy()
|
188
185
|
fullname = self.name + "." + task.name
|
186
|
+
rundir = task.rundir
|
189
187
|
|
190
188
|
if task.uses is not None:
|
191
189
|
self._log.debug("Uses: %s" % task.uses)
|
192
190
|
base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
|
193
191
|
base_params = base_ctor_t.mkTaskParams()
|
194
192
|
|
195
|
-
if passthrough is None:
|
196
|
-
passthrough = base_ctor_t.passthrough
|
197
|
-
if consumes is None:
|
198
|
-
consumes = base_ctor_t.consumes
|
199
193
|
|
200
194
|
if base_ctor_t is None:
|
201
195
|
self._log.error("Failed to load task ctor %s" % task.uses)
|
202
196
|
else:
|
203
197
|
self._log.debug("No 'uses' specified")
|
204
198
|
|
205
|
-
|
206
|
-
passthrough = PassthroughE.No
|
207
|
-
if consumes is None:
|
208
|
-
consumes = ConsumesE.All
|
199
|
+
passthrough, consumes, needs = self._getPTConsumesNeeds(task, base_ctor_t)
|
209
200
|
|
210
201
|
# Determine the implementation constructor first
|
211
202
|
if task.pytask is not None:
|
@@ -242,7 +233,8 @@ class PackageDef(BaseModel):
|
|
242
233
|
passthrough=passthrough,
|
243
234
|
consumes=consumes,
|
244
235
|
needs=needs, # TODO: need to determine the needs
|
245
|
-
task=callable
|
236
|
+
task=callable,
|
237
|
+
rundir=rundir)
|
246
238
|
elif base_ctor_t is not None:
|
247
239
|
# Use the existing (base) to create the implementation
|
248
240
|
ctor_t = TaskNodeCtorProxy(
|
@@ -252,6 +244,7 @@ class PackageDef(BaseModel):
|
|
252
244
|
passthrough=passthrough,
|
253
245
|
consumes=consumes,
|
254
246
|
needs=needs,
|
247
|
+
rundir=rundir,
|
255
248
|
uses=base_ctor_t)
|
256
249
|
else:
|
257
250
|
self._log.debug("Use 'Null' as the class implementation")
|
@@ -262,6 +255,7 @@ class PackageDef(BaseModel):
|
|
262
255
|
passthrough=passthrough,
|
263
256
|
consumes=consumes,
|
264
257
|
needs=needs,
|
258
|
+
rundir=rundir,
|
265
259
|
task=TaskNull)
|
266
260
|
|
267
261
|
self._log.debug("<-- %s::mkTaskCtor %s" % (self.name, task.name))
|
@@ -273,9 +267,7 @@ class PackageDef(BaseModel):
|
|
273
267
|
ctor_t : TaskNodeCtor = None
|
274
268
|
base_params : BaseModel = None
|
275
269
|
callable = None
|
276
|
-
|
277
|
-
consumes = [] if task.consumes is None else task.consumes.copy()
|
278
|
-
needs = [] if task.needs is None else task.needs.copy()
|
270
|
+
|
279
271
|
fullname = self.name + "." + task.name
|
280
272
|
|
281
273
|
|
@@ -284,13 +276,11 @@ class PackageDef(BaseModel):
|
|
284
276
|
base_ctor_t = self.getTaskCtor(session, task.uses, tasks_m)
|
285
277
|
base_params = base_ctor_t.mkTaskParams()
|
286
278
|
|
287
|
-
# Once we have passthrough, we can't turn it off
|
288
|
-
passthrough |= base_ctor_t.passthrough
|
289
|
-
consumes.extend(base_ctor_t.consumes)
|
290
|
-
|
291
279
|
if base_ctor_t is None:
|
292
280
|
self._log.error("Failed to load task ctor %s" % task.uses)
|
293
281
|
|
282
|
+
passthrough, consumes, needs = self._getPTConsumesNeeds(task, base_ctor_t)
|
283
|
+
|
294
284
|
# Determine if we need to use a new
|
295
285
|
paramT = self._getParamT(session, task, base_params)
|
296
286
|
|
@@ -321,6 +311,25 @@ class PackageDef(BaseModel):
|
|
321
311
|
|
322
312
|
self._log.debug("<-- %s::mkTaskCtor %s (%d)" % (self.name, task.name, len(ctor_t.tasks)))
|
323
313
|
return ctor_t
|
314
|
+
|
315
|
+
def _getPTConsumesNeeds(self, task, base_ctor_t):
|
316
|
+
passthrough = task.passthrough
|
317
|
+
consumes = task.consumes.copy() if isinstance(task.consumes, list) else task.consumes
|
318
|
+
needs = [] if task.needs is None else task.needs.copy()
|
319
|
+
|
320
|
+
if self.uses is not None:
|
321
|
+
if passthrough is None:
|
322
|
+
passthrough = base_ctor_t.passthrough
|
323
|
+
if consumes is None:
|
324
|
+
consumes = base_ctor_t.consumes
|
325
|
+
|
326
|
+
if passthrough is None:
|
327
|
+
passthrough = PassthroughE.No
|
328
|
+
if consumes is None:
|
329
|
+
consumes = ConsumesE.All
|
330
|
+
|
331
|
+
|
332
|
+
return (passthrough, consumes, needs)
|
324
333
|
|
325
334
|
def _getParamT(self, session, task, base_t : BaseModel):
|
326
335
|
self._log.debug("--> _getParamT %s" % task.fullname)
|
@@ -25,6 +25,7 @@ import logging
|
|
25
25
|
from .package import Package
|
26
26
|
from .package_def import PackageDef, PackageSpec
|
27
27
|
from .pkg_rgy import PkgRgy
|
28
|
+
from .task_def import RundirE
|
28
29
|
from .task_node import TaskNode
|
29
30
|
from .task_node_ctor import TaskNodeCtor
|
30
31
|
from typing import Dict, List, Union
|
@@ -37,6 +38,7 @@ class TaskNamespaceScope(object):
|
|
37
38
|
class CompoundTaskCtxt(object):
|
38
39
|
parent : 'TaskGraphBuilder'
|
39
40
|
task : 'TaskNode'
|
41
|
+
rundir : RundirE
|
40
42
|
task_m : Dict[str,TaskNode] = dc.field(default_factory=dict)
|
41
43
|
uses_s : List[Dict[str, TaskNode]] = dc.field(default_factory=list)
|
42
44
|
|
@@ -53,6 +55,7 @@ class TaskGraphBuilder(object):
|
|
53
55
|
_override_m : Dict[str,str] = dc.field(default_factory=dict)
|
54
56
|
_ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
|
55
57
|
_compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
|
58
|
+
_rundir_s : List[str] = dc.field(default_factory=list)
|
56
59
|
_uses_count : int = 0
|
57
60
|
|
58
61
|
_logger : logging.Logger = None
|
@@ -97,7 +100,7 @@ class TaskGraphBuilder(object):
|
|
97
100
|
for subpkg in pkg.subpkg_m.values():
|
98
101
|
self._registerPackages(subpkg, visited)
|
99
102
|
|
100
|
-
|
103
|
+
|
101
104
|
def push_package(self, pkg : Package, add=False):
|
102
105
|
self._pkg_s.append(pkg)
|
103
106
|
if add:
|
@@ -109,6 +112,18 @@ class TaskGraphBuilder(object):
|
|
109
112
|
def package(self):
|
110
113
|
return self._pkg_s[-1]
|
111
114
|
|
115
|
+
def enter_rundir(self, rundir : str):
|
116
|
+
self._rundir_s.append(rundir)
|
117
|
+
|
118
|
+
def get_rundir(self, rundir=None):
|
119
|
+
ret = self._rundir_s.copy()
|
120
|
+
if rundir is not None:
|
121
|
+
ret.append(rundir)
|
122
|
+
return ret
|
123
|
+
|
124
|
+
def leave_rundir(self):
|
125
|
+
self._rundir_s.pop()
|
126
|
+
|
112
127
|
def enter_uses(self):
|
113
128
|
self._uses_count += 1
|
114
129
|
|
@@ -117,9 +132,13 @@ class TaskGraphBuilder(object):
|
|
117
132
|
|
118
133
|
def leave_uses(self):
|
119
134
|
self._uses_count -= 1
|
120
|
-
|
121
|
-
def enter_compound(self, task : TaskNode):
|
122
|
-
self._compound_task_ctxt_s.append(CompoundTaskCtxt(
|
135
|
+
|
136
|
+
def enter_compound(self, task : TaskNode, rundir=None):
|
137
|
+
self._compound_task_ctxt_s.append(CompoundTaskCtxt(
|
138
|
+
parent=self, task=task, rundir=rundir))
|
139
|
+
|
140
|
+
if rundir is None or rundir == RundirE.Unique:
|
141
|
+
self._rundir_s.append(task.name)
|
123
142
|
|
124
143
|
def get_name_prefix(self):
|
125
144
|
if len(self._compound_task_ctxt_s) > 0:
|
@@ -151,6 +170,7 @@ class TaskGraphBuilder(object):
|
|
151
170
|
|
152
171
|
def addTask(self, name, task : TaskNode):
|
153
172
|
self._logger.debug("--> addTask: %s" % name)
|
173
|
+
|
154
174
|
if len(self._compound_task_ctxt_s) == 0:
|
155
175
|
self._task_m[name] = task
|
156
176
|
else:
|
@@ -179,7 +199,9 @@ class TaskGraphBuilder(object):
|
|
179
199
|
return task
|
180
200
|
|
181
201
|
def leave_compound(self, task : TaskNode):
|
182
|
-
self._compound_task_ctxt_s.pop()
|
202
|
+
ctxt = self._compound_task_ctxt_s.pop()
|
203
|
+
if ctxt.rundir is None or ctxt.rundir == RundirE.Unique:
|
204
|
+
self._rundir_s.pop()
|
183
205
|
|
184
206
|
def mkTaskGraph(self, task : str) -> TaskNode:
|
185
207
|
self._pkg_s.clear()
|
dv_flow/mgr/task_node.py
CHANGED
@@ -48,8 +48,8 @@ class TaskNode(object):
|
|
48
48
|
passthrough : bool = False
|
49
49
|
consumes : List[Any] = dc.field(default_factory=list)
|
50
50
|
needs : List[Tuple['TaskNode',bool]] = dc.field(default_factory=list)
|
51
|
-
rundir : str = dc.field(default=None)
|
52
|
-
rundir_t : RundirE = dc.field(default=RundirE.Unique)
|
51
|
+
rundir : List[str] = dc.field(default=None)
|
52
|
+
# rundir_t : RundirE = dc.field(default=RundirE.Unique)
|
53
53
|
output : TaskDataOutput = dc.field(default=None)
|
54
54
|
result : TaskDataResult = dc.field(default=None)
|
55
55
|
start : float = dc.field(default=None)
|
@@ -58,6 +58,8 @@ class TaskNode(object):
|
|
58
58
|
_log : ClassVar = logging.getLogger("TaskNode")
|
59
59
|
|
60
60
|
def __post_init__(self):
|
61
|
+
# Give this a default run directory based on the task name
|
62
|
+
self.rundir = [self.name]
|
61
63
|
if self.needs is None:
|
62
64
|
self.needs = []
|
63
65
|
else:
|
@@ -69,7 +71,7 @@ class TaskNode(object):
|
|
69
71
|
runner,
|
70
72
|
rundir,
|
71
73
|
memento : Any = None) -> 'TaskDataResult':
|
72
|
-
|
74
|
+
raise NotImplementedError("do_run not implemented for %s" % self.__class__.__name__)
|
73
75
|
|
74
76
|
def __hash__(self):
|
75
77
|
return id(self)
|
@@ -20,12 +20,17 @@
|
|
20
20
|
#*
|
21
21
|
#****************************************************************************
|
22
22
|
import dataclasses as dc
|
23
|
+
from pydantic import BaseModel
|
24
|
+
from .task_def import ConsumesE
|
23
25
|
from .task_node import TaskNode
|
24
26
|
from .task_node_leaf import TaskNodeLeaf
|
25
27
|
from .task_data import TaskDataResult, TaskDataInput, TaskDataOutput
|
26
28
|
from .task_runner import TaskRunner
|
27
29
|
from typing import Any, List
|
28
30
|
|
31
|
+
class NullParams(BaseModel):
|
32
|
+
pass
|
33
|
+
|
29
34
|
@dc.dataclass
|
30
35
|
class TaskNodeCompound(TaskNode):
|
31
36
|
"""A Compound task node is the 'out' node in the subgraph"""
|
@@ -33,17 +38,55 @@ class TaskNodeCompound(TaskNode):
|
|
33
38
|
input : TaskNode = None
|
34
39
|
|
35
40
|
def __post_init__(self):
|
41
|
+
async def null_run(runner, input):
|
42
|
+
return TaskDataResult()
|
43
|
+
|
36
44
|
self.input = TaskNodeLeaf(
|
37
|
-
self.name + ".in",
|
45
|
+
name=self.name + ".in",
|
38
46
|
srcdir=self.srcdir,
|
39
|
-
params=
|
47
|
+
params=NullParams())
|
48
|
+
self.input.task = null_run
|
40
49
|
return super().__post_init__()
|
41
50
|
|
42
51
|
async def do_run(self,
|
43
52
|
runner : TaskRunner,
|
44
53
|
rundir,
|
45
54
|
memento : Any=None) -> TaskDataResult:
|
46
|
-
|
55
|
+
self._log.debug("Compound task %s (%d)" % (self.name, len(self.needs)))
|
56
|
+
|
57
|
+
add_s = set()
|
58
|
+
output = []
|
59
|
+
status = 0
|
60
|
+
changed = False
|
61
|
+
|
62
|
+
for n in self.needs:
|
63
|
+
status |= n[0].result.status
|
64
|
+
changed |= n[0].output.changed
|
65
|
+
for o in n[0].output.output:
|
66
|
+
o_id = (o.src, o.seq)
|
67
|
+
if not o_id in add_s:
|
68
|
+
if self.consumes is not None or self.consumes == ConsumesE.All:
|
69
|
+
add_s.add(o_id)
|
70
|
+
output.append(o)
|
71
|
+
elif isinstance(self.consumes, list) and self._matches(o, self.consumes):
|
72
|
+
add_s.add(o_id)
|
73
|
+
output.append(o)
|
74
|
+
|
75
|
+
self.result = TaskDataResult(
|
76
|
+
status=status,
|
77
|
+
changed=changed,
|
78
|
+
output=output
|
79
|
+
)
|
80
|
+
|
81
|
+
# TODO: Handle passthrough and deps
|
82
|
+
|
83
|
+
self.output = TaskDataOutput(
|
84
|
+
changed=changed,
|
85
|
+
output=output,
|
86
|
+
dep_m={})
|
87
|
+
|
88
|
+
return 0
|
47
89
|
|
48
90
|
def __hash__(self):
|
49
|
-
return id(self)
|
91
|
+
return id(self)
|
92
|
+
|
@@ -25,7 +25,7 @@ import dataclasses as dc
|
|
25
25
|
import logging
|
26
26
|
from pydantic import BaseModel
|
27
27
|
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
28
|
-
from .task_def import TaskDef
|
28
|
+
from .task_def import TaskDef, RundirE
|
29
29
|
from .task_data import TaskDataOutput, TaskDataResult
|
30
30
|
from .task_node import TaskNode
|
31
31
|
from .task_node_ctor import TaskNodeCtor
|
@@ -44,14 +44,18 @@ class TaskNodeCtorCompound(TaskNodeCtor):
|
|
44
44
|
if srcdir is None:
|
45
45
|
srcdir = self.srcdir
|
46
46
|
|
47
|
+
if params is None:
|
48
|
+
raise Exception("params is None")
|
49
|
+
|
47
50
|
node = TaskNodeCompound(
|
48
51
|
name=name,
|
49
52
|
srcdir=srcdir,
|
50
53
|
params=params,
|
51
54
|
needs=needs)
|
55
|
+
# Use the compound task's rundir
|
56
|
+
node.input.rundir = builder.get_rundir()
|
52
57
|
|
53
|
-
|
54
|
-
builder.enter_compound(node)
|
58
|
+
builder.enter_compound(node, self.task_def.rundir)
|
55
59
|
builder.addTask("in", node.input)
|
56
60
|
|
57
61
|
self._buildSubGraph(builder, node)
|
@@ -28,15 +28,14 @@ import logging
|
|
28
28
|
import toposort
|
29
29
|
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
30
30
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
31
|
-
from .
|
32
|
-
from .param_ref_eval import ParamRefEval
|
33
|
-
from .param import Param
|
31
|
+
from .task_def import RundirE
|
34
32
|
from .task_node_ctor import TaskNodeCtor
|
35
33
|
|
36
34
|
@dc.dataclass
|
37
35
|
class TaskNodeCtorDefBase(TaskNodeCtor):
|
38
36
|
"""Task defines its own needs, that will need to be filled in"""
|
39
37
|
needs : List['str']
|
38
|
+
rundir : RundirE
|
40
39
|
|
41
40
|
def __post_init__(self):
|
42
41
|
if self.needs is None:
|
@@ -28,9 +28,6 @@ import logging
|
|
28
28
|
import toposort
|
29
29
|
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
30
30
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
31
|
-
from .task_params_ctor import TaskParamsCtor
|
32
|
-
from .param_ref_eval import ParamRefEval
|
33
|
-
from .param import Param
|
34
31
|
from .task_node import TaskNode
|
35
32
|
from .task_node_ctor import TaskNodeCtor
|
36
33
|
from .task_node_ctor_def_base import TaskNodeCtorDefBase
|
@@ -28,12 +28,9 @@ import logging
|
|
28
28
|
import toposort
|
29
29
|
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
30
30
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
31
|
-
from .
|
32
|
-
from .param_ref_eval import ParamRefEval
|
33
|
-
from .param import Param
|
31
|
+
from .task_def import RundirE
|
34
32
|
from .task_node import TaskNode
|
35
33
|
from .task_node_leaf import TaskNodeLeaf
|
36
|
-
from .task_node_ctor import TaskNodeCtor
|
37
34
|
from .task_node_ctor_def_base import TaskNodeCtorDefBase
|
38
35
|
|
39
36
|
@dc.dataclass
|
@@ -49,6 +46,9 @@ class TaskNodeCtorTask(TaskNodeCtorDefBase):
|
|
49
46
|
if srcdir is None:
|
50
47
|
srcdir = self.srcdir
|
51
48
|
|
49
|
+
if params is None:
|
50
|
+
raise Exception("params is None")
|
51
|
+
|
52
52
|
node = TaskNodeLeaf(
|
53
53
|
name=name,
|
54
54
|
srcdir=srcdir,
|
@@ -58,6 +58,7 @@ class TaskNodeCtorTask(TaskNodeCtorDefBase):
|
|
58
58
|
node.passthrough = self.passthrough
|
59
59
|
node.consumes = self.consumes
|
60
60
|
node.task = self.task
|
61
|
+
node.rundir = builder.get_rundir()
|
61
62
|
builder.addTask(name, node)
|
62
63
|
|
63
64
|
self._log.debug("<-- mkTaskNode")
|
@@ -28,19 +28,20 @@ import logging
|
|
28
28
|
import toposort
|
29
29
|
from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
30
30
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
31
|
-
from .task_params_ctor import TaskParamsCtor
|
32
|
-
from .param_ref_eval import ParamRefEval
|
33
31
|
from .param import Param
|
34
32
|
from .task_node import TaskNode
|
35
33
|
from .task_node_leaf import TaskNodeLeaf
|
36
34
|
from .task_node_ctor import TaskNodeCtor
|
37
|
-
from .task_node_ctor_def_base import TaskNodeCtorDefBase
|
38
35
|
|
39
36
|
@dc.dataclass
|
40
37
|
class TaskNodeCtorWrapper(TaskNodeCtor):
|
41
38
|
T : Any
|
39
|
+
_count : int = 0
|
42
40
|
|
43
41
|
def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> TaskNode:
|
42
|
+
if params is None:
|
43
|
+
raise Exception("params is None")
|
44
|
+
|
44
45
|
node = TaskNodeLeaf(
|
45
46
|
name=name,
|
46
47
|
srcdir=srcdir,
|
@@ -49,6 +50,14 @@ class TaskNodeCtorWrapper(TaskNodeCtor):
|
|
49
50
|
needs=needs)
|
50
51
|
node.passthrough = self.passthrough
|
51
52
|
node.consumes = self.consumes
|
53
|
+
if name is None:
|
54
|
+
name = "%s_%d" % (self.name, self._count)
|
55
|
+
self._count += 1
|
56
|
+
|
57
|
+
if builder is not None:
|
58
|
+
node.rundir = builder.get_rundir(name)
|
59
|
+
else:
|
60
|
+
node.rundir = [name]
|
52
61
|
return node
|
53
62
|
|
54
63
|
def mkTaskParams(self, params : Dict = None) -> Any:
|
dv_flow/mgr/task_node_leaf.py
CHANGED
@@ -9,7 +9,6 @@ from typing import Any, Callable, ClassVar, Dict, List, Tuple
|
|
9
9
|
from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
|
10
10
|
from .task_def import ConsumesE, PassthroughE
|
11
11
|
from .task_node import TaskNode
|
12
|
-
from .task_params_ctor import TaskParamsCtor
|
13
12
|
from .param_ref_eval import ParamRefEval
|
14
13
|
from .param import Param
|
15
14
|
|
@@ -28,6 +27,9 @@ class TaskNodeLeaf(TaskNode):
|
|
28
27
|
|
29
28
|
self.rundir = rundir
|
30
29
|
|
30
|
+
if self.params is None:
|
31
|
+
raise Exception("params is None (%s)" % str(self))
|
32
|
+
|
31
33
|
# TODO: Form dep-map from inputs
|
32
34
|
|
33
35
|
dep_m = {}
|
dv_flow/mgr/task_runner.py
CHANGED
@@ -106,7 +106,7 @@ class TaskSetRunner(TaskRunner):
|
|
106
106
|
if self._log.isEnabledFor(logging.DEBUG):
|
107
107
|
self._log.debug("Order:")
|
108
108
|
for active_s in order:
|
109
|
-
self._log.debug("- {%s}", ",".join(t.name for t in active_s))
|
109
|
+
self._log.debug("- {%s}", ",".join(str(t.name) for t in active_s))
|
110
110
|
|
111
111
|
active_task_l = []
|
112
112
|
done_task_s = set()
|
@@ -117,33 +117,28 @@ class TaskSetRunner(TaskRunner):
|
|
117
117
|
while len(active_task_l) >= self.nproc and t not in done_task_s:
|
118
118
|
# Wait for at least one job to complete
|
119
119
|
done, pending = await asyncio.wait(at[1] for at in active_task_l)
|
120
|
-
|
121
|
-
for i in range(len(active_task_l)):
|
122
|
-
if active_task_l[i][1] == d:
|
123
|
-
tt = active_task_l[i][0]
|
124
|
-
tt.end = datetime.now()
|
125
|
-
if tt.result.memento is not None:
|
126
|
-
dst_memento[tt.name] = tt.result.memento.model_dump()
|
127
|
-
else:
|
128
|
-
dst_memento[tt.name] = None
|
129
|
-
self.status |= tt.result.status
|
130
|
-
self._notify(tt, "leave")
|
131
|
-
done_task_s.add(tt)
|
132
|
-
active_task_l.pop(i)
|
133
|
-
break
|
120
|
+
self._completeTasks(active_task_l, done_task_s, done, dst_memento)
|
134
121
|
|
135
122
|
if self.status == 0 and t not in done_task_s:
|
136
123
|
memento = src_memento.get(t.name, None)
|
137
|
-
dirname = t.name
|
124
|
+
# dirname = t.name
|
138
125
|
invalid_chars_pattern = r'[\/:*?"<>|#%&{}\$\\!\'`;=@+]'
|
139
126
|
|
140
|
-
|
141
|
-
|
142
|
-
|
127
|
+
# TaskNode rundir is a list of path elements relative
|
128
|
+
# to the root rundir
|
129
|
+
rundir = self.rundir
|
143
130
|
|
144
|
-
|
145
|
-
|
146
|
-
rundir =
|
131
|
+
for rundir_e in t.rundir:
|
132
|
+
rundir_e = re.sub(invalid_chars_pattern, '_', rundir_e)
|
133
|
+
rundir = os.path.join(rundir, rundir_e)
|
134
|
+
|
135
|
+
# if t.rundir_t == RundirE.Unique:
|
136
|
+
# # Replace invalid characters with the replacement string.
|
137
|
+
# dirname = re.sub(invalid_chars_pattern, '_', dirname)
|
138
|
+
|
139
|
+
# rundir = os.path.join(self.rundir, dirname)
|
140
|
+
# else:
|
141
|
+
# rundir = self.rundir
|
147
142
|
|
148
143
|
if not os.path.isdir(rundir):
|
149
144
|
os.makedirs(rundir, exist_ok=True)
|
@@ -162,19 +157,10 @@ class TaskSetRunner(TaskRunner):
|
|
162
157
|
|
163
158
|
# All pending tasks in the task-group have been launched
|
164
159
|
# Wait for them to all complete
|
165
|
-
|
160
|
+
while len(active_task_l):
|
166
161
|
# TODO: Shouldn't gather here -- reach to each completion
|
167
|
-
|
168
|
-
|
169
|
-
for tt in active_task_l:
|
170
|
-
tt[0].end = datetime.now()
|
171
|
-
if tt[0].result.memento is not None:
|
172
|
-
dst_memento[tt[0].name] = tt[0].result.memento.model_dump()
|
173
|
-
else:
|
174
|
-
dst_memento[tt[0].name] = None
|
175
|
-
self.status |= tt[0].result.status
|
176
|
-
self._notify(tt[0], "leave")
|
177
|
-
active_task_l.clear()
|
162
|
+
done, pending = await asyncio.wait(at[1] for at in active_task_l)
|
163
|
+
self._completeTasks(active_task_l, done_task_s, done, dst_memento)
|
178
164
|
|
179
165
|
if self.status != 0:
|
180
166
|
self._log.debug("Exiting due to status: %d", self.status)
|
@@ -196,6 +182,25 @@ class TaskSetRunner(TaskRunner):
|
|
196
182
|
else:
|
197
183
|
return None
|
198
184
|
|
185
|
+
def _completeTasks(self, active_task_l, done_task_s, done_l, dst_memento):
|
186
|
+
for d in done_l:
|
187
|
+
for i in range(len(active_task_l)):
|
188
|
+
if active_task_l[i][1] == d:
|
189
|
+
tt = active_task_l[i][0]
|
190
|
+
tt.end = datetime.now()
|
191
|
+
if tt.result is None:
|
192
|
+
raise Exception("Task %s did not produce a result" % tt.name)
|
193
|
+
if tt.result.memento is not None:
|
194
|
+
dst_memento[tt.name] = tt.result.memento.model_dump()
|
195
|
+
else:
|
196
|
+
dst_memento[tt.name] = None
|
197
|
+
self.status |= tt.result.status
|
198
|
+
self._notify(tt, "leave")
|
199
|
+
done_task_s.add(tt)
|
200
|
+
active_task_l.pop(i)
|
201
|
+
break
|
202
|
+
pass
|
203
|
+
|
199
204
|
def buildDepMap(self, task : Union[TaskNode, List[TaskNode]]) -> Dict[TaskNode, Set[TaskNode]]:
|
200
205
|
tasks = task if isinstance(task, list) else [task]
|
201
206
|
dep_m = {}
|
{dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/RECORD
RENAMED
@@ -7,7 +7,7 @@ dv_flow/mgr/fileset.py,sha256=4izm4-qldWLiZxWDUa5Em0iFizzS-S-O0YU6zvtbUUs,1264
|
|
7
7
|
dv_flow/mgr/fragment_def.py,sha256=AHx3ITbiNbGCzlDk5FM8Iimm0GJnWXDDRAoRC1b41jM,1647
|
8
8
|
dv_flow/mgr/out,sha256=d8GGBi3J43fhdLBlnsUbzBfRe0TD0QTP3nOTz54l2bI,200
|
9
9
|
dv_flow/mgr/package.py,sha256=9FhOu1hKYKzlD88zehwrI6KAcs1E7TbXDnq4v54dqDQ,1701
|
10
|
-
dv_flow/mgr/package_def.py,sha256=
|
10
|
+
dv_flow/mgr/package_def.py,sha256=pyuwgTszJ9n4r3WXuITSkrJuIoN6RwS6saX-_XmHgGk,20880
|
11
11
|
dv_flow/mgr/package_import_spec.py,sha256=aZMpnS9a5NFY76_pYXEuO3-Mkc_xFzy73fdrUe_54Dc,1760
|
12
12
|
dv_flow/mgr/param.py,sha256=kkxMRGf6mPjSZJsjgLKH2vJL62Sn0ZESvjBLkEYOp20,1386
|
13
13
|
dv_flow/mgr/param_def.py,sha256=mkAw3DanIxcVWRYeh9lUAfUMkpqDFly_Ku_iobr4ix8,1745
|
@@ -16,22 +16,22 @@ dv_flow/mgr/parsetab.py,sha256=I-p3nC60t9jiNtPhKyl_sE92SiP96zJLnNdydcLy33g,3780
|
|
16
16
|
dv_flow/mgr/pkg_rgy.py,sha256=d1nIjRm3ymMNJT-yiMDxCS6bFisTPvLMqh5VrfsHVKM,5404
|
17
17
|
dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
|
18
18
|
dv_flow/mgr/task_def.py,sha256=9_VJfY8hfZI7x9LFqDxALurg6oWMQQw6rdsppcjTweM,3669
|
19
|
-
dv_flow/mgr/task_graph_builder.py,sha256=
|
19
|
+
dv_flow/mgr/task_graph_builder.py,sha256=J5C9PhVmiYuQRlEXkWVCzsX06rWUgxrWrFc5sIwUhbw,15029
|
20
20
|
dv_flow/mgr/task_graph_dot_writer.py,sha256=GxqiYwQJbFgUQdnPCS9vpIYmYFbSXwnXCSbGmjbxC3M,2418
|
21
21
|
dv_flow/mgr/task_listener_log.py,sha256=hrJEjSRXXoovDTcD1Cmhi3Spzw7uEJ-WP6tv6JUaa6s,4060
|
22
|
-
dv_flow/mgr/task_node.py,sha256=
|
23
|
-
dv_flow/mgr/task_node_compound.py,sha256=
|
22
|
+
dv_flow/mgr/task_node.py,sha256=twCozRjXcWWeQXJBSOmKM51qpJMEPr5sj9mMK_QiE6A,3728
|
23
|
+
dv_flow/mgr/task_node_compound.py,sha256=mNu4nf9hVqu2698ue5fpE3FeAOkvJH0Ke2W9V0G3-As,2975
|
24
24
|
dv_flow/mgr/task_node_ctor.py,sha256=COFGvm5PR2B92H3uW1yhDIUCmppo9U4IfOcv_Jrsreo,3952
|
25
|
-
dv_flow/mgr/task_node_ctor_compound.py,sha256=
|
26
|
-
dv_flow/mgr/task_node_ctor_compound_proxy.py,sha256=
|
27
|
-
dv_flow/mgr/task_node_ctor_def_base.py,sha256=
|
28
|
-
dv_flow/mgr/task_node_ctor_proxy.py,sha256=
|
29
|
-
dv_flow/mgr/task_node_ctor_task.py,sha256=
|
30
|
-
dv_flow/mgr/task_node_ctor_wrapper.py,sha256=
|
31
|
-
dv_flow/mgr/task_node_leaf.py,sha256=
|
25
|
+
dv_flow/mgr/task_node_ctor_compound.py,sha256=WBSdREZEATEN9dbamZo7P3oN87E7wu7r3pcUCLqDRt4,4200
|
26
|
+
dv_flow/mgr/task_node_ctor_compound_proxy.py,sha256=D8x54nD8Pd-2-_mr1syhqVeSFfIVf100ldi3bdzmSfI,2073
|
27
|
+
dv_flow/mgr/task_node_ctor_def_base.py,sha256=_8QQHKDkONio_ve0Z409yxC0AMO8ocNBPDjRiNED1FI,1503
|
28
|
+
dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v2PA6VA,1906
|
29
|
+
dv_flow/mgr/task_node_ctor_task.py,sha256=d49g90TyPCMFR8BuWWqp4ym-MW5vGSdDR0V47Ru28JY,2232
|
30
|
+
dv_flow/mgr/task_node_ctor_wrapper.py,sha256=Nb1CVcPHZofnb-iLWDHQWAxlTOdbRrnR9DdSxY8yOec,3626
|
31
|
+
dv_flow/mgr/task_node_leaf.py,sha256=iEz-O4943SupriqO0aI5Ai7MIvkFRMkPitN8G85-G9g,7327
|
32
32
|
dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
|
33
33
|
dv_flow/mgr/task_params_ctor.py,sha256=BPkbnoCtzhCxc1g8CJ6VimCcm5UAu92PXeDMhQ4lYsQ,1957
|
34
|
-
dv_flow/mgr/task_runner.py,sha256
|
34
|
+
dv_flow/mgr/task_runner.py,sha256=-919VntXAe2XSuFW2dFpgvUre-NkILBnDBbAKBZYn5w,9594
|
35
35
|
dv_flow/mgr/type.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
36
36
|
dv_flow/mgr/type_def.py,sha256=NDeyplKrPnWwEv4yHkhXEMK9d9j39b5MOeLB-1Mchqo,1095
|
37
37
|
dv_flow/mgr/cmds/cmd_graph.py,sha256=Ykw__EdwamDBZZKxQZVbtMtFl7koq5dJMShTBXSM2pk,2672
|
@@ -49,9 +49,9 @@ dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,
|
|
49
49
|
dv_flow/mgr/util/util.py,sha256=yg9oTPRiO87mkCSOQpOtlG9vyKPQzY3qp4OJkEMbWyY,1443
|
50
50
|
dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
51
|
dv_flow/mgr/util/cmds/cmd_schema.py,sha256=lrEI-Jwb8j4I4yYOn9hq7_7NYbK8leVxLesrHyEWm-E,1879
|
52
|
-
dv_flow_mgr-0.0.1.
|
53
|
-
dv_flow_mgr-0.0.1.
|
54
|
-
dv_flow_mgr-0.0.1.
|
55
|
-
dv_flow_mgr-0.0.1.
|
56
|
-
dv_flow_mgr-0.0.1.
|
57
|
-
dv_flow_mgr-0.0.1.
|
52
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
53
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/METADATA,sha256=61gFcfnbRS1QgZKLPgpwMtYTkObC6pF2R5l35Mqz6M8,13336
|
54
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
55
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
56
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
57
|
+
dv_flow_mgr-0.0.1.14147547123a1.dist-info/RECORD,,
|
{dv_flow_mgr-0.0.1.14116344455a1.dist-info → dv_flow_mgr-0.0.1.14147547123a1.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|