dv-flow-mgr 1.0.0.14389547107a1__py3-none-any.whl → 1.0.0.14392445998a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dv_flow/mgr/task_graph_builder.py +85 -277
- dv_flow/mgr/task_graph_dot_writer.py +90 -30
- dv_flow/mgr/task_node.py +5 -0
- dv_flow/mgr/task_node_compound.py +6 -0
- dv_flow/mgr/task_node_leaf.py +1 -1
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/METADATA +1 -1
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/RECORD +11 -11
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/WHEEL +0 -0
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/entry_points.txt +0 -0
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/licenses/LICENSE +0 -0
- {dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/top_level.txt +0 -0
@@ -71,6 +71,7 @@ class TaskGraphBuilder(object):
|
|
71
71
|
_ns_scope_s : List[TaskNamespaceScope] = dc.field(default_factory=list)
|
72
72
|
_compound_task_ctxt_s : List[CompoundTaskCtxt] = dc.field(default_factory=list)
|
73
73
|
_task_rundir_s : List[List[str]] = dc.field(default_factory=list)
|
74
|
+
_task_node_s : List[TaskNode] = dc.field(default_factory=list)
|
74
75
|
_uses_count : int = 0
|
75
76
|
|
76
77
|
_log : logging.Logger = None
|
@@ -203,19 +204,7 @@ class TaskGraphBuilder(object):
|
|
203
204
|
|
204
205
|
def mkTaskGraph(self, task : str, rundir=None) -> TaskNode:
|
205
206
|
return self.mkTaskNode(task, rundir=rundir)
|
206
|
-
# self._task_node_m.clear()
|
207
|
-
|
208
|
-
# if rundir is not None:
|
209
|
-
# self._rundir_s.append(rundir)
|
210
|
-
|
211
|
-
# ret = self._mkTaskGraph(task)
|
212
|
-
|
213
|
-
# if rundir is not None:
|
214
|
-
# self._rundir_s.pop()
|
215
|
-
|
216
|
-
# return ret
|
217
207
|
|
218
|
-
|
219
208
|
def mkTaskNode(self, task_t, name=None, srcdir=None, needs=None, **kwargs):
|
220
209
|
self._log.debug("--> mkTaskNode: %s" % task_t)
|
221
210
|
|
@@ -250,37 +239,59 @@ class TaskGraphBuilder(object):
|
|
250
239
|
break
|
251
240
|
return task
|
252
241
|
|
253
|
-
def _mkTaskNode(self, task : Task, hierarchical=False):
|
242
|
+
def _mkTaskNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False):
|
254
243
|
|
255
244
|
if not hierarchical:
|
256
245
|
self._task_rundir_s.append([])
|
257
246
|
|
258
247
|
# Determine how to build this node
|
259
|
-
if
|
260
|
-
ret = self._mkTaskCompoundNode(
|
248
|
+
if self._isCompound(task):
|
249
|
+
ret = self._mkTaskCompoundNode(
|
250
|
+
task,
|
251
|
+
name=name,
|
252
|
+
srcdir=srcdir,
|
253
|
+
params=params,
|
254
|
+
hierarchical=hierarchical)
|
261
255
|
else:
|
262
|
-
ret = self._mkTaskLeafNode(
|
256
|
+
ret = self._mkTaskLeafNode(
|
257
|
+
task,
|
258
|
+
name=name,
|
259
|
+
srcdir=srcdir,
|
260
|
+
params=params,
|
261
|
+
hierarchical=hierarchical)
|
263
262
|
|
264
263
|
if not hierarchical:
|
265
264
|
self._task_rundir_s.pop()
|
266
265
|
|
267
266
|
return ret
|
268
267
|
|
268
|
+
def _isCompound(self, task):
|
269
|
+
if task.subtasks is not None and len(task.subtasks):
|
270
|
+
return True
|
271
|
+
elif task.uses is not None:
|
272
|
+
return self._isCompound(task.uses)
|
273
|
+
|
269
274
|
def _getTaskNode(self, name):
|
270
275
|
if name in self._task_node_m.keys():
|
271
276
|
return self._task_node_m[name]
|
272
277
|
else:
|
273
278
|
return self.mkTaskNode(name)
|
274
279
|
|
275
|
-
def _mkTaskLeafNode(self, task : Task, name=None) -> TaskNode:
|
280
|
+
def _mkTaskLeafNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
|
276
281
|
self._log.debug("--> _mkTaskLeafNode %s" % task.name)
|
277
|
-
|
282
|
+
|
283
|
+
if name is None:
|
284
|
+
name = task.name
|
285
|
+
|
286
|
+
if srcdir is None:
|
287
|
+
srcdir = os.path.dirname(task.srcinfo.file)
|
288
|
+
|
289
|
+
if params is None:
|
290
|
+
params = task.paramT()
|
278
291
|
|
279
292
|
if task.rundir == RundirE.Unique:
|
280
293
|
self.enter_rundir(task.name)
|
281
294
|
|
282
|
-
if name is None:
|
283
|
-
name = task.name
|
284
295
|
|
285
296
|
callable = None
|
286
297
|
if task.run is not None:
|
@@ -296,13 +307,16 @@ class TaskGraphBuilder(object):
|
|
296
307
|
node = TaskNodeLeaf(
|
297
308
|
name=name,
|
298
309
|
srcdir=srcdir,
|
299
|
-
params=
|
310
|
+
params=params,
|
300
311
|
passthrough=task.passthrough,
|
301
312
|
consumes=task.consumes,
|
302
313
|
task=callable(task.run))
|
303
314
|
self._task_node_m[name] = node
|
304
315
|
node.rundir = self.get_rundir()
|
305
316
|
|
317
|
+
if len(self._task_node_s):
|
318
|
+
node.parent = self._task_node_s[-1]
|
319
|
+
|
306
320
|
# Now, link up the needs
|
307
321
|
self._log.debug("--> processing needs")
|
308
322
|
self._gatherNeeds(task, node)
|
@@ -314,23 +328,46 @@ class TaskGraphBuilder(object):
|
|
314
328
|
self._log.debug("<-- _mkTaskLeafNode %s" % task.name)
|
315
329
|
return node
|
316
330
|
|
317
|
-
def _mkTaskCompoundNode(self, task : Task, name=None) -> TaskNode:
|
331
|
+
def _mkTaskCompoundNode(self, task : Task, name=None, srcdir=None, params=None, hierarchical=False) -> TaskNode:
|
318
332
|
self._log.debug("--> _mkTaskCompoundNode %s" % task.name)
|
319
|
-
srcdir = os.path.dirname(task.srcinfo.file)
|
320
333
|
|
321
334
|
if name is None:
|
322
335
|
name = task.name
|
323
336
|
|
337
|
+
if srcdir is None:
|
338
|
+
srcdir = os.path.dirname(task.srcinfo.file)
|
339
|
+
|
340
|
+
if params is None:
|
341
|
+
params = task.paramT()
|
342
|
+
|
324
343
|
if task.rundir == RundirE.Unique:
|
325
344
|
self.enter_rundir(task.name)
|
326
345
|
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
346
|
+
if task.uses is not None:
|
347
|
+
# This is a compound task that is based on
|
348
|
+
# another. Create the base implementation
|
349
|
+
node = self._mkTaskNode(
|
350
|
+
task.uses,
|
351
|
+
name=name,
|
352
|
+
srcdir=srcdir,
|
353
|
+
params=params,
|
354
|
+
hierarchical=True)
|
355
|
+
|
356
|
+
if not isinstance(node, TaskNodeCompound):
|
357
|
+
# TODO: need to enclose the leaf node in a compound wrapper
|
358
|
+
raise Exception("Task %s is not compound" % task.uses)
|
359
|
+
else:
|
360
|
+
# Node represents the terminal node of the sub-DAG
|
361
|
+
node = TaskNodeCompound(
|
362
|
+
name=name,
|
363
|
+
srcdir=srcdir,
|
364
|
+
params=params)
|
365
|
+
|
366
|
+
if len(self._task_node_s):
|
367
|
+
node.parent = self._task_node_s[-1]
|
368
|
+
|
333
369
|
self._task_node_m[name] = node
|
370
|
+
self._task_node_s.append(node)
|
334
371
|
|
335
372
|
node.rundir = self.get_rundir()
|
336
373
|
|
@@ -339,8 +376,13 @@ class TaskGraphBuilder(object):
|
|
339
376
|
node.input.rundir = self.get_rundir()
|
340
377
|
self.leave_rundir()
|
341
378
|
|
342
|
-
self._log.debug("--> processing needs")
|
343
|
-
|
379
|
+
self._log.debug("--> processing needs (%s)" % task.name)
|
380
|
+
for need in task.needs:
|
381
|
+
need_n = self._getTaskNode(need.name)
|
382
|
+
self._log.debug("Add need %s" % need_n.name)
|
383
|
+
if need_n is None:
|
384
|
+
raise Exception("Failed to find need %s" % need.name)
|
385
|
+
node.input.needs.append((need_n, False))
|
344
386
|
self._log.debug("<-- processing needs")
|
345
387
|
|
346
388
|
# TODO: handle strategy
|
@@ -349,11 +391,17 @@ class TaskGraphBuilder(object):
|
|
349
391
|
# For now, build out local tasks and link up the needs
|
350
392
|
tasks = []
|
351
393
|
for t in task.subtasks:
|
352
|
-
nn = self._mkTaskNode(t, True)
|
353
|
-
tasks.append(
|
394
|
+
nn = self._mkTaskNode(t, hierarchical=True)
|
395
|
+
node.tasks.append(nn)
|
396
|
+
# tasks.append((t, self._getTaskNode(t.name)))
|
397
|
+
tasks.append((t, nn))
|
398
|
+
|
399
|
+
# Pop the node stack, since we're done constructing the body
|
400
|
+
self._task_node_s.pop()
|
354
401
|
|
355
402
|
# Fill in 'needs'
|
356
403
|
for t, tn in tasks:
|
404
|
+
self._log.debug("Process node %s" % t.name)
|
357
405
|
|
358
406
|
referenced = None
|
359
407
|
for tt in task.subtasks:
|
@@ -362,7 +410,9 @@ class TaskGraphBuilder(object):
|
|
362
410
|
break
|
363
411
|
|
364
412
|
refs_internal = None
|
365
|
-
|
413
|
+
# Assess how this task is connected to others in the compound node
|
414
|
+
for nn,_ in tn.first.needs:
|
415
|
+
self._log.debug("Need: %s" % nn.name)
|
366
416
|
for _,tnn in tasks:
|
367
417
|
if nn == tnn:
|
368
418
|
refs_internal = tnn
|
@@ -376,7 +426,7 @@ class TaskGraphBuilder(object):
|
|
376
426
|
self._log.debug("Node %s doesn't reference any internal node" % t.name)
|
377
427
|
tn.needs.append((node.input, False))
|
378
428
|
else:
|
379
|
-
self._log.debug("Node references internal node %s" % refs_internal.name)
|
429
|
+
self._log.debug("Node %s references internal node %s" % (t.name, refs_internal.name))
|
380
430
|
|
381
431
|
if referenced is not None:
|
382
432
|
# Add this task as a dependency of the output
|
@@ -403,34 +453,6 @@ class TaskGraphBuilder(object):
|
|
403
453
|
node.needs.append((need_n, False))
|
404
454
|
self._log.debug("<-- _gatherNeeds %s" % task_t.name)
|
405
455
|
|
406
|
-
def getTaskCtor(self, spec : Union[str,'TaskSpec'], pkg : PackageDef = None) -> 'TaskNodeCtor':
|
407
|
-
from .task_def import TaskSpec
|
408
|
-
if type(spec) == str:
|
409
|
-
spec = TaskSpec(spec)
|
410
|
-
|
411
|
-
self._log.debug("--> getTaskCtor %s" % spec.name)
|
412
|
-
spec_e = spec.name.split(".")
|
413
|
-
task_name = spec_e[-1]
|
414
|
-
|
415
|
-
# if len(spec_e) == 1:
|
416
|
-
# # Just have a task name. Use the current package
|
417
|
-
# if len(self._pkg_s) == 0:
|
418
|
-
# raise Exception("No package context for task %s" % spec.name)
|
419
|
-
# pkg = self._pkg_s[-1]
|
420
|
-
# else:
|
421
|
-
# pkg_name = ".".join(spec_e[0:-1])
|
422
|
-
|
423
|
-
# try:
|
424
|
-
# pkg = self.getPackage(PackageSpec(pkg_name))
|
425
|
-
# except Exception as e:
|
426
|
-
# self._log.critical("Failed to find package %s while looking for task %s" % (pkg_name, spec.name))
|
427
|
-
# raise e
|
428
|
-
|
429
|
-
ctor = pkg.getTaskCtor(task_name)
|
430
|
-
|
431
|
-
self._log.debug("<-- getTaskCtor %s" % spec.name)
|
432
|
-
return ctor
|
433
|
-
|
434
456
|
def error(self, msg, loc=None):
|
435
457
|
if loc is not None:
|
436
458
|
marker = TaskMarker(msg=msg, severity=SeverityE.Error, loc=loc)
|
@@ -441,217 +463,3 @@ class TaskGraphBuilder(object):
|
|
441
463
|
def marker(self, marker):
|
442
464
|
self.marker_l(marker)
|
443
465
|
|
444
|
-
def _getTaskCtor(self, task : Task) -> TaskNodeCtor:
|
445
|
-
if task in self._task_ctor_m.keys():
|
446
|
-
ctor = self._task_ctor_m[task]
|
447
|
-
else:
|
448
|
-
ctor = self._mkTaskCtor(task)
|
449
|
-
self._task_ctor_m[task] = ctor
|
450
|
-
return ctor
|
451
|
-
|
452
|
-
def _mkTaskCtor(self, task):
|
453
|
-
srcdir = os.path.dirname(task.srcinfo.file)
|
454
|
-
self._log.debug("--> mkTaskCtor %s (srcdir: %s)" % (task.name, srcdir))
|
455
|
-
|
456
|
-
if len(task.subtasks) > 0:
|
457
|
-
self._log.debug("Task has a body")
|
458
|
-
# Compound task
|
459
|
-
self._log.debug("Task specifies sub-task implementation")
|
460
|
-
ctor = self._mkCompoundTaskCtor(task)
|
461
|
-
else:
|
462
|
-
self._log.debug("Task doesn't specify a body")
|
463
|
-
# Shell task or 'null'
|
464
|
-
ctor = self._mkLeafTaskCtor(task)
|
465
|
-
|
466
|
-
if ctor is None:
|
467
|
-
raise Exception()
|
468
|
-
|
469
|
-
return ctor
|
470
|
-
|
471
|
-
def _mkLeafTaskCtor(self, task) -> TaskNodeCtor:
|
472
|
-
self._log.debug("--> _mkLeafTaskCtor")
|
473
|
-
srcdir = os.path.dirname(task.srcinfo.file)
|
474
|
-
base_ctor_t : TaskNodeCtor = None
|
475
|
-
ctor_t : TaskNodeCtor = None
|
476
|
-
base_params = None
|
477
|
-
callable = None
|
478
|
-
# fullname = self.name + "." + task.name
|
479
|
-
# rundir = task.rundir
|
480
|
-
|
481
|
-
# TODO: should we have the ctor look this up itself?
|
482
|
-
# Want to confirm that the value can be found.
|
483
|
-
# Defer final resolution until actual graph building (post-config)
|
484
|
-
if task.uses is not None:
|
485
|
-
self._log.debug("Uses: %s" % task.uses.name)
|
486
|
-
|
487
|
-
base_ctor_t = self._getTaskCtor(task.uses)
|
488
|
-
|
489
|
-
if base_ctor_t is None:
|
490
|
-
self._log.error("Failed to load task ctor %s" % task.uses)
|
491
|
-
# base_params = base_ctor_t.mkTaskParams()
|
492
|
-
else:
|
493
|
-
self._log.debug("No 'uses' specified %s" % task.name)
|
494
|
-
|
495
|
-
self._log.debug("%d needs" % len(task.needs))
|
496
|
-
|
497
|
-
# Determine the implementation constructor first
|
498
|
-
if task.run is not None:
|
499
|
-
shell = task.shell if task.shell is not None else "shell"
|
500
|
-
|
501
|
-
if shell in self._shell_m.keys():
|
502
|
-
self._log.debug("Use shell implementation")
|
503
|
-
callable = self._shell_m[shell]
|
504
|
-
else:
|
505
|
-
self._log.debug("Shell %s not found" % shell)
|
506
|
-
raise Exception("Shell %s not found" % shell)
|
507
|
-
|
508
|
-
# if taskdef.body.pytask is not None:
|
509
|
-
# # Built-in impl
|
510
|
-
# # Now, lookup the class
|
511
|
-
# self._log.debug("Use PyTask implementation")
|
512
|
-
# last_dot = taskdef.body.pytask.rfind('.')
|
513
|
-
# clsname = taskdef.body.pytask[last_dot+1:]
|
514
|
-
# modname = taskdef.body.pytask[:last_dot]
|
515
|
-
|
516
|
-
# try:
|
517
|
-
# if modname not in sys.modules:
|
518
|
-
# if srcdir not in sys.path:
|
519
|
-
# sys.path.append(srcdir)
|
520
|
-
# mod = importlib.import_module(modname)
|
521
|
-
# else:
|
522
|
-
# mod = sys.modules[modname]
|
523
|
-
# except ModuleNotFoundError as e:
|
524
|
-
# raise Exception("Failed to import module %s (_basedir=%s): %s" % (
|
525
|
-
# modname, self._basedir, str(e)))
|
526
|
-
|
527
|
-
# if not hasattr(mod, clsname):
|
528
|
-
# raise Exception("Method %s not found in module %s" % (clsname, modname))
|
529
|
-
# callable = getattr(mod, clsname)
|
530
|
-
# elif taskdef.body.run is not None:
|
531
|
-
# callable = self._getRunCallable(taskdef)
|
532
|
-
else:
|
533
|
-
# TODO: use null task
|
534
|
-
pass
|
535
|
-
|
536
|
-
# Determine if we need to use a new
|
537
|
-
if task.paramT is None:
|
538
|
-
raise Exception()
|
539
|
-
paramT = task.paramT
|
540
|
-
needs = []
|
541
|
-
|
542
|
-
# TODO:
|
543
|
-
rundir : RundirE = task.rundir
|
544
|
-
|
545
|
-
if callable is not None:
|
546
|
-
ctor_t = TaskNodeCtorTask(
|
547
|
-
name=task.name,
|
548
|
-
srcdir=srcdir,
|
549
|
-
paramT=task.paramT, # TODO: need to determine the parameter type
|
550
|
-
passthrough=task.passthrough,
|
551
|
-
consumes=task.consumes,
|
552
|
-
needs=needs, # TODO: need to determine the needs
|
553
|
-
rundir=rundir,
|
554
|
-
task=callable)
|
555
|
-
elif base_ctor_t is not None:
|
556
|
-
# Use the existing (base) to create the implementation
|
557
|
-
ctor_t = TaskNodeCtorProxy(
|
558
|
-
name=task.name,
|
559
|
-
srcdir=srcdir,
|
560
|
-
paramT=task.paramT, # TODO: need to determine the parameter type
|
561
|
-
passthrough=task.passthrough,
|
562
|
-
consumes=task.consumes,
|
563
|
-
needs=needs,
|
564
|
-
rundir=rundir,
|
565
|
-
uses=base_ctor_t)
|
566
|
-
else:
|
567
|
-
self._log.debug("Use 'Null' as the class implementation")
|
568
|
-
ctor_t = TaskNodeCtorTask(
|
569
|
-
name=task.name,
|
570
|
-
srcdir=srcdir,
|
571
|
-
paramT=paramT,
|
572
|
-
passthrough=task.passthrough,
|
573
|
-
consumes=task.consumes,
|
574
|
-
needs=needs,
|
575
|
-
rundir=rundir,
|
576
|
-
task=TaskNull)
|
577
|
-
|
578
|
-
self._log.debug("<-- mkTaskCtor %s" % task.name)
|
579
|
-
return ctor_t
|
580
|
-
|
581
|
-
def _getRunCallable(self, task):
|
582
|
-
self._log.debug("--> _getRunCallable %s" % task.name)
|
583
|
-
callable = None
|
584
|
-
if task.run is not None and task.shell == "python":
|
585
|
-
# Evaluate a Python script
|
586
|
-
pass
|
587
|
-
else:
|
588
|
-
# run a shell script
|
589
|
-
shell = None
|
590
|
-
body = task.run.strip()
|
591
|
-
|
592
|
-
callable = ShellCallable(body=body, shell=shell)
|
593
|
-
pass
|
594
|
-
return callable
|
595
|
-
|
596
|
-
def _mkCompoundTaskCtor(self, task) -> TaskNodeCtor:
|
597
|
-
self._log.debug("--> _mkCompoundTaskCtor %s" % task.name)
|
598
|
-
srcdir = os.path.dirname(task.srcinfo.file)
|
599
|
-
base_ctor_t : TaskNodeCtor = None
|
600
|
-
ctor_t : TaskNodeCtor = None
|
601
|
-
base_params = None
|
602
|
-
callable = None
|
603
|
-
|
604
|
-
# fullname = self._getScopeFullname()
|
605
|
-
fullname = task.name
|
606
|
-
|
607
|
-
if task.uses is not None:
|
608
|
-
self._log.debug("Uses: %s" % task.uses)
|
609
|
-
base_ctor_t = task.uses.ctor
|
610
|
-
base_params = base_ctor_t.mkTaskParams()
|
611
|
-
|
612
|
-
if base_ctor_t is None:
|
613
|
-
self._log.error("Failed to load task ctor %s" % task.uses)
|
614
|
-
|
615
|
-
# TODO: should build during loading
|
616
|
-
# passthrough, consumes, needs = self._getPTConsumesNeeds(taskdef, base_ctor_t)
|
617
|
-
passthrough = []
|
618
|
-
consumes = []
|
619
|
-
needs = []
|
620
|
-
|
621
|
-
# Determine if we need to use a new
|
622
|
-
# paramT = self._getParamT(taskdef, base_params)
|
623
|
-
paramT = task.paramT
|
624
|
-
|
625
|
-
if base_ctor_t is not None:
|
626
|
-
ctor_t = TaskNodeCtorCompoundProxy(
|
627
|
-
name=fullname,
|
628
|
-
srcdir=srcdir,
|
629
|
-
paramT=paramT,
|
630
|
-
passthrough=passthrough,
|
631
|
-
consumes=consumes,
|
632
|
-
needs=needs,
|
633
|
-
task=task,
|
634
|
-
uses=base_ctor_t)
|
635
|
-
else:
|
636
|
-
self._log.debug("No 'uses' specified")
|
637
|
-
ctor_t = TaskNodeCtorCompound(
|
638
|
-
name=fullname,
|
639
|
-
srcdir=srcdir,
|
640
|
-
paramT=paramT,
|
641
|
-
passthrough=passthrough,
|
642
|
-
consumes=consumes,
|
643
|
-
needs=needs,
|
644
|
-
task=task)
|
645
|
-
|
646
|
-
for st in task.subtasks:
|
647
|
-
ctor = self._getTaskCtor(st)
|
648
|
-
if ctor is None:
|
649
|
-
raise Exception("ctor for %s is None" % st.name)
|
650
|
-
ctor_t.tasks.append(st)
|
651
|
-
|
652
|
-
# for t in task.subtasks:
|
653
|
-
# ctor_t.tasks.append(self._mkTaskCtor(t, srcdir))
|
654
|
-
|
655
|
-
|
656
|
-
self._log.debug("<-- mkCompoundTaskCtor %s (%d)" % (task.name, len(ctor_t.tasks)))
|
657
|
-
return ctor_t
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import dataclasses as dc
|
2
2
|
import logging
|
3
3
|
import sys
|
4
|
-
from typing import ClassVar, Dict, TextIO
|
4
|
+
from typing import ClassVar, Dict, Set, TextIO
|
5
5
|
from .task_node import TaskNode
|
6
6
|
from .task_node_compound import TaskNodeCompound
|
7
7
|
|
@@ -10,6 +10,7 @@ class TaskGraphDotWriter(object):
|
|
10
10
|
fp : TextIO = dc.field(default=None)
|
11
11
|
_ind : str = ""
|
12
12
|
_node_id_m : Dict[TaskNode, str] = dc.field(default_factory=dict)
|
13
|
+
_processed_needs : Set[TaskNode] = dc.field(default_factory=set)
|
13
14
|
_node_id : int = 1
|
14
15
|
_cluster_id : int = 1
|
15
16
|
_log : ClassVar = logging.getLogger("TaskGraphDotWriter")
|
@@ -22,48 +23,107 @@ class TaskGraphDotWriter(object):
|
|
22
23
|
else:
|
23
24
|
self.fp = open(filename, "w")
|
24
25
|
self.println("digraph G {")
|
25
|
-
|
26
|
+
# First, build-out all nodes
|
27
|
+
self.build_node(node)
|
28
|
+
self.process_needs(node)
|
26
29
|
self.println("}")
|
27
30
|
|
28
31
|
self.fp.close()
|
29
32
|
self._log.debug("<-- TaskGraphDotWriter::write")
|
30
33
|
|
31
|
-
def
|
32
|
-
self._log.debug("-->
|
33
|
-
node_id = self._node_id
|
34
|
-
self._node_id += 1
|
35
|
-
node_name = "n%d" % self._node_id
|
36
|
-
self._node_id_m[node] = node_name
|
34
|
+
def build_node(self, node):
|
35
|
+
self._log.debug("--> build_node %s (%d)" % (node.name, len(node.needs),))
|
37
36
|
|
38
37
|
if isinstance(node, TaskNodeCompound):
|
39
|
-
self.
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
self.
|
45
|
-
self.process_node(node.input)
|
46
|
-
|
47
|
-
self.println("%s[label=\"%s.out\"];" % (
|
48
|
-
node_name,
|
49
|
-
node.name))
|
38
|
+
self._log.debug("-- compound node")
|
39
|
+
# Find the root and build out any expanded sub-nodes
|
40
|
+
root = node
|
41
|
+
while root.parent is not None:
|
42
|
+
root = root.parent
|
43
|
+
self.build_compound_node(root)
|
50
44
|
else:
|
51
|
-
|
52
|
-
|
53
|
-
|
45
|
+
# Leaf node
|
46
|
+
self._log.debug("-- leaf node")
|
47
|
+
node_id = self._node_id
|
48
|
+
self._node_id += 1
|
49
|
+
node_name = "n%d" % node_id
|
50
|
+
self._node_id_m[node] = node_name
|
51
|
+
self.println("%s[label=\"%s\"];" % (node_name, node.name))
|
52
|
+
self._log.debug("<-- build_node %s (%d)" % (node.name, len(node.needs),))
|
53
|
+
|
54
|
+
def process_needs(self, node):
|
55
|
+
self._log.debug("--> process_needs %s (%d)" % (node.name, len(node.needs),))
|
56
|
+
|
57
|
+
# if isinstance(node, TaskNodeCompound):
|
58
|
+
# self.println("subgraph cluster_%d {" % self._cluster_id)
|
59
|
+
# self._cluster_id += 1
|
60
|
+
# self.inc_ind()
|
61
|
+
# self.println("label=\"%s\";" % node.name)
|
62
|
+
# self.println("color=blue;")
|
63
|
+
# self.println("style=dashed;")
|
64
|
+
# self.process_node(node.input)
|
54
65
|
|
55
|
-
|
56
|
-
|
57
|
-
|
66
|
+
# self.println("%s[label=\"%s.out\"];" % (
|
67
|
+
# node_name,
|
68
|
+
# node.name))
|
69
|
+
# else:
|
70
|
+
# self.println("%s[label=\"%s\"];" % (
|
71
|
+
# node_name,
|
72
|
+
# node.name))
|
73
|
+
|
74
|
+
for dep,_ in node.needs:
|
75
|
+
if dep not in self._node_id_m.keys():
|
76
|
+
self.build_node(dep)
|
77
|
+
if dep not in self._node_id_m.keys():
|
78
|
+
self._log.error("Dep-node not built: %s" % dep.name)
|
79
|
+
if node not in self._node_id_m.keys():
|
80
|
+
self.build_node(node)
|
81
|
+
if node not in self._node_id_m.keys():
|
82
|
+
self._log.error("Dep-node not built: %s" % node.name)
|
58
83
|
self.println("%s -> %s;" % (
|
59
|
-
self._node_id_m[dep
|
84
|
+
self._node_id_m[dep],
|
60
85
|
self._node_id_m[node]))
|
86
|
+
if dep not in self._processed_needs:
|
87
|
+
self._processed_needs.add(dep)
|
88
|
+
self.process_needs(dep)
|
61
89
|
|
62
|
-
|
63
|
-
|
64
|
-
|
90
|
+
self._log.debug("<-- process_needs %s (%d)" % (node.name, len(node.needs),))
|
91
|
+
|
92
|
+
def build_compound_node(self, node):
|
93
|
+
"""Hierarchical build of a compound root node"""
|
94
|
+
|
95
|
+
self._log.debug("--> build_compound_node %s (%d)" % (node.name, len(node.tasks),))
|
96
|
+
|
97
|
+
id = self._cluster_id
|
98
|
+
self._cluster_id += 1
|
99
|
+
self.println("subgraph cluster_%d {" % id)
|
100
|
+
self.inc_ind()
|
101
|
+
self.println("label=\"%s\";" % node.name)
|
102
|
+
self.println("color=blue;")
|
103
|
+
self.println("style=dashed;")
|
104
|
+
|
105
|
+
task_node_id = self._node_id
|
106
|
+
self._node_id += 1
|
107
|
+
task_node_name = "n%d" % task_node_id
|
108
|
+
self.println("%s[label=\"%s\"];" % (task_node_name, node.name))
|
109
|
+
self._node_id_m[node] = task_node_name
|
110
|
+
|
111
|
+
for n in node.tasks:
|
112
|
+
if isinstance(n, TaskNodeCompound):
|
113
|
+
# Recurse
|
114
|
+
self.build_compound_node(n)
|
115
|
+
else:
|
116
|
+
# Leaf node
|
117
|
+
node_id = self._node_id
|
118
|
+
self._node_id += 1
|
119
|
+
node_name = "n%d" % node_id
|
120
|
+
self._node_id_m[n] = node_name
|
121
|
+
leaf_name = n.name[n.name.rfind(".") + 1:]
|
122
|
+
self.println("%s[label=\"%s\"];" % (node_name, leaf_name))
|
123
|
+
self.dec_ind()
|
124
|
+
self.println("}")
|
65
125
|
|
66
|
-
self._log.debug("<--
|
126
|
+
self._log.debug("<-- build_compound_node %s (%d)" % (node.name, len(node.tasks),))
|
67
127
|
|
68
128
|
def println(self, l):
|
69
129
|
self.fp.write("%s%s\n" % (self._ind, l))
|
dv_flow/mgr/task_node.py
CHANGED
@@ -56,6 +56,7 @@ class TaskNode(object):
|
|
56
56
|
start : float = dc.field(default=None)
|
57
57
|
end : float = dc.field(default=None)
|
58
58
|
save_exec_data : bool = dc.field(default=True)
|
59
|
+
parent : 'TaskNode' = dc.field(default=None)
|
59
60
|
|
60
61
|
_log : ClassVar = logging.getLogger("TaskNode")
|
61
62
|
|
@@ -69,6 +70,10 @@ class TaskNode(object):
|
|
69
70
|
if not isinstance(need, tuple):
|
70
71
|
self.needs[i] = (need, False)
|
71
72
|
|
73
|
+
@property
|
74
|
+
def first(self):
|
75
|
+
return self
|
76
|
+
|
72
77
|
async def do_run(self,
|
73
78
|
runner,
|
74
79
|
rundir,
|
@@ -46,8 +46,14 @@ class TaskNodeCompound(TaskNode):
|
|
46
46
|
srcdir=self.srcdir,
|
47
47
|
params=NullParams())
|
48
48
|
self.input.task = null_run
|
49
|
+
self.tasks.append(self.input)
|
50
|
+
|
49
51
|
return super().__post_init__()
|
50
52
|
|
53
|
+
@property
|
54
|
+
def first(self):
|
55
|
+
return self.input
|
56
|
+
|
51
57
|
async def do_run(self,
|
52
58
|
runner : TaskRunner,
|
53
59
|
rundir,
|
dv_flow/mgr/task_node_leaf.py
CHANGED
@@ -98,7 +98,7 @@ class TaskNodeLeaf(TaskNode):
|
|
98
98
|
else:
|
99
99
|
self._log.debug("consumes(unknown): %s" % str(self.consumes))
|
100
100
|
|
101
|
-
for name,field in self.params.model_fields.items():
|
101
|
+
for name,field in type(self.params).model_fields.items():
|
102
102
|
value = getattr(self.params, name)
|
103
103
|
if type(value) == str:
|
104
104
|
if value.find("${{") != -1:
|
{dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/RECORD
RENAMED
@@ -31,11 +31,11 @@ dv_flow/mgr/srcinfo.py,sha256=xrfp-relVr7hYNbxOjY5jqX4H0nNMm9HK5AAM6_Piyk,366
|
|
31
31
|
dv_flow/mgr/task.py,sha256=Xe1sCCTN4yJwhB2Aw_rywdlmVPY2CDnuwZisULuOXhw,2036
|
32
32
|
dv_flow/mgr/task_data.py,sha256=lN7Iq8YTitEMGG4rZqYQi6Ri2HuPgBQ5oGQbW-63T8c,12436
|
33
33
|
dv_flow/mgr/task_def.py,sha256=VAjkFay7KxXfZoloZfbiPr6WhURztV2Dxm6-REMQweo,4912
|
34
|
-
dv_flow/mgr/task_graph_builder.py,sha256=
|
35
|
-
dv_flow/mgr/task_graph_dot_writer.py,sha256=
|
34
|
+
dv_flow/mgr/task_graph_builder.py,sha256=h0jiGY4VQI-wLJevawHl40wTILiMulA2eZpi0jFJ82A,16739
|
35
|
+
dv_flow/mgr/task_graph_dot_writer.py,sha256=fZlNR-ZUFsI5FpH1KFCWJcSpyLSKUh8IhHO9Eavc60c,4925
|
36
36
|
dv_flow/mgr/task_listener_log.py,sha256=QH7hAVO1agYJoOx16XqjWv1c-jk5zb6p6A-hic7OMU0,4150
|
37
|
-
dv_flow/mgr/task_node.py,sha256=
|
38
|
-
dv_flow/mgr/task_node_compound.py,sha256=
|
37
|
+
dv_flow/mgr/task_node.py,sha256=hPlCcped3nCNgOYj4oysVcxu4cHOIUCBUqAevOr20Lc,5086
|
38
|
+
dv_flow/mgr/task_node_compound.py,sha256=kmhfcHeqNwYE-XIRh8dsdU_zvLpAzHwVTdr4S3eAtN8,3080
|
39
39
|
dv_flow/mgr/task_node_ctor.py,sha256=YsoVMX5WbpbzcHvEK7ps_ZRV-J7MZ3F8NNozQw7vbog,4418
|
40
40
|
dv_flow/mgr/task_node_ctor_compound.py,sha256=290JdcTnL3b3Gv7s_wRLjdM02ezKhc9QnxZE0mv72i8,4379
|
41
41
|
dv_flow/mgr/task_node_ctor_compound_proxy.py,sha256=D8x54nD8Pd-2-_mr1syhqVeSFfIVf100ldi3bdzmSfI,2073
|
@@ -43,7 +43,7 @@ dv_flow/mgr/task_node_ctor_def_base.py,sha256=_8QQHKDkONio_ve0Z409yxC0AMO8ocNBPD
|
|
43
43
|
dv_flow/mgr/task_node_ctor_proxy.py,sha256=ViOFJ64JM4-CGFZNl89BghFuKSQ66kZVqSj4v2PA6VA,1906
|
44
44
|
dv_flow/mgr/task_node_ctor_task.py,sha256=d49g90TyPCMFR8BuWWqp4ym-MW5vGSdDR0V47Ru28JY,2232
|
45
45
|
dv_flow/mgr/task_node_ctor_wrapper.py,sha256=Nb1CVcPHZofnb-iLWDHQWAxlTOdbRrnR9DdSxY8yOec,3626
|
46
|
-
dv_flow/mgr/task_node_leaf.py,sha256=
|
46
|
+
dv_flow/mgr/task_node_leaf.py,sha256=QZru3SisLZwykpfATJPo8qQJbRwArw0GBvRlLtLB8bs,8658
|
47
47
|
dv_flow/mgr/task_output.py,sha256=ZwyvwnYj_gHOEFAEOH3m24Xfc4Cn77hb1j7LkX8_3C4,1086
|
48
48
|
dv_flow/mgr/task_params_ctor.py,sha256=qlrzibGAFHmbqOu88jEoh1wOSFHu68Gwdgc259-50e8,1915
|
49
49
|
dv_flow/mgr/task_run_ctxt.py,sha256=w1EOG9R16j0eT8fNcOCqlfyFrw277onsSur607eNprY,2811
|
@@ -66,9 +66,9 @@ dv_flow/mgr/util/__main__.py,sha256=F0LXpCDpYTPalSo0dc1h_qZkip5v1AZYYh-vcYbh5s0,
|
|
66
66
|
dv_flow/mgr/util/util.py,sha256=cBNt3JJ0PGLlUQFTtBLi12i2j_9gNgSBtKdwS3VfF5Y,1566
|
67
67
|
dv_flow/mgr/util/cmds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
68
68
|
dv_flow/mgr/util/cmds/cmd_schema.py,sha256=IJzZdxCSEgIQ79LpYiM7UqJ9RJ-7yraqmBN2XVgAgXA,1752
|
69
|
-
dv_flow_mgr-1.0.0.
|
70
|
-
dv_flow_mgr-1.0.0.
|
71
|
-
dv_flow_mgr-1.0.0.
|
72
|
-
dv_flow_mgr-1.0.0.
|
73
|
-
dv_flow_mgr-1.0.0.
|
74
|
-
dv_flow_mgr-1.0.0.
|
69
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
70
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/METADATA,sha256=B60agWgNKhS9gxExLemROgmyWl_S8SpiF2OqDjCyAKg,13336
|
71
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
72
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/entry_points.txt,sha256=1roy8wAFM48LabOvr6jiOw0MUs-qE8X3Vf8YykPazxk,50
|
73
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/top_level.txt,sha256=amfVTkggzYPtWwLqNmRukfz1Buu0pGS2SrYBBLhXm04,8
|
74
|
+
dv_flow_mgr-1.0.0.14392445998a1.dist-info/RECORD,,
|
{dv_flow_mgr-1.0.0.14389547107a1.dist-info → dv_flow_mgr-1.0.0.14392445998a1.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|