vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/algo/components.py
CHANGED
|
@@ -50,7 +50,7 @@ from bronx.syntax.decorators import nicedeco
|
|
|
50
50
|
import footprints
|
|
51
51
|
from taylorism import Boss
|
|
52
52
|
import vortex
|
|
53
|
-
|
|
53
|
+
import vortex.config as config
|
|
54
54
|
from vortex.algo import mpitools
|
|
55
55
|
from vortex.syntax.stdattrs import DelayedEnvValue
|
|
56
56
|
from vortex.tools.parallelism import ParallelResultParser
|
|
@@ -63,11 +63,13 @@ logger = loggers.getLogger(__name__)
|
|
|
63
63
|
|
|
64
64
|
class AlgoComponentError(Exception):
|
|
65
65
|
"""Generic exception class for Algo Components."""
|
|
66
|
+
|
|
66
67
|
pass
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
class AlgoComponentAssertionError(AlgoComponentError):
|
|
70
71
|
"""Assertion exception class for Algo Components."""
|
|
72
|
+
|
|
71
73
|
pass
|
|
72
74
|
|
|
73
75
|
|
|
@@ -83,8 +85,12 @@ class DelayedAlgoComponentError(AlgoComponentError):
|
|
|
83
85
|
|
|
84
86
|
def __str__(self):
|
|
85
87
|
outstr = "One or several errors occurred during the run. In order of appearance:\n"
|
|
86
|
-
outstr += "\n".join(
|
|
87
|
-
|
|
88
|
+
outstr += "\n".join(
|
|
89
|
+
[
|
|
90
|
+
"{:3d}. {!s} (type: {!s})".format(i + 1, exc, type(exc))
|
|
91
|
+
for i, exc in enumerate(self)
|
|
92
|
+
]
|
|
93
|
+
)
|
|
88
94
|
return outstr
|
|
89
95
|
|
|
90
96
|
|
|
@@ -102,10 +108,14 @@ def _clsmtd_mixin_locked(f):
|
|
|
102
108
|
This is a utility decorator (for class methods) : it ensures that the method can only
|
|
103
109
|
be called on a bare :class:`AlgoComponentDecoMixin` class.
|
|
104
110
|
"""
|
|
111
|
+
|
|
105
112
|
def wrapped_clsmethod(cls, *kargs, **kwargs):
|
|
106
113
|
if issubclass(cls, AlgoComponent):
|
|
107
|
-
raise RuntimeError(
|
|
114
|
+
raise RuntimeError(
|
|
115
|
+
"This class method should not be called once the mixin is in use."
|
|
116
|
+
)
|
|
108
117
|
return f(cls, *kargs, **kwargs)
|
|
118
|
+
|
|
109
119
|
return wrapped_clsmethod
|
|
110
120
|
|
|
111
121
|
|
|
@@ -114,45 +124,69 @@ def algo_component_deco_mixin_autodoc(cls):
|
|
|
114
124
|
Decorator that adds an automatic documentation on any :class:`AlgoComponentDecoMixin`
|
|
115
125
|
class.
|
|
116
126
|
"""
|
|
117
|
-
extradoc =
|
|
127
|
+
extradoc = ""
|
|
118
128
|
|
|
119
129
|
# Document extra footprints
|
|
120
130
|
if cls.MIXIN_AUTO_FPTWEAK and cls._MIXIN_EXTRA_FOOTPRINTS:
|
|
121
|
-
extradoc +=
|
|
131
|
+
extradoc += "\nThe following footprints will be applied to the target classes:\n\n"
|
|
122
132
|
for fp in cls._MIXIN_EXTRA_FOOTPRINTS:
|
|
123
133
|
if isinstance(fp, footprints.Footprint):
|
|
124
|
-
extradoc += footprints.doc.format_docstring(
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
extradoc +=
|
|
134
|
+
extradoc += footprints.doc.format_docstring(
|
|
135
|
+
fp, footprints.setup.docstrings, abstractfpobj=True
|
|
136
|
+
)
|
|
137
|
+
extradoc += "\n"
|
|
128
138
|
|
|
129
139
|
# Document decorating classes
|
|
130
140
|
if cls.MIXIN_AUTO_DECO:
|
|
131
|
-
for what, desc in (
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
for what, desc in (
|
|
142
|
+
("PREPARE_PREHOOKS", "before the original ``prepare`` method"),
|
|
143
|
+
("PREPARE_HOOKS", "after the original ``prepare`` method"),
|
|
144
|
+
("POSTFIX_PREHOOKS", "before the original ``postfix`` method"),
|
|
145
|
+
("POSTFIX_HOOKS", "after the original ``postfix`` method"),
|
|
146
|
+
("SPAWN_HOOKS", "after the original ``spawn_hook`` method"),
|
|
147
|
+
(
|
|
148
|
+
"CLI_OPTS_EXTEND",
|
|
149
|
+
"to alter the result of the ``spawn_command_options`` method",
|
|
150
|
+
),
|
|
151
|
+
(
|
|
152
|
+
"STDIN_OPTS_EXTEND",
|
|
153
|
+
"to alter the result of the ``spawn_stdin_options`` method",
|
|
154
|
+
),
|
|
155
|
+
(
|
|
156
|
+
"_MIXIN_EXECUTE_OVERWRITE",
|
|
157
|
+
"instead of the original ``execute`` method",
|
|
158
|
+
),
|
|
159
|
+
(
|
|
160
|
+
"MPIBINS_HOOKS",
|
|
161
|
+
"to alter the result of the ``_bootstrap_mpibins_hack`` method",
|
|
162
|
+
),
|
|
163
|
+
(
|
|
164
|
+
"MPIENVELOPE_HOOKS",
|
|
165
|
+
"to alter the result of the ``_bootstrap_mpienvelope_hack`` method",
|
|
166
|
+
),
|
|
167
|
+
):
|
|
168
|
+
what = "_MIXIN_{:s}".format(what)
|
|
142
169
|
if getattr(cls, what, ()):
|
|
143
|
-
extradoc +=
|
|
144
|
-
|
|
145
|
-
|
|
170
|
+
extradoc += "\nThe following method(s) will be called {:s}:\n\n".format(
|
|
171
|
+
desc
|
|
172
|
+
)
|
|
173
|
+
extradoc += "\n".join(
|
|
174
|
+
" * {!r}".format(cb) for cb in getattr(cls, what)
|
|
175
|
+
)
|
|
176
|
+
extradoc += "\n"
|
|
146
177
|
|
|
147
178
|
if extradoc:
|
|
148
|
-
extradoc = (
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
179
|
+
extradoc = (
|
|
180
|
+
"\n .. note:: The following documentation is automatically generated. "
|
|
181
|
+
+ "From a developer point of view, using the present mixin class "
|
|
182
|
+
+ "will result in the following actions:\n"
|
|
183
|
+
+ " \n".join(
|
|
184
|
+
[" " + t if t else "" for t in extradoc.split("\n")]
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
if isinstance(getattr(cls, "__doc__", None), str):
|
|
189
|
+
cls.__doc__ += "\n" + extradoc
|
|
156
190
|
else:
|
|
157
191
|
cls.__doc__ = extradoc
|
|
158
192
|
|
|
@@ -253,8 +287,11 @@ class AlgoComponentDecoMixin:
|
|
|
253
287
|
def __new__(cls, *args, **kwargs):
|
|
254
288
|
if not issubclass(cls, AlgoComponent):
|
|
255
289
|
# This class cannot be instanciated by itself !
|
|
256
|
-
raise RuntimeError(
|
|
257
|
-
|
|
290
|
+
raise RuntimeError(
|
|
291
|
+
"< {0.__name__:s} > is a mixin class: it cannot be instantiated.".format(
|
|
292
|
+
cls
|
|
293
|
+
)
|
|
294
|
+
)
|
|
258
295
|
else:
|
|
259
296
|
return super().__new__(cls)
|
|
260
297
|
|
|
@@ -268,18 +305,24 @@ class AlgoComponentDecoMixin:
|
|
|
268
305
|
|
|
269
306
|
@classmethod
|
|
270
307
|
@_clsmtd_mixin_locked
|
|
271
|
-
def _get_algo_wrapped(
|
|
308
|
+
def _get_algo_wrapped(
|
|
309
|
+
cls, targetcls, targetmtd, hooks, prehooks=(), reentering=False
|
|
310
|
+
):
|
|
272
311
|
"""Wraps **targetcls**'s **targetmtd** method."""
|
|
273
312
|
orig_mtd = getattr(targetcls, targetmtd)
|
|
274
313
|
if prehooks and reentering:
|
|
275
|
-
raise ValueError(
|
|
314
|
+
raise ValueError(
|
|
315
|
+
"Conflicting values between prehooks and reenterin."
|
|
316
|
+
)
|
|
276
317
|
|
|
277
318
|
def wrapped_method(self, *kargs, **kwargs):
|
|
278
319
|
for phook in prehooks:
|
|
279
320
|
phook(self, *kargs, **kwargs)
|
|
280
321
|
rv = orig_mtd(self, *kargs, **kwargs)
|
|
281
322
|
if reentering:
|
|
282
|
-
kargs = [
|
|
323
|
+
kargs = [
|
|
324
|
+
rv,
|
|
325
|
+
] + list(kargs)
|
|
283
326
|
for phook in hooks:
|
|
284
327
|
rv = phook(self, *kargs, **kwargs)
|
|
285
328
|
if reentering:
|
|
@@ -288,9 +331,11 @@ class AlgoComponentDecoMixin:
|
|
|
288
331
|
return rv
|
|
289
332
|
|
|
290
333
|
wrapped_method.__name__ = orig_mtd.__name__
|
|
291
|
-
wrapped_method.__doc__ = (
|
|
292
|
-
|
|
293
|
-
|
|
334
|
+
wrapped_method.__doc__ = (orig_mtd.__doc__ or "").rstrip(
|
|
335
|
+
"\n"
|
|
336
|
+
) + "\n\nDecorated by :class:`{0.__module__:s}{0.__name__:s}`.".format(
|
|
337
|
+
cls
|
|
338
|
+
)
|
|
294
339
|
wrapped_method.__dict__.update(orig_mtd.__dict__)
|
|
295
340
|
return wrapped_method
|
|
296
341
|
|
|
@@ -302,33 +347,36 @@ class AlgoComponentDecoMixin:
|
|
|
302
347
|
:class:`AlgoComponent` class.
|
|
303
348
|
"""
|
|
304
349
|
if not issubclass(targetcls, AlgoComponent):
|
|
305
|
-
raise RuntimeError(
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
cls._MIXIN_STDIN_OPTS_EXTEND, (),
|
|
328
|
-
True), ]:
|
|
350
|
+
raise RuntimeError(
|
|
351
|
+
"This class can only be mixed in AlgoComponent classes."
|
|
352
|
+
)
|
|
353
|
+
for targetmtd, hooks, prehooks, reenter in [
|
|
354
|
+
(
|
|
355
|
+
"prepare",
|
|
356
|
+
cls._MIXIN_PREPARE_HOOKS,
|
|
357
|
+
cls._MIXIN_PREPARE_PREHOOKS,
|
|
358
|
+
False,
|
|
359
|
+
),
|
|
360
|
+
("fail_execute", cls._MIXIN_FAIL_EXECUTE_HOOKS, (), False),
|
|
361
|
+
("execute_finalise", cls._MIXIN_EXECUTE_FINALISE_HOOKS, (), False),
|
|
362
|
+
(
|
|
363
|
+
"postfix",
|
|
364
|
+
cls._MIXIN_POSTFIX_HOOKS,
|
|
365
|
+
cls._MIXIN_POSTFIX_PREHOOKS,
|
|
366
|
+
False,
|
|
367
|
+
),
|
|
368
|
+
("spawn_hook", cls._MIXIN_SPAWN_HOOKS, (), False),
|
|
369
|
+
("spawn_command_options", cls._MIXIN_CLI_OPTS_EXTEND, (), True),
|
|
370
|
+
("spawn_stdin_options", cls._MIXIN_STDIN_OPTS_EXTEND, (), True),
|
|
371
|
+
]:
|
|
329
372
|
if hooks or prehooks:
|
|
330
|
-
setattr(
|
|
331
|
-
|
|
373
|
+
setattr(
|
|
374
|
+
targetcls,
|
|
375
|
+
targetmtd,
|
|
376
|
+
cls._get_algo_wrapped(
|
|
377
|
+
targetcls, targetmtd, hooks, prehooks, reenter
|
|
378
|
+
),
|
|
379
|
+
)
|
|
332
380
|
return targetcls
|
|
333
381
|
|
|
334
382
|
@classmethod
|
|
@@ -339,7 +387,7 @@ class AlgoComponentDecoMixin:
|
|
|
339
387
|
@classmethod
|
|
340
388
|
def mixin_execute_companion(cls):
|
|
341
389
|
"""Find on which class "super" should be called (if_MIXIN_EXECUTE_OVERWRITE is used)."""
|
|
342
|
-
comp = getattr(cls,
|
|
390
|
+
comp = getattr(cls, "_algo_meta_execute_companion", ())
|
|
343
391
|
if not comp:
|
|
344
392
|
raise RuntimeError("unable to find a suitable companion class")
|
|
345
393
|
return comp
|
|
@@ -396,19 +444,32 @@ class AlgoComponentMpiDecoMixin(AlgoComponentDecoMixin):
|
|
|
396
444
|
"""
|
|
397
445
|
targetcls = AlgoComponentDecoMixin.mixin_algo_deco(targetcls)
|
|
398
446
|
if not issubclass(targetcls, Parallel):
|
|
399
|
-
raise RuntimeError(
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
447
|
+
raise RuntimeError(
|
|
448
|
+
"This class can only be mixed in Parallel classes."
|
|
449
|
+
)
|
|
450
|
+
for targetmtd, hooks, prehooks, reenter in [
|
|
451
|
+
("_bootstrap_mpibins_hack", cls._MIXIN_MPIBINS_HOOKS, (), True),
|
|
452
|
+
(
|
|
453
|
+
"_bootstrap_mpienvelope_hack",
|
|
454
|
+
cls._MIXIN_MPIENVELOPE_HOOKS,
|
|
455
|
+
(),
|
|
456
|
+
True,
|
|
457
|
+
),
|
|
458
|
+
(
|
|
459
|
+
"_bootstrap_mpienvelope_posthack",
|
|
460
|
+
cls._MIXIN_MPIENVELOPE_POSTHOOKS,
|
|
461
|
+
(),
|
|
462
|
+
True,
|
|
463
|
+
),
|
|
464
|
+
]:
|
|
409
465
|
if hooks or prehooks:
|
|
410
|
-
setattr(
|
|
411
|
-
|
|
466
|
+
setattr(
|
|
467
|
+
targetcls,
|
|
468
|
+
targetmtd,
|
|
469
|
+
cls._get_algo_wrapped(
|
|
470
|
+
targetcls, targetmtd, hooks, prehooks, reenter
|
|
471
|
+
),
|
|
472
|
+
)
|
|
412
473
|
return targetcls
|
|
413
474
|
|
|
414
475
|
|
|
@@ -423,31 +484,47 @@ class AlgoComponentMeta(footprints.FootprintBaseMeta):
|
|
|
423
484
|
def __new__(cls, n, b, d):
|
|
424
485
|
# Mixin candidates: a mixin must only be dealt with once hence the
|
|
425
486
|
# condition on issubclass(base, AlgoComponent)
|
|
426
|
-
candidates = [
|
|
427
|
-
|
|
428
|
-
|
|
487
|
+
candidates = [
|
|
488
|
+
base
|
|
489
|
+
for base in b
|
|
490
|
+
if (
|
|
491
|
+
issubclass(base, AlgoComponentDecoMixin)
|
|
492
|
+
and not issubclass(base, AlgoComponent)
|
|
493
|
+
)
|
|
494
|
+
]
|
|
429
495
|
# Tweak footprints
|
|
430
496
|
todobases = [base for base in candidates if base.MIXIN_AUTO_FPTWEAK]
|
|
431
497
|
if todobases:
|
|
432
|
-
fplocal = d.get(
|
|
498
|
+
fplocal = d.get("_footprint", list())
|
|
433
499
|
if not isinstance(fplocal, list):
|
|
434
|
-
fplocal = [
|
|
500
|
+
fplocal = [
|
|
501
|
+
fplocal,
|
|
502
|
+
]
|
|
435
503
|
for base in todobases:
|
|
436
504
|
base.mixin_tweak_footprint(fplocal)
|
|
437
|
-
d[
|
|
505
|
+
d["_footprint"] = fplocal
|
|
438
506
|
# Overwrite the execute method...
|
|
439
|
-
todobases_exc = [
|
|
507
|
+
todobases_exc = [
|
|
508
|
+
base
|
|
509
|
+
for base in candidates
|
|
510
|
+
if base.mixin_execute_overwrite() is not None
|
|
511
|
+
]
|
|
440
512
|
if len(todobases_exc) > 1:
|
|
441
|
-
raise RuntimeError(
|
|
442
|
-
|
|
513
|
+
raise RuntimeError(
|
|
514
|
+
"Cannot overwrite < execute > multiple times: {:s}".format(
|
|
515
|
+
",".join([base.__name__ for base in todobases_exc])
|
|
516
|
+
)
|
|
517
|
+
)
|
|
443
518
|
if todobases_exc:
|
|
444
|
-
if
|
|
445
|
-
raise RuntimeError(
|
|
446
|
-
|
|
519
|
+
if "execute" in d:
|
|
520
|
+
raise RuntimeError(
|
|
521
|
+
"< execute > is already defined in the target class: cannot proceed"
|
|
522
|
+
)
|
|
523
|
+
d["execute"] = todobases_exc[0].mixin_execute_overwrite()
|
|
447
524
|
# Create the class as usual
|
|
448
525
|
fpcls = super().__new__(cls, n, b, d)
|
|
449
526
|
if todobases_exc:
|
|
450
|
-
setattr(fpcls,
|
|
527
|
+
setattr(fpcls, "_algo_meta_execute_companion", fpcls)
|
|
451
528
|
# Apply decorators
|
|
452
529
|
todobases = [base for base in candidates if base.MIXIN_AUTO_DECO]
|
|
453
530
|
for base in reversed(todobases):
|
|
@@ -463,84 +540,86 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
463
540
|
_SERVERSYNC_STOPONEXIT = True
|
|
464
541
|
|
|
465
542
|
_abstract = True
|
|
466
|
-
_collector = (
|
|
543
|
+
_collector = ("component",)
|
|
467
544
|
_footprint = dict(
|
|
468
|
-
info
|
|
469
|
-
attr
|
|
470
|
-
engine
|
|
471
|
-
info
|
|
472
|
-
values
|
|
545
|
+
info="Abstract algo component",
|
|
546
|
+
attr=dict(
|
|
547
|
+
engine=dict(
|
|
548
|
+
info="The way the executable should be run.",
|
|
549
|
+
values=[
|
|
550
|
+
"algo",
|
|
551
|
+
],
|
|
473
552
|
),
|
|
474
|
-
flyput
|
|
475
|
-
info
|
|
476
|
-
type
|
|
477
|
-
optional
|
|
478
|
-
default
|
|
479
|
-
access
|
|
480
|
-
doc_visibility
|
|
481
|
-
doc_zorder
|
|
553
|
+
flyput=dict(
|
|
554
|
+
info="Activate a background job in charge off on the fly processing.",
|
|
555
|
+
type=bool,
|
|
556
|
+
optional=True,
|
|
557
|
+
default=False,
|
|
558
|
+
access="rwx",
|
|
559
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
560
|
+
doc_zorder=-99,
|
|
482
561
|
),
|
|
483
|
-
flypoll
|
|
484
|
-
info
|
|
485
|
-
optional
|
|
486
|
-
default
|
|
487
|
-
access
|
|
488
|
-
doc_visibility
|
|
489
|
-
doc_zorder
|
|
562
|
+
flypoll=dict(
|
|
563
|
+
info="The system method called by the flyput background job.",
|
|
564
|
+
optional=True,
|
|
565
|
+
default="io_poll",
|
|
566
|
+
access="rwx",
|
|
567
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
568
|
+
doc_zorder=-99,
|
|
490
569
|
),
|
|
491
|
-
flyargs
|
|
492
|
-
info
|
|
493
|
-
type
|
|
494
|
-
optional
|
|
495
|
-
default
|
|
496
|
-
doc_visibility
|
|
497
|
-
doc_zorder
|
|
570
|
+
flyargs=dict(
|
|
571
|
+
info="Arguments for the *flypoll* method.",
|
|
572
|
+
type=footprints.FPTuple,
|
|
573
|
+
optional=True,
|
|
574
|
+
default=footprints.FPTuple(),
|
|
575
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
576
|
+
doc_zorder=-99,
|
|
498
577
|
),
|
|
499
|
-
flymapping
|
|
500
|
-
info
|
|
501
|
-
optional
|
|
502
|
-
default
|
|
503
|
-
access
|
|
504
|
-
doc_visibility
|
|
505
|
-
doc_zorder
|
|
578
|
+
flymapping=dict(
|
|
579
|
+
info="Allow renaming of output files during on the fly processing.",
|
|
580
|
+
optional=True,
|
|
581
|
+
default=False,
|
|
582
|
+
access="rwx",
|
|
583
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
584
|
+
doc_zorder=-99,
|
|
506
585
|
),
|
|
507
|
-
timeout
|
|
508
|
-
info
|
|
509
|
-
type
|
|
510
|
-
optional
|
|
511
|
-
default
|
|
512
|
-
doc_zorder
|
|
586
|
+
timeout=dict(
|
|
587
|
+
info="Default timeout (in sec.) used when waiting for an expected resource.",
|
|
588
|
+
type=int,
|
|
589
|
+
optional=True,
|
|
590
|
+
default=180,
|
|
591
|
+
doc_zorder=-50,
|
|
513
592
|
),
|
|
514
|
-
server_run
|
|
515
|
-
info
|
|
516
|
-
type
|
|
517
|
-
optional
|
|
518
|
-
values
|
|
519
|
-
default
|
|
520
|
-
access
|
|
521
|
-
doc_visibility
|
|
593
|
+
server_run=dict(
|
|
594
|
+
info="Run the executable as a server.",
|
|
595
|
+
type=bool,
|
|
596
|
+
optional=True,
|
|
597
|
+
values=[False],
|
|
598
|
+
default=False,
|
|
599
|
+
access="rwx",
|
|
600
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
522
601
|
),
|
|
523
|
-
serversync_method
|
|
524
|
-
info
|
|
525
|
-
optional
|
|
526
|
-
doc_visibility
|
|
602
|
+
serversync_method=dict(
|
|
603
|
+
info="The method that is used to synchronise with the server.",
|
|
604
|
+
optional=True,
|
|
605
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
527
606
|
),
|
|
528
|
-
serversync_medium
|
|
529
|
-
info
|
|
530
|
-
optional
|
|
531
|
-
doc_visibility
|
|
607
|
+
serversync_medium=dict(
|
|
608
|
+
info="The medium that is used to synchronise with the server.",
|
|
609
|
+
optional=True,
|
|
610
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
532
611
|
),
|
|
533
|
-
extendpypath
|
|
534
|
-
info
|
|
535
|
-
type
|
|
536
|
-
default
|
|
537
|
-
optional
|
|
612
|
+
extendpypath=dict(
|
|
613
|
+
info="The list of things to be prepended in the python's path.",
|
|
614
|
+
type=footprints.FPList,
|
|
615
|
+
default=footprints.FPList([]),
|
|
616
|
+
optional=True,
|
|
538
617
|
),
|
|
539
|
-
)
|
|
618
|
+
),
|
|
540
619
|
)
|
|
541
620
|
|
|
542
621
|
def __init__(self, *args, **kw):
|
|
543
|
-
logger.debug(
|
|
622
|
+
logger.debug("Algo component init %s", self.__class__)
|
|
544
623
|
self._fslog = list()
|
|
545
624
|
self._promises = None
|
|
546
625
|
self._expected = None
|
|
@@ -552,7 +631,7 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
552
631
|
@property
|
|
553
632
|
def realkind(self):
|
|
554
633
|
"""Default kind is ``algo``."""
|
|
555
|
-
return
|
|
634
|
+
return "algo"
|
|
556
635
|
|
|
557
636
|
@property
|
|
558
637
|
def fslog(self):
|
|
@@ -561,7 +640,7 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
561
640
|
|
|
562
641
|
def fstag(self):
|
|
563
642
|
"""Defines a tag specific to the current algo component."""
|
|
564
|
-
return
|
|
643
|
+
return "-".join((self.realkind, self.engine))
|
|
565
644
|
|
|
566
645
|
def fsstamp(self, opts):
|
|
567
646
|
"""Ask the current context to put a stamp on file system."""
|
|
@@ -576,7 +655,8 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
576
655
|
"""Build and return list of actual promises of the current component."""
|
|
577
656
|
if self._promises is None:
|
|
578
657
|
self._promises = [
|
|
579
|
-
x
|
|
658
|
+
x
|
|
659
|
+
for x in self.context.sequence.outputs()
|
|
580
660
|
if x.rh.provider.expected
|
|
581
661
|
]
|
|
582
662
|
return self._promises
|
|
@@ -586,7 +666,8 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
586
666
|
"""Return the list of really expected inputs."""
|
|
587
667
|
if self._expected is None:
|
|
588
668
|
self._expected = [
|
|
589
|
-
x
|
|
669
|
+
x
|
|
670
|
+
for x in self.context.sequence.effective_inputs()
|
|
590
671
|
if x.rh.is_expected()
|
|
591
672
|
]
|
|
592
673
|
return self._expected
|
|
@@ -596,42 +677,42 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
596
677
|
logger.error("An exception is delayed")
|
|
597
678
|
if traceback:
|
|
598
679
|
(exc_type, exc_value, exc_traceback) = sys.exc_info()
|
|
599
|
-
print(
|
|
600
|
-
print(
|
|
601
|
-
print(
|
|
680
|
+
print("Exception type: {!s}".format(exc_type))
|
|
681
|
+
print("Exception info: {!s}".format(exc_value))
|
|
682
|
+
print("Traceback:")
|
|
602
683
|
print("\n".join(py_traceback.format_tb(exc_traceback)))
|
|
603
684
|
self._delayed_excs.append(exc)
|
|
604
685
|
|
|
605
|
-
def algoassert(self, assertion, msg=
|
|
686
|
+
def algoassert(self, assertion, msg=""):
|
|
606
687
|
if not assertion:
|
|
607
688
|
raise AlgoComponentAssertionError(msg)
|
|
608
689
|
|
|
609
|
-
def grab(self, sec, comment=
|
|
690
|
+
def grab(self, sec, comment="resource", sleep=10, timeout=None):
|
|
610
691
|
"""Wait for a given resource and get it if expected."""
|
|
611
692
|
local = sec.rh.container.localpath()
|
|
612
|
-
self.system.header(
|
|
693
|
+
self.system.header("Wait for " + comment + " ... [" + local + "]")
|
|
613
694
|
if timeout is None:
|
|
614
695
|
timeout = self.timeout
|
|
615
696
|
if sec.rh.wait(timeout=timeout, sleep=sleep):
|
|
616
697
|
if sec.rh.is_expected():
|
|
617
698
|
sec.get(incache=True)
|
|
618
699
|
elif sec.fatal:
|
|
619
|
-
logger.critical(
|
|
620
|
-
raise ValueError(
|
|
700
|
+
logger.critical("Missing expected resource <%s>", local)
|
|
701
|
+
raise ValueError("Could not get " + local)
|
|
621
702
|
else:
|
|
622
|
-
logger.error(
|
|
703
|
+
logger.error("Missing expected resource <%s>", local)
|
|
623
704
|
|
|
624
705
|
def export(self, packenv):
|
|
625
706
|
"""Export environment variables in given pack."""
|
|
626
|
-
for k, v in from_config(section=packenv).items():
|
|
707
|
+
for k, v in config.from_config(section=packenv).items():
|
|
627
708
|
if k not in self.env:
|
|
628
|
-
logger.info(
|
|
709
|
+
logger.info("Setting %s env %s = %s", packenv.upper(), k, v)
|
|
629
710
|
self.env[k] = v
|
|
630
711
|
|
|
631
712
|
def prepare(self, rh, opts):
|
|
632
713
|
"""Set some defaults env values."""
|
|
633
|
-
if
|
|
634
|
-
self.export(
|
|
714
|
+
if config.is_defined(section="env"):
|
|
715
|
+
self.export("env")
|
|
635
716
|
|
|
636
717
|
def absexcutable(self, xfile):
|
|
637
718
|
"""Retuns the absolute pathname of the ``xfile`` executable."""
|
|
@@ -640,15 +721,17 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
640
721
|
|
|
641
722
|
def flyput_method(self):
|
|
642
723
|
"""Check out what could be a valid io_poll command."""
|
|
643
|
-
return getattr(
|
|
724
|
+
return getattr(
|
|
725
|
+
self, "io_poll_method", getattr(self.system, self.flypoll, None)
|
|
726
|
+
)
|
|
644
727
|
|
|
645
728
|
def flyput_args(self):
|
|
646
729
|
"""Return actual io_poll prefixes."""
|
|
647
|
-
return getattr(self,
|
|
730
|
+
return getattr(self, "io_poll_args", tuple(self.flyargs))
|
|
648
731
|
|
|
649
732
|
def flyput_kwargs(self):
|
|
650
733
|
"""Return actual io_poll prefixes."""
|
|
651
|
-
return getattr(self,
|
|
734
|
+
return getattr(self, "io_poll_kwargs", dict())
|
|
652
735
|
|
|
653
736
|
def flyput_check(self):
|
|
654
737
|
"""Check default args for io_poll command."""
|
|
@@ -658,38 +741,54 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
658
741
|
return self.flyput_args()
|
|
659
742
|
else:
|
|
660
743
|
for arg in self.flyput_args():
|
|
661
|
-
logger.info(
|
|
662
|
-
if any(
|
|
744
|
+
logger.info("Check arg <%s>", arg)
|
|
745
|
+
if any(
|
|
746
|
+
[
|
|
747
|
+
x.rh.container.basename.startswith(arg)
|
|
748
|
+
for x in self.promises
|
|
749
|
+
]
|
|
750
|
+
):
|
|
663
751
|
logger.info(
|
|
664
|
-
|
|
665
|
-
str(
|
|
666
|
-
|
|
752
|
+
"Match some promise %s",
|
|
753
|
+
str(
|
|
754
|
+
[
|
|
755
|
+
x.rh.container.basename
|
|
756
|
+
for x in self.promises
|
|
757
|
+
if x.rh.container.basename.startswith(arg)
|
|
758
|
+
]
|
|
759
|
+
),
|
|
667
760
|
)
|
|
668
761
|
actual_args.append(arg)
|
|
669
762
|
else:
|
|
670
|
-
logger.info(
|
|
671
|
-
|
|
763
|
+
logger.info(
|
|
764
|
+
"Do not match any promise %s",
|
|
765
|
+
str([x.rh.container.basename for x in self.promises]),
|
|
766
|
+
)
|
|
672
767
|
return actual_args
|
|
673
768
|
|
|
674
769
|
def flyput_sleep(self):
|
|
675
770
|
"""Return a sleeping time in seconds between io_poll commands."""
|
|
676
|
-
return getattr(
|
|
771
|
+
return getattr(
|
|
772
|
+
self, "io_poll_sleep", self.env.get("IO_POLL_SLEEP", 20)
|
|
773
|
+
)
|
|
677
774
|
|
|
678
775
|
def flyput_outputmapping(self, item):
|
|
679
776
|
"""Map output to another filename."""
|
|
680
|
-
return item,
|
|
777
|
+
return item, "unknown"
|
|
681
778
|
|
|
682
|
-
def _flyput_job_internal_search(
|
|
779
|
+
def _flyput_job_internal_search(
|
|
780
|
+
self, io_poll_method, io_poll_args, io_poll_kwargs
|
|
781
|
+
):
|
|
683
782
|
data = list()
|
|
684
783
|
for arg in io_poll_args:
|
|
685
|
-
logger.info(
|
|
784
|
+
logger.info("Polling check arg %s", arg)
|
|
686
785
|
rc = io_poll_method(arg, **io_poll_kwargs)
|
|
687
786
|
try:
|
|
688
787
|
data.extend(rc.result)
|
|
689
788
|
except AttributeError:
|
|
690
789
|
data.extend(rc)
|
|
691
790
|
data = [x for x in data if x]
|
|
692
|
-
logger.info(
|
|
791
|
+
logger.info("Polling retrieved data %s", str(data))
|
|
693
792
|
return data
|
|
694
793
|
|
|
695
794
|
def _flyput_job_internal_put(self, data):
|
|
@@ -697,27 +796,46 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
697
796
|
if self.flymapping:
|
|
698
797
|
mappeddata, mappedfmt = self.flyput_outputmapping(thisdata)
|
|
699
798
|
if not mappeddata:
|
|
700
|
-
raise AlgoComponentError(
|
|
799
|
+
raise AlgoComponentError(
|
|
800
|
+
"The mapping method failed for {:s}.".format(thisdata)
|
|
801
|
+
)
|
|
701
802
|
if thisdata != mappeddata:
|
|
702
|
-
logger.info(
|
|
703
|
-
|
|
704
|
-
|
|
803
|
+
logger.info(
|
|
804
|
+
"Linking <%s> to <%s> (fmt=%s) before put",
|
|
805
|
+
thisdata,
|
|
806
|
+
mappeddata,
|
|
807
|
+
mappedfmt,
|
|
808
|
+
)
|
|
809
|
+
self.system.cp(
|
|
810
|
+
thisdata, mappeddata, intent="in", fmt=mappedfmt
|
|
811
|
+
)
|
|
705
812
|
else:
|
|
706
813
|
mappeddata = thisdata
|
|
707
|
-
candidates = [
|
|
708
|
-
|
|
814
|
+
candidates = [
|
|
815
|
+
x
|
|
816
|
+
for x in self.promises
|
|
817
|
+
if x.rh.container.abspath
|
|
818
|
+
== self.system.path.abspath(mappeddata)
|
|
819
|
+
]
|
|
709
820
|
if candidates:
|
|
710
|
-
logger.info(
|
|
821
|
+
logger.info("Polled data is promised <%s>", mappeddata)
|
|
711
822
|
bingo = candidates.pop()
|
|
712
823
|
bingo.put(incache=True)
|
|
713
824
|
else:
|
|
714
|
-
logger.warning(
|
|
715
|
-
|
|
716
|
-
def flyput_job(
|
|
717
|
-
|
|
825
|
+
logger.warning("Polled data not promised <%s>", mappeddata)
|
|
826
|
+
|
|
827
|
+
def flyput_job(
|
|
828
|
+
self,
|
|
829
|
+
io_poll_method,
|
|
830
|
+
io_poll_args,
|
|
831
|
+
io_poll_kwargs,
|
|
832
|
+
event_complete,
|
|
833
|
+
event_free,
|
|
834
|
+
queue_context,
|
|
835
|
+
):
|
|
718
836
|
"""Poll new data resources."""
|
|
719
|
-
logger.info(
|
|
720
|
-
logger.info(
|
|
837
|
+
logger.info("Polling with method %s", str(io_poll_method))
|
|
838
|
+
logger.info("Polling with args %s", str(io_poll_args))
|
|
721
839
|
|
|
722
840
|
time_sleep = self.flyput_sleep()
|
|
723
841
|
redo = True
|
|
@@ -728,29 +846,33 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
728
846
|
while redo and not event_complete.is_set():
|
|
729
847
|
event_free.clear()
|
|
730
848
|
try:
|
|
731
|
-
data = self._flyput_job_internal_search(
|
|
732
|
-
|
|
849
|
+
data = self._flyput_job_internal_search(
|
|
850
|
+
io_poll_method, io_poll_args, io_poll_kwargs
|
|
851
|
+
)
|
|
733
852
|
self._flyput_job_internal_put(data)
|
|
734
853
|
except Exception as trouble:
|
|
735
|
-
logger.error(
|
|
736
|
-
|
|
854
|
+
logger.error(
|
|
855
|
+
"Polling trouble: %s. %s",
|
|
856
|
+
str(trouble),
|
|
857
|
+
py_traceback.format_exc(),
|
|
858
|
+
)
|
|
737
859
|
redo = False
|
|
738
860
|
finally:
|
|
739
861
|
event_free.set()
|
|
740
862
|
if redo and not data and not event_complete.is_set():
|
|
741
|
-
logger.info(
|
|
863
|
+
logger.info("Get asleep for %d seconds...", time_sleep)
|
|
742
864
|
self.system.sleep(time_sleep)
|
|
743
865
|
|
|
744
866
|
# Stop recording and send back the results
|
|
745
867
|
ctxrec.unregister()
|
|
746
|
-
logger.info(
|
|
868
|
+
logger.info("Sending the Context recorder to the master process.")
|
|
747
869
|
queue_context.put(ctxrec)
|
|
748
870
|
queue_context.close()
|
|
749
871
|
|
|
750
872
|
if redo:
|
|
751
|
-
logger.info(
|
|
873
|
+
logger.info("Polling exit on complete event")
|
|
752
874
|
else:
|
|
753
|
-
logger.warning(
|
|
875
|
+
logger.warning("Polling exit on abort")
|
|
754
876
|
|
|
755
877
|
def flyput_begin(self):
|
|
756
878
|
"""Launch a co-process to handle promises."""
|
|
@@ -760,22 +882,24 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
760
882
|
return nope
|
|
761
883
|
|
|
762
884
|
sh = self.system
|
|
763
|
-
sh.subtitle(
|
|
885
|
+
sh.subtitle("On the fly - Begin")
|
|
764
886
|
|
|
765
887
|
if not self.promises:
|
|
766
|
-
logger.info(
|
|
888
|
+
logger.info("No promise, no co-process")
|
|
767
889
|
return nope
|
|
768
890
|
|
|
769
891
|
# Find out a polling method
|
|
770
892
|
io_poll_method = self.flyput_method()
|
|
771
893
|
if not io_poll_method:
|
|
772
|
-
logger.error(
|
|
894
|
+
logger.error(
|
|
895
|
+
"No method or shell function defined for polling data"
|
|
896
|
+
)
|
|
773
897
|
return nope
|
|
774
898
|
|
|
775
899
|
# Be sure that some default args could match local promises names
|
|
776
900
|
io_poll_args = self.flyput_check()
|
|
777
901
|
if not io_poll_args:
|
|
778
|
-
logger.error(
|
|
902
|
+
logger.error("Could not check default arguments for polling data")
|
|
779
903
|
return nope
|
|
780
904
|
|
|
781
905
|
# Additional named attributes
|
|
@@ -789,7 +913,14 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
789
913
|
p_io = multiprocessing.Process(
|
|
790
914
|
name=self.footprint_clsname(),
|
|
791
915
|
target=self.flyput_job,
|
|
792
|
-
args=(
|
|
916
|
+
args=(
|
|
917
|
+
io_poll_method,
|
|
918
|
+
io_poll_args,
|
|
919
|
+
io_poll_kwargs,
|
|
920
|
+
event_stop,
|
|
921
|
+
event_free,
|
|
922
|
+
queue_ctx,
|
|
923
|
+
),
|
|
793
924
|
)
|
|
794
925
|
|
|
795
926
|
# The co-process is started
|
|
@@ -802,16 +933,17 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
802
933
|
# Find out a polling method
|
|
803
934
|
io_poll_method = self.flyput_method()
|
|
804
935
|
if not io_poll_method:
|
|
805
|
-
raise AlgoComponentError(
|
|
936
|
+
raise AlgoComponentError("Unable to find an io_poll_method")
|
|
806
937
|
# Find out some polling prefixes
|
|
807
938
|
io_poll_args = self.flyput_check()
|
|
808
939
|
if not io_poll_args:
|
|
809
|
-
raise AlgoComponentError(
|
|
940
|
+
raise AlgoComponentError("Unable to find an io_poll_args")
|
|
810
941
|
# Additional named attributes
|
|
811
942
|
io_poll_kwargs = self.flyput_kwargs()
|
|
812
943
|
# Starting polling each of the prefixes
|
|
813
|
-
return self._flyput_job_internal_search(
|
|
814
|
-
|
|
944
|
+
return self._flyput_job_internal_search(
|
|
945
|
+
io_poll_method, io_poll_args, io_poll_kwargs
|
|
946
|
+
)
|
|
815
947
|
|
|
816
948
|
def manual_flypolling_job(self):
|
|
817
949
|
"""Call the flyput method and deal with promised files."""
|
|
@@ -821,7 +953,7 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
821
953
|
def flyput_end(self, p_io, e_complete, e_free, queue_ctx):
|
|
822
954
|
"""Wait for the co-process in charge of promises."""
|
|
823
955
|
e_complete.set()
|
|
824
|
-
logger.info(
|
|
956
|
+
logger.info("Waiting for polling process... <%s>", p_io.pid)
|
|
825
957
|
t0 = date.now()
|
|
826
958
|
e_free.wait(60)
|
|
827
959
|
# Get the Queue and update the context
|
|
@@ -839,11 +971,14 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
839
971
|
p_io.join(30)
|
|
840
972
|
t1 = date.now()
|
|
841
973
|
waiting = t1 - t0
|
|
842
|
-
logger.info(
|
|
974
|
+
logger.info(
|
|
975
|
+
"Waiting for polling process took %f seconds",
|
|
976
|
+
waiting.total_seconds(),
|
|
977
|
+
)
|
|
843
978
|
if p_io.is_alive():
|
|
844
|
-
logger.warning(
|
|
979
|
+
logger.warning("Force termination of polling process")
|
|
845
980
|
p_io.terminate()
|
|
846
|
-
logger.info(
|
|
981
|
+
logger.info("Polling still alive ? %s", str(p_io.is_alive()))
|
|
847
982
|
return not p_io.is_alive()
|
|
848
983
|
|
|
849
984
|
def server_begin(self, rh, opts):
|
|
@@ -852,7 +987,7 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
852
987
|
self._server_process = multiprocessing.Process(
|
|
853
988
|
name=self.footprint_clsname(),
|
|
854
989
|
target=self.server_job,
|
|
855
|
-
args=(rh, opts)
|
|
990
|
+
args=(rh, opts),
|
|
856
991
|
)
|
|
857
992
|
self._server_process.start()
|
|
858
993
|
|
|
@@ -867,17 +1002,19 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
867
1002
|
self.execute_single(rh, opts)
|
|
868
1003
|
except Exception:
|
|
869
1004
|
(exc_type, exc_value, exc_traceback) = sys.exc_info()
|
|
870
|
-
print(
|
|
871
|
-
print(
|
|
872
|
-
print(
|
|
1005
|
+
print("Exception type: {!s}".format(exc_type))
|
|
1006
|
+
print("Exception info: {!s}".format(exc_value))
|
|
1007
|
+
print("Traceback:")
|
|
873
1008
|
print("\n".join(py_traceback.format_tb(exc_traceback)))
|
|
874
1009
|
# Alert the main process of the error
|
|
875
1010
|
self._server_event.set()
|
|
876
1011
|
|
|
877
1012
|
def server_alive(self):
|
|
878
1013
|
"""Is the server still running ?"""
|
|
879
|
-
return (
|
|
880
|
-
|
|
1014
|
+
return (
|
|
1015
|
+
self._server_process is not None
|
|
1016
|
+
and self._server_process.is_alive()
|
|
1017
|
+
)
|
|
881
1018
|
|
|
882
1019
|
def server_end(self):
|
|
883
1020
|
"""End the server.
|
|
@@ -887,39 +1024,55 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
887
1024
|
"""
|
|
888
1025
|
rc = False
|
|
889
1026
|
# This test should always succeed...
|
|
890
|
-
if
|
|
1027
|
+
if (
|
|
1028
|
+
self._server_synctool is not None
|
|
1029
|
+
and self._server_process is not None
|
|
1030
|
+
):
|
|
891
1031
|
# Is the process still running ?
|
|
892
1032
|
if self._server_process.is_alive():
|
|
893
1033
|
# Try to stop it nicely
|
|
894
|
-
if
|
|
1034
|
+
if (
|
|
1035
|
+
self._SERVERSYNC_STOPONEXIT
|
|
1036
|
+
and self._server_synctool.trigger_stop()
|
|
1037
|
+
):
|
|
895
1038
|
t0 = date.now()
|
|
896
1039
|
self._server_process.join(30)
|
|
897
1040
|
waiting = date.now() - t0
|
|
898
|
-
logger.info(
|
|
899
|
-
|
|
1041
|
+
logger.info(
|
|
1042
|
+
"Waiting for the server to stop took %f seconds",
|
|
1043
|
+
waiting.total_seconds(),
|
|
1044
|
+
)
|
|
900
1045
|
rc = not self._server_event.is_set()
|
|
901
1046
|
# Be less nice if needed...
|
|
902
|
-
if (
|
|
903
|
-
|
|
1047
|
+
if (
|
|
1048
|
+
not self._SERVERSYNC_STOPONEXIT
|
|
1049
|
+
) or self._server_process.is_alive():
|
|
1050
|
+
logger.warning("Force termination of the server process")
|
|
904
1051
|
self._server_process.terminate()
|
|
905
|
-
self.system.sleep(
|
|
1052
|
+
self.system.sleep(
|
|
1053
|
+
1
|
|
1054
|
+
) # Allow some time for the process to terminate
|
|
906
1055
|
if not self._SERVERSYNC_STOPONEXIT:
|
|
907
1056
|
rc = False
|
|
908
1057
|
else:
|
|
909
1058
|
rc = not self._server_event.is_set()
|
|
910
|
-
logger.info(
|
|
1059
|
+
logger.info(
|
|
1060
|
+
"Server still alive ? %s", str(self._server_process.is_alive())
|
|
1061
|
+
)
|
|
911
1062
|
# We are done with the server
|
|
912
1063
|
self._server_synctool = None
|
|
913
1064
|
self._server_process = None
|
|
914
1065
|
del self._server_event
|
|
915
1066
|
# Check the rc
|
|
916
1067
|
if not rc:
|
|
917
|
-
raise AlgoComponentError(
|
|
1068
|
+
raise AlgoComponentError("The server process ended badly.")
|
|
918
1069
|
return rc
|
|
919
1070
|
|
|
920
1071
|
def spawn_pre_dirlisting(self):
|
|
921
1072
|
"""Print a directory listing just before run."""
|
|
922
|
-
self.system.subtitle(
|
|
1073
|
+
self.system.subtitle(
|
|
1074
|
+
"{:s} : directory listing (pre-execution)".format(self.realkind)
|
|
1075
|
+
)
|
|
923
1076
|
self.system.dir(output=False, fatal=False)
|
|
924
1077
|
|
|
925
1078
|
def spawn_hook(self):
|
|
@@ -936,24 +1089,27 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
936
1089
|
"""
|
|
937
1090
|
sh = self.system
|
|
938
1091
|
|
|
939
|
-
if self.env.true(
|
|
940
|
-
sh.subtitle(
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
1092
|
+
if self.env.true("vortex_debug_env"):
|
|
1093
|
+
sh.subtitle(
|
|
1094
|
+
"{:s} : dump environment (os bound: {!s})".format(
|
|
1095
|
+
self.realkind, self.env.osbound()
|
|
1096
|
+
)
|
|
1097
|
+
)
|
|
944
1098
|
self.env.osdump()
|
|
945
1099
|
|
|
946
1100
|
# On-the-fly coprocessing initialisation
|
|
947
1101
|
p_io, e_complete, e_free, q_ctx = self.flyput_begin()
|
|
948
1102
|
|
|
949
|
-
sh.remove(
|
|
950
|
-
sh.softlink(
|
|
1103
|
+
sh.remove("core")
|
|
1104
|
+
sh.softlink("/dev/null", "core")
|
|
951
1105
|
self.spawn_hook()
|
|
952
1106
|
self.target.spawn_hook(sh)
|
|
953
1107
|
self.spawn_pre_dirlisting()
|
|
954
|
-
sh.subtitle(
|
|
1108
|
+
sh.subtitle("{:s} : start execution".format(self.realkind))
|
|
955
1109
|
try:
|
|
956
|
-
sh.spawn(
|
|
1110
|
+
sh.spawn(
|
|
1111
|
+
args, output=False, stdin=stdin, fatal=opts.get("fatal", True)
|
|
1112
|
+
)
|
|
957
1113
|
finally:
|
|
958
1114
|
# On-the-fly coprocessing cleaning
|
|
959
1115
|
if p_io:
|
|
@@ -977,8 +1133,8 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
977
1133
|
opts = self.spawn_stdin_options()
|
|
978
1134
|
stdin_text = rh.resource.stdin_text(**opts)
|
|
979
1135
|
if stdin_text is not None:
|
|
980
|
-
plocale = locale.getlocale()[1] or
|
|
981
|
-
tmpfh = tempfile.TemporaryFile(dir=self.system.pwd(), mode=
|
|
1136
|
+
plocale = locale.getlocale()[1] or "ascii"
|
|
1137
|
+
tmpfh = tempfile.TemporaryFile(dir=self.system.pwd(), mode="w+b")
|
|
982
1138
|
if isinstance(stdin_text, str):
|
|
983
1139
|
tmpfh.write(stdin_text.encode(plocale))
|
|
984
1140
|
else:
|
|
@@ -1002,13 +1158,15 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1002
1158
|
# First time here ?
|
|
1003
1159
|
if self._server_synctool is None:
|
|
1004
1160
|
if self.serversync_method is None:
|
|
1005
|
-
raise ValueError(
|
|
1161
|
+
raise ValueError("The serversync_method must be provided.")
|
|
1006
1162
|
self._server_synctool = footprints.proxy.serversynctool(
|
|
1007
1163
|
method=self.serversync_method,
|
|
1008
1164
|
medium=self.serversync_medium,
|
|
1009
1165
|
raiseonexit=self._SERVERSYNC_RAISEONEXIT,
|
|
1010
1166
|
)
|
|
1011
|
-
self._server_synctool.set_servercheck_callback(
|
|
1167
|
+
self._server_synctool.set_servercheck_callback(
|
|
1168
|
+
self.server_alive
|
|
1169
|
+
)
|
|
1012
1170
|
self.server_begin(rh, opts)
|
|
1013
1171
|
# Wait for the first request
|
|
1014
1172
|
self._server_synctool.trigger_wait()
|
|
@@ -1034,7 +1192,9 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1034
1192
|
self.server_end()
|
|
1035
1193
|
|
|
1036
1194
|
def postfix_post_dirlisting(self):
|
|
1037
|
-
self.system.subtitle(
|
|
1195
|
+
self.system.subtitle(
|
|
1196
|
+
"{:s} : directory listing (post-run)".format(self.realkind)
|
|
1197
|
+
)
|
|
1038
1198
|
self.system.dir(output=False, fatal=False)
|
|
1039
1199
|
|
|
1040
1200
|
def postfix(self, rh, opts):
|
|
@@ -1043,7 +1203,7 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1043
1203
|
|
|
1044
1204
|
def dumplog(self, opts):
|
|
1045
1205
|
"""Dump to local file the internal log of the current algo component."""
|
|
1046
|
-
self.system.pickle_dump(self.fslog,
|
|
1206
|
+
self.system.pickle_dump(self.fslog, "log." + self.fstag())
|
|
1047
1207
|
|
|
1048
1208
|
def delayed_exceptions(self, opts):
|
|
1049
1209
|
"""Gather all the delayed exceptions and raises one if necessary."""
|
|
@@ -1063,13 +1223,15 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1063
1223
|
"""A shortcut to avoid next steps of the run."""
|
|
1064
1224
|
|
|
1065
1225
|
def fastexit(self, *args, **kw):
|
|
1066
|
-
logger.warning(
|
|
1226
|
+
logger.warning(
|
|
1227
|
+
"Run <%s> skipped because abort occurred [%s]", step, msg
|
|
1228
|
+
)
|
|
1067
1229
|
|
|
1068
1230
|
return fastexit
|
|
1069
1231
|
|
|
1070
|
-
def abort(self, msg=
|
|
1232
|
+
def abort(self, msg="Not documented"):
|
|
1071
1233
|
"""A shortcut to avoid next steps of the run."""
|
|
1072
|
-
for step in (
|
|
1234
|
+
for step in ("prepare", "execute", "postfix"):
|
|
1073
1235
|
setattr(self, step, self.abortfabrik(step, msg))
|
|
1074
1236
|
|
|
1075
1237
|
def run(self, rh=None, **kw):
|
|
@@ -1080,33 +1242,34 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1080
1242
|
self.ticket = vortex.sessions.current()
|
|
1081
1243
|
self.context = self.ticket.context
|
|
1082
1244
|
self.system = self.context.system
|
|
1083
|
-
self.target = kw.pop(
|
|
1245
|
+
self.target = kw.pop("target", None)
|
|
1084
1246
|
if self.target is None:
|
|
1085
1247
|
self.target = self.system.default_target
|
|
1086
1248
|
|
|
1087
1249
|
# Before trying to do anything, check the executable
|
|
1088
1250
|
if not self.valid_executable(rh):
|
|
1089
|
-
logger.warning(
|
|
1251
|
+
logger.warning(
|
|
1252
|
+
"Resource %s is not a valid executable", rh.resource
|
|
1253
|
+
)
|
|
1090
1254
|
return False
|
|
1091
1255
|
|
|
1092
1256
|
# A cloned environment will be bound to the OS
|
|
1093
1257
|
self.env = self.context.env.clone()
|
|
1094
1258
|
with self.env:
|
|
1095
|
-
|
|
1096
1259
|
# The actual "run" recipe
|
|
1097
|
-
self.prepare(rh, kw)
|
|
1098
|
-
self.fsstamp(kw)
|
|
1260
|
+
self.prepare(rh, kw) # 1
|
|
1261
|
+
self.fsstamp(kw) # 2
|
|
1099
1262
|
try:
|
|
1100
|
-
self.execute(rh, kw)
|
|
1263
|
+
self.execute(rh, kw) # 3
|
|
1101
1264
|
except Exception as e:
|
|
1102
1265
|
self.fail_execute(e, rh, kw) # 3.1
|
|
1103
1266
|
raise
|
|
1104
1267
|
finally:
|
|
1105
|
-
self.execute_finalise(kw)
|
|
1106
|
-
self.fscheck(kw)
|
|
1107
|
-
self.postfix(rh, kw)
|
|
1108
|
-
self.dumplog(kw)
|
|
1109
|
-
self.delayed_exceptions(kw)
|
|
1268
|
+
self.execute_finalise(kw) # 3.2
|
|
1269
|
+
self.fscheck(kw) # 4
|
|
1270
|
+
self.postfix(rh, kw) # 5
|
|
1271
|
+
self.dumplog(kw) # 6
|
|
1272
|
+
self.delayed_exceptions(kw) # 7
|
|
1110
1273
|
|
|
1111
1274
|
# Free local references
|
|
1112
1275
|
self.env = None
|
|
@@ -1116,30 +1279,41 @@ class AlgoComponent(footprints.FootprintBase, metaclass=AlgoComponentMeta):
|
|
|
1116
1279
|
|
|
1117
1280
|
def quickview(self, nb=0, indent=0):
|
|
1118
1281
|
"""Standard glance to objects."""
|
|
1119
|
-
tab =
|
|
1120
|
-
print(
|
|
1121
|
-
for subobj in (
|
|
1282
|
+
tab = " " * indent
|
|
1283
|
+
print("{}{:02d}. {:s}".format(tab, nb, repr(self)))
|
|
1284
|
+
for subobj in ("kind", "engine", "interpreter"):
|
|
1122
1285
|
obj = getattr(self, subobj, None)
|
|
1123
1286
|
if obj:
|
|
1124
|
-
print(
|
|
1287
|
+
print("{} {:s}: {!s}".format(tab, subobj, obj))
|
|
1125
1288
|
print()
|
|
1126
1289
|
|
|
1127
|
-
def setlink(
|
|
1290
|
+
def setlink(
|
|
1291
|
+
self,
|
|
1292
|
+
initrole=None,
|
|
1293
|
+
initkind=None,
|
|
1294
|
+
initname=None,
|
|
1295
|
+
inittest=lambda x: True,
|
|
1296
|
+
):
|
|
1128
1297
|
"""Set a symbolic link for actual resource playing defined role."""
|
|
1129
1298
|
initsec = [
|
|
1130
|
-
x
|
|
1299
|
+
x
|
|
1300
|
+
for x in self.context.sequence.effective_inputs(
|
|
1301
|
+
role=initrole, kind=initkind
|
|
1302
|
+
)
|
|
1131
1303
|
if inittest(x.rh)
|
|
1132
1304
|
]
|
|
1133
1305
|
|
|
1134
1306
|
if not initsec:
|
|
1135
1307
|
logger.warning(
|
|
1136
|
-
|
|
1137
|
-
initrole,
|
|
1308
|
+
"Could not find logical role %s with kind %s - assuming already renamed",
|
|
1309
|
+
initrole,
|
|
1310
|
+
initkind,
|
|
1138
1311
|
)
|
|
1139
1312
|
|
|
1140
1313
|
if len(initsec) > 1:
|
|
1141
|
-
logger.warning(
|
|
1142
|
-
|
|
1314
|
+
logger.warning(
|
|
1315
|
+
"More than one role %s with kind %s", initrole, initkind
|
|
1316
|
+
)
|
|
1143
1317
|
|
|
1144
1318
|
if initname is not None:
|
|
1145
1319
|
for l in [x.rh.container.localpath() for x in initsec]:
|
|
@@ -1186,25 +1360,19 @@ class PythonFunction(AlgoComponent):
|
|
|
1186
1360
|
"""
|
|
1187
1361
|
|
|
1188
1362
|
_footprint = dict(
|
|
1189
|
-
info
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
),
|
|
1195
|
-
func_name = dict(
|
|
1196
|
-
info="The function's name"
|
|
1197
|
-
),
|
|
1198
|
-
func_kwargs = dict(
|
|
1363
|
+
info="Execute a Python function in a given module",
|
|
1364
|
+
attr=dict(
|
|
1365
|
+
engine=dict(values=["function"]),
|
|
1366
|
+
func_name=dict(info="The function's name"),
|
|
1367
|
+
func_kwargs=dict(
|
|
1199
1368
|
info=(
|
|
1200
|
-
"A dictionary containing the function's "
|
|
1201
|
-
"keyword arguments"
|
|
1369
|
+
"A dictionary containing the function's keyword arguments"
|
|
1202
1370
|
),
|
|
1203
1371
|
type=footprints.FPDict,
|
|
1204
1372
|
default=footprints.FPDict({}),
|
|
1205
1373
|
optional=True,
|
|
1206
1374
|
),
|
|
1207
|
-
)
|
|
1375
|
+
),
|
|
1208
1376
|
)
|
|
1209
1377
|
|
|
1210
1378
|
def prepare(self, rh, opts):
|
|
@@ -1221,7 +1389,8 @@ class PythonFunction(AlgoComponent):
|
|
|
1221
1389
|
|
|
1222
1390
|
def execute(self, rh, opts):
|
|
1223
1391
|
self.func(
|
|
1224
|
-
self.context.sequence,
|
|
1392
|
+
self.context.sequence,
|
|
1393
|
+
**self.func_kwargs,
|
|
1225
1394
|
)
|
|
1226
1395
|
|
|
1227
1396
|
def execute_finalise(self, opts):
|
|
@@ -1255,7 +1424,13 @@ class xExecutableAlgoComponent(ExecutableAlgoComponent):
|
|
|
1255
1424
|
rc = super().valid_executable(rh)
|
|
1256
1425
|
if rc:
|
|
1257
1426
|
# Ensure that the input file is executable
|
|
1258
|
-
xrh =
|
|
1427
|
+
xrh = (
|
|
1428
|
+
rh
|
|
1429
|
+
if isinstance(rh, (list, tuple))
|
|
1430
|
+
else [
|
|
1431
|
+
rh,
|
|
1432
|
+
]
|
|
1433
|
+
)
|
|
1259
1434
|
for arh in xrh:
|
|
1260
1435
|
self.system.xperm(arh.container.localpath(), force=True)
|
|
1261
1436
|
return rc
|
|
@@ -1273,23 +1448,23 @@ class TaylorRun(AlgoComponent):
|
|
|
1273
1448
|
|
|
1274
1449
|
_abstract = True
|
|
1275
1450
|
_footprint = dict(
|
|
1276
|
-
info
|
|
1277
|
-
attr
|
|
1278
|
-
kind
|
|
1279
|
-
verbose
|
|
1280
|
-
info
|
|
1281
|
-
type
|
|
1282
|
-
default
|
|
1283
|
-
optional
|
|
1284
|
-
doc_zorder
|
|
1451
|
+
info="Abstract algo component based on the taylorism package.",
|
|
1452
|
+
attr=dict(
|
|
1453
|
+
kind=dict(),
|
|
1454
|
+
verbose=dict(
|
|
1455
|
+
info="Run in verbose mode",
|
|
1456
|
+
type=bool,
|
|
1457
|
+
default=False,
|
|
1458
|
+
optional=True,
|
|
1459
|
+
doc_zorder=-50,
|
|
1285
1460
|
),
|
|
1286
|
-
ntasks
|
|
1287
|
-
info
|
|
1288
|
-
type
|
|
1289
|
-
default
|
|
1290
|
-
optional
|
|
1461
|
+
ntasks=dict(
|
|
1462
|
+
info="The maximum number of parallel tasks",
|
|
1463
|
+
type=int,
|
|
1464
|
+
default=DelayedEnvValue("VORTEX_SUBMIT_TASKS", 1),
|
|
1465
|
+
optional=True,
|
|
1291
1466
|
),
|
|
1292
|
-
)
|
|
1467
|
+
),
|
|
1293
1468
|
)
|
|
1294
1469
|
|
|
1295
1470
|
def __init__(self, *kargs, **kwargs):
|
|
@@ -1303,8 +1478,12 @@ class TaylorRun(AlgoComponent):
|
|
|
1303
1478
|
def _default_pre_execute(self, rh, opts):
|
|
1304
1479
|
"""Various initialisations. In particular it creates the task scheduler (Boss)."""
|
|
1305
1480
|
# Start the task scheduler
|
|
1306
|
-
self._boss = Boss(
|
|
1307
|
-
|
|
1481
|
+
self._boss = Boss(
|
|
1482
|
+
verbose=self.verbose,
|
|
1483
|
+
scheduler=footprints.proxy.scheduler(
|
|
1484
|
+
limit="threads", max_threads=self.ntasks
|
|
1485
|
+
),
|
|
1486
|
+
)
|
|
1308
1487
|
self._boss.make_them_work()
|
|
1309
1488
|
|
|
1310
1489
|
def _add_instructions(self, common_i, individual_i):
|
|
@@ -1313,12 +1492,16 @@ class TaylorRun(AlgoComponent):
|
|
|
1313
1492
|
|
|
1314
1493
|
def _default_post_execute(self, rh, opts):
|
|
1315
1494
|
"""Summarise the results of the various tasks that were run."""
|
|
1316
|
-
logger.info(
|
|
1495
|
+
logger.info(
|
|
1496
|
+
"All the input files were dealt with: now waiting for the parallel processing to finish"
|
|
1497
|
+
)
|
|
1317
1498
|
self._boss.wait_till_finished()
|
|
1318
|
-
logger.info(
|
|
1499
|
+
logger.info(
|
|
1500
|
+
"The parallel processing has finished. here are the results:"
|
|
1501
|
+
)
|
|
1319
1502
|
report = self._boss.get_report()
|
|
1320
1503
|
prp = ParallelResultParser(self.context)
|
|
1321
|
-
for r in report[
|
|
1504
|
+
for r in report["workers_report"]:
|
|
1322
1505
|
rc = prp(r)
|
|
1323
1506
|
if isinstance(rc, Exception):
|
|
1324
1507
|
self.delayed_exception_add(rc, traceback=False)
|
|
@@ -1328,8 +1511,10 @@ class TaylorRun(AlgoComponent):
|
|
|
1328
1511
|
def _default_rc_action(self, rh, opts, report, rc):
|
|
1329
1512
|
"""How should we process the return code ?"""
|
|
1330
1513
|
if not rc:
|
|
1331
|
-
logger.warning(
|
|
1332
|
-
|
|
1514
|
+
logger.warning(
|
|
1515
|
+
"Apparently something went sideways with this task (rc=%s).",
|
|
1516
|
+
str(rc),
|
|
1517
|
+
)
|
|
1333
1518
|
|
|
1334
1519
|
def execute(self, rh, opts):
|
|
1335
1520
|
"""
|
|
@@ -1359,28 +1544,28 @@ class Expresso(ExecutableAlgoComponent):
|
|
|
1359
1544
|
"""Run a script resource in the good environment."""
|
|
1360
1545
|
|
|
1361
1546
|
_footprint = dict(
|
|
1362
|
-
info
|
|
1363
|
-
attr
|
|
1364
|
-
interpreter
|
|
1365
|
-
info
|
|
1366
|
-
values
|
|
1367
|
-
),
|
|
1368
|
-
interpreter_path = dict(
|
|
1369
|
-
info = 'The interpreter command.',
|
|
1370
|
-
optional = True,
|
|
1547
|
+
info="AlgoComponent that simply runs a script",
|
|
1548
|
+
attr=dict(
|
|
1549
|
+
interpreter=dict(
|
|
1550
|
+
info="The interpreter needed to run the script.",
|
|
1551
|
+
values=["current", "awk", "ksh", "bash", "perl", "python"],
|
|
1371
1552
|
),
|
|
1372
|
-
|
|
1373
|
-
|
|
1553
|
+
interpreter_path=dict(
|
|
1554
|
+
info="The interpreter command.",
|
|
1555
|
+
optional=True,
|
|
1374
1556
|
),
|
|
1375
|
-
|
|
1557
|
+
engine=dict(values=["exec", "launch"]),
|
|
1558
|
+
),
|
|
1376
1559
|
)
|
|
1377
1560
|
|
|
1378
1561
|
@property
|
|
1379
1562
|
def _actual_interpreter(self):
|
|
1380
1563
|
"""Return the interpreter command."""
|
|
1381
|
-
if self.interpreter ==
|
|
1564
|
+
if self.interpreter == "current":
|
|
1382
1565
|
if self.interpreter_path is not None:
|
|
1383
|
-
raise ValueError(
|
|
1566
|
+
raise ValueError(
|
|
1567
|
+
"*interpreter=current* and *interpreter_path* attributes are incompatible"
|
|
1568
|
+
)
|
|
1384
1569
|
return sys.executable
|
|
1385
1570
|
else:
|
|
1386
1571
|
if self.interpreter_path is None:
|
|
@@ -1389,15 +1574,20 @@ class Expresso(ExecutableAlgoComponent):
|
|
|
1389
1574
|
if self.system.xperm(self.interpreter_path):
|
|
1390
1575
|
return self.interpreter_path
|
|
1391
1576
|
else:
|
|
1392
|
-
raise AlgoComponentError(
|
|
1393
|
-
|
|
1577
|
+
raise AlgoComponentError(
|
|
1578
|
+
"The '{:s}' interpreter is not executable".format(
|
|
1579
|
+
self.interpreter_path
|
|
1580
|
+
)
|
|
1581
|
+
)
|
|
1394
1582
|
|
|
1395
1583
|
def _interpreter_args_fix(self, rh, opts):
|
|
1396
1584
|
absexec = self.absexcutable(rh.container.localpath())
|
|
1397
|
-
if self.interpreter ==
|
|
1398
|
-
return [
|
|
1585
|
+
if self.interpreter == "awk":
|
|
1586
|
+
return ["-f", absexec]
|
|
1399
1587
|
else:
|
|
1400
|
-
return [
|
|
1588
|
+
return [
|
|
1589
|
+
absexec,
|
|
1590
|
+
]
|
|
1401
1591
|
|
|
1402
1592
|
def execute_single(self, rh, opts):
|
|
1403
1593
|
"""
|
|
@@ -1405,19 +1595,23 @@ class Expresso(ExecutableAlgoComponent):
|
|
|
1405
1595
|
using the resource command_line method as args.
|
|
1406
1596
|
"""
|
|
1407
1597
|
# Generic config
|
|
1408
|
-
args = [
|
|
1598
|
+
args = [
|
|
1599
|
+
self._actual_interpreter,
|
|
1600
|
+
]
|
|
1409
1601
|
args.extend(self._interpreter_args_fix(rh, opts))
|
|
1410
1602
|
args.extend(self.spawn_command_line(rh))
|
|
1411
|
-
logger.info(
|
|
1603
|
+
logger.info("Run script %s", args)
|
|
1412
1604
|
rh_stdin = self.spawn_stdin(rh)
|
|
1413
1605
|
if rh_stdin is not None:
|
|
1414
|
-
plocale = locale.getlocale()[1] or
|
|
1415
|
-
logger.info(
|
|
1606
|
+
plocale = locale.getlocale()[1] or "ascii"
|
|
1607
|
+
logger.info(
|
|
1608
|
+
"Script stdin:\n%s", rh_stdin.read().decode(plocale, "replace")
|
|
1609
|
+
)
|
|
1416
1610
|
rh_stdin.seek(0)
|
|
1417
1611
|
# Python path stuff
|
|
1418
|
-
newpypath =
|
|
1419
|
-
if
|
|
1420
|
-
newpypath +=
|
|
1612
|
+
newpypath = ":".join(self.extendpypath)
|
|
1613
|
+
if "pythonpath" in self.env:
|
|
1614
|
+
newpypath += ":{:s}".format(self.env.pythonpath)
|
|
1421
1615
|
# launching the program...
|
|
1422
1616
|
with self.env.delta_context(pythonpath=newpypath):
|
|
1423
1617
|
self.spawn(args, opts, stdin=rh_stdin)
|
|
@@ -1435,26 +1629,24 @@ class ParaExpresso(TaylorRun):
|
|
|
1435
1629
|
|
|
1436
1630
|
_abstract = True
|
|
1437
1631
|
_footprint = dict(
|
|
1438
|
-
info
|
|
1439
|
-
attr
|
|
1440
|
-
interpreter
|
|
1441
|
-
info
|
|
1442
|
-
values
|
|
1443
|
-
),
|
|
1444
|
-
engine = dict(
|
|
1445
|
-
values = ['exec', 'launch']
|
|
1632
|
+
info="AlgoComponent that simply runs a script using the taylorism package.",
|
|
1633
|
+
attr=dict(
|
|
1634
|
+
interpreter=dict(
|
|
1635
|
+
info="The interpreter needed to run the script.",
|
|
1636
|
+
values=["current", "awk", "ksh", "bash", "perl", "python"],
|
|
1446
1637
|
),
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1638
|
+
engine=dict(values=["exec", "launch"]),
|
|
1639
|
+
interpreter_path=dict(
|
|
1640
|
+
info="The full path to the interpreter.",
|
|
1641
|
+
optional=True,
|
|
1450
1642
|
),
|
|
1451
|
-
extendpypath
|
|
1452
|
-
info
|
|
1453
|
-
type
|
|
1454
|
-
default
|
|
1455
|
-
optional
|
|
1643
|
+
extendpypath=dict(
|
|
1644
|
+
info="The list of things to be prepended in the python's path.",
|
|
1645
|
+
type=footprints.FPList,
|
|
1646
|
+
default=footprints.FPList([]),
|
|
1647
|
+
optional=True,
|
|
1456
1648
|
),
|
|
1457
|
-
)
|
|
1649
|
+
),
|
|
1458
1650
|
)
|
|
1459
1651
|
|
|
1460
1652
|
def valid_executable(self, rh):
|
|
@@ -1466,25 +1658,32 @@ class ParaExpresso(TaylorRun):
|
|
|
1466
1658
|
|
|
1467
1659
|
def _interpreter_args_fix(self, rh, opts):
|
|
1468
1660
|
absexec = self.absexcutable(rh.container.localpath())
|
|
1469
|
-
if self.interpreter ==
|
|
1470
|
-
return [
|
|
1661
|
+
if self.interpreter == "awk":
|
|
1662
|
+
return ["-f", absexec]
|
|
1471
1663
|
else:
|
|
1472
|
-
return [
|
|
1664
|
+
return [
|
|
1665
|
+
absexec,
|
|
1666
|
+
]
|
|
1473
1667
|
|
|
1474
1668
|
def _default_common_instructions(self, rh, opts):
|
|
1475
1669
|
"""Create a common instruction dictionary that will be used by the workers."""
|
|
1476
1670
|
ddict = super()._default_common_instructions(rh, opts)
|
|
1477
|
-
actual_interpreter =
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1671
|
+
actual_interpreter = (
|
|
1672
|
+
sys.executable
|
|
1673
|
+
if self.interpreter == "current"
|
|
1674
|
+
else self.interpreter
|
|
1675
|
+
)
|
|
1676
|
+
ddict["progname"] = actual_interpreter
|
|
1677
|
+
ddict["progargs"] = footprints.FPList(
|
|
1678
|
+
self._interpreter_args_fix(rh, opts) + self.spawn_command_line(rh)
|
|
1679
|
+
)
|
|
1680
|
+
ddict["progenvdelta"] = footprints.FPDict()
|
|
1482
1681
|
# Deal with the python path
|
|
1483
|
-
newpypath =
|
|
1484
|
-
if
|
|
1485
|
-
self.env.pythonpath +=
|
|
1682
|
+
newpypath = ":".join(self.extendpypath)
|
|
1683
|
+
if "pythonpath" in self.env:
|
|
1684
|
+
self.env.pythonpath += ":{:s}".format(newpypath)
|
|
1486
1685
|
if newpypath:
|
|
1487
|
-
ddict[
|
|
1686
|
+
ddict["progenvdelta"]["pythonpath"] = newpypath
|
|
1488
1687
|
return ddict
|
|
1489
1688
|
|
|
1490
1689
|
|
|
@@ -1495,12 +1694,8 @@ class BlindRun(xExecutableAlgoComponent):
|
|
|
1495
1694
|
"""
|
|
1496
1695
|
|
|
1497
1696
|
_footprint = dict(
|
|
1498
|
-
info
|
|
1499
|
-
attr
|
|
1500
|
-
engine = dict(
|
|
1501
|
-
values = ['blind']
|
|
1502
|
-
)
|
|
1503
|
-
)
|
|
1697
|
+
info="AlgoComponent that simply runs a serial binary",
|
|
1698
|
+
attr=dict(engine=dict(values=["blind"])),
|
|
1504
1699
|
)
|
|
1505
1700
|
|
|
1506
1701
|
def execute_single(self, rh, opts):
|
|
@@ -1511,12 +1706,15 @@ class BlindRun(xExecutableAlgoComponent):
|
|
|
1511
1706
|
|
|
1512
1707
|
args = [self.absexcutable(rh.container.localpath())]
|
|
1513
1708
|
args.extend(self.spawn_command_line(rh))
|
|
1514
|
-
logger.info(
|
|
1709
|
+
logger.info("BlindRun executable resource %s", args)
|
|
1515
1710
|
rh_stdin = self.spawn_stdin(rh)
|
|
1516
1711
|
if rh_stdin is not None:
|
|
1517
|
-
plocale = locale.getlocale()[1] or
|
|
1518
|
-
logger.info(
|
|
1519
|
-
|
|
1712
|
+
plocale = locale.getlocale()[1] or "ascii"
|
|
1713
|
+
logger.info(
|
|
1714
|
+
"BlindRun executable stdin (fileno:%d):\n%s",
|
|
1715
|
+
rh_stdin.fileno(),
|
|
1716
|
+
rh_stdin.read().decode(plocale, "replace"),
|
|
1717
|
+
)
|
|
1520
1718
|
rh_stdin.seek(0)
|
|
1521
1719
|
self.spawn(args, opts, stdin=rh_stdin)
|
|
1522
1720
|
|
|
@@ -1533,26 +1731,26 @@ class ParaBlindRun(TaylorRun):
|
|
|
1533
1731
|
|
|
1534
1732
|
_abstract = True
|
|
1535
1733
|
_footprint = dict(
|
|
1536
|
-
info
|
|
1537
|
-
attr
|
|
1538
|
-
engine
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1734
|
+
info="Abstract AlgoComponent that runs a serial binary using the taylorism package.",
|
|
1735
|
+
attr=dict(
|
|
1736
|
+
engine=dict(values=["blind"]),
|
|
1737
|
+
taskset=dict(
|
|
1738
|
+
info="Topology/Method to set up the CPU affinity of the child task.",
|
|
1739
|
+
default=None,
|
|
1740
|
+
optional=True,
|
|
1741
|
+
values=[
|
|
1742
|
+
"{:s}{:s}".format(t, m)
|
|
1743
|
+
for t in ("raw", "socketpacked", "numapacked")
|
|
1744
|
+
for m in ("", "_taskset", "_gomp", "_omp", "_ompverbose")
|
|
1745
|
+
],
|
|
1548
1746
|
),
|
|
1549
|
-
taskset_bsize
|
|
1550
|
-
info
|
|
1551
|
-
type
|
|
1552
|
-
default
|
|
1553
|
-
optional
|
|
1747
|
+
taskset_bsize=dict(
|
|
1748
|
+
info="The number of threads used by one task",
|
|
1749
|
+
type=int,
|
|
1750
|
+
default=1,
|
|
1751
|
+
optional=True,
|
|
1554
1752
|
),
|
|
1555
|
-
)
|
|
1753
|
+
),
|
|
1556
1754
|
)
|
|
1557
1755
|
|
|
1558
1756
|
def valid_executable(self, rh):
|
|
@@ -1563,7 +1761,13 @@ class ParaBlindRun(TaylorRun):
|
|
|
1563
1761
|
rc = rh is not None
|
|
1564
1762
|
if rc:
|
|
1565
1763
|
# Ensure that the input file is executable
|
|
1566
|
-
xrh =
|
|
1764
|
+
xrh = (
|
|
1765
|
+
rh
|
|
1766
|
+
if isinstance(rh, (list, tuple))
|
|
1767
|
+
else [
|
|
1768
|
+
rh,
|
|
1769
|
+
]
|
|
1770
|
+
)
|
|
1567
1771
|
for arh in xrh:
|
|
1568
1772
|
self.system.xperm(arh.container.localpath(), force=True)
|
|
1569
1773
|
return rc
|
|
@@ -1571,10 +1775,10 @@ class ParaBlindRun(TaylorRun):
|
|
|
1571
1775
|
def _default_common_instructions(self, rh, opts):
|
|
1572
1776
|
"""Create a common instruction dictionary that will be used by the workers."""
|
|
1573
1777
|
ddict = super()._default_common_instructions(rh, opts)
|
|
1574
|
-
ddict[
|
|
1575
|
-
ddict[
|
|
1576
|
-
ddict[
|
|
1577
|
-
ddict[
|
|
1778
|
+
ddict["progname"] = self.absexcutable(rh.container.localpath())
|
|
1779
|
+
ddict["progargs"] = footprints.FPList(self.spawn_command_line(rh))
|
|
1780
|
+
ddict["progtaskset"] = self.taskset
|
|
1781
|
+
ddict["progtaskset_bsize"] = self.taskset_bsize
|
|
1578
1782
|
return ddict
|
|
1579
1783
|
|
|
1580
1784
|
|
|
@@ -1584,56 +1788,60 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1584
1788
|
"""
|
|
1585
1789
|
|
|
1586
1790
|
_footprint = dict(
|
|
1587
|
-
info
|
|
1588
|
-
attr
|
|
1589
|
-
engine
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
type = mpitools.MpiTool,
|
|
1596
|
-
doc_visibility = footprints.doc.visibility.GURU,
|
|
1791
|
+
info="AlgoComponent that simply runs an MPI binary",
|
|
1792
|
+
attr=dict(
|
|
1793
|
+
engine=dict(values=["parallel"]),
|
|
1794
|
+
mpitool=dict(
|
|
1795
|
+
info="The object used to launch the parallel program",
|
|
1796
|
+
optional=True,
|
|
1797
|
+
type=mpitools.MpiTool,
|
|
1798
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1597
1799
|
),
|
|
1598
|
-
mpiname
|
|
1599
|
-
info
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1800
|
+
mpiname=dict(
|
|
1801
|
+
info=(
|
|
1802
|
+
"The mpiname of a class in the mpitool collector "
|
|
1803
|
+
+ "(used only if *mpitool* is not provided)"
|
|
1804
|
+
),
|
|
1805
|
+
optional=True,
|
|
1806
|
+
alias=["mpi"],
|
|
1807
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1604
1808
|
),
|
|
1605
|
-
mpiverbose
|
|
1606
|
-
info
|
|
1607
|
-
optional
|
|
1608
|
-
default
|
|
1609
|
-
doc_visibility
|
|
1809
|
+
mpiverbose=dict(
|
|
1810
|
+
info="Boost logging verbosity in mpitools",
|
|
1811
|
+
optional=True,
|
|
1812
|
+
default=False,
|
|
1813
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1610
1814
|
),
|
|
1611
|
-
binaries
|
|
1612
|
-
info
|
|
1613
|
-
optional
|
|
1614
|
-
type
|
|
1615
|
-
doc_visibility
|
|
1815
|
+
binaries=dict(
|
|
1816
|
+
info="List of MpiBinaryDescription objects",
|
|
1817
|
+
optional=True,
|
|
1818
|
+
type=footprints.FPList,
|
|
1819
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1616
1820
|
),
|
|
1617
|
-
binarysingle
|
|
1618
|
-
info
|
|
1619
|
-
optional
|
|
1620
|
-
default
|
|
1621
|
-
doc_visibility
|
|
1821
|
+
binarysingle=dict(
|
|
1822
|
+
info="If *binaries* is missing, the default binary role for single binaries",
|
|
1823
|
+
optional=True,
|
|
1824
|
+
default="basicsingle",
|
|
1825
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1622
1826
|
),
|
|
1623
|
-
binarymulti
|
|
1624
|
-
info
|
|
1625
|
-
type
|
|
1626
|
-
optional
|
|
1627
|
-
default
|
|
1628
|
-
|
|
1827
|
+
binarymulti=dict(
|
|
1828
|
+
info="If *binaries* is missing, the default binary role for multiple binaries",
|
|
1829
|
+
type=footprints.FPList,
|
|
1830
|
+
optional=True,
|
|
1831
|
+
default=footprints.FPList(
|
|
1832
|
+
[
|
|
1833
|
+
"basic",
|
|
1834
|
+
]
|
|
1835
|
+
),
|
|
1836
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
1629
1837
|
),
|
|
1630
|
-
)
|
|
1838
|
+
),
|
|
1631
1839
|
)
|
|
1632
1840
|
|
|
1633
1841
|
def _mpitool_attributes(self, opts):
|
|
1634
1842
|
"""Return the dictionary of attributes needed to create the mpitool object."""
|
|
1635
1843
|
# Read the appropriate configuration in the target file
|
|
1636
|
-
conf_dict = from_config(section="mpitool")
|
|
1844
|
+
conf_dict = config.from_config(section="mpitool")
|
|
1637
1845
|
if self.mpiname:
|
|
1638
1846
|
conf_dict["mpiname"] = self.mpiname
|
|
1639
1847
|
# Make "mpirun" the default mpi command name
|
|
@@ -1642,7 +1850,7 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1642
1850
|
possible_attrs = functools.reduce(
|
|
1643
1851
|
lambda s, t: s | t,
|
|
1644
1852
|
[
|
|
1645
|
-
set(cls.
|
|
1853
|
+
set(cls.footprint_retrieve().attr.keys())
|
|
1646
1854
|
for cls in footprints.proxy.mpitools
|
|
1647
1855
|
],
|
|
1648
1856
|
)
|
|
@@ -1650,7 +1858,7 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1650
1858
|
if nonkeys:
|
|
1651
1859
|
msg = (
|
|
1652
1860
|
"The following keywords are unknown configuration"
|
|
1653
|
-
|
|
1861
|
+
'keys for section "mpitool":\n'
|
|
1654
1862
|
)
|
|
1655
1863
|
|
|
1656
1864
|
raise ValueError(msg + "\n".join(nonkeys))
|
|
@@ -1674,56 +1882,68 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1674
1882
|
|
|
1675
1883
|
# Rh is a list binaries...
|
|
1676
1884
|
if not isinstance(rh, collections.abc.Iterable):
|
|
1677
|
-
rh = [
|
|
1885
|
+
rh = [
|
|
1886
|
+
rh,
|
|
1887
|
+
]
|
|
1678
1888
|
|
|
1679
1889
|
# Find the MPI launcher
|
|
1680
1890
|
mpi = self.mpitool
|
|
1681
1891
|
if not mpi:
|
|
1682
1892
|
mpi = footprints.proxy.mpitool(
|
|
1683
|
-
sysname=self.system.sysname,
|
|
1684
|
-
** self._mpitool_attributes(opts)
|
|
1893
|
+
sysname=self.system.sysname, **self._mpitool_attributes(opts)
|
|
1685
1894
|
)
|
|
1686
1895
|
if not mpi:
|
|
1687
|
-
logger.critical(
|
|
1688
|
-
|
|
1896
|
+
logger.critical(
|
|
1897
|
+
"Component %s could not find any mpitool",
|
|
1898
|
+
self.footprint_clsname(),
|
|
1899
|
+
)
|
|
1900
|
+
raise AttributeError("No valid mpitool attr could be found.")
|
|
1689
1901
|
|
|
1690
1902
|
# Setup various useful things (env, system, ...)
|
|
1691
1903
|
mpi.import_basics(self)
|
|
1692
1904
|
|
|
1693
|
-
mpi_opts = opts.get(
|
|
1905
|
+
mpi_opts = opts.get("mpiopts", dict())
|
|
1694
1906
|
|
|
1695
1907
|
envelope = []
|
|
1696
|
-
use_envelope =
|
|
1908
|
+
use_envelope = "envelope" in mpi_opts
|
|
1697
1909
|
if use_envelope:
|
|
1698
|
-
envelope = mpi_opts.pop(
|
|
1699
|
-
if envelope ==
|
|
1700
|
-
blockspec = dict(
|
|
1701
|
-
|
|
1702
|
-
|
|
1910
|
+
envelope = mpi_opts.pop("envelope")
|
|
1911
|
+
if envelope == "auto":
|
|
1912
|
+
blockspec = dict(
|
|
1913
|
+
nn=self.env.get("VORTEX_SUBMIT_NODES", 1),
|
|
1914
|
+
)
|
|
1915
|
+
if "VORTEX_SUBMIT_TASKS" in self.env:
|
|
1916
|
+
blockspec["nnp"] = self.env.get("VORTEX_SUBMIT_TASKS")
|
|
1703
1917
|
else:
|
|
1704
|
-
raise ValueError(
|
|
1705
|
-
|
|
1918
|
+
raise ValueError(
|
|
1919
|
+
"when envelope='auto', VORTEX_SUBMIT_TASKS must be set up."
|
|
1920
|
+
)
|
|
1921
|
+
envelope = [
|
|
1922
|
+
blockspec,
|
|
1923
|
+
]
|
|
1706
1924
|
elif isinstance(envelope, dict):
|
|
1707
|
-
envelope = [
|
|
1925
|
+
envelope = [
|
|
1926
|
+
envelope,
|
|
1927
|
+
]
|
|
1708
1928
|
elif isinstance(envelope, (list, tuple)):
|
|
1709
1929
|
pass
|
|
1710
1930
|
else:
|
|
1711
|
-
raise AttributeError(
|
|
1931
|
+
raise AttributeError("Invalid envelope specification")
|
|
1712
1932
|
if envelope:
|
|
1713
|
-
envelope_ntasks = sum([d[
|
|
1933
|
+
envelope_ntasks = sum([d["nn"] * d["nnp"] for d in envelope])
|
|
1714
1934
|
if not envelope:
|
|
1715
1935
|
use_envelope = False
|
|
1716
1936
|
|
|
1717
1937
|
if not use_envelope:
|
|
1718
1938
|
# Some MPI presets
|
|
1719
1939
|
mpi_desc = dict()
|
|
1720
|
-
for mpi_k in (
|
|
1721
|
-
mpi_kenv =
|
|
1940
|
+
for mpi_k in ("tasks", "openmp"):
|
|
1941
|
+
mpi_kenv = "VORTEX_SUBMIT_" + mpi_k.upper()
|
|
1722
1942
|
if mpi_kenv in self.env:
|
|
1723
1943
|
mpi_desc[mpi_k] = self.env.get(mpi_kenv)
|
|
1724
1944
|
|
|
1725
1945
|
# Binaries may be grouped together on the same nodes
|
|
1726
|
-
bin_groups = mpi_opts.pop(
|
|
1946
|
+
bin_groups = mpi_opts.pop("groups", [])
|
|
1727
1947
|
|
|
1728
1948
|
# Find out the command line
|
|
1729
1949
|
bargs = self.spawn_command_line(rh)
|
|
@@ -1733,65 +1953,89 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1733
1953
|
|
|
1734
1954
|
# The usual case: no indications, 1 binary + a potential ioserver
|
|
1735
1955
|
if len(rh) == 1 and not self.binaries:
|
|
1736
|
-
|
|
1737
1956
|
# In such a case, defining group does not makes sense
|
|
1738
|
-
self.algoassert(
|
|
1739
|
-
|
|
1957
|
+
self.algoassert(
|
|
1958
|
+
not bin_groups,
|
|
1959
|
+
"With only one binary, groups should not be defined",
|
|
1960
|
+
)
|
|
1740
1961
|
|
|
1741
1962
|
# The main program
|
|
1742
|
-
allowbind = mpi_opts.pop(
|
|
1743
|
-
distribution = mpi_opts.pop(
|
|
1744
|
-
|
|
1963
|
+
allowbind = mpi_opts.pop("allowbind", True)
|
|
1964
|
+
distribution = mpi_opts.pop(
|
|
1965
|
+
"distribution",
|
|
1966
|
+
self.env.get("VORTEX_MPIBIN_DEF_DISTRIBUTION", None),
|
|
1967
|
+
)
|
|
1745
1968
|
if use_envelope:
|
|
1746
1969
|
master = footprints.proxy.mpibinary(
|
|
1747
1970
|
kind=self.binarysingle,
|
|
1748
1971
|
ranks=envelope_ntasks,
|
|
1749
|
-
openmp=self.env.get(
|
|
1972
|
+
openmp=self.env.get("VORTEX_SUBMIT_OPENMP", None),
|
|
1750
1973
|
allowbind=allowbind,
|
|
1751
|
-
distribution=distribution
|
|
1974
|
+
distribution=distribution,
|
|
1975
|
+
)
|
|
1752
1976
|
else:
|
|
1753
1977
|
master = footprints.proxy.mpibinary(
|
|
1754
1978
|
kind=self.binarysingle,
|
|
1755
|
-
nodes=self.env.get(
|
|
1979
|
+
nodes=self.env.get("VORTEX_SUBMIT_NODES", 1),
|
|
1756
1980
|
allowbind=allowbind,
|
|
1757
1981
|
distribution=distribution,
|
|
1758
|
-
**mpi_desc
|
|
1982
|
+
**mpi_desc,
|
|
1983
|
+
)
|
|
1759
1984
|
master.options = mpi_opts
|
|
1760
1985
|
master.master = self.absexcutable(rh[0].container.localpath())
|
|
1761
1986
|
master.arguments = bargs[0]
|
|
1762
|
-
bins = [
|
|
1987
|
+
bins = [
|
|
1988
|
+
master,
|
|
1989
|
+
]
|
|
1763
1990
|
# Source files ?
|
|
1764
|
-
if hasattr(rh[0].resource,
|
|
1765
|
-
sources.extend(
|
|
1991
|
+
if hasattr(rh[0].resource, "guess_binary_sources"):
|
|
1992
|
+
sources.extend(
|
|
1993
|
+
rh[0].resource.guess_binary_sources(rh[0].provider)
|
|
1994
|
+
)
|
|
1766
1995
|
|
|
1767
1996
|
# Multiple binaries are to be launched: no IO server support here.
|
|
1768
1997
|
elif len(rh) > 1 and not self.binaries:
|
|
1769
|
-
|
|
1770
1998
|
# Binary roles
|
|
1771
1999
|
if len(self.binarymulti) == 1:
|
|
1772
2000
|
bnames = self.binarymulti * len(rh)
|
|
1773
2001
|
else:
|
|
1774
2002
|
if len(self.binarymulti) != len(rh):
|
|
1775
|
-
raise ParallelInconsistencyAlgoComponentError(
|
|
2003
|
+
raise ParallelInconsistencyAlgoComponentError(
|
|
2004
|
+
"self.binarymulti"
|
|
2005
|
+
)
|
|
1776
2006
|
bnames = self.binarymulti
|
|
1777
2007
|
|
|
1778
2008
|
# Check mpiopts shape
|
|
1779
2009
|
for k, v in mpi_opts.items():
|
|
1780
2010
|
if not isinstance(v, collections.abc.Iterable):
|
|
1781
|
-
raise ValueError(
|
|
2011
|
+
raise ValueError(
|
|
2012
|
+
"In such a case, mpiopts must be Iterable"
|
|
2013
|
+
)
|
|
1782
2014
|
if len(v) != len(rh):
|
|
1783
|
-
raise ParallelInconsistencyAlgoComponentError(
|
|
2015
|
+
raise ParallelInconsistencyAlgoComponentError(
|
|
2016
|
+
"mpiopts[{:s}]".format(k)
|
|
2017
|
+
)
|
|
1784
2018
|
# Check bin_group shape
|
|
1785
2019
|
if bin_groups:
|
|
1786
2020
|
if len(bin_groups) != len(rh):
|
|
1787
|
-
raise ParallelInconsistencyAlgoComponentError(
|
|
2021
|
+
raise ParallelInconsistencyAlgoComponentError("bin_group")
|
|
1788
2022
|
|
|
1789
2023
|
# Create MpiBinaryDescription objects
|
|
1790
2024
|
bins = list()
|
|
1791
|
-
allowbinds = mpi_opts.pop(
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
2025
|
+
allowbinds = mpi_opts.pop(
|
|
2026
|
+
"allowbind",
|
|
2027
|
+
[
|
|
2028
|
+
True,
|
|
2029
|
+
]
|
|
2030
|
+
* len(rh),
|
|
2031
|
+
)
|
|
2032
|
+
distributions = mpi_opts.pop(
|
|
2033
|
+
"distribution",
|
|
2034
|
+
[
|
|
2035
|
+
self.env.get("VORTEX_MPIBIN_DEF_DISTRIBUTION", None),
|
|
2036
|
+
]
|
|
2037
|
+
* len(rh),
|
|
2038
|
+
)
|
|
1795
2039
|
for i, r in enumerate(rh):
|
|
1796
2040
|
if use_envelope:
|
|
1797
2041
|
bins.append(
|
|
@@ -1805,10 +2049,10 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1805
2049
|
bins.append(
|
|
1806
2050
|
footprints.proxy.mpibinary(
|
|
1807
2051
|
kind=bnames[i],
|
|
1808
|
-
nodes=self.env.get(
|
|
2052
|
+
nodes=self.env.get("VORTEX_SUBMIT_NODES", 1),
|
|
1809
2053
|
allowbind=allowbinds[i],
|
|
1810
2054
|
distribution=distributions[i],
|
|
1811
|
-
**mpi_desc
|
|
2055
|
+
**mpi_desc,
|
|
1812
2056
|
)
|
|
1813
2057
|
)
|
|
1814
2058
|
# Reshape mpiopts
|
|
@@ -1818,7 +2062,7 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1818
2062
|
bins[i].master = self.absexcutable(r.container.localpath())
|
|
1819
2063
|
bins[i].arguments = bargs[i]
|
|
1820
2064
|
# Source files ?
|
|
1821
|
-
if hasattr(r.resource,
|
|
2065
|
+
if hasattr(r.resource, "guess_binary_sources"):
|
|
1822
2066
|
sources.extend(r.resource.guess_binary_sources(r.provider))
|
|
1823
2067
|
|
|
1824
2068
|
# Nothing to do: binary descriptions are provided by the user
|
|
@@ -1836,8 +2080,12 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1836
2080
|
mpi.envelope = envelope
|
|
1837
2081
|
|
|
1838
2082
|
# The binaries description
|
|
1839
|
-
mpi.binaries = self._bootstrap_mpibins_hack(
|
|
1840
|
-
|
|
2083
|
+
mpi.binaries = self._bootstrap_mpibins_hack(
|
|
2084
|
+
bins, rh, opts, use_envelope
|
|
2085
|
+
)
|
|
2086
|
+
upd_envelope = self._bootstrap_mpienvelope_posthack(
|
|
2087
|
+
envelope, rh, opts, mpi
|
|
2088
|
+
)
|
|
1841
2089
|
if upd_envelope:
|
|
1842
2090
|
mpi.envelope = upd_envelope
|
|
1843
2091
|
|
|
@@ -1850,14 +2098,20 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1850
2098
|
mpibins_total = sum([m.nprocs for m in mpi.binaries])
|
|
1851
2099
|
if not envelope_ntasks == mpibins_total:
|
|
1852
2100
|
raise AlgoComponentError(
|
|
1853
|
-
(
|
|
1854
|
-
|
|
1855
|
-
|
|
2101
|
+
(
|
|
2102
|
+
"The number of requested ranks ({:d}) must be equal "
|
|
2103
|
+
"to the number of processes available in the envelope ({:d})"
|
|
2104
|
+
).format(mpibins_total, envelope_ntasks)
|
|
2105
|
+
)
|
|
1856
2106
|
|
|
1857
2107
|
args = mpi.mkcmdline()
|
|
1858
2108
|
for b in mpi.binaries:
|
|
1859
|
-
logger.info(
|
|
1860
|
-
|
|
2109
|
+
logger.info(
|
|
2110
|
+
"Run %s in parallel mode. Args: %s.",
|
|
2111
|
+
b.master,
|
|
2112
|
+
" ".join(b.arguments),
|
|
2113
|
+
)
|
|
2114
|
+
logger.info("Full MPI command line: %s", " ".join(args))
|
|
1861
2115
|
|
|
1862
2116
|
# Setup various useful things (env, system, ...)
|
|
1863
2117
|
mpi.import_basics(self)
|
|
@@ -1868,7 +2122,9 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1868
2122
|
def _tweak_mpitools_logging(self):
|
|
1869
2123
|
if self.mpiverbose:
|
|
1870
2124
|
m_loggers = dict()
|
|
1871
|
-
for m_logger_name in [
|
|
2125
|
+
for m_logger_name in [
|
|
2126
|
+
l for l in loggers.lognames if "mpitools" in l
|
|
2127
|
+
]:
|
|
1872
2128
|
m_logger = loggers.getLogger(m_logger_name)
|
|
1873
2129
|
m_loggers[m_logger] = m_logger.level
|
|
1874
2130
|
m_logger.setLevel(logging.DEBUG)
|
|
@@ -1887,10 +2143,9 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1887
2143
|
contain indications on the number of nodes, tasks, ...
|
|
1888
2144
|
"""
|
|
1889
2145
|
|
|
1890
|
-
self.system.subtitle(
|
|
2146
|
+
self.system.subtitle("{:s} : parallel engine".format(self.realkind))
|
|
1891
2147
|
|
|
1892
2148
|
with self._tweak_mpitools_logging():
|
|
1893
|
-
|
|
1894
2149
|
# Return a mpitool object and the mpicommand line
|
|
1895
2150
|
mpi, args = self._bootstrap_mpitool(rh, opts)
|
|
1896
2151
|
|
|
@@ -1908,91 +2163,125 @@ class Parallel(xExecutableAlgoComponent):
|
|
|
1908
2163
|
class ParallelIoServerMixin(AlgoComponentMpiDecoMixin):
|
|
1909
2164
|
"""Adds an IOServer capabilities (footprints attributes + MPI bianries alteration)."""
|
|
1910
2165
|
|
|
1911
|
-
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
2166
|
+
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
2167
|
+
footprints.Footprint(
|
|
2168
|
+
info="Abstract IoServer footprints' attributes.",
|
|
2169
|
+
attr=dict(
|
|
2170
|
+
ioserver=dict(
|
|
2171
|
+
info="The object used to launch the IOserver part of the binary.",
|
|
2172
|
+
type=mpitools.MpiBinaryIOServer,
|
|
2173
|
+
optional=True,
|
|
2174
|
+
default=None,
|
|
2175
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
2176
|
+
),
|
|
2177
|
+
ioname=dict(
|
|
2178
|
+
info=(
|
|
2179
|
+
"The binary_kind of a class in the mpibinary collector "
|
|
2180
|
+
+ "(used only if *ioserver* is not provided)"
|
|
2181
|
+
),
|
|
2182
|
+
optional=True,
|
|
2183
|
+
default="ioserv",
|
|
2184
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
2185
|
+
),
|
|
2186
|
+
iolocation=dict(
|
|
2187
|
+
info="Location of the IO server within the binary list",
|
|
2188
|
+
type=int,
|
|
2189
|
+
default=-1,
|
|
2190
|
+
optional=True,
|
|
2191
|
+
),
|
|
1927
2192
|
),
|
|
1928
|
-
|
|
1929
|
-
info='Location of the IO server within the binary list',
|
|
1930
|
-
type=int,
|
|
1931
|
-
default=-1,
|
|
1932
|
-
optional=True
|
|
1933
|
-
)
|
|
1934
|
-
)),
|
|
2193
|
+
),
|
|
1935
2194
|
]
|
|
1936
2195
|
|
|
1937
|
-
def _bootstrap_mpibins_ioserver_hack(
|
|
2196
|
+
def _bootstrap_mpibins_ioserver_hack(
|
|
2197
|
+
self, bins, bins0, rh, opts, use_envelope
|
|
2198
|
+
):
|
|
1938
2199
|
"""If requested, adds an extra binary that will act as an IOServer."""
|
|
1939
2200
|
master = bins[-1]
|
|
1940
2201
|
# A potential IO server
|
|
1941
2202
|
io = self.ioserver
|
|
1942
|
-
if not io and int(self.env.get(
|
|
2203
|
+
if not io and int(self.env.get("VORTEX_IOSERVER_NODES", -1)) >= 0:
|
|
1943
2204
|
io = footprints.proxy.mpibinary(
|
|
1944
2205
|
kind=self.ioname,
|
|
1945
2206
|
nodes=self.env.VORTEX_IOSERVER_NODES,
|
|
1946
|
-
tasks=(
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
2207
|
+
tasks=(
|
|
2208
|
+
self.env.VORTEX_IOSERVER_TASKS
|
|
2209
|
+
or master.options.get("nnp", master.tasks)
|
|
2210
|
+
),
|
|
2211
|
+
openmp=(
|
|
2212
|
+
self.env.VORTEX_IOSERVER_OPENMP
|
|
2213
|
+
or master.options.get("openmp", master.openmp)
|
|
2214
|
+
),
|
|
2215
|
+
iolocation=self.iolocation,
|
|
2216
|
+
)
|
|
2217
|
+
io.options = {
|
|
2218
|
+
x[3:]: opts[x] for x in opts.keys() if x.startswith("io_")
|
|
2219
|
+
}
|
|
1953
2220
|
io.master = master.master
|
|
1954
2221
|
io.arguments = master.arguments
|
|
1955
|
-
if
|
|
2222
|
+
if (
|
|
2223
|
+
not io
|
|
2224
|
+
and int(self.env.get("VORTEX_IOSERVER_COMPANION_TASKS", -1)) >= 0
|
|
2225
|
+
):
|
|
1956
2226
|
io = footprints.proxy.mpibinary(
|
|
1957
2227
|
kind=self.ioname,
|
|
1958
|
-
nodes=master.options.get(
|
|
2228
|
+
nodes=master.options.get("nn", master.nodes),
|
|
1959
2229
|
tasks=self.env.VORTEX_IOSERVER_COMPANION_TASKS,
|
|
1960
|
-
openmp=(
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
2230
|
+
openmp=(
|
|
2231
|
+
self.env.VORTEX_IOSERVER_OPENMP
|
|
2232
|
+
or master.options.get("openmp", master.openmp)
|
|
2233
|
+
),
|
|
2234
|
+
)
|
|
2235
|
+
io.options = {
|
|
2236
|
+
x[3:]: opts[x] for x in opts.keys() if x.startswith("io_")
|
|
2237
|
+
}
|
|
1964
2238
|
io.master = master.master
|
|
1965
2239
|
io.arguments = master.arguments
|
|
1966
2240
|
if master.group is not None:
|
|
1967
2241
|
# The master binary is already in a group ! Use it.
|
|
1968
2242
|
io.group = master.group
|
|
1969
2243
|
else:
|
|
1970
|
-
io.group =
|
|
1971
|
-
master.group =
|
|
1972
|
-
if
|
|
1973
|
-
|
|
2244
|
+
io.group = "auto_masterwithio"
|
|
2245
|
+
master.group = "auto_masterwithio"
|
|
2246
|
+
if (
|
|
2247
|
+
not io
|
|
2248
|
+
and self.env.get("VORTEX_IOSERVER_INCORE_TASKS", None) is not None
|
|
2249
|
+
):
|
|
2250
|
+
if hasattr(master, "incore_iotasks"):
|
|
1974
2251
|
master.incore_iotasks = self.env.VORTEX_IOSERVER_INCORE_TASKS
|
|
1975
|
-
if
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
if hasattr(master,
|
|
2252
|
+
if (
|
|
2253
|
+
not io
|
|
2254
|
+
and self.env.get("VORTEX_IOSERVER_INCORE_FIXER", None) is not None
|
|
2255
|
+
):
|
|
2256
|
+
if hasattr(master, "incore_iotasks_fixer"):
|
|
2257
|
+
master.incore_iotasks_fixer = (
|
|
2258
|
+
self.env.VORTEX_IOSERVER_INCORE_FIXER
|
|
2259
|
+
)
|
|
2260
|
+
if (
|
|
2261
|
+
not io
|
|
2262
|
+
and self.env.get("VORTEX_IOSERVER_INCORE_DIST", None) is not None
|
|
2263
|
+
):
|
|
2264
|
+
if hasattr(master, "incore_iodist"):
|
|
1980
2265
|
master.incore_iodist = self.env.VORTEX_IOSERVER_INCORE_DIST
|
|
1981
2266
|
if io:
|
|
1982
2267
|
rh.append(rh[0])
|
|
1983
2268
|
if master.group is None:
|
|
1984
|
-
if
|
|
1985
|
-
master.options[
|
|
2269
|
+
if "nn" in master.options:
|
|
2270
|
+
master.options["nn"] = (
|
|
2271
|
+
master.options["nn"] - io.options["nn"]
|
|
2272
|
+
)
|
|
1986
2273
|
else:
|
|
1987
|
-
logger.warning(
|
|
1988
|
-
|
|
2274
|
+
logger.warning(
|
|
2275
|
+
'The "nn" option is not available in the master binary '
|
|
2276
|
+
+ "mpi options. Consequently it can be fixed..."
|
|
2277
|
+
)
|
|
1989
2278
|
if self.iolocation >= 0:
|
|
1990
2279
|
bins.insert(self.iolocation, io)
|
|
1991
2280
|
else:
|
|
1992
2281
|
bins.append(io)
|
|
1993
2282
|
return bins
|
|
1994
2283
|
|
|
1995
|
-
_MIXIN_MPIBINS_HOOKS = (_bootstrap_mpibins_ioserver_hack,
|
|
2284
|
+
_MIXIN_MPIBINS_HOOKS = (_bootstrap_mpibins_ioserver_hack,)
|
|
1996
2285
|
|
|
1997
2286
|
|
|
1998
2287
|
@algo_component_deco_mixin_autodoc
|
|
@@ -2006,38 +2295,44 @@ class ParallelOpenPalmMixin(AlgoComponentMpiDecoMixin):
|
|
|
2006
2295
|
provided using the **openpalm_driver** footprint's argument.
|
|
2007
2296
|
"""
|
|
2008
2297
|
|
|
2009
|
-
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2298
|
+
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
2299
|
+
footprints.Footprint(
|
|
2300
|
+
info="Abstract OpenPALM footprints' attributes.",
|
|
2301
|
+
attr=dict(
|
|
2302
|
+
openpalm_driver=dict(
|
|
2303
|
+
info=(
|
|
2304
|
+
"The path to the OpenPALM driver binary. "
|
|
2305
|
+
+ "When omitted, the input sequence is looked up "
|
|
2306
|
+
+ "for section with ``role=OpenPALM Driver``."
|
|
2307
|
+
),
|
|
2308
|
+
optional=True,
|
|
2309
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
2310
|
+
),
|
|
2311
|
+
openpalm_overcommit=dict(
|
|
2312
|
+
info=(
|
|
2313
|
+
"Run the OpenPALM driver on the first node in addition "
|
|
2314
|
+
+ "to existing tasks. Otherwise dedicated tasks are used."
|
|
2315
|
+
),
|
|
2316
|
+
type=bool,
|
|
2317
|
+
default=True,
|
|
2318
|
+
optional=True,
|
|
2319
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
2320
|
+
),
|
|
2321
|
+
openpalm_binddriver=dict(
|
|
2322
|
+
info="Try to bind the OpenPALM driver binary.",
|
|
2323
|
+
type=bool,
|
|
2324
|
+
optional=True,
|
|
2325
|
+
default=False,
|
|
2326
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
2327
|
+
),
|
|
2328
|
+
openpalm_binkind=dict(
|
|
2329
|
+
info="The binary kind for the OpenPALM driver.",
|
|
2330
|
+
optional=True,
|
|
2331
|
+
default="basic",
|
|
2332
|
+
doc_visibility=footprints.doc.visibility.GURU,
|
|
2333
|
+
),
|
|
2039
2334
|
),
|
|
2040
|
-
)
|
|
2335
|
+
),
|
|
2041
2336
|
]
|
|
2042
2337
|
|
|
2043
2338
|
@property
|
|
@@ -2045,19 +2340,28 @@ class ParallelOpenPalmMixin(AlgoComponentMpiDecoMixin):
|
|
|
2045
2340
|
"""Returns the OpenPALM's driver location."""
|
|
2046
2341
|
path = self.openpalm_driver
|
|
2047
2342
|
if path is None:
|
|
2048
|
-
drivers = self.context.sequence.effective_inputs(
|
|
2343
|
+
drivers = self.context.sequence.effective_inputs(
|
|
2344
|
+
role="OpenPALMDriver"
|
|
2345
|
+
)
|
|
2049
2346
|
if not drivers:
|
|
2050
|
-
raise AlgoComponentError(
|
|
2347
|
+
raise AlgoComponentError("No OpenPALM driver was provided.")
|
|
2051
2348
|
elif len(drivers) > 1:
|
|
2052
|
-
raise AlgoComponentError(
|
|
2349
|
+
raise AlgoComponentError(
|
|
2350
|
+
"Several OpenPALM driver were provided."
|
|
2351
|
+
)
|
|
2053
2352
|
path = drivers[0].rh.container.localpath()
|
|
2054
2353
|
else:
|
|
2055
2354
|
if not self.system.path.exists(path):
|
|
2056
|
-
raise AlgoComponentError(
|
|
2057
|
-
|
|
2355
|
+
raise AlgoComponentError(
|
|
2356
|
+
"No OpenPALM driver was provider ({:s} does not exists).".format(
|
|
2357
|
+
path
|
|
2358
|
+
)
|
|
2359
|
+
)
|
|
2058
2360
|
return path
|
|
2059
2361
|
|
|
2060
|
-
def _bootstrap_mpibins_openpalm_hack(
|
|
2362
|
+
def _bootstrap_mpibins_openpalm_hack(
|
|
2363
|
+
self, bins, bins0, rh, opts, use_envelope
|
|
2364
|
+
):
|
|
2061
2365
|
"""Adds the OpenPALM driver to the binary list."""
|
|
2062
2366
|
single_bin = len(bins) == 1
|
|
2063
2367
|
master = bins[0]
|
|
@@ -2066,12 +2370,16 @@ class ParallelOpenPalmMixin(AlgoComponentMpiDecoMixin):
|
|
|
2066
2370
|
nodes=1,
|
|
2067
2371
|
tasks=self.env.VORTEX_OPENPALM_DRV_TASKS or 1,
|
|
2068
2372
|
openmp=self.env.VORTEX_OPENPALM_DRV_OPENMP or 1,
|
|
2069
|
-
allowbind=opts.pop(
|
|
2070
|
-
|
|
2071
|
-
|
|
2373
|
+
allowbind=opts.pop(
|
|
2374
|
+
"palmdrv_bind",
|
|
2375
|
+
self.env.get(
|
|
2376
|
+
"VORTEX_OPENPALM_DRV_BIND", self.openpalm_binddriver
|
|
2377
|
+
),
|
|
2378
|
+
),
|
|
2072
2379
|
)
|
|
2073
|
-
driver.options = {
|
|
2074
|
-
|
|
2380
|
+
driver.options = {
|
|
2381
|
+
x[8:]: opts[x] for x in opts.keys() if x.startswith("palmdrv_")
|
|
2382
|
+
}
|
|
2075
2383
|
driver.master = self._actual_openpalm_driver
|
|
2076
2384
|
self.system.xperm(driver.master, force=True)
|
|
2077
2385
|
bins.insert(0, driver)
|
|
@@ -2080,57 +2388,75 @@ class ParallelOpenPalmMixin(AlgoComponentMpiDecoMixin):
|
|
|
2080
2388
|
# the driver
|
|
2081
2389
|
# NB: If multiple binaries are provided, the user must do this by
|
|
2082
2390
|
# himself (i.e. leave enough room for the driver's task).
|
|
2083
|
-
if
|
|
2084
|
-
master.options[
|
|
2391
|
+
if "nn" in master.options:
|
|
2392
|
+
master.options["nn"] = master.options["nn"] - 1
|
|
2085
2393
|
else:
|
|
2086
2394
|
# Ok, tweak nprocs instead (an envelope might be defined)
|
|
2087
2395
|
try:
|
|
2088
2396
|
nprocs = master.nprocs
|
|
2089
2397
|
except mpitools.MpiException:
|
|
2090
|
-
logger.error(
|
|
2091
|
-
|
|
2092
|
-
|
|
2398
|
+
logger.error(
|
|
2399
|
+
'Neither the "nn" option nor the nprocs is '
|
|
2400
|
+
+ "available for the master binary. Consequently "
|
|
2401
|
+
+ "it can be fixed..."
|
|
2402
|
+
)
|
|
2093
2403
|
else:
|
|
2094
|
-
master.options[
|
|
2404
|
+
master.options["np"] = nprocs - driver.nprocs
|
|
2095
2405
|
return bins
|
|
2096
2406
|
|
|
2097
|
-
_MIXIN_MPIBINS_HOOKS = (_bootstrap_mpibins_openpalm_hack,
|
|
2407
|
+
_MIXIN_MPIBINS_HOOKS = (_bootstrap_mpibins_openpalm_hack,)
|
|
2098
2408
|
|
|
2099
|
-
def _bootstrap_mpienvelope_openpalm_posthack(
|
|
2409
|
+
def _bootstrap_mpienvelope_openpalm_posthack(
|
|
2410
|
+
self, env, env0, rh, opts, mpi
|
|
2411
|
+
):
|
|
2100
2412
|
"""
|
|
2101
2413
|
Tweak the MPI envelope in order to execute the OpenPALM driver on the
|
|
2102
2414
|
appropriate node.
|
|
2103
2415
|
"""
|
|
2104
|
-
master = mpi.binaries[
|
|
2416
|
+
master = mpi.binaries[
|
|
2417
|
+
1
|
|
2418
|
+
] # The first "real" program that will be launched
|
|
2105
2419
|
driver = mpi.binaries[0] # The OpenPALM driver
|
|
2106
2420
|
if self.openpalm_overcommit:
|
|
2107
2421
|
# Execute the driver on the first compute node
|
|
2108
2422
|
if env or env0:
|
|
2109
2423
|
env = env or copy.deepcopy(env0)
|
|
2110
2424
|
# An envelope is already defined... update it
|
|
2111
|
-
if not (
|
|
2112
|
-
raise AlgoComponentError(
|
|
2113
|
-
|
|
2114
|
-
|
|
2425
|
+
if not ("nn" in env[0] and "nnp" in env[0]):
|
|
2426
|
+
raise AlgoComponentError(
|
|
2427
|
+
"'nn' and 'nnp' must be defined in the envelope"
|
|
2428
|
+
)
|
|
2429
|
+
if env[0]["nn"] > 1:
|
|
2430
|
+
env[0]["nn"] -= 1
|
|
2115
2431
|
newenv = copy.copy(env[0])
|
|
2116
|
-
newenv[
|
|
2117
|
-
newenv[
|
|
2432
|
+
newenv["nn"] = 1
|
|
2433
|
+
newenv["nnp"] += driver.nprocs
|
|
2118
2434
|
env.insert(0, newenv)
|
|
2119
2435
|
else:
|
|
2120
|
-
env[0][
|
|
2436
|
+
env[0]["nnp"] += driver.nprocs
|
|
2121
2437
|
else:
|
|
2122
2438
|
# Setup a new envelope
|
|
2123
|
-
if not (
|
|
2124
|
-
raise AlgoComponentError(
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2439
|
+
if not ("nn" in master.options and "nnp" in master.options):
|
|
2440
|
+
raise AlgoComponentError(
|
|
2441
|
+
"'nn' and 'nnp' must be defined for the master executable"
|
|
2442
|
+
)
|
|
2443
|
+
env = [
|
|
2444
|
+
dict(
|
|
2445
|
+
nn=1,
|
|
2446
|
+
nnp=master.options["nnp"] + driver.nprocs,
|
|
2447
|
+
openmp=master.options.get("openmp", 1),
|
|
2448
|
+
)
|
|
2449
|
+
]
|
|
2450
|
+
if master.options["nn"] > 1:
|
|
2451
|
+
env.append(
|
|
2452
|
+
dict(
|
|
2453
|
+
nn=master.options["nn"] - 1,
|
|
2454
|
+
nnp=master.options["nnp"],
|
|
2455
|
+
openmp=master.options.get("openmp", 1),
|
|
2456
|
+
)
|
|
2457
|
+
)
|
|
2132
2458
|
if len(mpi.binaries) > 2:
|
|
2133
2459
|
env.extend([b.options for b in mpi.binaries[2:]])
|
|
2134
2460
|
return env
|
|
2135
2461
|
|
|
2136
|
-
_MIXIN_MPIENVELOPE_POSTHOOKS = (_bootstrap_mpienvelope_openpalm_posthack,
|
|
2462
|
+
_MIXIN_MPIENVELOPE_POSTHOOKS = (_bootstrap_mpienvelope_openpalm_posthack,)
|