vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +59 -45
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +940 -614
- vortex/algo/mpitools.py +802 -497
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +428 -225
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +544 -413
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +420 -271
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +210 -122
- vortex/nwp/algo/monitoring.py +132 -76
- vortex/nwp/algo/mpitools.py +321 -191
- vortex/nwp/algo/odbtools.py +617 -353
- vortex/nwp/algo/oopsroot.py +449 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +125 -59
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +311 -149
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +632 -298
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +402 -333
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1211 -631
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +377 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
- vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/tools/env.py
CHANGED
|
@@ -19,10 +19,12 @@ __all__ = []
|
|
|
19
19
|
logger = loggers.getLogger(__name__)
|
|
20
20
|
|
|
21
21
|
#: Pre-compiled evaluation mostly used by :class:`Environment` method (true).
|
|
22
|
-
vartrue = re.compile(
|
|
22
|
+
vartrue = re.compile(
|
|
23
|
+
r"^\s*(?:[1-9]\d*|ok|on|true|yes|y)\s*$", flags=re.IGNORECASE
|
|
24
|
+
)
|
|
23
25
|
|
|
24
26
|
#: Pre-compiled evaluation mostly used by :class:`Environment` method (false).
|
|
25
|
-
varfalse = re.compile(r
|
|
27
|
+
varfalse = re.compile(r"^\s*(?:0|ko|off|false|no|n)\s*$", flags=re.IGNORECASE)
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
def current():
|
|
@@ -51,8 +53,16 @@ class Environment:
|
|
|
51
53
|
|
|
52
54
|
_current_active = None
|
|
53
55
|
|
|
54
|
-
def __init__(
|
|
55
|
-
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
env=None,
|
|
59
|
+
active=False,
|
|
60
|
+
clear=False,
|
|
61
|
+
verbose=False,
|
|
62
|
+
noexport=[],
|
|
63
|
+
contextlock=None,
|
|
64
|
+
history=True,
|
|
65
|
+
):
|
|
56
66
|
"""
|
|
57
67
|
:param Environment env: An existing Environment used to initialise this object.
|
|
58
68
|
:param bool active: Is this new environment activated when created.
|
|
@@ -69,18 +79,18 @@ class Environment:
|
|
|
69
79
|
:class:`~bronx.stdtypes.history.PrivateHistory` object that will be
|
|
70
80
|
accessible through the :attr:`history` property.
|
|
71
81
|
"""
|
|
72
|
-
self.__dict__[
|
|
73
|
-
self.__dict__[
|
|
74
|
-
self.__dict__[
|
|
75
|
-
self.__dict__[
|
|
76
|
-
self.__dict__[
|
|
77
|
-
self.__dict__[
|
|
78
|
-
self.__dict__[
|
|
82
|
+
self.__dict__["_history"] = PrivateHistory() if history else None
|
|
83
|
+
self.__dict__["_verbose"] = verbose
|
|
84
|
+
self.__dict__["_frozen"] = collections.deque()
|
|
85
|
+
self.__dict__["_pool"] = dict()
|
|
86
|
+
self.__dict__["_mods"] = set()
|
|
87
|
+
self.__dict__["_sh"] = None
|
|
88
|
+
self.__dict__["_os"] = list()
|
|
79
89
|
if env is not None and isinstance(env, Environment):
|
|
80
90
|
self._env_clone_internals(env, contextlock)
|
|
81
91
|
if verbose:
|
|
82
92
|
try:
|
|
83
|
-
self.__dict__[
|
|
93
|
+
self.__dict__["_sh"] = env._sh
|
|
84
94
|
except AttributeError:
|
|
85
95
|
pass
|
|
86
96
|
else:
|
|
@@ -88,20 +98,22 @@ class Environment:
|
|
|
88
98
|
active = False
|
|
89
99
|
else:
|
|
90
100
|
if self._current_active is not None:
|
|
91
|
-
self._env_clone_internals(
|
|
101
|
+
self._env_clone_internals(
|
|
102
|
+
self._current_active, contextlock
|
|
103
|
+
)
|
|
92
104
|
else:
|
|
93
105
|
self._pool.update(os.environ)
|
|
94
|
-
self.__dict__[
|
|
106
|
+
self.__dict__["_noexport"] = [x.upper() for x in noexport]
|
|
95
107
|
self.active(active)
|
|
96
108
|
|
|
97
109
|
def _env_clone_internals(self, env, contextlock):
|
|
98
|
-
self.__dict__[
|
|
99
|
-
self.__dict__[
|
|
110
|
+
self.__dict__["_os"] = env.osstack()
|
|
111
|
+
self.__dict__["_os"].append(env)
|
|
100
112
|
self._pool.update(env.items())
|
|
101
113
|
if contextlock is not None:
|
|
102
|
-
self.__dict__[
|
|
114
|
+
self.__dict__["_contextlock"] = contextlock
|
|
103
115
|
else:
|
|
104
|
-
self.__dict__[
|
|
116
|
+
self.__dict__["_contextlock"] = env.contextlock
|
|
105
117
|
|
|
106
118
|
@property
|
|
107
119
|
def history(self):
|
|
@@ -116,7 +128,9 @@ class Environment:
|
|
|
116
128
|
self.history.append(var, value, traceback.format_stack()[:-1])
|
|
117
129
|
|
|
118
130
|
def __str__(self):
|
|
119
|
-
return
|
|
131
|
+
return "{:s} | including {:d} variables>".format(
|
|
132
|
+
repr(self).rstrip(">"), len(self)
|
|
133
|
+
)
|
|
120
134
|
|
|
121
135
|
@classmethod
|
|
122
136
|
def current(cls):
|
|
@@ -139,11 +153,11 @@ class Environment:
|
|
|
139
153
|
"""Dump the specified ``value`` as a string (utility function)."""
|
|
140
154
|
if isinstance(value, str):
|
|
141
155
|
obj = str(value)
|
|
142
|
-
elif hasattr(value,
|
|
156
|
+
elif hasattr(value, "export_dict"):
|
|
143
157
|
obj = value.export_dict()
|
|
144
|
-
elif hasattr(value,
|
|
158
|
+
elif hasattr(value, "footprint_export"):
|
|
145
159
|
obj = value.footprint_export()
|
|
146
|
-
elif hasattr(value,
|
|
160
|
+
elif hasattr(value, "__dict__"):
|
|
147
161
|
obj = vars(value)
|
|
148
162
|
else:
|
|
149
163
|
obj = value
|
|
@@ -169,7 +183,9 @@ class Environment:
|
|
|
169
183
|
os.environ[upvar] = actualvalue
|
|
170
184
|
if self.verbose():
|
|
171
185
|
if self.osbound() and self._sh:
|
|
172
|
-
self._sh.stderr(
|
|
186
|
+
self._sh.stderr(
|
|
187
|
+
"export", "{:s}={:s}".format(upvar, actualvalue)
|
|
188
|
+
)
|
|
173
189
|
logger.debug('Env export %s="%s"', upvar, actualvalue)
|
|
174
190
|
|
|
175
191
|
def __setitem__(self, varname, value):
|
|
@@ -195,7 +211,7 @@ class Environment:
|
|
|
195
211
|
return self.getvar(varname)
|
|
196
212
|
|
|
197
213
|
def __getattr__(self, varname):
|
|
198
|
-
if varname.startswith(
|
|
214
|
+
if varname.startswith("_"):
|
|
199
215
|
raise AttributeError
|
|
200
216
|
else:
|
|
201
217
|
return self.getvar(varname)
|
|
@@ -217,9 +233,9 @@ class Environment:
|
|
|
217
233
|
if seen and self.osbound():
|
|
218
234
|
del os.environ[varname.upper()]
|
|
219
235
|
if self.verbose() and self._sh:
|
|
220
|
-
self._sh.stderr(
|
|
236
|
+
self._sh.stderr("unset", "{:s}".format(varname.upper()))
|
|
221
237
|
if seen:
|
|
222
|
-
self._record(varname.upper(),
|
|
238
|
+
self._record(varname.upper(), "!!deleted!!")
|
|
223
239
|
|
|
224
240
|
def __delitem__(self, varname):
|
|
225
241
|
self.delvar(varname)
|
|
@@ -268,9 +284,11 @@ class Environment:
|
|
|
268
284
|
return self._pool.items()
|
|
269
285
|
|
|
270
286
|
def __eq__(self, other):
|
|
271
|
-
return (
|
|
272
|
-
|
|
273
|
-
|
|
287
|
+
return (
|
|
288
|
+
isinstance(other, type(self))
|
|
289
|
+
and set(self.keys()) == set(other.keys)
|
|
290
|
+
and all([self[k] == other[k] for k in self.keys])
|
|
291
|
+
)
|
|
274
292
|
|
|
275
293
|
def __ne__(self, other):
|
|
276
294
|
return not self == other
|
|
@@ -342,7 +360,9 @@ class Environment:
|
|
|
342
360
|
try:
|
|
343
361
|
eclone.verbose(self._verbose, self._sh)
|
|
344
362
|
except AttributeError:
|
|
345
|
-
logger.debug(
|
|
363
|
+
logger.debug(
|
|
364
|
+
"Could not find verbose attributes while cloning env..."
|
|
365
|
+
)
|
|
346
366
|
return eclone
|
|
347
367
|
|
|
348
368
|
def __enter__(self):
|
|
@@ -365,10 +385,10 @@ class Environment:
|
|
|
365
385
|
def verbose(self, switch=None, sh=None, fromenv=None):
|
|
366
386
|
"""Switch on or off the verbose mode. Returns actual value."""
|
|
367
387
|
if switch is not None:
|
|
368
|
-
self.__dict__[
|
|
388
|
+
self.__dict__["_verbose"] = bool(switch)
|
|
369
389
|
if sh is not None:
|
|
370
|
-
self.__dict__[
|
|
371
|
-
return self.__dict__[
|
|
390
|
+
self.__dict__["_sh"] = sh
|
|
391
|
+
return self.__dict__["_verbose"]
|
|
372
392
|
|
|
373
393
|
def active(self, *args):
|
|
374
394
|
"""
|
|
@@ -383,19 +403,23 @@ class Environment:
|
|
|
383
403
|
if args and type(args[0]) is bool:
|
|
384
404
|
active = args[0]
|
|
385
405
|
if previous_act and not active and self._os:
|
|
386
|
-
self._record(
|
|
406
|
+
self._record("!!OS_BINDING!!", "Broken...")
|
|
387
407
|
self.__class__._current_active = self._os[-1]
|
|
388
408
|
osrewind = self.__class__._current_active
|
|
389
409
|
if not previous_act and active:
|
|
390
410
|
if self.contextlock is not None and not self.contextlock.active:
|
|
391
|
-
raise RuntimeError(
|
|
392
|
-
|
|
393
|
-
|
|
411
|
+
raise RuntimeError(
|
|
412
|
+
"It's not allowed to switch to an Environment "
|
|
413
|
+
+ "that belongs to an inactive context"
|
|
414
|
+
)
|
|
415
|
+
self._record("!!OS_BINDING!!", "Acquiring...")
|
|
394
416
|
self.__class__._current_active = self
|
|
395
417
|
osrewind = self.__class__._current_active
|
|
396
418
|
if osrewind:
|
|
397
419
|
os.environ.clear()
|
|
398
|
-
for k in filter(
|
|
420
|
+
for k in filter(
|
|
421
|
+
lambda x: x not in osrewind._noexport, osrewind._pool.keys()
|
|
422
|
+
):
|
|
399
423
|
os.environ[k] = osrewind.native(k)
|
|
400
424
|
return active
|
|
401
425
|
|
|
@@ -436,11 +460,15 @@ class Environment:
|
|
|
436
460
|
|
|
437
461
|
def mkautolist(self, prefix):
|
|
438
462
|
"""Return a list of variable starting with the ``prefix`` string."""
|
|
439
|
-
return [
|
|
463
|
+
return [
|
|
464
|
+
var + '="' + self.get(var, "") + '"'
|
|
465
|
+
for var in self.keys()
|
|
466
|
+
if var.startswith(prefix)
|
|
467
|
+
]
|
|
440
468
|
|
|
441
469
|
def trueshell(self):
|
|
442
470
|
"""Extract the actual shell name according to env variable SHELL."""
|
|
443
|
-
return re.sub(
|
|
471
|
+
return re.sub("^.*/", "", self.getvar("shell"))
|
|
444
472
|
|
|
445
473
|
def true(self, varname):
|
|
446
474
|
"""Extended boolean positive test of the variable given as argument."""
|
|
@@ -461,41 +489,41 @@ class Environment:
|
|
|
461
489
|
:param str value: The value that will be inserted in the path
|
|
462
490
|
:param int pos: Where in the path, to insert ``value`` (by default, at the end)
|
|
463
491
|
"""
|
|
464
|
-
mypath = self.getvar(var).split(
|
|
492
|
+
mypath = self.getvar(var).split(":") if self.getvar(var) else []
|
|
465
493
|
value = str(value)
|
|
466
494
|
while value in mypath:
|
|
467
495
|
mypath.remove(value)
|
|
468
496
|
if pos is None:
|
|
469
497
|
pos = len(mypath)
|
|
470
498
|
mypath.insert(pos, value)
|
|
471
|
-
self.setvar(var,
|
|
499
|
+
self.setvar(var, ":".join(mypath))
|
|
472
500
|
|
|
473
501
|
def rmgenericpath(self, var, value):
|
|
474
502
|
"""Remove the specified ``value`` from a PATH like variable."""
|
|
475
|
-
mypath = self.getvar(var).split(
|
|
503
|
+
mypath = self.getvar(var).split(":") if self.getvar(var) else []
|
|
476
504
|
while value in mypath:
|
|
477
505
|
mypath.remove(value)
|
|
478
|
-
self.setvar(var,
|
|
506
|
+
self.setvar(var, ":".join(mypath))
|
|
479
507
|
|
|
480
508
|
def setbinpath(self, value, pos=None):
|
|
481
509
|
"""
|
|
482
510
|
Insert a new path ``value`` to the bin search path (i.e. the PATH
|
|
483
511
|
environment variable) at given position.
|
|
484
512
|
"""
|
|
485
|
-
self.setgenericpath(
|
|
513
|
+
self.setgenericpath("PATH", value, pos)
|
|
486
514
|
|
|
487
515
|
def rmbinpath(self, value):
|
|
488
516
|
"""
|
|
489
517
|
Remove the specified ``value`` from the bin path (i.e. the PATH
|
|
490
518
|
environment variable).
|
|
491
519
|
"""
|
|
492
|
-
self.rmgenericpath(
|
|
520
|
+
self.rmgenericpath("PATH", value)
|
|
493
521
|
|
|
494
522
|
|
|
495
523
|
collections.abc.Mapping.register(Environment)
|
|
496
524
|
|
|
497
525
|
|
|
498
|
-
class EnvironmentDeltaContext
|
|
526
|
+
class EnvironmentDeltaContext:
|
|
499
527
|
"""Context that will apply a delta on the Environment and rewind it on exit."""
|
|
500
528
|
|
|
501
529
|
def __init__(self, env, **kw):
|